diff --git a/.gitignore b/.gitignore index 5f262c46179..cf904522d4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,13 @@ .*.swp *.o -*.rdb *.log -redis-cli -redis-server +dump.rdb redis-benchmark -redis-check-dump redis-check-aof +redis-check-dump +redis-cli +redis-sentinel +redis-server doc-tools release misc/* @@ -16,7 +17,6 @@ SHORT_TERM_TODO release.h src/transfer.sh src/configs -src/redis-server.dSYM redis.ds src/redis.conf deps/lua/src/lua @@ -24,3 +24,4 @@ deps/lua/src/luac deps/lua/src/liblua.a .make-* .prerequisites +*.dSYM diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 8a225b7179c..69b239965ba 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,81 +1,727 @@ -Redis 2.6 release notes +Redis 2.8 release notes +======================= -Migrating from 2.4 to 2.6 +** IMPORTANT ** Check the 'Migrating from 2.6 to 2.8' section at the end of + this file for information about what changed between 2.6 and + 2.8 and how this may affect your application. + +-------------------------------------------------------------------------------- +Upgrade urgency levels: + +LOW: No need to upgrade unless there are new features you want to use. +MODERATE: Program an upgrade of the server, but it's not urgent. +HIGH: There is a critical bug that may affect a subset of users. Upgrade! +CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. +-------------------------------------------------------------------------------- + +--[ Redis 2.8.24 ] Release date: 18 Dec 2015 + +Upgrade urgency: MODERATE. We fixed a crash that happens very rarely, so + updating does not hurt, but most users are unlikely to + experience this condition because it requires some odd + timing. + +* [FIX] lua_struct.c/getnum security issue fixed. (Luca Bruno discovered it, + patched by Sun He and Chris Lamb) +* [FIX] Fix a race condition in processCommand() because of interactions + with freeMemoryIfNeeded(). Details in issue #2948 and especially + in the commit message d999f5a. (Race found analytically by + Oran Agra, patch by Salvatore Sanfilippo) + +* [NEW] Log offending memory access address on SIGSEGV/SIGBUS (Salvatore + Sanfilippo) + +--[ Redis 2.8.23 ] Release date: 15 Oct 2015 + +Upgrade urgency: MODERATE, the most important thing is a fix in the replication + code that may make the slave hanging forever if the master + remains with an open socket even if it is no longer able to + reply. + +* [FIX] MOVE now moves the TTL as well. A bug lasting forever... finally + fixed thanks to Andy Grunwald that reported it. + (reported by Andy Grunwald, fixed by Salvatore Sanfilippo) +* [FIX] Fix a false positive in HSTRLEN test. +* [FIX] Fix a bug in redis-cli --pipe mode that was not able to read back + replies from the server incrementally. Now a mass import will use + a lot less memory, and you can use --pipe to do incremental streaming. + (reported by Twitter user @fsaintjacques, fixed by Salvatore + Sanfilippo) +* [FIX] Slave detection of master timeout. (fixed by Kevin McGehee, refactoring + and regression test by Salvatore Sanfilippo) + +--[ Redis 2.8.22 ] Release date: 8 Sep 2015 + +Upgrade urgency: HIGH for Redis and Sentinel. However note that in order to + fix certain replication bugs, the replication internals were + modified in a very heavy way. So while this release is + conceptually saner, it may contain regressions. For this + reason, before the release, QA activities were performed by + me (antirez) and Redis Labs and no evident bug was found. + +* [FIX] A number of bugs related to replication PSYNC and the (yet experimental) + diskless replication feature were fixed. The bugs could lead to + inconsistency between masters and slaves. (Salvatore Sanfilippo, Oran + Agra fixed the issue found by Yuval Inbar) +* [FIX] A replication bug in the context of PSYNC partial resynchonization was + found and fixed. This bug happens even when diskless replication is off + in the case different slaves connect at different times while the master + is creating an RDB file, and later a partial resynchronization is + attempted by a slave that connected not as the first one. (Salvatore + Sanfilippo, Oran Agra) +* [FIX] Chained replication and PSYNC interactions leading to potential stale + chained slaves data set, see issue #2694. (Salvatore Sanfilippo fixed + an issue reported by "GeorgeBJ" user at Github) +* [FIX] redis-cli --scan iteration fixed when returned cursor overflows + 32 bit signed integer. (Ofir Luzon, Yuval Inbar) +* [FIX] Sentinel: fixed a bug during the master switch process, where for a + failed conditional check, the new configuration is rewritten, during + a small window of time, in a corrupted way where the master is + also reported to be one of the slaves. This bug is rare to trigger + but apparently it happens in the wild, and the effect is to see + a replication loop where the master will try to replicate with itself. + A detailed explanation of the bug and its effects can be found in + the commit message here: https://github.com/antirez/redis/commit/c20218eb5770b2cafb12bc7092313b8358fedc0a. + The bug was found by Jan-Erik Rediger using a static analyzer and + fixed by Salvatore Sanfilippo. +* [FIX] Sentinel lack of arity checks for certain commands. + (Rogerio Goncalves, Salvatore Sanfilippo) + +* [NEW] Replication internals rewritten in order to be more resistant to bugs. + The replication handshake in the slave side was rewritten as a non + blocking state machine. (Salvatore Sanfilippo, Oran Agra) +* [NEW] New "replication capabilities" feature introduced in order to signal + from the master to the slave what are the features supported, so that + the master can choose the kind of replication to start (diskless or + not) when master and slave are of different versions. (Oran Agra, + Salvatore Sanfilippo) +* [NEW] Log clients details when SLAVEOF command is received. (Salvatore + Sanfilippo with inputs from Nick Craver and Marc Gravell). + +--[ Redis 2.8.21 ] Release date: 4 Jun 2015 + +Upgrade urgency: HIGH for Redis because of a security issue. + LOW for Sentinel. + +* [FIX] Critical security issue fix by Ben Murphy: http://t.co/LpGTyZmfS7 +* [FIX] SMOVE reply fixed when src and dst keys are the same. (Glenn Nethercutt) +* [FIX] Lua cmsgpack lib updated to support str8 type. (Sebastian Waisbrot) + +* [NEW] Senitnel: CKQUORUM and FLUSHCONFIG commands back ported. + (Salvatore Sanfilippo and Bill Anderson) + +--[ Redis 2.8.20 ] Release date: 5 May 2015 + +Upgrade urgency: LOW for Redis, MODERATE for Sentinel. + +* [FIX] Sentinel memory leak due to hiredis fixed. (Salvatore Sanfilippo) +* [FIX] Sentinel memory leak on duplicated instance. (Charsyam) +* [FIX] Redis crash on Lua reaching output buffer limits. (Yossi Gottlieb) +* [FIX] Sentinel flushes config on +slave events. (Bill Anderson) +* [FIX] Fixes to diskless replication. (Oran Agra) +* [FIX] Redis (non clustered & clustered) replication bug involving blocking + operations: see issue #2473. (Salvatore Sanfilippo) +* [FIX] Config: missing activerehashing option support in CONFIG SET added. + (Salvatore Sanfilippo, thx to Bill Anderson) +* [FIX] AOF bug unlikely to happen in practice and mostly harmless: child + process segfaults when parent is not reachable via pipe. (Sun He) +* [FIX] Scripting engine now reports an error when misused with Lua debug + hooks, instead of crashing. (Salvatore Sanfilippo) +* [FIX] INFO loading stats: three fixes. +* [FIX] Fixed memory leaks in rdbSaveToSlavesSockets(). (Alon Diamant) + +* [NEW] Redis-cli --latency-dist backported from unstable. + (Salvatore Sanfilippo) + +--[ Redis 2.8.19 ] Release date: 16 Dec 2014 + +# UPGRADE URGENCY: LOW for both Redis and Sentinel. This release mostly + fixes small issues. + +02d465c Don't log admin commands in MONITOR. (antirez) +4d8f426 List of commands flagged as admin commands modified. (antirez) +e47e460 Lua cmsgpack lib updated to latest version. (antirez) +5509c14 Add symlink to redis-sentinel during make install (Rhommel Lamas) +7de1ef7 SORT: Don't sort Set elements if not needed. (antirez) +e945a54 Fix zero-ordering SORT when called against lists (Matt Stancliff) +d81c383 Update redis_init_script.tpl (Ben Dowling) +dba57ea FIXED redis-benchmark's idle mode.With idle mode shouldn't create write event (zhanghailei) +888ea17 zipmap.c: update comments above (Sun He) +86ebc13 replaced // comments #2150 (Deepak Verma) +3d73f08 redis-benchmark AUTH command to be discarded after the first send #2150 (azure provisioned user) +76d53a6 sds.c: Correct two spelling mistakes in comments (Sun He) +4848cf9 sds.c/sdscatvprintf: set va_end to finish va_list cpy (Sun He) +d2f584f sds.c: Correct some comments (Sun He) +2ed3f09 Update whatisdoing.sh (Serghei Iakovlev) +77b997d Include stropts only if __sun is defined. (antirez) +d409371 Fix implicit declaration of ioctl on Solaris (Jan-Erik Rediger) +23b96c0 Silence _BSD_SOURCE warnings in glibc 2.20 and forward (Johan Bergström) +a47a042 Mark whatisdoing.sh as deprecated in top-comment. (antirez) +b5737d2 getting pid fixes (Serghei Iakovlev) +a598e08 sparkline.c: AddSample skip Empty label (Sun He) +7d480ab sparkline.c: mov label-ini into the AddSample Function (Sun He) +2f3c860 Only ignore sigpipe in interactive mode (Jan-Erik Rediger) +0c211a1 Simplify lua_cmsgpack macro and fix build on old Linux distros. (antirez) + +--[ Redis 2.8.18 ] Release date: 4 Dec 2014 + +# UPGRADE URGENCY: LOW for both Redis and Sentinel. This release mostly + adds new features to Redis, and contains non critical + fixes. + +* [FIX] Linenoise updated to be more VT100 compatible. (Salvatore Sanfilippo) +* [FIX] A number of typos fixed inside comments. (Various authors) +* [FIX] redis-cli no longer quits after long timeouts. (Matt Stancliff) +* [FIX] Test framework improved to detect never terminating scripts, cleanup + instances on crashes. (Salvatore Sanfilippo) +* [FIX] PFCOUNT can be used on slaves now. (Salvatore Sanfilippo) +* [FIX] ZSCAN no longer report very small scores as 0. (Matt Stancliff, + Michael Grunder, Salvatore Sanfilippo) +* [FIX] Don't show the ASCII logo if syslog is enabled. Redis is now + an Enterprise Grade product. (Salvatore Sanfilippo) + +* [NEW] EXPERIMENTAL: Diskless replication, for more info check the doc at + http://redis.io/topics/replication. (Salvatore Sanfilippo). +* [NEW] Transparent Huge Pages detection and reporting in logs and + LATENCY DOCTOR output. (Salvatore Sanfilippo) +* [NEW] Many Lua scripting enhancements: Bitops API, cjson upgrade and tests, + cmsgpack upgrade. (Matt Stancliff) +* [NEW] Total and instantaneous Network bandwidth tracking in INFO. +* [NEW] DEBUG POPULATE two args form implemented (old form still works). + The second argument is the key prefix. Default is "key:" (Salvatore + Sanfilippo) +* [NEW] Check that tcp-backlog is matched by /proc/sys/net/core/somaxconn, and + warn about it if not. (Salvatore Sanfilippo) + +--[ Redis 2.8.17 ] Release date: 19 Sep 2014 + +# UPGRADE URGENCY: HIGH for Redis Sentinel. + LOW for Redis Server (unmodified compared to 2.8.16). + +* [FIX] Resolved a memory leak in the hiredis library causing a memory leak + in Redis Sentinel when a monitored instance or another Sentinel is + unavailable. Every reconnection attempt will leak a small amount of + memory, but in the long run the process can reach a considerable size. + +--[ Redis 2.8.16 ] Release date: 16 Sep 2014 + +# UPGRADE URGENCY: HIGH for Redis if you are using 2.8.15 + AOF. + LOW for Sentinel. + +* [FIX] The ability to load truncated AOF files introduced with Redis 2.8.15 + contains a bug fixed in this release: after loading the file was not + truncated to the last valid command, so the new commands are appended + after a non well formed command. This means that: + + 1) The first AOF rewrite triggered by the server will automatically + fix the problem. + 2) However, if the server is restarted before the rewrite, Redis may + not be able to load the file and you need to manually fix it. + + In order to fix a corrupted file you should start the redis-check-aof + utility WITHOUT the --fix option, just to check the offset where the + corruption is found. Around the offset reported by the check utility + you'll find, inside your AOF file, a command which is not complete + according to the Redis protocol. Just remove this incomplete command + leafing the file unaltered before and after the offending command, + and restart the server. + + IMPORTANT #1: Redis 2.8.15 is the only stable version of Redis with + this bug so probably no actual real-world problem happened since the + problem is automatically fixed at the first automatic AOF rewrite. + + IMPORTANT #2: Before upgrading to Redis 2.8.16, if you are using Redis + 2.8.15 with AOF enabled, make sure to trigger a manual AOF rewrite + using the BGREWRITEAOF command. + +* [FIX] SAVE is no longer propagated to AOF / slaves. + +--[ Redis 2.8.15 ] Release date: 12 Sep 2014 + +# UPGRADE URGENCY: LOW for Redis, HIGH for Sentinel. + +* [FIX] Sentinel critical bug fixed: the absolute majority was computed in a + wrong way because of a programming error. Now the implementation does + what the specification says and the majority to authorize a failover + (that should not be confused with the ODOWN quorum) is the majority of + *all* the Sentinels ever seen for a given master, regardless of their + current state. +* [FIX] GETRANGE test no longer fails for 32 bit builds (Matt Stancliff). +* [FIX] Limit SCAN latency when the hash table is in an odd state (very few + populted buckets because rehashing is in progress). (Xiaost and + Salvatore Sanfilippo) + +* [NEW] Redis is now able to load truncated AOF files without requiring a + redis-check-aof utility run. The default now is to load truncated + (but apparently not corrupted) AOFs, you can change this in redis.conf. + (Salvatore Sanfilippo). +* [NEW] Sentinel: ability to announce itself with an arbitrary IP/port to work + in the context of natted networks. However this is probably still + not enough since there is no equivalent mechanism for slaves listed + in the master INFO output. (Dara Kong and Salvatore Sanfilippo) + +--[ Redis 2.8.14 ] Release date: 1 Sep 2014 + +# UPGRADE URGENCY: HIGH for Lua scripting users, the server could crash because + of a bug introduced in Redis 2.8.10, otherwise LOW. + LOW for Redis Sentinel. + +* [FIX] Don't prevent use of shared integers if maxmemory policy is non-LRU. + (Salvatore Sanfilippo) +* [FIX] Fail SYNC if background save child aborted due to a signal. + (Yossi Gottlieb) +* [FIX] Different small redis-cli fixes. (Dov Murik, Charsyam, cubicdaiya, + Kashif Rasul, Jan-Erik Rediger, Matt Stancliff) +* [FIX] AIX compilation fixes. (Siah Lyimo) +* [FIX] A number of other smaller issues. +* [FIX] Improved SIGINT handling (Matt Stancliff, Salvatore Sanfilippo) +* [FIX] Use unsigned types in SDS header to raise limit to 4GB. + (Matt Stancliff, Salvatore Sanfilippo) +* [FIX] Handle signed/unsigned comparisons with more care around the code. + (Salvatore Sanfilippo) +* [FIX] Colorized test output fixed to don't change the background color. + (Mariano Pérez Rodríguez) +* [FIX] More Sentinel IPv6 fixes. (Eiichi Sato) +* [FIX] Deny CLIENT command in scripts. (Matt Stancliff) +* [FIX] Allow datasets with more than 2 billion of keys, initial work. +* [FIX] Fix a Lua scripting crash by storing the length of the static + argv when first allocated. (Paddy Byers) + +* [NEW] Pub/Sub PING. (Salvatore Sanfilippo) +* [NEW] Much faster ZUNIONSTORE. (Kyle Hubert, Salvatore Sanfilippo) +* [NEW] Faster ll2string() implementation. (Salvatore Sanfilippo) +* [NEW] **WARNING, minor API change**: PUBSUB NUMSUB: return type modified + to integer. (Matt Stancliff) +* [NEW] redis-benchmark support for AUTH. (CharSyam) + +--[ Redis 2.8.13 ] Release date: 14 Jul 2014 + +# UPGRADE URGENCY: LOW for Redis and Sentinel, this is a features enhancement + release mostly. Since this release introduces the latency + monitoring feature, Redis 2.8 users experiencing latency + issues are strongly encouraged to upgrade. + +* [FIX] CLIENT KILL minor backward compatibility fixes. (Salvatore Sanfilippo) +* [FIX] Enable HAVE_ATOMIC for PowerPC. (Matt Stancliff) +* [FIX] More robust PSYNC and AOF rewrites tests. (Salvatore Sanfilippo) +* [FIX] Solaris build fixed. (Matt Stancliff, Salvatore Sanfilippo) + +* [NEW] The new latency monitoring feature, as documented at + http://redis.io/topics/latency-monitor (Salvatore Sanfilippo) +* [NEW] The COMMAND command, exposing the Redis command table + as an API. (Matt Stancliff) +* [NEW] Update used memory with C11 __atomic. (Matt Stancliff) + +--[ Redis 2.8.12 ] Release date: 23 Jun 2014 + +# UPGRADE URGENCY: HIGH for Redis, CRITICAL for Sentinel. + +# WARNING: This release contains a non backward compatible semantical change + to Lua script that should affect an almost zero number of users. + +* [FIX / BREAKS BACKWARD COMPATIBILITY] Using SELECT inside Lua scripts no + longer makes the selected DB to be set in the calling client. + So Lua can still use SELECT, but the client calling the script will + remain set to the original DB. Thix fixes an issue with Redis + replication of Lua scripts that called SELECT without reverting the + selected DB to the original one. (Salvatore Sanfilippo) +* [FIX] Sentinel failover was instalbe if the master was detected as available + during the failover (especially during manual failovers) because + of an implementation error (lack of checking of + SRI_PROMOTED flag). (Salvatore Sanfilippo) +* [FIX] Cancel SHUTDOWN if initial AOF is being written. (Matt Stancliff) +* [FIX] Sentinel: bind source address for outcoming connections. (Matt + Stancliff). +* [FIX] Less timing sensitive Sentinel tests. (Salvatore Sanfilippo). + +* [NEW] redis-cli --intrinsic-latency stopped with SIGINT still reports + stats (Matt Stancliff) +* [NEW] Sentinels broadcast an HELLO message ASAP after a failover in order to + reach a consistent state faster (before it relied for periodic HELLO + messages). (Salvatore Sanfilippo). +* [NEW] Jemalloc updated to 3.6.0. (Salvatore Sanfilippo) +* [NEW] CLIENT LIST speedup. (Salvatore Sanfilippo) +* [NEW] CLIENT LIST new unique incremental ID to every client. (Salvatore + Sanfilippo) +* [NEW] ROLE command added. (Salvatore Sanfilippo) +* [NEW] CLIENT KILL new form to kill by client type and ID (see doc at + redis.io for more info). (Salvatore Sanfilippo) +* [NEW] Sentinel now disconnects clients when instances are reconfigured + (see http://redis.io/topics/sentinel-clients). (Salvatore Sanfilippo) +* [NEW] Hiredis update to latest version. (Matt Stancliff) + +--[ Redis 2.8.11 ] Release date: 11 Jun 2014 + +# UPGRADE URGENCY: HIGH if you use Lua scripting, LOW otherwise. + +* [FIX] A previous fix for Lua -> Redis numerical precision enhancement + introduced a new problem. In Redis 2.8.10 commands called from Lua + passing a string that "looks like" a very large number, may actually + use as argument the string converted as a float. This bug is now + fixed. +* [FIX] Now commands other than *PUSH* adding elements to a list will be able + to awake clients blocked in a blocking POP operation. +* [FIX] Cygwin compilation fixes. + +--[ Redis 2.8.10 ] Release date: 5 Jun 2014 + +# UPGRADE URGENCY: HIGH if you use min-slaves-to-write option. + +* [FIX] IMPORTANT! A min-slaves-to-write option active in a slave totally + prevented the slave from acception the master stream of commands. + This release includes testes for min-slaves-to-write, and a fix + for this issue. +* [FIX] Sometimes DEL returned 1 for already expired keys. Fixed. +* [FIX] Fix test false positive because new osx 'leaks' output. +* [FIX] PFCOUNT HLL cache invalidation fixed: no wrong value was reported + but the cache was not used at all, leading to lower performances. +* [FIX] Accept(2) multiple clients per readable-event invocation, and better + processing of I/O while loading or busy running a timedout script. + Basically now the LOADING / BUSY errors are reported at a decent + speed. +* [FIX] A softwaer watchdog crash fixed. +* [FIX] Fixed a Lua -> Redis numerical precision loss. + +* [NEW] Lua scripting engine speed improved. +* [NEW] Sentinel generates one new event for humans to understand better + what is happening during a failover: +config-update-from. + Also the time at which a failover will be re-attempted is logged. + +--[ Redis 2.8.9 ] Release date: 22 Apr 2014 + +# UPGRADE URGENCY: LOW, only new features introduced, no bugs fixed. + +* [NEW] The HyperLogLog data structure. You can read more about it + in this blog post. http://antirez.com/news/75 +* [NEW] The Sorted Set data type has now support for lexicographic range + queries, check the new commands ZRANGEBYLEX, ZLEXCOUNT and + ZREMRANGEBYLEX, which are documented at http://redis.io. + +--[ Redis 2.8.8 ] Release date: 25 Mar 2014 + +# UPGRADE URGENCY: HIGH for Redis, LOW for Sentinel. There is a potentially + critical bug fix causing data loss in Redis but it requires + a combination of disk full and the use of the + SHUTDOWN command. + +* [FIX] Fixed data loss when SHUTDOWN was used with a disk full condition. +* [FIX] Fixed a memory leak in the SORT syntax error processing. +* [FIX] When Sentinel down-after-milliseconds parameter is modified at runtime + now it gets propagated to all the slaves and sentinel instances + of the master. +* [FIX] `install_server.sh` script finally fixed. +* [FIX] Different fixes to maxclients handling. + +* [NEW] Sentinels are now able to send update messages in a peer-to-peer + fashion even if no Redis instances are available. Now the Sentinel + liveness property that the most updated configuration in a given + partition is propagated to all the Sentinels is extended to partitions + without reachable instances. +* [NEW] Sentinel safety properties are now ensured in a crash-recovery system + model since some state is persisted on disk before replying to other + nodes, and reloaded at startup. +* [NEW] Sentinel now uses CLIENT SETNAME so that it is easy to identify + Sentinels using CLIENT LIST among other clients. +* [NEW] Sentinel failure detection and reconnection code improved. +* [NEW] Use all 24 bits (instead of 22) for the Redis objects LRU field. + Note that the new LRU algorithm using eviction pools was not backported + from unstable for safery / code maturity concerns. +* [NEW] Majory speedup for the INFO command (it is now 6 times faster). +* [NEW] More Sentinel unit tests. +* [NEW] New command DEBUG ERROR returns the specified error. Example: + DEBUG ERROR "LOADING database". This is handy to write Redis client + libraries unit tests. +* [NEW] redis-cli now supports multi-line editing via updated linenoise lib. + +Thanks to Matt Stancliff and Jan-Erik Rediger for the work done in the context +of this release. + +--[ Redis 2.8.7 ] Release date: 5 Mar 2014 + +# UPGRADE URGENCY: LOW for Redis, LOW for Sentinel. However this release adds + new features so users may want to upgrade in order to + exploit the new functionalities. + +* [FIX] Sometimes the absolute config file path was obtained in a wrong way. + This happened when there was a "dir" directive inside the config file + and at the same time the configuration file was given as a relative + path to redis-server or redis-sentinel executables. +* [FIX] redis-cli: Automatically enter --slave mode when SYNC or PSYNC are + called during an interactive session. +* [FIX] Sentinel "IDONTKNOW" error removed as it does not made sense with the + new Sentinel design. This error was actually a fix for a design error + in the first implementation of Sentinel. +* [FIX] Sentinel: added a missing exit() call to abort after config file + checks at startup. This error was introduced with an improvement in + a previous 2.8 release. +* [FIX] BITCOUNT: fixed unaligned access causing issues in sparc and other + archs not capable of dealing with unaligned accesses. This also makes + the code faster in archs where unaligned accesses are allowed. +* [FIX] Sentinel: better nodes fail over start time desynchronization to avoid + split-brain during the voting process needed to get authorization to + fail over. This means the system is less likely to need to retry + and will fail over faster. No changes in behavior / correctness. +* [FIX] Force INFO used_memory_peak to match peak memory. This generated some + confusion among users even if it was not an actual bug. + +* [NEW] Sentinel unit tests and framework. More tests needed and units must + be improved in order to have less false positives, but it is a start + and features a debugging console that is useful to fix tests or to + inspect bugs causing tests failures. +* [NEW] New Sentinel events: +/-monitor and +set used to monitor when an + instance to monitor is added or removed, or when a configuration + is modified via SENTINEL SET. +* [NEW] Redis-cli updated to use SCAN instead of random sampling via + RANDOMKEY in order to implement --bigkeys feature. Moreover the + implementation now supports pipelining and reports more information + at the end of the scan. Much faster, much better. A special thank + you to Michael Grunder for this improvement. +* [NEW] redis-cli now supports a new --intrinsic-latency mode that is able + to meter the latency of a system due to kernel / hypervisor. + How to use it is explained at http://redis.io/topics/latency. +* [NEW] New command BITPOS: find first bit set or clear in a bitmap. +* [NEW] CONFIG REWRITE calls are now logged. + +--[ Redis 2.8.6 ] Release date: 13 Feb 2014 + +# UPGRADE URGENCY: HIGH for Redis, LOW for Sentinel. Redis users using Lua + scripts with EVALSHA and attached slaves and/or AOF + persistence should consider upgrading ASAP. + +* [FIX] Fixed an critical EVALSHA script cache bug: scripts executed may not + propagate to AOF / Slaves correctly under certain conditions. + See issue #1549 at Github for more information. +* [FIX] Fixed multiple bugs resulting into closing the link with master or slave + during replication without good reasons. This will result in useless + resynchronizations, or infinite loops where the replication link can't + be established. +* [FIX] Don't count the time needed to populate the buffers of clients waiting + in MONITOR mode when populating the Slow Log entries. + +* [NEW] AOF write errors (like no space on device) no longer abort Redis if the + fsync policy is none or every second. The database enters a read-only + mode where every write is refused with an error. Normal operations are + restored as soon as Redis is able to append again data to the AOF file. +* [NEW] Sentinel now accepts SHUTDOWN command. + +--[ Redis 2.8.5 ] Release date: 4 Feb 2014 + +# UPGRADE URGENCY: HIGH for Redis, LOW for Sentinel. Redis users using Lua + scripts with expires, and Redis users relying on the + ability of Redis to block writes on RDB saving errors + should plan to upgrade ASAP. + +* [FIX] Fixed a replication bug caused by Lua scripts + expired keys: keys could + expire in the middle of scripts causing non-deterministic behavior. +* [FIX] MISCONFIG error if condition fixed, the server was no longer able + to stop writes on RDB misconfiguration after this error was introduced. +* [FIX] REDIS_AOF_REWRITE_MIN_SIZE is now 64mb like example redis.conf default. +* [FIX] Perform fflush() before fsync() in rio.c (bug without actual effects). +* [FIX] Don't log MONITOR clients as disconnecting slaves. +* [FIX] SENTINEL MASTER arity check fixed. Crashed the Sentinel instance when + the command was given without arguments. + +* [NEW] Allow CONFIG and SHUTDOWN while in stale-slave state. +* [NEW] Support for configurable TCP listen(2) backlog size. +* [NEW] redis-cli supports SCAN via the --scan and --pattern options. +* [NEW] SENTINEL SET master quorum via runtime API implemented. + +--[ Redis 2.8.4 ] Release date: 13 Jan 2014 + +# UPGRADE URGENCY: MODERATE for Redis and Sentinel. + +* [FIX] Makefile compatibility with non common make variants improved. +* [FIX] SDIFF crash in very unlikely to trigger state fixed. +* [FIX] Config rewriting fixed: don't wipe options unknown to the rewrite + process. +* [FIX] Set TCP port to 0 works again to disable TCP networking. +* [FIX] Fixed replication with old Redis instances as masters by not + sending REPLCONF ACK to them. +* [FIX] Fix keyspace notifications rewrite and CONFIG GET output. +* [FIX] Fix RESTORE TTL handling in 32 bit systems (32 bit overflow). + +* [NEW] Sentinel now has a run time configuration API. +* [NEW] Log when we lost connection with master or slave. +* [NEW] When instance is turned from slave to master now inherits the + old master replication offset when possible. This improves the + Sentinel failover procedure. + +--[ Redis 2.8.3 ] Release date: 11 Dec 2013 + +# UPGRADE URGENCY: MODERATE for Redis, HIGH for Sentinel. + +* [FIX] Sentinel instance role sampling fixed, the system is now more + reliable during failover and when reconfiguring instances with + non matching configuration. +* [FIX] Inline requests are now handled even when terminated with just LF. +* [FIX] Replication timeout handling greatly improved, now the slave is able + to ping the master while removing the old data from memory, and while + loading the new RDB file. This avoid false timeouts sensed by + masters. +* [FIX] Fixed a replication bug involving 32 bit instances and big datasets + hard to compress that resulted into more than 2GB of RDB file sent. +* [FIX] Return error for inline requests with unbalanced quotes. +* [FIX] Publish the slave replication offset even when disconnected from the + master if there is still a cached master instance. + +--[ Redis 2.8.2 ] Release date: 2 Dec 2013 + +# UPGRADE URGENCY: MODERATE for both Redis and Sentinel. + +* [FIX] Sentinel better desynchronization to avoid split-brain elections + where no Sentinel managed to get elected. +* [FIX] Stop accepting writes on "MISCONF" error only if master, not slave. +* [FIX] Reply to PING with an error on "MISCONF" errors. + +--[ Redis 2.8.1 ] Release date: 25 Nov 2013 + +# UPGRADE URGENCY: LOW for Redis, CRITICAL for Senitnel. You don't need to + upgrade your Redis instances but it is highly recommended + to upgrade and restart all the Sentinel processes. + +* [FIX] Fixed a bug in "new Sentinel" config propagation. +* [FIX] Fixed a false positive in Redis tests. + +--[ Redis 2.8.0 ] Release date: 22 Nov 2013 + +# UPGRADE URGENCY: LOW, unless you want to upgrade to new Sentinel code. + +* [FIX] Fixed an error in rdbWriteRaw() that should have no practical impact. +* [NEW] Log the new master when SLAVEOF command is used. +* [NEW] Sentinel code synchronized with the unstable branch, the new Sentinel + is a reimplementation that uses more reliable algorithms. + +--[ Redis 2.8 Release Candidate 6 (2.7.106) ] Release date: 6 Nov 2013 + +This is the 6th release candidate of Redis 2.8 (official version is 2.7.106). + +# UPGRADE URGENCY: LOW, only new features back ported, no fixes. + +* [NEW] SCAN, SSCAN, HSCAN, ZSCAN commands. + +--[ Redis 2.8 Release Candidate 5 (2.7.105) ] Release date: 9 Oct 2013 + +This is the 5th release candidate of Redis 2.8 (official version is 2.7.105). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH because of many non critical replication bugs fixed. + +* [FIX] redis-cli: don't crash with --bigkeys when the key no longer exist. +* [FIX] Allow AUTH / PING when disconnected from slave and serve-stale-data is no. +* [FIX] PSYNC: safer handling of PSYNC requests with offsets in the future. +* [FIX] Replication: Fix master timeout detection. +* [FIX] Replication: Correctly install the write handler after successful PSYNC. + +--[ Redis 2.8 Release Candidate 4 (2.7.104) ] Release date: 30 Aug 2013 + +This is the fourth release candidate of Redis 2.8 (official version is 2.7.104). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH because of the EVAL memory leak. + +* [FIX] Fixed a serious EVAL memory leak in the Lua stack. +* [FIX] Fixed server startup when no IPv6 address exists in any interface. +* [FIX] Send MISCONFIG error when BGSAVE fails because can't fork. +* [FIX] Memory efficiency with large (> a few kbytes) values improved considerably. +* [NEW] DEBUG SDSLEN for sds memory debugging. + +--[ Redis 2.8 Release Candidate 3 (2.7.103) ] Release date: 19 Aug 2013 + +This is the third release candidate of Redis 2.8 (official version is 2.7.103). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH + +* [FIX] Improved expired keys collection algorithm. Even under heavy load keys + to be expired can't accumulate because of lack of CPU time. +* [FIX] Replication speed regression fixed (issue #1238). +* [FIX] Fixed an hard to trigger PSYNC bug. +* [FIX] Fixed Issue #1240, ZUNIONSTORE could lead to wrong result. +* [NEW] Add per-db average TTL information in INFO output. +* [NEW] redis-benchmark improvements. +* [NEW] dict.c API wrong usage detection. + +--[ Redis 2.8 Release Candidate 2 (2.7.102) ] Release date: 30 Jul 2013 + +This is the second release candidate of Redis 2.8 (official version is 2.7.102). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH + +* [FIX] Fixed a critical replication bug, see issue #1221. +* [NEW] The new inline protocol now accepts quoted strings like, for example + you can now type in a telnet session: set 'foo bar' "hello world\n". + +--[ Redis 2.8 Release Candidate 1 (2.7.101) ] Release date: 18 Jul 2013 + +This is the first release candidate of Redis 2.8 (official version is 2.7.101). + +The following is a list of improvements in Redis 2.8, compared to Redis 2.6. + +* [NEW] Slaves are now able to partially resynchronize with the master, so most + of the times a full resynchronization with the RDB creation in the master + side is not needed when the master-slave link is disconnected for a short + amount of time. +* [NEW] Experimental IPv6 support. +* [NEW] Slaves explicitly ping masters now, a master is able to detect a timed out + slave independently. +* [NEW] Masters can stop accepting writes if not enough slaves with a given + maximum latency are connected. +* [NEW] Keyspace changes notifications via Pub/Sub. +* [NEW] CONFIG SET maxclients is now available. +* [NEW] Ability to bind multiple IP addresses. +* [NEW] Set process names so that you can recognize, in the "ps" command output, + the listening port of an instance, or if it is a saving child. +* [NEW] Automatic memory check on crash. +* [NEW] CONFIG REWRITE is able to materialize the changes in the configuration + operated using CONFIG SET into the redis.conf file. +* [NEW] More NetBSD friendly code base. +* [NEW] PUBSUB command for Pub/Sub introspection capabilities. +* [NEW] EVALSHA can now be replicated as such, without requiring to be expanded + to a full EVAL for the replication link. +* [NEW] Better Lua scripts error reporting. +* [NEW] SDIFF performance improved. +* [FIX] A number of bugfixes. + +Migrating from 2.6 to 2.8 ========================= -Redis 2.4 is mostly a strict subset of 2.6. However there are a few things +Redis 2.6 is mostly a strict subset of 2.8. However there are a few things that you should be aware of: -* You can't use .rdb and AOF files generated with 2.6 into a 2.4 instance. -* 2.4 slaves can be attached to 2.6 masters, but not the contrary, and only - for the time needed to perform the version upgrade. +The following commands changed behavior: -There are also a few API differences, that are unlikely to cause problems, -but it is better to keep them in mind: + * The TTL and PTTL commands now return -2 if the key does not exist and + -1 if it exists but has no associated expire. Redis 2.6 and previous + versions used to return -1 for both the conditions. + * SORT with ALPHA now sorts according to local collation locale if no STORE + option is used. + * ZADD/ZINCRBY are now able to accept a bigger range of values as valid + scores, that is, all the values you may end having as a result of + calling ZINCRBY multiple times. + * Many errors are now prefixed by a more specific error code instead of + the generic -ERR, for example -WRONGTYPE, -NOAUTH, ... + * PUBLISH called inside Lua scripts is now correctly propagated to slaves. -* SORT now will refuse to sort in numerical mode elements that can't be parsed - as numbers. -* EXPIREs now all have millisecond resolution (but this is very unlikely to - break code that was not conceived exploting the previous resolution error - in some way.) -* INFO output is a bit different now, and contains empty lines and comments - starting with '#'. All the major clients should be already fixed to work - with the new INFO format. +The following redis.conf and CONFIG GET / SET parameters changed: -Also the following redis.conf and CONFIG GET / SET parameters changed name: + * logfile now uses the empty string in order to log to standard output, + so 'logfile stdout' is now invalid, use 'logfile ""' instead. - * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries - * hash-max-zipmap-value, now replaced by hash-max-ziplist-value - * glueoutputbuf was no completely removed as it does not make sense +The following INFO fields changed format in a non-backward compatible way: ---------- -CHANGELOG ---------- + * The list of slaves in INFO is now in field=value format. -What's new in Redis 2.6.0 -========================= +Replication: -UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to - upgrade after some testing of the application with the new - Redis version. - -* Server side Lua scripting, see http://redis.io/commands/eval -* Virtual Memory removed (was deprecated in 2.4) -* Hardcoded limits about max number of clients removed. -* AOF low level semantics is generally more sane, and especially when used - in slaves. -* Milliseconds resolution expires, also added new commands with milliseconds - precision (PEXPIRE, PTTL, ...). -* Clinets max output buffer soft and hard limits. You can specifiy different - limits for different classes of clients (normal,pubsub,slave). -* AOF is now able to rewrite aggregate data types using variadic commands, - often producing an AOF that is faster to save, load, and is smaller in size. -* Every redis.conf directive is now accepted as a command line option for the - redis-server binary, with the same name and number of arguments. -* Hash table seed randomization for protection against collisions attacks. -* Performances improved when writing large objects to Redis. -* Significant parts of the core refactored or rewritten. New internal APIs - and core changes allowed to develop Redis Cluster on top of the new code, - however for 2.6 all the cluster code was removed, and will be released with - Redis 3.0 when it is more complete and stable. -* Redis ASCII art logo added at startup. -* Crash report on memory violation or failed asserts improved significantly - to make debugging of hard to catch bugs simpler. -* redis-benchmark improvements: ability to run selected tests, - CSV output, faster, better help. -* redis-cli improvements: --eval for comfortable development of Lua scripts. -* SHUTDOWN now supports two optional arguments: "SAVE" and "NOSAVE". -* INFO output split into sections, the command is now able to just show - pecific sections. -* New statistics about how many time a command was called, and how much - execution time it used (INFO commandstats). -* More predictable SORT behavior in edge cases. -* INCRBYFLOAT and HINCRBYFLOAT commands. + Redis 2.8 can be used as slave for Redis 2.6, but doing this is only + a good idea for the short amount of time needed to upgrade your servers. + We suggest to update both master and slaves at about the same time. -------------------------------------------------------------------------------- -Credits: Where not specified the implementation and design are done by -Salvatore Sanfilippo and Pieter Noordhuis. Thanks to VMware for making all +Credits: Where not specified the implementation and design is done by +Salvatore Sanfilippo. Thanks to VMware and Pivotal for making all this possible. Also many thanks to all the other contributors and the amazing community we have. diff --git a/BUGS b/BUGS index 96d52bf8bd2..a8e936892f5 100644 --- a/BUGS +++ b/BUGS @@ -1 +1 @@ -Plese check https://github.com/antirez/redis/issues +Please check https://github.com/antirez/redis/issues diff --git a/CONTRIBUTING b/CONTRIBUTING index 623c5a6ec07..f7b6836f701 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -1,8 +1,29 @@ -1. Enter irc.freenode.org #redis and start talking with 'antirez' and/or 'pietern' to check if there is interest for such a feature and to understand the probability of it being merged. We'll try hard to keep Redis simple... so you'll likely encounter high resistance. +Note: by contributing code to the Redis project in any form, including sending +a pull request via Github, a code fragment or patch via private email or +public discussion groups, you agree to release your code under the terms +of the BSD license that you can find in the COPYING file included in the Redis +source distribution. You will include BSD license in the COPYING file within +each source file that you contribute. -2. Drop a message to the Redis Google Group with a proposal of semantics/API. +# IMPORTANT: HOW TO USE REDIS GITHUB ISSUES -3. If steps 1 and 2 are ok, use the following procedure to submit a patch: +* Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature + requests. Everything else belongs to the Redis Google Group. + + PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected + bugs in the Github issues system. We'll be very happy to help you and provide + all the support in the Redis Google Group. + + Redis Google Group address: + + https://groups.google.com/forum/?fromgroups#!forum/redis-db + +# How to provide a patch for a new feature + +1. Drop a message to the Redis Google Group with a proposal of semantics/API. + +2. If in step 1 you get an acknowledge from the project leaders, use the + following procedure to submit a patch: a. Fork Redis on github ( http://help.github.com/fork-a-repo/ ) b. Create a topic branch (git checkout -b my_branch) diff --git a/COPYING b/COPYING index 3e704e3ebd4..ac68e012bca 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (c) 2006-2009, Salvatore Sanfilippo +Copyright (c) 2006-2015, Salvatore Sanfilippo All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Changelog b/Changelog deleted file mode 100644 index f72746663d6..00000000000 --- a/Changelog +++ /dev/null @@ -1,1032 +0,0 @@ -2010-07-01 gitignore modified (antirez) -2010-06-22 redis.c split into many different C files. (antirez) -2010-06-16 more pub/sub tests (Pieter Noordhuis) -2010-06-15 initial basic pub/sub tests (Pieter Noordhuis) -2010-06-15 fix BLPOP/BRPOP to use the wrapped function for list length (Pieter Noordhuis) -2010-06-15 tests for BLPOP/BRPOP via an option in the tcl client that defers reading the reply (Pieter Noordhuis) -2010-06-14 TODO updated (antirez) -2010-06-14 Merge branch 'ltrim-tests' of git://github.com/pietern/redis (antirez) -2010-06-14 rename "list" to "linkedlist" to be more verbose (Pieter Noordhuis) -2010-06-14 allow running the test suite against an external Redis instance, without auto spawning (antirez) -2010-06-14 change ltrim tests to cover all min/max cases and add stronger stresser (Pieter Noordhuis) -2010-06-13 Fixed deps in makefile and mkreleasehdr.sh script to really take advantage of the new trick to avoid recompilation of redis.c on git sha1 or dirty status change (antirez) -2010-06-13 hopefully faster recompiling with a trick (antirez) -2010-06-13 fixed a bug in rdbLoadObject abount specially encoded objects (antirez) -2010-06-13 use raw strings when loading a hash from the rdb into a zipmap (Pieter Noordhuis) -2010-06-12 Merge branch 'expire' of git://github.com/pietern/redis (antirez) -2010-06-11 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-11 LPUSHX, RPUSHX, LINSERT only work on non-empty lists, so there are no clients waiting for a push (Pieter Noordhuis) -2010-06-11 make LINSERT return -1 when the value could not be inserted (Pieter Noordhuis) -2010-06-11 check if the list encoding needs to be changed on LPUSHX, RPUSHX, LINSERT (Pieter Noordhuis) -2010-06-11 make sure the value to insert is string encoded (Pieter Noordhuis) -2010-06-11 rename vars, move arguments, add comments (Pieter Noordhuis) -2010-06-11 always iterate from head to tail on LINSERT (Pieter Noordhuis) -2010-06-11 use REDIS_TAIL to insert AFTER an entry and REDIS_HEAD to insert BEFORE an entry (Pieter Noordhuis) -2010-06-11 move listTypeInsert to be grouped with other wrapper functions (Pieter Noordhuis) -2010-06-11 squashed merge from robey/twitter3: LINSERT BEFORE|AFTER, LPUSHX, RPUSHX (Pieter Noordhuis) -2010-06-09 remove pop function and the sds dependency; can be implemented using get+delete (Pieter Noordhuis) -2010-06-07 compute swappability for ziplist encoded lists (Pieter Noordhuis) -2010-06-07 reuse the sds from the main dictionary in the expiration dictionary (Pieter Noordhuis) -2010-06-07 TODO updated (antirez) -2010-06-07 encode integers while loading an hash (antirez) -2010-06-05 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-05 fixed two leaks for the dual encoded lists (Pieter Noordhuis) -2010-06-04 TODO updated (antirez) -2010-06-04 DISCSARD now unwatches all keys, as it should (antirez) -2010-06-04 generated tests for different encodings to avoid test code duplication (Pieter Noordhuis) -2010-06-04 refactor list tests to test both encodings; implemented assert functions (Pieter Noordhuis) -2010-06-04 renamed hash wrapper functions to match wrapper function naming convention: "Type" (Pieter Noordhuis) -2010-06-04 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-04 Merge branch 'smallkeys' (antirez) -2010-06-04 safety assert in listTypeNext (Pieter Noordhuis) -2010-06-04 renamed list wrapper functions to be more verbose (Pieter Noordhuis) -2010-06-04 add thresholds for converting a ziplist to a real list (Pieter Noordhuis) -2010-06-04 merge antirez/smallkeys (Pieter Noordhuis) -2010-06-03 test restored (antirez) -2010-06-03 memory leak introduced in the latest big changes fixed (antirez) -2010-06-03 Fixed VM bugs introduced with the top level keys as sds strings changes (antirez) -2010-06-03 top level keys are no longer redis objects but sds strings. There are still a few bugs to fix when VM is enabled (antirez) -2010-06-03 update Makefile to include ziplist.o (Pieter Noordhuis) -2010-06-03 use ziplists in SORT STORE until the thresholds are determined (Pieter Noordhuis) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis (antirez) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis into smallkeys (antirez) -2010-06-03 tag memory leak check on kill server as "leaks" (Pieter Noordhuis) -2010-06-03 tag test with sleep() as slow (Pieter Noordhuis) -2010-06-03 make sure the config it returned when called without code (Pieter Noordhuis) -2010-06-03 tag more slow tests (Pieter Noordhuis) -2010-06-03 change how arguments are passed from the AOF tests (Pieter Noordhuis) -2010-06-03 scope res variable outside test (Pieter Noordhuis) -2010-06-02 tags for existing tests (Pieter Noordhuis) -2010-06-02 pass tags to filter and match via arguments (Pieter Noordhuis) -2010-06-02 basic support to tag tests (Pieter Noordhuis) -2010-06-02 changed how server.tcl accepts options to support more directives without requiring more arguments to the proc (Pieter Noordhuis) -2010-06-02 removed obsolete code (Pieter Noordhuis) -2010-06-02 catch exceptions in the server proc, to be able to kill the entire chain of running servers (Pieter Noordhuis) -2010-06-02 Merge branch 'master' into smallkeys (antirez) -2010-06-02 smarter swapout policy on AOF too (antirez) -2010-06-02 better swapout policy while loading RDB file (antirez) -2010-06-02 minor code comment change (antirez) -2010-06-01 use integer types from stdint.h to be more verbose on the size in bytes of encoded elements. update list length to use 2 bytes instead of 1. (Pieter Noordhuis) -2010-06-01 added stress test for heavy i/o in ziplists (Pieter Noordhuis) -2010-06-01 fix signedness errors in ziplist testing code (Pieter Noordhuis) -2010-06-01 minor code movements and free object pull restored to 1 million (antirez) -2010-06-01 TODO updated with syslog plans for 2.2 (antirez) -2010-06-01 Debug message was printing stuff that are sometimes not initialized/valid (antirez) -2010-06-01 Merge branch 'smallkeys' of github.com:antirez/redis into smallkeys (antirez) -2010-06-01 fixed a few comments (antirez) -2010-06-01 fixed bugs introduced in the rewrite of the new VM engine (antirez) -2010-05-31 support rewriting the AOF with dual list encoding (Pieter Noordhuis) -2010-05-31 small refactor of fwrite* commands for AOF rewrite to allow writing a bulk long long (Pieter Noordhuis) -2010-05-31 use list wrapper functions in computing the dataset digest (Pieter Noordhuis) -2010-05-31 ziplistNext should work as expected when called with a pointer to ZIP_END (Pieter Noordhuis) -2010-05-31 update SORT to work with the dual list encoding (Pieter Noordhuis) -2010-05-31 function to create a new ziplist encoded list (Pieter Noordhuis) -2010-05-31 fixed missing incrRefCount (antirez) -2010-05-31 support rdb saving/loading with dual list encoding (Pieter Noordhuis) -2010-05-31 fixed signedness and disambiguate variable names (Pieter Noordhuis) -2010-05-31 added rdb save function to directly save long long values (Pieter Noordhuis) -2010-05-31 update RPOPLPUSH to support dual encoding (Pieter Noordhuis) -2010-05-31 update list iteration semantic to work as expected (i.e. "while(lNext(..))") (Pieter Noordhuis) -2010-05-31 ziplistDelete no longer needs a direction now ziplistPrev is fixed (Pieter Noordhuis) -2010-05-31 ziplistPrev should return the tail when the argument is ZIP_END (Pieter Noordhuis) -2010-05-31 first step of VM rewrite. blocking VM tests passing, more work needed in the async side (antirez) -2010-05-31 Merge branch 'no-appendfsync-on-rewrite' (antirez) -2010-05-30 fix LREM to remove *all* occurances when a zero argument is given (Pieter Noordhuis) -2010-05-30 fixed LINDEX to always return bulk response (Pieter Noordhuis) -2010-05-30 the tail offset must be an integer pointer to hold a 32-bit offset (Pieter Noordhuis) -2010-05-30 update LREM to support dual encoding via extra iteration primitives (Pieter Noordhuis) -2010-05-30 support dual encoding in LTRIM (Pieter Noordhuis) -2010-05-30 update LRANGE to use basic iteration code to support dual encoding (Pieter Noordhuis) -2010-05-30 inline support for dual encoding in the LINDEX and LSET commands (Pieter Noordhuis) -2010-05-30 generic pop and length function for ziplist encoding (Pieter Noordhuis) -2010-05-30 generic push function that supports the dual encoding (Pieter Noordhuis) -2010-05-30 change delete function to accept a direction argument, so "p" can be properly updated (Pieter Noordhuis) -2010-05-30 expose extra functionality from ziplist.c (Pieter Noordhuis) -2010-05-30 code style consistency fixes (Pieter Noordhuis) -2010-05-29 ziplistIndex now accepts negative indices (Pieter Noordhuis) -2010-05-29 fix compile warnings (Pieter Noordhuis) -2010-05-29 use simpler encoding for the length of the previous entry (Pieter Noordhuis) -2010-05-29 replace functions to get pointers to head and tail by macros (Pieter Noordhuis) -2010-05-29 function to insert an element at an arbitrary position in the list (Pieter Noordhuis) -2010-05-29 extract a generic delete function that can be used in pop and delete(range) (Pieter Noordhuis) -2010-05-29 use the entry struct in zipRawEntryLength (Pieter Noordhuis) -2010-05-29 rename argument names to s* to disambiguate from e* (Pieter Noordhuis) -2010-05-29 change ziplistRepr to use the entry struct (Pieter Noordhuis) -2010-05-29 modify compare function to check if the encoding is equal before comparing (Pieter Noordhuis) -2010-05-29 use a struct to retrieve all details for an entry (Pieter Noordhuis) -2010-05-29 initial implementation for making the ziplist doubly linked (Pieter Noordhuis) -2010-05-29 fix some warnings (Pieter Noordhuis) -2010-05-29 add function to retrieve ziplist size in bytes (Pieter Noordhuis) -2010-05-22 fix compare function of ziplist to only load integer from ziplist when it is encoded as integer (Pieter Noordhuis) -2010-05-22 add function to retrieve length of ziplist (Pieter Noordhuis) -2010-05-22 re-introduce ZIP_BIGLEN for clarity (Pieter Noordhuis) -2010-05-22 added header ziplist.h (Pieter Noordhuis) -2010-05-22 code to compare strings with entries in ziplist, regardless of their encoding (Pieter Noordhuis) -2010-05-22 updated iteration code to work well with different encodings (Pieter Noordhuis) -2010-05-22 move code from zip.c to ziplist.c (Pieter Noordhuis) -2010-05-22 partial revert of c80df5 because ziplist functions are starting to divert too much from zipmap functions (Pieter Noordhuis) -2010-05-22 initial work for integer encoding in ziplists (Pieter Noordhuis) -2010-05-22 move length housekeeping to a macro (Pieter Noordhuis) -2010-05-21 allow entries to be deleted in place when iterating over a ziplist (Pieter Noordhuis) -2010-05-21 allow pointer to be stored to current element when iterating over ziplist (Pieter Noordhuis) -2010-05-21 rename ziplistDelete to ziplistDeleteRange (Pieter Noordhuis) -2010-05-21 code to delete an inner range from the ziplist (Pieter Noordhuis) -2010-05-21 check if *value is non-NULL before setting it (Pieter Noordhuis) -2010-05-21 change iteration code to avoid allocating a new sds for each traversed entry (Pieter Noordhuis) -2010-05-21 code to iterate over a ziplist (Pieter Noordhuis) -2010-05-21 implementation for a ziplist with push and pop support (Pieter Noordhuis) -2010-05-21 extracted general methods to zip.c for reuse in other zip* structures (Pieter Noordhuis) -2010-05-28 command table size calculated with sizeof (antirez) -2010-05-28 use qsort and bsearch to lookup commands in O(log(N)) instead of O(N) (Pieter Noordhuis) -2010-05-28 Merge branch 'cli-stdin' of git://github.com/pietern/redis (antirez) -2010-05-28 Fixed ZINCR Nan bugs leading to server crash and added tests (antirez) -2010-05-28 redis.conf new features the new option, a minor typo preventing the compilation fixed (antirez) -2010-05-28 don't fsync after a rewrite if appendfsync is set to no. use aof_fsycn instead of fsync where appropriate (antirez) -2010-05-28 added new option no-appendfsync-on-rewrite to avoid blocking on fsync() in the main thread while a background process is doing big I/O (antirez) -2010-05-28 Added Git sha1 and dirty status in redis-server -v output (antirez) -2010-05-28 changed the message in the Makefile with the new command like to run the test suite (antirez) -2010-05-27 Fixed typo. (Vincent Palmer) -2010-05-27 new multi/exec tests (antirez) -2010-05-26 build command outside while loop (Pieter Noordhuis) -2010-05-26 require the flag "-c" to be used for redis-cli to read the last argument from stdin (Pieter Noordhuis) -2010-05-26 Merge branch 'master' into nested-multi (antirez) -2010-05-26 Fix EXEC bug that was leaving the client in dirty status when used with WATCH (antirez) -2010-05-26 raise error on nested MULTI and WATCH inside multi (antirez) -2010-05-25 allow regular sets to be passed to zunionstore/zinterstore (Pieter Noordhuis) -2010-05-25 Version is now 2.1.1 (antirez) -2010-05-25 RENAME is now WATCH-aware (antirez) -2010-05-25 TODO updated (antirez) -2010-05-25 WATCH is now able to detect keys removed by FLUSHALL and FLUSHDB (antirez) -2010-05-25 WATCH tests (antirez) -2010-05-25 minor bug fixed in WATCH (antirez) -2010-05-25 WATCH for MULTI/EXEC (CAS alike concurrency) (antirez) -2010-05-25 gitignore updated (antirez) -2010-05-21 Master is now already unfreezed, unstable, and ready to hacking sessions! (antirez) -2010-05-21 Merge branch 'solaris' of git://github.com/pietern/redis (antirez) -2010-05-21 Changelog updated (antirez) -2010-05-21 redis version is now 1.3.14 (aka 2.0.0 RC1) (antirez) -2010-05-21 html doc updated (antirez) -2010-05-21 by default test with valgrind does not show full leak info (antirez) -2010-05-21 minor fix for the skiplist code, resulting in a false positive with valgrind, and in general into a useless small allocation (antirez) -2010-05-21 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-21 tests suite initial support for valgrind, fixed the old test suite until the new one is able to target a specific host/port (antirez) -2010-05-21 include solaris fixes in sha1.c (Pieter Noordhuis) -2010-05-20 Don't exit with error in tests temp file cleanup if there are no files to clean (antirez) -2010-05-20 fix memory leak on 32-bit builds (Pieter Noordhuis) -2010-05-20 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-05-20 Fix for DEBUG DIGEST (antirez) -2010-05-20 Merge branch 'test_vm' of git://github.com/pietern/redis (antirez) -2010-05-20 code to enable running tests with the vm enabled (Pieter Noordhuis) -2010-05-20 minor change to shutdown (antirez) -2010-05-20 shutdown on SIGTERM (antirez) -2010-05-20 Merge http://github.com/ngmoco/redis (antirez) -2010-05-20 fix compile error on solaris (Pieter Noordhuis) -2010-05-20 added regression for zipmap bug (antirez) -2010-05-20 fix lookup of keys with length larger than ZIPMAP_BIGLEN (Pieter Noordhuis) -2010-05-19 TODO updated (antirez) -2010-05-19 initial tests for AOF (and small changes to server.tcl to support these) (Pieter Noordhuis) -2010-05-19 Merge branch 'master' into integration (Pieter Noordhuis) -2010-05-19 Fix for 'CONFIG SET appendonly no' (antirez) -2010-05-19 It's now possible to turn off and on the AOF via CONFIG (antirez) -2010-05-18 git hash 00000000 in reelase.h when git is not found enabled again after some shell scripting fix that is now compatible with most shells (antirez) -2010-05-18 build fixed when simpler shells are used to create release.h (antirez) -2010-05-18 use git diff when generating release.h to check for dirty status (antirez) -2010-05-18 Solaris fixes (antirez) -2010-05-18 html doc rebuild (antirez) -2010-05-18 buliding of release.h moved into an external script. Avoided recompialtion of redis.c if git sha1 is the same as the previous one (antirez) -2010-05-17 create release.h in make process and add this information to INFO listing (Pieter Noordhuis) -2010-05-16 Redis version is now 1.3.12 (antirez) -2010-05-16 redis version is now 1.3.11 (antirez) -2010-05-16 random refactoring and speedups (antirez) -2010-05-16 faster INCR with very little efforts... (antirez) -2010-05-15 print warnings in redis log when a test raises an exception (very likely to be caused by something like a failed assertion) (Pieter Noordhuis) -2010-05-15 Merge branch 'redis-cli-fix' of http://github.com/tizoc/redis (antirez) -2010-05-15 added pid info to the check memory leaks test, so that those tests don't appear to be duplicated (antirez) -2010-05-15 Merge branch 'integration' of git://github.com/pietern/redis (antirez) -2010-05-14 more endianess detection fix for SHA1 (antirez) -2010-05-14 fixed a warning seen with some GCC version under Linux (antirez) -2010-05-14 initial rough integration test for replication (Pieter Noordhuis) -2010-05-14 store entire server object on the stack instead of just the client (Pieter Noordhuis) -2010-05-14 proc to retrieve values from INFO properties (Pieter Noordhuis) -2010-05-14 one more fix for endianess detection (antirez) -2010-05-14 Fixed sha1.c compilation on Linux, due to endianess detection lameness (antirez) -2010-05-14 ZUNION,ZINTER -> ZUNIONSTORE,ZINTERSTORE (antirez) -2010-05-14 minor fixes to the new test suite, html doc updated (antirez) -2010-05-14 wait for redis-server to be settled and ready for connections (Pieter Noordhuis) -2010-05-14 fix cleaning up tmp folder (Pieter Noordhuis) -2010-05-14 update makefile to use the new test suite (Pieter Noordhuis) -2010-05-14 check for memory leaks before killing a server (Pieter Noordhuis) -2010-05-14 extract code to kill a server to a separate proc (Pieter Noordhuis) -2010-05-14 start servers on different ports to prevent conflicts (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in new test suite (Pieter Noordhuis) -2010-05-14 split test suite into multiple files; runs redis-server in isolation (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in the test instead of a function that was doing a similar work, but in a much slower and buggy way (antirez) -2010-05-14 Don't rely on cliReadReply being able to return on shutdown (Bruno Deferrari) -2010-05-14 If command is a shutdown, ignore errors on reply (Bruno Deferrari) -2010-05-14 DEBUG DIGEST implemented, in order to improve the ability to test persistence and replication consistency (antirez) -2010-05-13 Add SIGTERM shutdown handling. (Ashley Martens) -2010-05-13 makefile deps updated (antirez) -2010-05-13 conflicts resolved (antirez) -2010-05-13 feed SETEX as SET and EXPIREAT to AOF (Pieter Noordhuis) -2010-05-13 very strong speedup in saving time performance when there are many integers in the dataset. Instead of decoding the object before to pass them to the rdbSaveObject layer we check asap if the object is integer encoded and can be written on disk as an integer. (antirez) -2010-05-13 include limits.h otherwise no double precison macros (antirez) -2010-05-13 explicitly checks with ifdefs if our floating point and long long assumptions are verified (antirez) -2010-05-13 Yet another version of the double saving code, with comments explaining what's happening there (antirez) -2010-05-12 added overflow check in the double -> long long conversion trick to avoid integer overflows. I think this was not needed in practical terms, but it is safer (antirez) -2010-05-12 use withscores when performing the dataset digest (antirez) -2010-05-12 If a float can be casted to a long long without rounding loss, we can use the integer conversion function to write the score on disk. This is a seriuous speedup (antirez) -2010-05-12 fixed compilation warnings in the AOF sanity check tool (antirez) -2010-05-12 Merge branch 'vm-speedup' (antirez) -2010-05-11 fix to return error when calling INCR on a non-string type (Pieter Noordhuis) -2010-05-11 load objects encoded from disk directly without useless conversion (antirez) -2010-05-11 fixed a problem leading to crashes, as keys can't be currently specially encoded, so we can't encode integers at object loading time... For now this can be fixed passing a few flags, or later can be fixed allowing encoded keys as well (antirez) -2010-05-11 long long to string conversion speedup applied in other places as well. Still the code has bugs, fixing right now... (antirez) -2010-05-11 hand written code to turn a long long into a string -> very big speed win (antirez) -2010-05-11 added specialized function to compare string objects for perfect match that is optimized for this task (antirez) -2010-05-11 better use of encoding inforamtion in dictEncObjKeyCompare (antirez) -2010-05-10 CONFIG now can change appendfsync policy at run time (antirez) -2010-05-10 CONFIG command now supports hot modification of RDB saving parameters. (antirez) -2010-05-10 while loading the rdb file don't add the key to the dictionary at all if it's already expired, instead of removing it just after the insertion. (antirez) -2010-05-10 Merge branch 'check-aof' of git://github.com/pietern/redis (antirez) -2010-05-08 minor changes to improve code readability (antirez) -2010-05-08 swap objects out directly while loading an RDB file if we detect we can't stay in the vm max memory limits anyway (antirez) -2010-05-07 change command names no longer used to zunion/zinter (Pieter Noordhuis) -2010-05-07 DEBUG POPULATE command for fast creation of test databases (antirez) -2010-05-07 update TODO (Pieter Noordhuis) -2010-05-07 swap arguments in blockClientOnSwappedKeys to be consistent (Pieter Noordhuis) -2010-05-07 added function that preloads all keys needed to execute a MULTI/EXEC block (Pieter Noordhuis) -2010-05-07 add sanity check to zunionInterBlockClientOnSwappedKeys, as the number of keys used is provided as argument to the function (Pieter Noordhuis) -2010-05-07 make prototype of custom function to preload keys from the vm match the prototype of waitForMultipleSwappedKeys (Pieter Noordhuis) -2010-05-07 extract preloading of multiple keys according to the command prototype to a separate function (Pieter Noordhuis) -2010-05-07 make append only filename configurable (Pieter Noordhuis) -2010-05-07 don't load value from VM for EXISTS (Pieter Noordhuis) -2010-05-07 swap file name pid expansion removed. Not suited for mission critical software... (antirez) -2010-05-07 Swap file is now locked (antirez) -2010-05-06 Merge branch 'master' into aof-speedup (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (Pieter Noordhuis) -2010-05-06 Merge branch 'master' into check-aof (Pieter Noordhuis) -2010-05-06 hincrby should report an error when called against a hash key that doesn't contain an integer (Pieter Noordhuis) -2010-05-06 AOF writes are now accumulated into a buffer and flushed into disk just before re-entering the event loop. A lot less writes but still this guarantees that AOF is written before the client gets a positive reply about a write operation, as no reply is trasnmitted before re-entering into the event loop. (antirez) -2010-05-06 clarified a few messages in redis.conf (antirez) -2010-05-05 ask for confirmation before AOF is truncated (Pieter Noordhuis) -2010-05-05 str can be free'd outside readString (Pieter Noordhuis) -2010-05-05 moved argument parsing around (Pieter Noordhuis) -2010-05-05 ignore redis-check-aof binary (Pieter Noordhuis) -2010-05-05 allow AOF to be fixed by truncating to the portion of the file that is valid (Pieter Noordhuis) -2010-05-05 tool to check if AOF is valid (Pieter Noordhuis) -2010-05-02 included fmacros.h in linenose.c to avoid compilation warnings on Linux (antirez) -2010-05-02 compilation fix for mac os x (antirez) -2010-05-02 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-02 On Linux now fdatasync() is used insetad of fsync() in order to flush the AOF file kernel buffers (antirez) -2010-04-30 More tests for APPEND and tests for SUBSTR (antirez) -2010-04-30 linenoise.c updated, now redis-cli can be used in a pipe (antirez) -2010-04-29 redis-cli minor fix (less segfault is better) (antirez) -2010-04-29 New MONITOR output format with timestamp, every command in a single line, string representations (antirez) -2010-04-29 redis-cli INFO output format is now raw again (antirez) -2010-04-29 Added more information about slave election in Redis Cluster alternative doc (antirez) -2010-04-29 Redis cluster version 2 (antirez) -2010-04-27 Fixed a redis-cli bug, was using free instead of zfree call (antirez) -2010-04-27 AOF is now rewritten on slave after SYNC with master. Thanks to @_km for finding this bug and any others' (antirez) -2010-04-27 redis-cli is now using only the new protocol (antirez) -2010-04-27 Minimal support for subscribe/psubscribe in redis-cli (antirez) -2010-04-26 don't output the newline when stdout is not a tty (antirez) -2010-04-26 redis-cli now is able to also output the string representation instead of the raw string. Much better for debugging (antirez) -2010-04-26 Initial support for quoted strings in redis-cli (antirez) -2010-04-23 SETEX implemented (antirez) -2010-04-23 Pub/Sub API change: now messages received via pattern matching have a different message type and an additional field representing the original pattern the message matched (antirez) -2010-04-22 typo fixed, reloaded (antirez) -2010-04-22 typo fixed (antirez) -2010-04-22 REDIS-CLUSTER doc updated (antirez) -2010-04-22 Virtual memory design document removed, no longer needed as we have a full specification and implementation (antirez) -2010-04-22 new units for bytes specification (antirez) -2010-04-22 Now in redis.conf it is possible to specify units where appropriate instead of amounts of bytes, like 2Gi or 4M and so forth (antirez) -2010-04-21 binary safe keys ready implementation of RANDOMKEYS (antirez) -2010-04-21 Now that's the right 1.3.10 (antirez) -2010-04-21 Revert "fsync always now uses O_DIRECT on Linux" (antirez) -2010-04-21 Revert "define __USE_GNU to get O_DIRECT" (antirez) -2010-04-21 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-04-21 Revert "version 1.3.10" (antirez) -2010-04-21 version 1.3.10 (antirez) -2010-04-20 define __USE_GNU to get O_DIRECT (antirez) -2010-04-20 fsync always now uses O_DIRECT on Linux (antirez) -2010-04-20 More precise memory used guesswork in zmalloc.c (antirez) -2010-04-19 Fix for MULTI/EXEC and Replication/AOF: now the block is correctly sent as MULTI/..writing operations../EXEC. Ok for slaves but more work needed for the AOF as it should be a write-all-or-nothing business (antirez) -2010-04-19 running the test using tcl8.5 directly instead of tclsh that too often it's a symlink to 8.4 (antirez) -2010-04-19 Added package require Tcl 8.5 in redis.tcl so it will show a clear error when the test suit is attempted to run under 8.4 (antirez) -2010-04-18 Fix for a SORT bug introduced with commit 16fa22f1, regression test added (antirez) -2010-04-18 Guru mediation -> meditation (antirez) -2010-04-16 check eptr inline (Pieter Noordhuis) -2010-04-16 refactor code that retrieves value from object or replies to client (Pieter Noordhuis) -2010-04-17 Merge branch 'hash' of git://github.com/pietern/redis (antirez) -2010-04-17 redisAssert(0) => redisPanic("something meaningful") (antirez) -2010-04-17 make sure that the resulting value in hincrby is encoded when possible (Pieter Noordhuis) -2010-04-17 increment dirty counter after hmset (Pieter Noordhuis) -2010-04-17 strip tryObjectEncoding from hashSet, to enable the arguments being encoded in-place (Pieter Noordhuis) -2010-04-17 Added support for Guru Mediation, and raising a guru mediation if refCount <= 0 but decrRefCount is called against such an object (antirez) -2010-04-16 fix small error and memory leaks in SORT (Pieter Noordhuis) -2010-04-16 SORT/GET test added (antirez) -2010-04-16 Added tests for GET/BY against hashes fields (antirez) -2010-04-16 Merge branch 'hash-refactor' of git://github.com/pietern/redis (antirez) -2010-04-16 check object type in lookupKeyByPattern (Pieter Noordhuis) -2010-04-16 make sortCommand aware that lookupKeyByPattern always increased the refcount of the returned value (Pieter Noordhuis) -2010-04-16 revert 0c390a to stop using tricks with o->refcount (Pieter Noordhuis) -2010-04-16 store the hash iterator on the heap instead of the stack (Pieter Noordhuis) -2010-04-16 drop inline directive (Pieter Noordhuis) -2010-04-16 rename hashReplace to hashSet (Pieter Noordhuis) -2010-04-16 added dictFetchValue() to dict.c to make hash table API a bit less verbose in the common cases (antirez) -2010-04-03 Don't set expire to keys with ttl=0, remove them immediately. (antirez) -2010-04-15 make sure that cmpobj is in decoded form when sorting by ALPHA (this solves edge case from previous commit where (!sortby && alpha) == 1) (Pieter Noordhuis) -2010-04-15 enable hash dereference in SORT on BY and GET (Pieter Noordhuis) -2010-04-15 use shared replies for hset (Pieter Noordhuis) -2010-04-15 set refcount of string objects retrieved from zipmaps to 0, so we don't have to touch the refcount of the objects inside dicts (Pieter Noordhuis) -2010-04-15 added HSETNX (Pieter Noordhuis) -2010-04-14 refactor of hash commands to use specialized api that abstracts zipmap and dict apis (Pieter Noordhuis) -2010-04-13 move retrieval of long up to prevent an empty hash from being created (Pieter Noordhuis) -2010-04-15 more advanced leaks detection in test redis (antirez) -2010-04-15 ability to select port/host from make test (antirez) -2010-04-15 Active rehashing (antirez) -2010-04-15 Incrementally rehahsing hash table! Thanks to Derek Collison and Pieter Noordhuis for feedbacks/help (antirez) -2010-04-14 Does not allow commands other than Pub/Sub commands when there is at least one pattern (antirez) -2010-04-13 Fixed a tiny memory leak when loading the configuration file. (Alex McHale) -2010-04-13 Merge branch 'hmget' of git://github.com/pietern/redis (antirez) -2010-03-29 Validate numeric inputs. (Alex McHale) -2010-03-24 Remove trailing whitespace. (Alex McHale) -2010-04-12 Now all the commands returning a multi bulk reply against non existing keys will return an empty multi bulk, not a nil one (antirez) -2010-04-12 implemented HMGET (Pieter Noordhuis) -2010-04-12 implemented HMSET (Pieter Noordhuis) -2010-04-12 Sharing of small integer objects: may save a lot of memory with datasets having many of this (antirez) -2010-04-10 dict.c fixed to play well with enabling/disabling of the hash table (antirez) -2010-04-09 removed a no longer true assert in the VM code (antirez) -2010-04-09 shareobjects feautres killed - no gains most of the time, but VM complexities (antirez) -2010-04-09 use directly the real key object in VM I/O jobs to match by pointer, and to handle different keys with the same name living in different DBs, but being at the same moment in the IO job queues (antirez) -2010-04-08 last change reverted as it was unstable... more testing needed (antirez) -2010-04-08 Prevent hash table resize while there are active child processes in order to play well with copy on write (antirez) -2010-04-08 Merge branch 'issue_218' of git://github.com/pietern/redis (antirez) -2010-04-08 -1 not needed... (antirez) -2010-04-08 Skiplist theoretical fix (antirez) -2010-04-07 Now when a child is terminated by a signal, the signal number is logged as well (antirez) -2010-04-07 First version of evented Redis Tcl client, that will be used for BLPOP and Pub/Sub tests (antirez) -2010-04-05 use long long reply type for HINCRBY (Pieter Noordhuis) -2010-04-05 last argument is never encoded for HINCRBY (Pieter Noordhuis) -2010-04-02 Now PUBLISH commands are replicated to slaves (antirez) -2010-04-01 use the right object when cleaning up after zunion/zinter (fixes issue 216) (Pieter Noordhuis) -2010-04-01 Merge branch 'zipmap' of git://github.com/pietern/redis (antirez) -2010-04-01 reduce code complexity because zipmapLen now is O(1) (Pieter Noordhuis) -2010-04-01 update the zipmap entry in-place instead of appending it (Pieter Noordhuis) -2010-04-01 updated zipmap documentation to match the implementation (Pieter Noordhuis) -2010-04-01 allow 4 free trailing bytes for each value (Pieter Noordhuis) -2010-04-01 Pub/Sub pattern matching capabilities (antirez) -2010-04-01 use function to determine length of a single entry (Pieter Noordhuis) -2010-03-31 Deny EXEC under out of memory (antirez) -2010-03-29 No timeouts nor other commands for clients in a Pub/Sub context (antirez) -2010-03-29 free hash table entries about no longer active classes, so that PUBSUB can be abused with millions of different classes (antirez) -2010-03-29 Fixed a refcount stuff leading to PUBSUB crashes (antirez) -2010-03-29 fmacros added to linenoise, avoiding all the nice warnings... (antirez) -2010-03-29 First pubsub fix (antirez) -2010-03-29 PUBSUB implemented (antirez) -2010-03-29 Redis version is now 1.3.8 (antirez) -2010-03-28 removed references in code to ZIPMAP_EMPTY (Pieter Noordhuis) -2010-03-28 use first byte of zipmap to store length (Pieter Noordhuis) -2010-03-28 implemented strategy that doesn't use free blocks in zipmaps (Pieter Noordhuis) -2010-03-26 Merge branch 'hincrby' of git://github.com/pietern/redis (antirez) -2010-03-26 removed unnecessary refcount increase that caused the HINCRBY memleak (Pieter Noordhuis) -2010-03-26 implements HINCRBY and tests (todo: find and fix small memleak) (Pieter Noordhuis) -2010-03-26 Removed a useless if spotted by Pieter Noordhuis (antirez) -2010-03-26 Fixed a critical replication bug: binary values issued with the multi bulk protocol caused a protocol desync with slaves. (antirez) -2010-03-24 Fixed the reply about denied write commands under maxmemory reached condition: now the error will no longer lead to a client-server protocol desync (antirez) -2010-03-24 CONFIG command implemened -- just a start but already useful (antirez) -2010-03-24 redis-cli prompt is now redis> (antirez) -2010-03-23 with --help states that you can use - as config file name to feed config via stdin (antirez) -2010-03-23 New INFO field: expired_keys (antirez) -2010-03-23 the Cron timer function is now called 10 times per second instead of 1 time per second to make Redis more responsibe to BGSAVE and to delete expired keys more incrementally (antirez) -2010-03-23 Use linenoise for line editing on redis-cli. (Michel Martens) -2010-03-23 Fix authentication for redis-cli on non-interactive mode. (Michel Martens) -2010-03-23 key deletion on empty value fix + some refactoring (antirez) -2010-03-23 Empty value trigger key removal in all the operations (antirez) -2010-03-22 Merged gnrfan patches fixing issues 191, 193, 194 (antirez) -2010-03-22 Merge branch 'issue_193' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Merge branch 'issue_191' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Redis master version is now 1.3.7 (antirez) -2010-03-19 support for include directive in config parser (Jeremy Zawodny) -2010-03-19 Removed a stupid overriding of config values due to a wrong cut&paste (antirez) -2010-03-19 VM hash type swappability implemented. Handling of failed pthread_create() call. (antirez) -2010-03-19 Solving issue #191 on Google Code: -v and --version should print the version of Redis (Antonio Ognio) -2010-03-19 Solves issue #194 on Google Code: --help parameter to redis-srver prints the usage message (Antonio Ognio) -2010-03-19 Fixing issue 193 (Antonio Ognio) -2010-03-18 increment server.dirty on HDEL (antirez) -2010-03-18 Redis 1.3.6 (antirez) -2010-03-18 test-redis.tcl dataset digest function Hash support (antirez) -2010-03-18 zipmap fix for large values (antirez) -2010-03-18 Optimization fixed and re-activated (antirez) -2010-03-18 reverted an optimization that makes Redis not stable (antirez) -2010-03-18 Fixed redis-cli auth code (antirez) -2010-03-17 HDEL fix, an optimization for comparison of objects in hash table lookups when they are integer encoding (antirez) -2010-03-17 Version is now 1.3.5 (antirez) -2010-03-17 Merged Pietern patch for VM key args helper function. Fixed an obvious bug in the redis-cli passwd auth stuff (antirez) -2010-03-17 Merge branch 'aggregates' of git://github.com/pietern/redis (antirez) -2010-03-17 Added Authentication to redis-cli.c using -a switch Update usage fixed Makefile to delete redis-check-dump during make clean (root) -2010-03-17 HEXISTS and tests implemented (antirez) -2010-03-17 More hash tests (antirez) -2010-03-17 better HSET test (antirez) -2010-03-17 Fixed a bug in HSET, a memory leak, and a theoretical bug in dict.c (antirez) -2010-03-17 More Hash tests (antirez) -2010-03-13 added preloading keys from VM when using ZINTER or ZUNION (Pieter Noordhuis) -2010-03-13 added explicit AGGREGATE [SUM|MIN|MAX] option to ZUNION/ZINTER (Pieter Noordhuis) -2010-03-16 HGET fix for integer encoded field against zipmap encoded hash (antirez) -2010-03-16 zrevrank support in redis-cli (antirez) -2010-03-16 HKEYS / HVALS / HGETALL (antirez) -2010-03-16 Solved a memory leak with Hashes (antirez) -2010-03-15 pretty big refactoring (antirez) -2010-03-15 An interesting refactoring + more expressive internal API (antirez) -2010-03-15 Fixed the same problem in ZREVRANK (antirez) -2010-03-15 Fixed a ZRANK bug (antirez) -2010-03-15 zipmap to hash conversion in HSET (antirez) -2010-03-14 max zipmap entries and max zipmap value parameters added into INFO output (antirez) -2010-03-14 HDEL and some improvement in DEBUG OBJECT command (antirez) -2010-03-14 Append only file support for hashes (antirez) -2010-03-13 utility to check rdb files for unprocessable opcodes (Pieter Noordhuis) -2010-03-12 A minor fix and a few debug messages removed (antirez) -2010-03-12 Applied the replication bug patch provided by Jeremy Zawodny, removing temp file collision after the slave got the dump.rdb file in the SYNC stage (antirez) -2010-03-11 Fix for HGET against non Hash type, debug messages used to understand a bit better a corrupted rdb file (antirez) -2010-03-09 fix: use zmalloc instead of malloc (Pieter Noordhuis) -2010-03-09 Merged zsetops branch from Pietern (antirez) -2010-03-09 Merged ZREMBYRANK from Pietern (antirez) -2010-03-09 Merged ZREVRANK from Pietern (antirez) -2010-03-09 use a struct to store both a dict and its weight for ZUNION and ZINTER, so qsort can be applied (Pieter Noordhuis) -2010-03-09 Hash auto conversion from zipmap to hash table, type fixed for hashes, hash loading from disk (antirez) -2010-03-09 replaced ZMERGE by ZUNION and ZINTER. note: key preloading by the VM does not yet work (Pieter Noordhuis) -2010-03-08 Hashes saving / fixes (antirez) -2010-03-08 use ZMERGE as starting point (Pieter Noordhuis) -2010-03-07 HSET fixes, now the new pointer is stored back in the object pointer field (antirez) -2010-03-07 added ZREVRANK (Pieter Noordhuis) -2010-03-06 Fix for replicaiton with over 2GB dump file initial SYNC stage (antirez) -2010-03-06 first implementation of HSET/HSET. More work needed (antirez) -2010-03-05 zipmaps functions to get, iterate, test for existence. Initial works for Hash data type (antirez) -2010-03-04 redis-benchmark now implements Set commands benchmarks (antirez) -2010-03-04 zipmap iteration code (antirez) -2010-03-04 moved code to delete a single node from a zset to a separate function (Pieter Noordhuis) -2010-03-04 rename zslDeleteRange to zslDeleteRangeByScore (to differentiate between deleting using score or rank) (Pieter Noordhuis) -2010-03-04 use 1-based rank across zsl*Rank functions consistently (Pieter Noordhuis) -2010-03-04 implemented ZREMBYRANK (Pieter Noordhuis) -2010-03-04 A fix for initialization of augmented skip lists (antirez) -2010-03-04 A fix for an invalid access when VM is disabled (antirez) -2010-03-04 Merge branch 'zsl-get-rank' of git://github.com/pietern/redis (antirez) -2010-03-04 redis-cli now runs in interactive mode if no command is provided (antirez) -2010-03-04 merged memory reduction patch (Pieter Noordhuis) -2010-03-04 Now list push commands return the length of the new list, thanks to Gustavo Picon (antirez) -2010-03-04 first check if starting point is trivial (head or tail) before applying log(N) search (Pieter Noordhuis) -2010-03-04 use rank to find starting point for ZRANGE and ZREVRANGE (Pieter Noordhuis) -2010-03-04 lookup rank of a zset entry in a different function (Pieter Noordhuis) -2010-03-04 SUBSTR fix for integer encoded vals (antirez) -2010-03-04 fix ZRANK (realize that rank is 1-based due to the skip list header) (Pieter Noordhuis) -2010-03-03 initial implementation of SUBSTR (antirez) -2010-03-03 TODO updated (antirez) -2010-03-03 fpurge call removed from redis-cli (antirez) -2010-03-03 ZRANK stress tester (antirez) -2010-03-03 use less memory as element->span[0] will always be 1; any level 0 skip list is essentially a linked list (Pieter Noordhuis) -2010-03-03 rank is very unlikely to overflow integer range (Pieter Noordhuis) -2010-03-03 x->backward never equals zsl->header (Pieter Noordhuis) -2010-03-03 initial implementation for augmented zsets and the zrank command (Pieter Noordhuis) -2010-03-03 zipampDel() implemented (antirez) -2010-03-03 added quit and exit commands to redis-cli in order to quit the interactive mode (antirez) -2010-03-03 Merge remote branch 'djanowski/interactive' (antirez) -2010-03-02 Add support for MULTI/EXEC. (Damian Janowski & Michel Martens) -2010-03-02 Remove trailing newline in interactive mode. (Damian Janowski & Michel Martens) -2010-03-02 minor fix for a Linux warning (antirez) -2010-03-02 Add interactive mode to redis-cli. (Michel Martens & Damian Janowski) -2010-03-02 Better to increment the version minor number when a VM bug is fixed... it will be simpler to understand what's going on when users will report problems with the INFO trace. (antirez) -2010-03-02 Fixed a subtle VM bug... was not flushing the buffer so the child process read truncated data (antirez) -2010-03-01 KEYS now returns a multi bulk reply (antirez) -2010-02-27 Add DISCARD command to discard queued MULTI commands. (antirez) -2010-03-01 Swappability bug due to a typo fixed thanks to code review by Felix Geisendörfer @felixge (antirez) -2010-02-28 minor fixes for zipmap.c (antirez) -2010-02-27 first zipmap fix of a long sequence in the days to come ;) (antirez) -2010-02-27 initial zipmap.c implementation (antirez) -2010-02-27 Bug #169 fixed (BLOP/BRPOP interrupted connections are not cleared from the queue) (antirez) -2010-02-22 Fixed 32bit make target to work on Linux out of the box (antirez) -2010-02-19 A problem with replication with multiple slaves connectiong to a single master fixed. It was due to a typo, and reported on github by the user micmac. Also the copyright year fixed from many files. (antirez) -2010-02-10 Saner VM defaults for redis.conf (antirez) -2010-02-09 VM now is able to block clients on swapped keys for all the commands (antirez) -2010-02-07 ZCOUNT and ZRANGEBYSCORE new tests (antirez) -2010-02-07 ZRANGEBYSCORE now supports open intervals, prefixing double values with a open paren. Added ZCOUNT that can count the elements inside an interval of scores, this supports open intervals too (antirez) -2010-02-07 WITHSCORES in ZRANGEBYSCORE thanks to Sam Hendley (antirez) -2010-02-06 Added "withscores" option to zrangebyscore command. Based on withscores support in zrange function, ugliest part was the argument parsing to handle using it with the limit option. (Sam Hendley) -2010-02-06 DEBUG OBJECT provide info about serialized object length even when VM is disabled (antirez) -2010-02-06 multi bulk requests in redis-benchmark, default fsync policy changed to everysec, added a prefix character for DEBUG logs (antirez) -2010-02-04 APPEND tests (antirez) -2010-02-04 APPEND command (antirez) -2010-02-02 Faster version of the function hashing possibly encoded objects, leading to a general speed gain when working with Sets of integers (antirez) -2010-02-02 faster Set loading time from .rdb file resizing the hash table to the right size before loading elements (antirez) -2010-02-02 Log time taken to load the DB at startup, in seconds (antirez) -2010-01-31 Fixed VM corruption due to child fclosing the VM file directly or indirectly calling exit(), now replaced with _exit() in all the sensible places. Masked a few signals from IO threads. (antirez) -2010-01-28 loading side of the threaded VM (antirez) -2010-01-26 TODO cahnges (antirez) -2010-01-23 Fixed memory human style memory reporting, removed server.usedmemory, now zmalloc_used_memory() is used always. (antirez) -2010-01-22 VM tuning thanks to redis-stat vmstat. Now it performs much better under high load (antirez) -2010-01-21 Changelog updated (antirez) -2010-01-21 REDIS_MAX_COMPLETED_JOBS_PROCESSED is now in percentage, not number of jobs. Moved a debugging message a few lines forward as it was called where a few logged parameters where invalid, leading to a crash (antirez) -2010-01-20 fixed a deadlock caused by too much finished processes in queue so that I/O clients writing to the wirte side of the pipe used to awake the main thread where blocking. Then a BGSAVE started waiting for the last active thread to finish, condition impossible because all the I/O threads where blocking on threads. Takes this as a note to myself... (antirez) -2010-01-20 ae.c event loop does no longer support exception notifications, as they are fully pointless. Also a theoretical bug that never happens in practice fixed. (antirez) -2010-01-19 commercial tools stuff removed from the Redis makefile. cotools are now migrated into a different repos (antirez) -2010-01-19 removed a bug in the function to cancel an I/O job (antirez) -2010-01-17 static symbols update (antirez) -2010-01-16 removed support for REDIS_HELGRIND_FRIENDLY since Helgrind 3.5.0 is friendly enough even with many threads created and destroyed (antirez) -2010-01-15 now redis-cli understands -h (antirez) -2010-01-15 Create swap file only if not exists (antirez) -2010-01-15 I hate warnings (antirez) -2010-01-15 fixed a minor memory leak in configuration file parsing (antirez) -2010-01-15 minor fix (antirez) -2010-01-15 support for named VM swap file. Fixed a few important interaction issues between the background saving processes and IO threads (antirez) -2010-01-15 fix for the just added new test (antirez) -2010-01-15 useless debugging messages removed (antirez) -2010-01-15 new test added (antirez) -2010-01-15 thread safe zmalloc used memory counter (antirez) -2010-01-15 A define to make Redis more helgrind friendly (antirez) -2010-01-15 removed a few races from threaded VM (antirez) -2010-01-14 Fixed a never experienced, theoretical bug that can actually happen in practice. Basically when a thread is working on a I/O Job we need to wait it to finish before to cancel the Job in vmCancelThreadedIOJob(), otherwise the thread may mess with an object that is being manipulated by the main thread as well. (antirez) -2010-01-14 Set the new threads stack size to a LZF friendly amount (antirez) -2010-01-13 access to already freed job structure fixed by statements reoredering (antirez) -2010-01-13 removed a useless debugging message (antirez) -2010-01-13 Wait zero active threads condition before to fork() for BGSAVE or BGREWRITEAOF (antirez) -2010-01-13 list API is now thread safe (antirez) -2010-01-13 minor TODO and debugging info changes (antirez) -2010-01-12 support for blocking VM in config file (antirez) -2010-01-12 more non blocking VM changes (antirez) -2010-01-12 fix for test #11 (antirez) -2010-01-12 a few more stuff in INFO about VM. Test #11 changed a bit in order to be less lame (antirez) -2010-01-12 Added a define to configure how many completed IO jobs the handler should process at every call. (antirez) -2010-01-11 Fixed a bug in the IO Job canceling funtion (antirez) -2010-01-11 more steps towards a working non blocking VM (antirez) -2010-01-11 converted random printfs in debug logs (antirez) -2010-01-11 removed a bug introduced with non blocking VM (antirez) -2010-01-11 a few non blocking VM bugs fixed (antirez) -2010-01-11 More work on non-blocking VM. Should work in a few days (antirez) -2010-01-11 More threaded I/O VM work + Redis init script (antirez) -2010-01-10 more work on VM threaded I/O. Still nothing of usable (antirez) -2010-01-09 non-blocking VM data structures, just a start (antirez) -2010-01-08 used_memory_human added to INFO output. Human readable amount of memory used. (antirez) -2010-01-07 Now DEBUG OBJECT plays well with swapped out objects (antirez) -2010-01-07 fflush VM swap file after object swapping (antirez) -2010-01-07 added the fmacros to enable support for fseeko() lseeko() with 64bit off_t (antirez) -2010-01-07 VM now swaps objects out while loading datasets not fitting into vm-max-memory bytes of RAM (antirez) -2010-01-07 added process id information in INFO (antirez) -2010-01-06 vm-enabled set to no by default in redis.conf (antirez) -2010-01-06 a new default redis.conf (antirez) -2010-01-06 VM stats in INFO command (antirez) -2010-01-06 Introduced a new log verbosity level, so now DEBUG is really for debugging. Refactored a bit maxmemory. When virtual memory is short in RAM free the objects freelist as well as swapping things out. (antirez) -2010-01-05 fixed a bug in bgsave when VM is off but still it was testing for obj->storage field (antirez) -2010-01-05 converted a few calls to assert() => redisAssert() to print stack trace (antirez) -2010-01-05 BGREWRITEAOF now works with swapping on (antirez) -2010-01-05 A first fix for SET key overwrite (antirez) -2010-01-05 SAVE now works with VM (antirez) -2010-01-05 swapping algorithm a bit more aggressive under low memory (antirez) -2010-01-05 basic VM mostly working! (antirez) -2010-01-05 New object field (one of the unused bytes) to hold the type of the swapped out value object in key objects (antirez) -2010-01-05 VM internals bugfixes, set 1 (antirez) -2010-01-05 load key from swap on key lookup (antirez) -2010-01-05 more object-level VM primitives (antirez) -2010-01-05 Redis objects swapping / loading (antirez) -2010-01-05 rdbLoadObject() as a separated function to load objects from disk. Dropped support for RDB version 0, I guess no longer has this legacy DBs around (antirez) -2010-01-04 VM low level pages handling (antirez) -2010-01-04 vm swap file creation, and some basic configuration (antirez) -2010-01-04 version marked 1.3.2 (antirez) -2010-01-04 saving code refactored a bit, added a function returning the number of bytes an object will use on disk (antirez) -2010-01-02 Now the PUSH side of RPOPLPUSH is able to unblock clients blocked on BLPOP (antirez) -2010-01-02 Version is now 1.3.1 (antirez) -2010-01-02 New vararg BLPOP able to block against multiple keys (antirez) -2009-12-29 fixed a problem with BLPOP timeout of zero, now it blocks forever (antirez) -2009-12-29 BLPOP timeouts implemented (antirez) -2009-12-29 first working implementation of BLPOP and BRPOP, still everything is to test well (antirez) -2009-12-29 a few more fixes, still broken (antirez) -2009-12-29 First fix, still broken (antirez) -2009-12-29 minor fix for Linux 64 bit (antirez) -2009-12-29 not yet working BLPOP implementation (antirez) -2009-12-27 AOFSYNC removed, got a better idea... (antirez) -2009-12-27 AOFSYNC command implemented (antirez) -2009-12-27 Version changed to 1.3.0, welcome to the new unstable (antirez) -2009-12-27 Now MULTI returns +OK as well (antirez) -2009-12-27 MULTI/EXEC first implementation (antirez) -2009-12-24 Fixed a minor bug in GETSET, now the SET part is not performed if the GET fails because the key does not contain a string value (antirez) -2009-12-23 html doc readded (antirez) -2009-12-23 ZRANGE WITHSCORES test added (antirez) -2009-12-23 version is now 1.1.94 (antirez) -2009-12-23 Add the command name in the unknown command error message. (antirez) -2009-12-22 ZRANGE, ZREVRANGE now support WITHSCORES options (antirez) -2009-12-22 html docs update (ZINCRBY added) (antirez) -2009-12-18 TODO list update (antirez) -2009-12-18 the pipelining test was ran against DB 1 for error, now it runs on DB 9 like all the other tests (antirez) -2009-12-18 still more tests (antirez) -2009-12-18 SORT STORE test added (antirez) -2009-12-18 Now SORT returns an empty bulk reply if the key does not exist (antirez) -2009-12-18 modified a bit the ZREVRANGE test to cover a few lines of code more (antirez) -2009-12-18 SHUTDOWN now does the right thing when append only is on, that is, fsync instead to save the snapshot. (antirez) -2009-12-18 Added a missing server.dirty increment in a non critical place, added more tests (antirez) -2009-12-18 LTRIM stress testing test added (antirez) -2009-12-18 LTRIM now returns +OK against non existing keys. More tests in test-redis.tcl (antirez) -2009-12-18 added sdstoupper() declaration in sds.h (antirez) -2009-12-18 Fixed sds.c bug #124 (antirez) -2009-12-16 LZF compression re-enabled by default, but with INIT_HTAB set to 0 to avoid the very costly memset initialization. Note that with this option set valgrind will output some false positive about lzf_c.c (antirez) -2009-12-16 lzf compression switched off by default now, with config file option to enable it in redis.conf (antirez) -2009-12-16 Regression for epoll bug in redis-test.tcl, version is now 1.1.93 (antirez) -2009-12-16 Fixed a lame epoll issue (antirez) -2009-12-15 html doc updated (antirez) -2009-12-15 version is now 1.1.92 (antirez) -2009-12-15 Two important fixes to append only file: zero length values and expires. A pretty neat new test to check consistency of randomly build datasets against snapshotting and AOF. (antirez) -2009-12-15 debug loadaof implemented in order to add more consistency tests in test-redis.tcl (antirez) -2009-12-15 Added a new test able to stress a lot the snapshotting engine (antirez) -2009-12-15 Unified handling of empty queries with normal queries. (antirez) -2009-12-15 Fixed some subtle bug in the command processing code almost impossible to spot in the real world, thanks to gcov (antirez) -2009-12-15 Regression test for SINTERSTORE added (antirez) -2009-12-15 Fixed issue #121 (antirez) -2009-12-14 a few more tests and ability to run a specific test in test-redis.tcl (antirez) -2009-12-13 Changed the reply of BGSAVE and BGREWRITEAOF from +OK to a more meaningful message that makes the user aware of an operation that just started and is not yet finished. (antirez) -2009-12-13 Set the master->slave logical client as authenticated on creation, so that if the slave requires a password replication works anyway (antirez) -2009-12-13 TODO update (antirez) -2009-12-12 bgrewriteaof_in_progress added to INFO (antirez) -2009-12-12 TODO list modified. What's planned for 1.4 is now written in the stone ;) (antirez) -2009-12-12 better handling of non blocking connect on redis-benchmark: EPIPE on read does not print an error message now (antirez) -2009-12-11 some change to redis-sha1.rb utility to make it more robust against non-meaningful changes in the dataset (antirez) -2009-12-10 redis-sha1.rb utility updated (antirez) -2009-12-10 a bit more verbose -ERR wrong number o arguments error, now gives info about the command name causing the error (antirez) -2009-12-10 TODO change and minor SETNX optimization (antirez) -2009-12-06 in rdbLoadDoubleValue now the buffer is nul terminated correctly. Thanks valgrind. (antirez) -2009-12-06 printf format warnings fixed by casting (antirez) -2009-12-06 Regression tests for SETNX and MSETNX bugs added (antirez) -2009-12-06 SETNX and MSETNX now respect the delete-on-write operation of EXPIREing keys (antirez) -2009-12-06 Fixed daemonization when using kqueue/kevent. Now the server initialization is performed *after* the daemonization (antirez) -2009-12-05 more HTML doc changes (antirez) -2009-12-05 HTML doc update (antirez) -2009-12-05 a few redis-cli format specified fixed (antirez) -2009-12-05 use __attribute__ format in sdscatprintf() when the compiler is GCC. Fixed format bugs resulting from the new warnings. (antirez) -2009-12-01 TODO update (antirez) -2009-12-01 compilation problem on 64bit mac os x 10.5 possibly fixed (antirez) -2009-12-01 virtual memory design doc typos (antirez) -2009-12-01 design documents added to the project (antirez) -2009-11-30 Fixed issued #85 (getDecodedObject: Assertion 1 != 1 failed. While sorting a set), added a smarter assert() function to dump the stacktrace, provided a macro to initalize Redis objects on the stack to avoid this kind of bugs. (antirez) -2009-11-30 fixed a subtle bug in redis-cli not having visible effects (antirez) -2009-11-29 TODO updated (antirez) -2009-11-29 Version chagned to 1.100, also known as the first first 2.0 beta version (antirez) -2009-11-29 more tests in test-redis.tcl, some minor fix (antirez) -2009-11-29 SORT support for sorted sets (antirez) -2009-11-28 Implemented LIMIT option in ZRANGEBYSCORE. We now enter feature-freeze (antirez) -2009-11-28 Changelog updated (antirez) -2009-11-28 html doc updated (antirez) -2009-11-28 enable kqueue/kevent only for Mac OS X 10.6.x as it seems that 10.5.x has a broken implementation of this syscalls. (antirez) -2009-11-28 TODO updated (antirez) -2009-11-28 ZRANGEBYSCORE fuzzy test (antirez) -2009-11-28 ZRANGEBYSCORE memory leak fixed, ZRANGEBYSCORE initial test added (antirez) -2009-11-28 INFO refactored. Stack trace on memory corruption now dumps the same information as the INFO command (antirez) -2009-11-28 ifdefs added to use kevent on Free Open and Net BSD as well. INFO and ae.c modified in order to report the multiplexing API in use (antirez) -2009-11-28 Enabled object encoding for multiple keys in MSET. Added a test for memory leaks in test-redis.tcl when running on Mac OS X (antirez) -2009-11-28 Merge branch 'kqueue' of git://github.com/mallipeddi/redis (antirez) -2009-11-28 Changes to TODO list, commented a function in redis.c (antirez) -2009-11-28 Added support for kqueue. (Harish Mallipeddi) -2009-11-27 TODO updated (antirez) -2009-11-26 zero length bulk data reading fixed in loadAppendOnlyFile() (antirez) -2009-11-26 append only file fixes (antirez) -2009-11-26 log rebuilding, random refactoring, work in progress please wait for an OK commit before to use this version (antirez) -2009-11-24 DEBUG RELOAD implemented, and test-redis.tcl modified to use it to check for persistence consistency. (antirez) -2009-11-24 Redis version set to 1.07 (antirez) -2009-11-24 sorted sets saving fixed (antirez) -2009-11-24 minor TODO change (antirez) -2009-11-24 minor fix to avoid a false valgrind warning. (antirez) -2009-11-23 epoll support enabled by default for Linux builds (antirez) -2009-11-23 epoll module for ae.c implemented. Some more testing needed (antirez) -2009-11-23 commented the HAVE_EPOLL test in config.h to allow compilation under Linux now that the epoll module is still missing (antirez) -2009-11-23 ae_select module added (antirez) -2009-11-23 ae.c now supports multiple polling API modules, even if only ae_select.c is implemented currently. Also adding and removing an event is now O(1). (antirez) -2009-11-23 ae.c initial refactoring for epoll implementation (antirez) -2009-11-21 version incremented up to 1.06 (antirez) -2009-11-21 TODO aesthetic changes (antirez) -2009-11-21 TODO updated with plans up to 1.5 (antirez) -2009-11-21 SRANDMEMBER test (antirez) -2009-11-21 Fixed a SORT memory leak that should never happen in practice (antirez) -2009-11-21 SORT GET # implemented, with a test (antirez) -2009-11-21 EXPIREAT test (antirez) -2009-11-20 EXPIRE tests (antirez) -2009-11-20 more RPOPLPUSH tests (antirez) -2009-11-20 RPOPLPUSH tests added (antirez) -2009-11-20 ZINCRBY return value fixed (antirez) -2009-11-20 ZINCRSCOREBY => ZINCRBY (antirez) -2009-11-19 ZINCRSCOREBY implemented (antirez) -2009-11-19 writev() finally uncommented again (antirez) -2009-11-19 redis-benchmark hopefully last bug with multi bulk reply fixed (antirez) -2009-11-19 debug mode in redis-bench (antirez) -2009-11-19 Use writev(2) if glue output buffers is disabled (antirez) -2009-11-19 benchmark.c fixes (antirez) -2009-11-18 more experiments with long replies, glue output buffer, and writev. (antirez) -2009-11-18 benchmarking with different number of LRANGE elements. Ability to change the glue output buffer limit by #define (antirez) -2009-11-18 more writev tests/work (antirez) -2009-11-18 redis-benchmark multi bulk reply support hopefully fixed (antirez) -2009-11-17 support for writev implemented but currently ifdef-ed in order to understan why I can't see the improvements expected. Btw code provided by Stefano Barbato (antirez) -2009-11-17 multi-bulk reply support for redis-bench, and as a result LRANGE is not tested, providing some number for the tuning of multi-bulk requests performances server-side (antirez) -2009-11-12 Solaris fix thanks to Alan Harder (antirez) -2009-11-12 Merge git://github.com/ianxm/redis (antirez) -2009-11-12 ZSCORE fixed, now returns NULL on missing key or missing element (antirez) -2009-11-12 Redis test will not fail the SAVE test even if a background save is in progress (antirez) -2009-11-12 LPOPPUSH renamed into RPOPLPUSH (antirez) -2009-11-11 can select db num (ian) -2009-11-11 Workaround for test-redis.tcl and Tcl 8.4.x about ZSCORE test (antirez) -2009-11-11 Removed a long time warning compiling with recent GCC on Linux (antirez) -2009-11-11 TODO updated (antirez) -2009-11-11 LPUSHPOP first implementation (antirez) -2009-11-10 Tcl script, make target, and redis.c changes to build the static symbol table automagically (antirez) -2009-11-10 Implemented a much better lazy expiring algorithm for EXPIRE (antirez) -2009-11-10 Fixed issue 92 in redis: redis-cli (nil) return value lacks CR/LF (antirez) -2009-11-10 Minor TODO change with new expiring algorithm description. New expiring algorithm moved since it'll go in 1.1 (antirez) -2009-11-04 redis-test is now a better Redis citizen, testing everything against DB 9 and 10 and only if this DBs are empty. (antirez) -2009-11-04 fixed a refcounting bug with SORT ... STORE leading to random crashes (root) -2009-11-04 masterauth option merged, thanks to Anthony Lauzon (antirez) -2009-11-03 ZSets double to string serialization fixed (antirez) -2009-11-03 client-libraries directory readded (antirez) -2009-11-03 redis.tcl put at toplevel since it's uesd for the test-redis.tcl script (antirez) -2009-11-03 client libs removed from Redis git (antirez) -2009-11-03 redis-cli now accepts a -r (repeat) switch. Still there is a memory leaks to fix (antirez) -2009-11-01 TODO updated again (antirez) -2009-11-01 TODO updated (antirez) -2009-11-01 redis-cli now makes clear when the returned string is an integer (antirez) -2009-11-01 SORT STORE option (antirez) -2009-11-01 now Redis prints DB stats just after the startup without to wait a second for the first report (antirez) -2009-11-01 another fix for append only mode, now read-only operations are not appended (antirez) -2009-11-01 appendfsync parsing in config file fixed. If you benchmarked Redis against different appendfsync options is time to try again ;) (antirez) -2009-11-01 append only file loading fixed (antirez) -2009-11-01 first version of append only file loading -- STILL BROKEN don't use it (antirez) -2009-10-31 Fixed Issue 83:Using TYPE on a zset results in a malformed response from the Redis server (antirez) -2009-10-31 Fixed compilation on Linux (antirez) -2009-10-30 append only mode is now able to translate EXPIRE into EXPIREAT transparently (antirez) -2009-10-30 appendfsync is now set to NO by default (antirez) -2009-10-30 support for appendonly mode no, always, everysec (antirez) -2009-10-30 first fix for append only mode (antirez) -2009-10-30 Initial implementation of append-only mode. Loading still not implemented. (antirez) -2009-10-30 EXPIRE behaviour changed a bit, a negative TTL or an EXPIREAT with unix time in the past will now delete the key. It seems saner to me than doing nothing. (antirez) -2009-10-30 EXPIREAT implemented, will be useful for the append-only mode (antirez) -2009-10-29 Fixed Issue 74 (ERR just returned on invalid password), now the error message is -ERR invalid password. (antirez) -2009-10-29 Fixed issue 72 (SLAVEOF shutdowns redis-server on malformed reply) (antirez) -2009-10-29 Fixed issue 77 (Incorrect time in log files) thanks to youwantalex (antirez) -2009-10-29 Fixed Issue 76 (redis-server crashes when it can't connect to MASTER and client connects to SLAVE) (antirez) -2009-10-29 ZREMRANGEBYSCORE implemented. Remove a range of elements with score between min and max (antirez) -2009-10-28 TODO changes and mostly theoretical minor skiplist change (antirez) -2009-10-28 ZLEN renamed ZCARD for consistency with SCARD (antirez) -2009-10-27 TODO reworked to reflect the real roadmap (antirez) -2009-10-27 Fix for 'make 32bit' (antirez) -2009-10-27 a fix for the solaris fix itself ;) (antirez) -2009-10-27 More Solaris fixes (antirez) -2009-10-27 A lot of ZSETs tests implemented, and a bug fixed thanks to this new tests (antirez) -2009-10-27 zmalloc Solaris fixes thanks to Alan Harder (antirez) -2009-10-27 ZSCORE implemented (antirez) -2009-10-26 fix for ZRANGEBYSCORE (antirez) -2009-10-26 ZRANGEBYSCORE implemented. Redis got range queries! (antirez) -2009-10-26 A trivial change makes the new implementation O(log(N)) instead of O(log(N))+O(M) when there are M repeated scores! (antirez) -2009-10-26 ZSET now saved on disk like any other type (antirez) -2009-10-26 double serialization routines implemented (antirez) -2009-10-26 ZSETs random fixes. Now the implementation appears to be pretty stable (antirez) -2009-10-26 another leak fixed. Can't find more for now, but still a bug in ZSETs to fix (antirez) -2009-10-26 ZSETs memory leak #1 solved, another one missing (antirez) -2009-10-26 Fix for skiplists backward link (antirez) -2009-10-26 Merged Solaris patches provided by Alan Harder (antirez) -2009-10-26 backward support to skiplists for ZREVRANGE, still broken, committing since I've to merge the Solaris patches (antirez) -2009-10-26 TODO updated (antirez) -2009-10-26 ZREM implemented (antirez) -2009-10-24 fix for ZADD in score update mode (antirez) -2009-10-24 some work on ZADD against existing element (score update), still broken... (antirez) -2009-10-23 zrange now starts to work. zadd still does not support update and will crash or leak or b000mmmmm (antirez) -2009-10-23 zrange initial hack (not working for now) (antirez) -2009-10-23 first skiplist fix, courtesy of valgrind (antirez) -2009-10-23 zset symbols added to stack trace code. ZSets will simply crash at the moment (antirez) -2009-10-23 more work on ZSETs and a new make target called 32bit to build i386 binaries on mac os x leopard (antirez) -2009-10-23 initial skiplist implementation. Most memory checks removed and zmalloc() modified to fail with an error message and abort. Anyway Redis is not designed to recover from out of memory conditions. (antirez) -2009-10-23 Fixed compilation in mac os x snow leopard when compiling a 32 bit binary. (antirez) -2009-10-22 version incremented to 1.050 to distinguish from 1.001 stable and next stable versions with minor fixes (antirez) -2009-10-21 TODO updated (antirez) -2009-10-21 SRANDMEMBER added (antirez) -2009-10-20 Imporant bug leading to data corruption fixed (NOT affecting stable distribution), Tcl client lib MSET/MSETNX implementation fixed, Added new tests for MSET and MSETNX in test-redis.tcl (antirez) -2009-10-17 added multi-bulk protocol support to redis-cli and support for MSET and MSETNX (antirez) -2009-10-17 MSET fixed, was not able to replace keys already set for a stupid bug (antirez) -2009-10-16 some dead code removed (antirez) -2009-10-16 multi bulk input protocol fixed (antirez) -2009-10-16 MSET and MSETNX commands implemented (antirez) -2009-10-07 undoed all the sds hacking that lead just to random bugs and no memory saving ;) (antirez) -2009-10-07 initial multi-bulk query protocol, this will allow MSET and other interesting features. (antirez) -2009-10-03 benchmark now outputs the right command line to shorten the TIME_WAIT interval on Mac OS X when keep alive is set (antirez) -2009-10-02 Issue 69 fixed. Object integer encoding now works with replication and MONITORing again. (antirez) -2009-09-18 LREM fixed, used to crash since the new object integer encoding is on the stage (antirez) -2009-09-17 maxmemory didn't worked in 64 systems for values > 4GB since it used to be an unsigned int. Fixed (antirez) -2009-09-10 incremented version number to 1.001, AKA Redis edge is no longer stable... (antirez) -2009-09-10 in-memory specialized object encoding (for now 32 signed integers only) (antirez) -2009-09-03 Latest doc changes for 1.0 (antirez) -2009-09-03 Redis 1.0.0 release (antirez) -2009-09-02 Redis version pushed to 1.0 (antirez) -2009-09-02 Ruby client lib updated to the latest git version (antirez) -2009-09-02 update-scala-client script added (antirez) -2009-09-02 Scala client added thanks to Alejanro Crosa (antirez) -2009-09-02 QuickStart added (antirez) -2009-09-01 Fixed crash with only space and newline as command (issue 61), thanks to a guy having as nick "fixxxerrr" (antirez) -2009-08-11 TODO list modified (antirez) -2009-07-24 more snow leopard related fixes (for 32bit systems) (antirez) -2009-07-24 fixed compilation with Snow Leopard, thanks to Lon Baker for providing SSH access to Snow Leopard box (antirez) -2009-07-22 Fixed NetBSD compile problems (antirez) -2009-07-17 now the size of the shared pool can be really modified via config, also the number of objects in the sharing pool is logged when the log level is set to debug. Thanks to Aman Gupta (antirez) -2009-07-05 added utils/redis-copy.rb, a script that is able to copy data from one Redis server to another one on the fly. (antirez) -2009-07-04 Applied three different patches thanks to Chris Lamb, one to fix compilation and get the IP register value on Linux IA64 and other systems. One in order to log the overcommit problem on the logs instead of the standard output when Redis is demonized. The latest in order to suggest a more consistent way in order to switch to 1 the memory overcommit Linux feature. (antirez) -2009-07-03 bugfix: EXPIRE now propagates to the Slave. (antirez) -2009-06-16 Redis version modified to 0.900 (antirez) -2009-06-16 update-ruby-client script already points to ezmobius repo (antirez) -2009-06-16 client libraries updated (antirez) -2009-06-16 Redis release candidate 1 (antirez) -2009-06-16 Better handling of background saving process killed or crashed (antirez) -2009-06-14 number of keys info in INFO command thanks to Diego Rosario Brogna (antirez) -2009-06-14 SPOP documented (antirez) -2009-06-14 Clojure library thanks to Ragnar Dahlén (antirez) -2009-06-10 It is now possible to specify - as config file name to read it from stdin (antirez) -2009-06-10 sync with jodosha redis-rb (antirez) -2009-06-10 Redis-rb sync (antirez) -2009-06-10 max inline request raised again to 1024*1024*256 bytes (antirez) -2009-06-10 max bytes in an inline command raised to 1024*1024 bytes, in order to allow for very large MGETs and still protect from client crashes (antirez) -2009-06-08 SPOP implemented. Hash table resizing for Sets and Expires too. Changed the resize policy to play better with RANDOMKEY and SPOP. (antirez) -2009-06-07 some minor changes to the backtrace code (antirez) -2009-06-07 enable backtrace capabilities only for Linux and MacOSX (antirez) -2009-06-07 Dump a backtrace on sigsegv/sigbus, original coded thanks to Diego Rosario Brogna, modified in order to work on different OSes and to enhance reliability (antirez) -2009-06-06 Merge git://github.com/dierbro/redis (antirez) -2009-06-06 add more output (hrothgar) -2009-06-06 store static function pointer for a useful stack trace (hrothgar) -2009-06-06 TODO updated (antirez) -2009-06-06 Makefile dependencies updated (antirez) -2009-06-05 Avoid a busy loop while sending very large replies against very fast links, this allows to be more responsive with other clients even under a KEY * against the loopback interface (antirez) -2009-06-05 Kill the background saving process before performing SHUTDOWN to avoid races (antirez) -2009-06-05 LREM now returns :0 for non existing keys (antirez) -2009-06-05 - put some order in code - better output (hrothgar) -2009-06-05 added config.h for #ifdef business isolation, added fstat64 for Mac OS X (antirez) -2009-06-04 remove die() :-) (hrothgar) -2009-06-04 add compile options to debug (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 macosx specific zmalloc.c, uses malloc_size function in order to avoid to waste memory and time to put an additional header (antirez) -2009-06-04 DEBUG OBJECT implemented (antirez) -2009-06-04 backtrace support removed: unreliable stack trace :( (antirez) -2009-06-04 initial backtrace dumping on sigsegv/sigbus + debug command (antirez) -2009-06-03 Python lib updated (antirez) -2009-06-03 shareobjectspoolsize implemented in reds.conf, in order to control the pool size when object sharing is on (antirez) -2009-05-30 Erlang client updated (antirez) -2009-05-30 Python client library updated (antirez) -2009-05-29 Redis-rb minor bool convertion fix (antirez) -2009-05-29 ruby library client is not Redis-rb merged with RubyRedis "engine" by Brian McKinney (antirez) -2009-05-28 __P completely removed from pqsort.c/h (antirez) -2009-05-28 another minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-27 maxmemory implemented (antirez) -2009-05-26 Redis git version modified to 0.101 in order to distinguish that from the latest tar.gz via INFO ;) (antirez) -2009-05-26 Redis 0.100 released (antirez) -2009-05-26 client libraries synched in git (antirez) -2009-05-26 ignore gcc warning about write() return code not checked. It is esplicitily this way since the "max number of clients reached" is a best-effort error (antirez) -2009-05-26 max bytes of a received command enlarged from 1k to 16k (antirez) -2009-05-26 RubyRedis: set TCP_NODELAY TCP socket option to to disable the neagle algorithm. Makes a huge difference under some OS, notably Linux (antirez) -2009-05-25 maxclients implemented, see redis.conf for details (antirez) -2009-05-25 INFO command now reports replication info (antirez) -2009-05-25 minor fix to RubyRedis about bulk commands sent without arguments (antirez) -2009-05-24 Warns if using the default config (antirez) -2009-05-24 Issue with redis-client used in scripts solved, now to check if the latest argument must come from standard input we do not check that stdin is or not a tty but the command arity (antirez) -2009-05-23 RubyRedis: now sets are returned as arrays again, and not as Set objects (antirez) -2009-05-23 SLAVEOF command documented (antirez) -2009-05-23 SLAVEOF command implemented for replication remote control (antirez) -2009-05-22 Fix: no connection timeout for the master! (antirez) -2009-05-22 replication slave timeout when receiving the initial bulk data set to 3600 seconds, now that replication is non-blocking the server must save the db before to start the async replication and this can take a lot of time with huge datasets (antirez) -2009-05-22 README tutorial now reflects the new proto (antirez) -2009-05-22 critical bug about glueoutputbuffers=yes fixed. Under load and with pipelining and clients disconnecting on the middle of the chat with the server, Redis could block. Now it's ok (antirez) -2009-05-22 TTL command doc added (antirez) -2009-05-22 TTL command implemented (antirez) -2009-05-22 S*STORE now return the cardinality of the resulting set (antirez) -2009-05-22 rubyredis more compatible with Redis-rb (antirez) -2009-05-21 minor indentation fix (antirez) -2009-05-21 timeout support and Redis-rb compatibility aliases implemented in RubyRedis (antirez) -2009-05-21 RubyRedis info postprocessor rewritten in a more functional way (antirez) -2009-05-21 dead code removed from RubyRedis (antirez) -2009-05-21 command postprocessing implemented into RubyRedis (antirez) -2009-05-20 Automagically reconnection of RubyRedis (antirez) -2009-05-20 RubyRedis: Array alike operators implemented (antirez) -2009-05-20 random testing code removed (antirez) -2009-05-20 RubyRedis DB selection forced at object creation (antirez) -2009-05-20 Initial version of an alternative Ruby client added (antirez) -2009-05-20 SDIFF / SDIFFSTORE added to doc (antirez) -2009-05-20 Aman Gupta changes merged (antirez) -2009-05-20 Merge git://github.com/tmm1/redis (antirez) -2009-05-19 Allow timeout=0 config to disable client timeouts (Aman Gupta) -2009-05-19 Partial qsort implemented in SORT command, only when both BY and LIMIT is used. minor fix for a warning compiling under Linux. (antirez) -2009-05-19 psort.c/h added. This is a partial qsort implementation that Redis will use when SORT+LIMIT is requested (antirez) -2009-05-17 Fix SINTER/UNIONSTORE to allow for &=/|= style operations (i.e. SINTERSTORE set1 set1 set2) (Aman Gupta) -2009-05-17 Optimize SDIFF to return as soon as the result set is empty (Aman Gupta) -2009-05-17 SDIFF/SDIFFSTORE implemnted unifying it with the implementation of SUNION/SUNIONSTORE (antirez) -2009-05-11 timestamp in log lines (antirez) -2009-05-11 Python client updated pushing from Ludo's repository (antirez) -2009-05-11 disconnect when we cannot read from the socket (Ludovico Magnocavallo) -2009-05-11 benchmark utility now supports random keys (antirez) -2009-05-10 minor doc changes (antirez) -2009-05-09 added tests for vararg DEL (antirez) -2009-05-09 DEL is now a vararg, IMPORTANT: memory leak fixed in loading DB code (antirez) -2009-05-09 doc changes (antirez) -2009-05-09 CPP client added thanks to Brian Hammond (antirez) -2009-05-06 Infinite number of arguments for MGET and all the other commands (antirez) -2009-05-04 Warns if /proc/sys/vm/overcommit_memory is set to 0 on Linux. Also make sure to don't resize the hash tables while the child process is saving in order to avoid copy-on-write of memory pages (antirez) -2009-04-30 zmalloc fix, return NULL or real malloc failure (antirez) -2009-04-30 more fixes for dict.c and the 150 million keys limit (antirez) -2009-04-30 dict.c modified to be able to handle more than 150,000,000 keys (antirez) -2009-04-29 fuzz stresser implemented in redis-test (antirez) -2009-04-29 fixed for HT resize check 32bits overflow (antirez) -2009-04-29 Check for fork() failure in background saving (antirez) -2009-04-29 fix for the LZF off-by-one bug added (antirez) -2009-04-28 print bytes used at exit on SHUTDOWN (antirez) -2009-04-28 SMOVE test added (antirez) -2009-04-28 SMOVE command implemented (antirez) -2009-04-28 less CPU usage in command parsing, case insensitive config directives (antirez) -2009-04-28 GETSET command doc added (antirez) -2009-04-28 GETSET tests (antirez) -2009-04-28 GETSET implemented (antirez) -2009-04-27 ability to specify a different file name for the DB (antirez) -2009-04-27 log file parsing code improved a bit (antirez) -2009-04-27 bgsave_in_progress field in INFO output (antirez) -2009-04-27 INCRBY/DECRBY now support 64bit increments, with tests (antirez) -2009-04-23 RANDOMKEY regression test added (antirez) -2009-04-23 dictGetRandomKey bug fixed, RANDOMKEY will not block the server anymore (antirez) -2009-04-22 FLUSHALL/FLUSHDB no longer sync on disk. Just increment the dirty counter by the number of elements removed, that will probably trigger a background saving operation (antirez) -2009-04-21 forgot to comment testing code in PHP lib. Now it is ok (antirez) -2009-04-21 PHP client ported to PHP5 and fixed (antirez) -2009-04-21 doc update (antirez) -2009-04-20 Non blocking replication (finally!). C-side linked lists API improved. (antirez) -2009-04-19 SUNION, SUNIONSTORE, Initial work on non blocking replication (antirez) -2009-04-10 Redis 0.091 released (antirez) -2009-04-10 SINTER/SINTERSTORE/SLEMENTS fix: misisng keys are now not errors, but just like empty sets (antirez) -2009-04-09 doc changes (antirez) -2009-04-08 TODO changes, minor change to default redis.conf (antirez) -2009-04-08 html doc updated (antirez) -2009-04-08 library clients update scripts (antirez) -2009-04-08 Ruby client updated (antirez) -2009-04-08 Lua client updated (antirez) -2009-04-08 Changelog updated (antirez) -2009-04-08 Merge git://github.com/ludoo/redis (antirez) -2009-04-08 add expire command to the php lib (Ludovico Magnocavallo) -2009-04-08 fix decode bug, add flush and info commands (Ludovico Magnocavallo) -2009-04-07 Rearrange redisObject struct to reduce memory usage in 64bit environments (as recommended http://groups.google.com/group/redis-db/msg/68f5a743f8f4e287) (Bob Potter) -2009-04-07 ruby19 compat: use each_line on string (Bob Potter) -2009-04-07 64bit fixes for usedmemory (Bob Potter) -2009-04-08 RANDOMKEY issue 26 fixed, generic test + regression added (antirez) -2009-04-06 Don't accept SAVE if BGSAVE is in progress (antirez) -2009-04-06 add expire command to the python lib (Ludovico Magnocavallo) -2009-04-03 persistent EXPIRE (antirez) -2009-04-03 dirty increment was missing in two points. TODO updated (antirez) -2009-04-02 LZF configured to initalize the HT in order to be determinsitic and play well with valgrind (antirez) -2009-04-02 fix select test (Ludovico Magnocavallo) -2009-04-02 fix trailing cr+nl in values (Ludovico Magnocavallo) -2009-04-02 compression/decompression of large values on disk now working (antirez) -2009-04-02 disable LZF compression since it's not able to load the DB for now, the load part is missing (antirez) -2009-04-02 new LZF files added (antirez) -2009-04-02 Fixed issue 23 about AUTH (antirez) -2009-04-02 Issue 22 fixed (antirez) -2009-04-01 non-lazy expired keys purging implemented (antirez) -2009-04-01 fastlz dependence removed (antirez) -2009-04-01 Initial implementation of EXPIRE (antirez) -2009-03-30 TODO updated (antirez) -2009-03-30 changelog added (antirez) -2009-03-28 redis-sha1 utility added (antirez) -2009-03-28 Integer encoding implemented in dump file. Doc updated (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 now Redis is C99-ok (antirez) -2009-03-27 IMPORTANT FIX: new dump format implementation was broken. Now it's ok but tests for the 32-bit case values are needed (antirez) -2009-03-27 ANSI-C compatibility changes (antirez) -2009-03-27 Ruby client library updated. Important changes in this new version! (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 AUTH merged from Brian Hammond fork, reworked a bit to fix minor problems (antirez) -2009-03-25 Adds AUTH command. (Brian Hammond) -2009-03-25 Nasty bug of the new DB format fixed, objects sharing implemented (antirez) -2009-03-25 doc update (antirez) -2009-03-25 Erlang client synched with Valentiono's repo (antirez) -2009-03-25 New file dump format, perl client library added (antirez) -2009-03-25 New protocol fix for LREM (antirez) -2009-03-24 two typos fixed (antirez) -2009-03-24 Now the Redis test uses the proper Tcl client library (antirez) -2009-03-24 Tcl client library (antirez) -2009-03-24 redis-benchmark sync with the new protocol (antirez) -2009-03-24 git mess :) (Ludovico Magnocavallo) -2009-03-24 sync python client to the new protocol (Ludovico Magnocavallo) -2009-03-24 protocol fix in SORT reply with null elements (antirez) -2009-03-24 protocol doc changed (antirez) -2009-03-24 Server replies now in the new format, test-redis.tcl and redis-cli modified accordingly (antirez) -2009-03-24 Python client library updated, thanks to Ludo! (antirez) -2009-03-24 random tested mode for test-redis.tcl, minor other stuff, version switched to 0.8 (antirez) -2009-03-23 Now MONITOR/SYNC cannot be issued multiple times (antirez) -2009-03-23 MONITOR command implemented. (antirez) -2009-03-23 lucsky changes imported. pid file path can now be configured, redis-cli fixes (antirez) -2009-03-23 Merge git://github.com/lucsky/redis (antirez) -2009-03-23 another missing free->zfree replacement fixed. Thanks to Ludo (antirez) -2009-03-23 Fixed redis-cli readLine loop to correctly handle EOF. (Luc Heinrich) -2009-03-23 Display the port on server startup. (Luc Heinrich) -2009-03-23 Allow to specify the pid file from the config file. (Luc Heinrich) -2009-03-23 Added gitignore file. (Luc Heinrich) -2009-03-22 MGET tests added (antirez) -2009-03-22 doc changes (antirez) -2009-03-22 added doc for MGET (antirez) -2009-03-22 redis-cli now checks the arity of vararg commnads (antirez) -2009-03-22 INFO fixed, MGET implemented, redis-cli implements INFO/MGET (antirez) -2009-03-22 first commit (antirez) \ No newline at end of file diff --git a/MANIFESTO b/MANIFESTO index ad94fd8ea71..2b719057ee7 100644 --- a/MANIFESTO +++ b/MANIFESTO @@ -4,17 +4,64 @@ Redis Manifesto =============== -1 - A DSL for Abstract Data Types. Redis is a DSL (Domain Specific Language) that manipulates abstract data types and implemented as a TCP daemon. Commands manipulate a key space where keys are binary-safe strings and values are different kinds of abstract data types. Every data type represents an abstract version of a fundamental data structure. For instance Redis Lists are an abstract representation of linked lists. In Redis, the essence of a data type isn't just the kind of operations that the data types support, but also the space and time complexity of the data type and the operations performed upon it. +1 - A DSL for Abstract Data Types. Redis is a DSL (Domain Specific Language) + that manipulates abstract data types and implemented as a TCP daemon. + Commands manipulate a key space where keys are binary-safe strings and + values are different kinds of abstract data types. Every data type + represents an abstract version of a fundamental data structure. For instance + Redis Lists are an abstract representation of linked lists. In Redis, the + essence of a data type isn't just the kind of operations that the data types + support, but also the space and time complexity of the data type and the + operations performed upon it. -2 - Memory storage is #1. The Redis data set, composed of defined key-value pairs, is primarily stored in the computer's memory. The amount of memory in all kinds of computers, including entry-level servers, is increasing significantly each year. Memory is fast, and allows Redis to have very predictable performance. Datasets composed of 10k or 40 millions keys will perform similarly. Complex data types like Redis Sorted Sets are easy to implement and manipulate in memory with good performance, making Redis very simple. Redis will continue to explore alternative options (where data can be optionally stored on disk, say) but the main goal of the project remains the development of an in-memory database. +2 - Memory storage is #1. The Redis data set, composed of defined key-value + pairs, is primarily stored in the computer's memory. The amount of memory in + all kinds of computers, including entry-level servers, is increasing + significantly each year. Memory is fast, and allows Redis to have very + predictable performance. Datasets composed of 10k or 40 millions keys will + perform similarly. Complex data types like Redis Sorted Sets are easy to + implement and manipulate in memory with good performance, making Redis very + simple. Redis will continue to explore alternative options (where data can + be optionally stored on disk, say) but the main goal of the project remains + the development of an in-memory database. -3 - Fundamental data structures for a fundamental API. The Redis API is a direct consequence of fundamental data structures. APIs can often be arbitrary but not an API that resembles the nature of fundamental data structures. If we ever meet intelligent life forms from another part of the universe, they'll likely know, understand and recognize the same basic data structures we have in our computer science books. Redis will avoid intermediate layers in API, so that the complexity is obvious and more complex operations can be performed as the sum of the basic operations. +3 - Fundamental data structures for a fundamental API. The Redis API is a direct + consequence of fundamental data structures. APIs can often be arbitrary but + not an API that resembles the nature of fundamental data structures. If we + ever meet intelligent life forms from another part of the universe, they'll + likely know, understand and recognize the same basic data structures we have + in our computer science books. Redis will avoid intermediate layers in API, + so that the complexity is obvious and more complex operations can be + performed as the sum of the basic operations. -4 - Code is like a poem; it's not just something we write to reach some practical result. Sometimes people that are far from the Redis philosophy suggest using other code written by other authors (frequently in other languages) in order to implement something Redis currently lacks. But to us this is like if Shakespeare decided to end Enrico IV using the Paradiso from the Divina Commedia. Is using any external code a bad idea? Not at all. Like in "One Thousand and One Nights" smaller self contained stories are embedded in a bigger story, we'll be happy to use beautiful self contained libraries when needed. At the same time, when writing the Redis story we're trying to write smaller stories that will fit in to other code. +4 - Code is like a poem; it's not just something we write to reach some + practical result. Sometimes people that are far from the Redis philosophy + suggest using other code written by other authors (frequently in other + languages) in order to implement something Redis currently lacks. But to us + this is like if Shakespeare decided to end Enrico IV using the Paradiso from + the Divina Commedia. Is using any external code a bad idea? Not at all. Like + in "One Thousand and One Nights" smaller self contained stories are embedded + in a bigger story, we'll be happy to use beautiful self contained libraries + when needed. At the same time, when writing the Redis story we're trying to + write smaller stories that will fit in to other code. -5 - We're against complexity. We believe designing systems is a fight against complexity. We'll accept to fight the complexity when it's worthwhile but we'll try hard to recognize when a small feature is not worth 1000s of lines of code. Most of the time the best way to fight complexity is by not creating it at all. +5 - We're against complexity. We believe designing systems is a fight against + complexity. We'll accept to fight the complexity when it's worthwhile but + we'll try hard to recognize when a small feature is not worth 1000s of lines + of code. Most of the time the best way to fight complexity is by not + creating it at all. -6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits naturally into a distributed version of Redis and 2) a more complex API that supports multi-key operations. Both are useful if used judiciously but there's no way to make the more complex multi-keys API distributed in an opaque way without violating our other principles. We don't want to provide the illusion of something that will work magically when actually it can't in all cases. Instead we'll provide commands to quickly migrate keys from one instance to another to perform multi-key operations and expose the tradeoffs to the user. - -7 - We optimize for joy. We believe writing code is a lot of hard work, and the only way it can be worth is by enjoying it. When there is no longer joy in writing code, the best thing to do is stop. To prevent this, we'll avoid taking paths that will make Redis less of a joy to develop. +6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits + naturally into a distributed version of Redis and 2) a more complex API that + supports multi-key operations. Both are useful if used judiciously but + there's no way to make the more complex multi-keys API distributed in an + opaque way without violating our other principles. We don't want to provide + the illusion of something that will work magically when actually it can't in + all cases. Instead we'll provide commands to quickly migrate keys from one + instance to another to perform multi-key operations and expose the tradeoffs + to the user. +7 - We optimize for joy. We believe writing code is a lot of hard work, and the + only way it can be worth is by enjoying it. When there is no longer joy in + writing code, the best thing to do is stop. To prevent this, we'll avoid + taking paths that will make Redis less of a joy to develop. diff --git a/Makefile b/Makefile index 4f08126df79..e614ede891f 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,7 @@ default: all .DEFAULT: cd src && $(MAKE) $@ +install: + cd src && $(MAKE) $@ + +.PHONY: install diff --git a/README b/README index bba2439c339..36911863114 100644 --- a/README +++ b/README @@ -7,6 +7,13 @@ documentation at http://redis.io Building Redis -------------- +Redis can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD. +We support big endian and little endian architectures. + +It may compile on Solaris derived systems (for instance SmartOS) but our +support for this platform is "best effort" and Redis is not guaranteed to +work as well as in Linux, OSX, and *BSD there. + It is as simple as: % make @@ -19,9 +26,39 @@ After building Redis is a good idea to test it, using: % make test -NOTE: if after building Redis with a 32 bit target you need to rebuild it - with a 64 bit target you need to perform a "make clean" in the root - directory of the Redis distribution. +Fixing build problems with dependencies or cached build options +—-------- +Redis has some dependencies which are included into the "deps" directory. +"make" does not rebuild dependencies automatically, even if something in the +source code of dependencies is changes. + +When you update the source code with `git pull` or when code inside the +dependencies tree is modified in any other way, make sure to use the following +command in order to really clean everything and rebuild from scratch: + + make distclean + +This will clean: jemalloc, lua, hiredis, linenoise. + +Also if you force certain build options like 32bit target, no C compiler +optimizations (for debugging purposes), and other similar build time options, +those options are cached indefinitely until you issue a "make distclean" +command. + +Fixing problems building 32 bit binaries +--------- + +If after building Redis with a 32 bit target you need to rebuild it +with a 64 bit target, or the other way around, you need to perform a +"make distclean" in the root directory of the Redis distribution. + +In case of build errors when trying to build a 32 bit binary of Redis, try +the following steps: + +* Install the packages libc6-dev-i386 (also try g++-multilib). +* Try using the following command line instead of "make 32bit": + + make CFLAGS="-m32 -march=native" LDFLAGS="-m32" Allocator --------- @@ -112,7 +149,7 @@ it the proper way for a production system, we have a script doing this for Ubuntu and Debian systems: % cd utils - % ./install_server + % ./install_server.sh The script will ask you a few questions and will setup everything you need to run Redis properly as a background daemon that will start again on @@ -121,4 +158,16 @@ system reboots. You'll be able to stop and start Redis using the script named /etc/init.d/redis_, for instance /etc/init.d/redis_6379. +Code contributions +--- + +Note: by contributing code to the Redis project in any form, including sending +a pull request via Github, a code fragment or patch via private email or +public discussion groups, you agree to release your code under the terms +of the BSD license that you can find in the COPYING file included in the Redis +source distribution. + +Please see the CONTRIBUTING file in this source distribution for more +information. + Enjoy! diff --git a/TODO b/TODO deleted file mode 100644 index 145ec5243e9..00000000000 --- a/TODO +++ /dev/null @@ -1,45 +0,0 @@ -Redis TODO ----------- - -WARNING: are you a possible Redis contributor? - Before implementing what is listed in this file - please drop a message in the Redis google group or chat with - antirez or pietern on irc.freenode.org #redis to check if the work - is already in progress and if the feature is still interesting for - us, and *how* exactly this can be implemented to have good changes - of a merge. Otherwise it is probably wasted work! Thank you - - -CLUSTER -======= - -* Implement rehashing and cluster check in redis-trib. -* Reimplement MIGRATE / RESTORE to use just in memory buffers (no disk at - all). This will require touching a lot of the RDB stuff around, but we may - hand with faster persistence for RDB. -* Implement the slave nodes semantics and election. -* Allow redis-trib to create a cluster-wide snapshot (using SYNC). -* Allow redis-trib to restore a cluster-wide snapshot (implement UPLOAD?). - -SCRIPTING -========= - -* SCRIPT FLUSH or alike to start a fresh interpreter? - -OPTIMIZATIONS -============= - -* SORT: Don't copy the list into a vector when BY argument is constant. -* Write the hash table size of every db in the dump, so that Redis can resize the hash table just one time when loading a big DB. -* Read-only mode for slaves. -* Redis big lists as linked lists of small ziplists? - Possibly a simple heuristic that join near nodes when some node gets smaller than the low_level, and split it into two if gets bigger than high_level. - -KNOWN BUGS -========== - -* #519: Slave may have expired keys that were never read in the master (so a DEL - is not sent in the replication channel) but are already expired since - a lot of time. Maybe after a given delay that is undoubtably greater than - the replication link latency we should expire this key on the slave on - access? diff --git a/deps/Makefile b/deps/Makefile index b881c814e7d..1f623ea7b11 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,17 +1,6 @@ # Redis dependency Makefile -UNAME_S:=$(shell sh -c 'uname -s 2> /dev/null || echo not') - -LUA_CFLAGS=-O2 -Wall $(ARCH) -ifeq ($(UNAME_S),SunOS) - # Make isinf() available - LUA_CFLAGS+= -D__C99FEATURES__=1 -endif - -JEMALLOC_CFLAGS= -ifeq ($(ARCH),-m32) - JEMALLOC_CFLAGS+=CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32" -endif +uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not') CCCOLOR="\033[34m" LINKCOLOR="\033[34;1m" @@ -23,37 +12,72 @@ ENDCOLOR="\033[0m" default: @echo "Explicit target required" -# Clean everything when ARCH is different -ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) -.make-arch: distclean -else -.make-arch: +.PHONY: default + +# Prerequisites target +.make-prerequisites: + @touch $@ + +# Clean everything when CFLAGS is different +ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(CFLAGS)) +.make-cflags: distclean + -(echo "$(CFLAGS)" > .make-cflags) +.make-prerequisites: .make-cflags endif -.make-arch: - -(echo $(ARCH) > .make-arch) +# Clean everything when LDFLAGS is different +ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(LDFLAGS)) +.make-ldflags: distclean + -(echo "$(LDFLAGS)" > .make-ldflags) +.make-prerequisites: .make-ldflags +endif distclean: -(cd hiredis && $(MAKE) clean) > /dev/null || true -(cd linenoise && $(MAKE) clean) > /dev/null || true -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true - -(rm -f .make-arch) + -(rm -f .make-*) + +.PHONY: distclean + +hiredis: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd hiredis && $(MAKE) static + +.PHONY: hiredis + +linenoise: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd linenoise && $(MAKE) + +.PHONY: linenoise + +ifeq ($(uname_S),SunOS) + # Make isinf() available + LUA_CFLAGS= -D__C99FEATURES__=1 +endif + +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL $(CFLAGS) +LUA_LDFLAGS+= $(LDFLAGS) +# lua's Makefile defines AR="ar rcu", which is unusual, and makes it more +# challenging to cross-compile lua (and redis). These defines make it easier +# to fit redis into cross-compilation environments, which typically set AR. +AR=ar +ARFLAGS=rcu -hiredis: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) - cd hiredis && $(MAKE) static ARCH="$(ARCH)" +lua: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd lua/src && $(MAKE) all CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(LUA_LDFLAGS)" AR="$(AR) $(ARFLAGS)" -linenoise: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) - cd linenoise && $(MAKE) ARCH="$(ARCH)" +.PHONY: lua -lua: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)lua$(ENDCOLOR) - cd lua && $(MAKE) CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(ARCH)" ansi +JEMALLOC_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS) +JEMALLOC_LDFLAGS= $(LDFLAGS) -jemalloc: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)jemalloc$(ENDCOLOR) - cd jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a +jemalloc: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd jemalloc && ./configure --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a -.PHONY: default conditional_clean hiredis linenoise lua jemalloc +.PHONY: jemalloc diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index 1a4d60d282e..0c166a02e61 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -1,5 +1,5 @@ /hiredis-test -/hiredis-example* +/examples/hiredis-example* /*.o /*.so /*.dylib diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml new file mode 100644 index 00000000000..030427ff4b6 --- /dev/null +++ b/deps/hiredis/.travis.yml @@ -0,0 +1,6 @@ +language: c +compiler: + - gcc + - clang + +script: make && make check diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index d41db8a6084..268b15cd561 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -1,3 +1,11 @@ +### 0.11.0 + +* Increase the maximum multi-bulk reply depth to 7. + +* Increase the read buffer size from 2k to 16k. + +* Use poll(2) instead of select(2) to support large fds (>= 1024). + ### 0.10.1 * Makefile overhaul. Important to check out if you override one or more diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 16b8767b147..ddcc4e4f69d 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -4,11 +4,24 @@ # This file is released under the BSD license, see the COPYING file OBJ=net.o hiredis.o sds.o async.o -BINS=hiredis-example hiredis-test +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev +TESTS=hiredis-test LIBNAME=libhiredis HIREDIS_MAJOR=0 -HIREDIS_MINOR=10 +HIREDIS_MINOR=11 + +# redis-server configuration used for testing +REDIS_PORT=56379 +REDIS_SERVER=redis-server +define REDIS_TEST_CONFIG + daemonize yes + pidfile /tmp/hiredis-test-redis.pid + port $(REDIS_PORT) + bind 127.0.0.1 + unixsocket /tmp/hiredis-test-redis.sock +endef +export REDIS_TEST_CONFIG # Fallback to gcc when $CC is not in $PATH. CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') @@ -41,12 +54,11 @@ ifeq ($(uname_S),Darwin) DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) endif -all: $(DYLIBNAME) $(BINS) +all: $(DYLIBNAME) # Deps (use make dep to generate this) net.o: net.c fmacros.h net.h hiredis.h async.o: async.c async.h hiredis.h sds.h dict.c dict.h -example.o: example.c hiredis.h hiredis.o: hiredis.c fmacros.h hiredis.h net.h sds.h sds.o: sds.c sds.h test.o: test.c hiredis.h @@ -61,36 +73,44 @@ dynamic: $(DYLIBNAME) static: $(STLIBNAME) # Binaries: -hiredis-example-libevent: example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -levent example-libevent.c $(STLIBNAME) +hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) -hiredis-example-libev: example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -lev example-libev.c $(STLIBNAME) +hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) ifndef AE_DIR hiredis-example-ae: @echo "Please specify AE_DIR (e.g. /src)" @false else -hiredis-example-ae: example-ae.c adapters/ae.h $(STLIBNAME) - $(CC) -o $@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I$(AE_DIR) $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o example-ae.c $(STLIBNAME) +hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME) endif -hiredis-%: %.o $(STLIBNAME) +ifndef LIBUV_DIR +hiredis-example-libuv: + @echo "Please specify LIBUV_DIR (e.g. ../libuv/)" + @false +else +hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread $(STLIBNAME) +endif + +hiredis-example: examples/example.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + +examples: $(EXAMPLES) + +hiredis-test: test.o $(STLIBNAME) $(CC) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) test: hiredis-test ./hiredis-test check: hiredis-test - echo \ - "daemonize yes\n" \ - "pidfile /tmp/hiredis-test-redis.pid\n" \ - "port 56379\n" \ - "bind 127.0.0.1\n" \ - "unixsocket /tmp/hiredis-test-redis.sock" \ - | redis-server - - ./hiredis-test -h 127.0.0.1 -p 56379 -s /tmp/hiredis-test-redis.sock || \ + @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - + ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ ( kill `cat /tmp/hiredis-test-redis.pid` && false ) kill `cat /tmp/hiredis-test-redis.pid` @@ -98,17 +118,15 @@ check: hiredis-test $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(BINS) hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: $(CC) -MM *.c # Installation related variables and target PREFIX?=/usr/local -INCLUDE_PATH?=include/hiredis -LIBRARY_PATH?=lib -INSTALL_INCLUDE_PATH= $(PREFIX)/$(INCLUDE_PATH) -INSTALL_LIBRARY_PATH= $(PREFIX)/$(LIBRARY_PATH) +INSTALL_INCLUDE_PATH= $(PREFIX)/include/hiredis +INSTALL_LIBRARY_PATH= $(PREFIX)/lib ifeq ($(uname_S),SunOS) INSTALL?= cp -r diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index a58101cc6e5..dba4a8c8e08 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) + # HIREDIS Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database. @@ -44,7 +46,7 @@ After trying to connect to Redis using `redisConnect` you should check the `err` field to see if establishing the connection was successful: redisContext *c = redisConnect("127.0.0.1", 6379); - if (c->err) { + if (c != NULL && c->err) { printf("Error: %s\n", c->errstr); // handle error } @@ -66,14 +68,14 @@ When you need to pass binary safe strings in a command, the `%b` specifier can b used. Together with a pointer to the string, it requires a `size_t` length argument of the string: - reply = redisCommand(context, "SET foo %b", value, valuelen); + reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen); Internally, Hiredis splits the command in different arguments and will convert it to the protocol used to communicate with Redis. One or more spaces separates arguments, so you can use the specifiers anywhere in an argument: - reply = redisCommand("SET key:%s %s", myid, value); + reply = redisCommand(context, "SET key:%s %s", myid, value); ### Using replies @@ -320,6 +322,10 @@ The reply parsing API consists of the following functions: int redisReaderFeed(redisReader *reader, const char *buf, size_t len); int redisReaderGetReply(redisReader *reader, void **reply); +The same set of functions are used internally by hiredis when creating a +normal Redis context, the above API just exposes it to the user for a direct +usage. + ### Usage The function `redisReaderCreate` creates a `redisReader` structure that holds a @@ -333,6 +339,9 @@ and a reply object (as described above) via `void **reply`. The returned status can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went wrong (either a protocol error, or an out of memory error). +The parser limits the level of nesting for multi bulk payloads to 7. If the +multi bulk nesting level is higher than this, the parser returns an error. + ### Customizing replies The function `redisReaderGetReply` creates `redisReply` and makes the function @@ -346,6 +355,29 @@ immediately after creating the `redisReader`. For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c) uses customized reply object functions to create Ruby objects. +### Reader max buffer + +Both when using the Reader API directly or when using it indirectly via a +normal Redis context, the redisReader structure uses a buffer in order to +accumulate data from the server. +Usually this buffer is destroyed when it is empty and is larger than 16 +kb in order to avoid wasting memory in unused buffers + +However when working with very big payloads destroying the buffer may slow +down performances considerably, so it is possible to modify the max size of +an idle buffer changing the value of the `maxbuf` field of the reader structure +to the desired value. The special value of 0 means that there is no maximum +value for an idle buffer, so the buffer will never get freed. + +For instance if you have a normal Redis context you can set the maximum idle +buffer to zero (unlimited) just with: + + context->reader->maxbuf = 0; + +This should be done only in order to maximize performances when working with +large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again +as soon as possible in order to prevent allocation of useless memory. + ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and diff --git a/deps/hiredis/adapters/ae.h b/deps/hiredis/adapters/ae.h index 65235f802aa..5c551c2ed74 100644 --- a/deps/hiredis/adapters/ae.h +++ b/deps/hiredis/adapters/ae.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_AE_H__ #define __HIREDIS_AE_H__ #include diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h index 534d7436017..2bf8d521fc2 100644 --- a/deps/hiredis/adapters/libev.h +++ b/deps/hiredis/adapters/libev.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_LIBEV_H__ #define __HIREDIS_LIBEV_H__ #include diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 4055ec0f112..1c2b271bbc1 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_LIBEVENT_H__ #define __HIREDIS_LIBEVENT_H__ #include diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h new file mode 100644 index 00000000000..a1967f4fd9e --- /dev/null +++ b/deps/hiredis/adapters/libuv.h @@ -0,0 +1,121 @@ +#ifndef __HIREDIS_LIBUV_H__ +#define __HIREDIS_LIBUV_H__ +#include +#include "../hiredis.h" +#include "../async.h" +#include + +typedef struct redisLibuvEvents { + redisAsyncContext* context; + uv_poll_t handle; + int events; +} redisLibuvEvents; + +int redisLibuvAttach(redisAsyncContext*, uv_loop_t*); + +static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + if (status != 0) { + return; + } + + if (events & UV_READABLE) { + redisAsyncHandleRead(p->context); + } + if (events & UV_WRITABLE) { + redisAsyncHandleWrite(p->context); + } +} + + +static void redisLibuvAddRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_READABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelRead(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_READABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void redisLibuvAddWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events |= UV_WRITABLE; + + uv_poll_start(&p->handle, p->events, redisLibuvPoll); +} + + +static void redisLibuvDelWrite(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + p->events &= ~UV_WRITABLE; + + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } +} + + +static void on_close(uv_handle_t* handle) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + + free(p); +} + + +static void redisLibuvCleanup(void *privdata) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + uv_close((uv_handle_t*)&p->handle, on_close); +} + + +static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { + redisContext *c = &(ac->c); + + if (ac->ev.data != NULL) { + return REDIS_ERR; + } + + ac->ev.addRead = redisLibuvAddRead; + ac->ev.delRead = redisLibuvDelRead; + ac->ev.addWrite = redisLibuvAddWrite; + ac->ev.delWrite = redisLibuvDelWrite; + ac->ev.cleanup = redisLibuvCleanup; + + redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); + + if (!p) { + return REDIS_ERR; + } + + memset(p, 0, sizeof(*p)); + + if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + return REDIS_ERR; + } + + ac->ev.data = p; + p->handle.data = p; + p->context = ac; + + return REDIS_OK; +} +#endif diff --git a/deps/hiredis/async.c b/deps/hiredis/async.c index f83e2f51af0..9cc35638f82 100644 --- a/deps/hiredis/async.c +++ b/deps/hiredis/async.c @@ -62,7 +62,8 @@ void __redisAppendCommand(redisContext *c, char *cmd, size_t len); /* Functions managing dictionary of callbacks for pub/sub. */ static unsigned int callbackHash(const void *key) { - return dictGenHashFunction((unsigned char*)key,sdslen((char*)key)); + return dictGenHashFunction((const unsigned char *)key, + sdslen((const sds)key)); } static void *callbackValDup(void *privdata, const void *src) { @@ -76,8 +77,8 @@ static int callbackKeyCompare(void *privdata, const void *key1, const void *key2 int l1, l2; ((void) privdata); - l1 = sdslen((sds)key1); - l2 = sdslen((sds)key2); + l1 = sdslen((const sds)key1); + l2 = sdslen((const sds)key2); if (l1 != l2) return 0; return memcmp(key1,key2,l1) == 0; } @@ -102,7 +103,12 @@ static dictType callbackDict = { }; static redisAsyncContext *redisAsyncInitialize(redisContext *c) { - redisAsyncContext *ac = realloc(c,sizeof(redisAsyncContext)); + redisAsyncContext *ac; + + ac = realloc(c,sizeof(redisAsyncContext)); + if (ac == NULL) + return NULL; + c = &(ac->c); /* The regular connect functions will always set the flag REDIS_CONNECTED. @@ -142,15 +148,45 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { } redisAsyncContext *redisAsyncConnect(const char *ip, int port) { - redisContext *c = redisConnectNonBlock(ip,port); + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectNonBlock(ip,port); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + + __redisAsyncCopyError(ac); + return ac; +} + +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, + const char *source_addr) { + redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); redisAsyncContext *ac = redisAsyncInitialize(c); __redisAsyncCopyError(ac); return ac; } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c = redisConnectUnixNonBlock(path); - redisAsyncContext *ac = redisAsyncInitialize(c); + redisContext *c; + redisAsyncContext *ac; + + c = redisConnectUnixNonBlock(path); + if (c == NULL) + return NULL; + + ac = redisAsyncInitialize(c); + if (ac == NULL) { + redisFree(c); + return NULL; + } + __redisAsyncCopyError(ac); return ac; } @@ -182,6 +218,9 @@ static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { /* Copy callback from stack to heap */ cb = malloc(sizeof(*cb)); + if (cb == NULL) + return REDIS_ERR_OOM; + if (source != NULL) { memcpy(cb,source,sizeof(*cb)); cb->next = NULL; @@ -360,7 +399,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, void redisProcessCallbacks(redisAsyncContext *ac) { redisContext *c = &(ac->c); - redisCallback cb; + redisCallback cb = {NULL, NULL, NULL}; void *reply = NULL; int status; @@ -373,6 +412,11 @@ void redisProcessCallbacks(redisAsyncContext *ac) { return; } + /* If monitor mode, repush callback */ + if(c->flags & REDIS_MONITORING) { + __redisPushCallback(&ac->replies,&cb); + } + /* When the connection is not being disconnected, simply stop * trying to get replies and wait for the next loop tick. */ break; @@ -381,22 +425,32 @@ void redisProcessCallbacks(redisAsyncContext *ac) { /* Even if the context is subscribed, pending regular callbacks will * get a reply before pub/sub messages arrive. */ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { - /* A spontaneous reply in a not-subscribed context can only be the - * error reply that is sent when a new connection exceeds the - * maximum number of allowed connections on the server side. This - * is seen as an error instead of a regular reply because the - * server closes the connection after sending it. To prevent the - * error from being overwritten by an EOF error the connection is - * closed here. See issue #43. */ - if ( !(c->flags & REDIS_SUBSCRIBED) && ((redisReply*)reply)->type == REDIS_REPLY_ERROR ) { + /* + * A spontaneous reply in a not-subscribed context can be the error + * reply that is sent when a new connection exceeds the maximum + * number of allowed connections on the server side. + * + * This is seen as an error instead of a regular reply because the + * server closes the connection after sending it. + * + * To prevent the error from being overwritten by an EOF error the + * connection is closed here. See issue #43. + * + * Another possibility is that the server is loading its dataset. + * In this case we also want to close the connection, and have the + * user wait until the server is ready to take our request. + */ + if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) { c->err = REDIS_ERR_OTHER; snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str); + c->reader->fn->freeObject(reply); __redisAsyncDisconnect(ac); return; } - /* No more regular callbacks and no errors, the context *must* be subscribed. */ - assert(c->flags & REDIS_SUBSCRIBED); - __redisGetSubscribeCallback(ac,reply,&cb); + /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */ + assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING)); + if(c->flags & REDIS_SUBSCRIBED) + __redisGetSubscribeCallback(ac,reply,&cb); } if (cb.fn != NULL) { @@ -428,7 +482,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { static int __redisAsyncHandleConnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); - if (redisCheckSocketError(c,c->fd) == REDIS_ERR) { + if (redisCheckSocketError(c) == REDIS_ERR) { /* Try again later when connect(2) is still in progress. */ if (errno == EINPROGRESS) return REDIS_OK; @@ -557,6 +611,10 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* (P)UNSUBSCRIBE does not have its own response: every channel or * pattern that is unsubscribed will receive a message. This means we * should not append a callback function for this command. */ + } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) { + /* Set monitor flag and push callback */ + c->flags |= REDIS_MONITORING; + __redisPushCallback(&ac->replies,&cb); } else { if (c->flags & REDIS_SUBSCRIBED) /* This will likely result in an error reply, but it needs to be diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 268274e8e70..8a2cf1ecd2d 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -102,6 +102,7 @@ typedef struct redisAsyncContext { /* Functions that proxy to hiredis */ redisAsyncContext *redisAsyncConnect(const char *ip, int port); +redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); diff --git a/deps/hiredis/example-ae.c b/deps/hiredis/examples/example-ae.c similarity index 89% rename from deps/hiredis/example-ae.c rename to deps/hiredis/examples/example-ae.c index 5ed34a3a685..8efa7306aec 100644 --- a/deps/hiredis/example-ae.c +++ b/deps/hiredis/examples/example-ae.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/ae.h" + +#include +#include +#include /* Put event loop in the global scope, so it can be explicitly stopped */ static aeEventLoop *loop; @@ -21,17 +22,22 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { void connectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { printf("Error: %s\n", c->errstr); + aeStop(loop); return; } + printf("Connected...\n"); } void disconnectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { printf("Error: %s\n", c->errstr); + aeStop(loop); return; } + printf("Disconnected...\n"); + aeStop(loop); } int main (int argc, char **argv) { @@ -44,7 +50,7 @@ int main (int argc, char **argv) { return 1; } - loop = aeCreateEventLoop(); + loop = aeCreateEventLoop(64); redisAeAttach(loop, c); redisAsyncSetConnectCallback(c,connectCallback); redisAsyncSetDisconnectCallback(c,disconnectCallback); diff --git a/deps/hiredis/example-libev.c b/deps/hiredis/examples/example-libev.c similarity index 95% rename from deps/hiredis/example-libev.c rename to deps/hiredis/examples/example-libev.c index 7894f1f487b..cc8b166ec62 100644 --- a/deps/hiredis/example-libev.c +++ b/deps/hiredis/examples/example-libev.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/libev.h" + +#include +#include +#include void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; diff --git a/deps/hiredis/example-libevent.c b/deps/hiredis/examples/example-libevent.c similarity index 95% rename from deps/hiredis/example-libevent.c rename to deps/hiredis/examples/example-libevent.c index 9da8e02bff7..d333c22b79e 100644 --- a/deps/hiredis/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -2,9 +2,10 @@ #include #include #include -#include "hiredis.h" -#include "async.h" -#include "adapters/libevent.h" + +#include +#include +#include void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c new file mode 100644 index 00000000000..a5462d41051 --- /dev/null +++ b/deps/hiredis/examples/example-libuv.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + uv_loop_t* loop = uv_default_loop(); + + redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + + redisLibuvAttach(c,loop); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + uv_run(loop, UV_RUN_DEFAULT); + return 0; +} diff --git a/deps/hiredis/example.c b/deps/hiredis/examples/example.c similarity index 73% rename from deps/hiredis/example.c rename to deps/hiredis/examples/example.c index 90ff9ed5e97..25226a807ea 100644 --- a/deps/hiredis/example.c +++ b/deps/hiredis/examples/example.c @@ -2,17 +2,24 @@ #include #include -#include "hiredis.h" +#include -int main(void) { +int main(int argc, char **argv) { unsigned int j; redisContext *c; redisReply *reply; + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout((char*)"127.0.0.2", 6379, timeout); - if (c->err) { - printf("Connection error: %s\n", c->errstr); + c = redisConnectWithTimeout(hostname, port, timeout); + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } exit(1); } @@ -27,7 +34,7 @@ int main(void) { freeReplyObject(reply); /* Set a key using binary safe API */ - reply = redisCommand(c,"SET %b %b", "bar", 3, "hello", 5); + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); printf("SET (binary API): %s\n", reply->str); freeReplyObject(reply); @@ -64,5 +71,8 @@ int main(void) { } freeReplyObject(reply); + /* Disconnects and frees the context */ + redisFree(c); + return 0; } diff --git a/deps/hiredis/fmacros.h b/deps/hiredis/fmacros.h index 21cd9cfee3d..6a41aa1761b 100644 --- a/deps/hiredis/fmacros.h +++ b/deps/hiredis/fmacros.h @@ -5,12 +5,20 @@ #define _BSD_SOURCE #endif +#if defined(_AIX) +#define _ALL_SOURCE +#endif + #if defined(__sun__) #define _POSIX_C_SOURCE 200112L -#elif defined(__linux__) +#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) #define _XOPEN_SOURCE 600 #else #define _XOPEN_SOURCE #endif +#if __APPLE__ && __MACH__ +#define _OSX +#endif + #endif diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index e6109db847c..2afee5666ec 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -446,7 +446,7 @@ static int processMultiBulkItem(redisReader *r) { long elements; int root = 0; - /* Set error for nested multi bulks with depth > 2 */ + /* Set error for nested multi bulks with depth > 7 */ if (r->ridx == 8) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "No support for nested multi bulk replies with depth > 7"); @@ -564,6 +564,7 @@ redisReader *redisReaderCreate(void) { r->errstr[0] = '\0'; r->fn = &defaultFunctions; r->buf = sdsempty(); + r->maxbuf = REDIS_READER_MAX_BUF; if (r->buf == NULL) { free(r); return NULL; @@ -590,9 +591,8 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) { /* Copy the provided buffer. */ if (buf != NULL && len >= 1) { -#if 0 /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && sdsavail(r->buf) > 16*1024) { + if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { sdsfree(r->buf); r->buf = sdsempty(); r->pos = 0; @@ -600,7 +600,6 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) { /* r->buf should not be NULL since we just free'd a larger one. */ assert(r->buf != NULL); } -#endif newbuf = sdscatlen(r->buf,buf,len); if (newbuf == NULL) { @@ -651,7 +650,7 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Discard part of the buffer when we've consumed at least 1k, to avoid * doing unnecessary calls to memmove() in sds.c. */ if (r->pos >= 1024) { - r->buf = sdsrange(r->buf,r->pos,-1); + sdsrange(r->buf,r->pos,-1); r->pos = 0; r->len = sdslen(r->buf); } @@ -918,7 +917,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { * %b represents a binary safe string * * When using %b you need to provide both the pointer to the string - * and the length in bytes. Examples: + * and the length in bytes as a size_t. Examples: * * len = redisFormatCommand(target, "GET %s", mykey); * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen); @@ -1011,58 +1010,122 @@ void redisFree(redisContext *c) { free(c); } +int redisFreeKeepFd(redisContext *c) { + int fd = c->fd; + c->fd = -1; + redisFree(c); + return fd; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectTcp(c,ip,port,NULL); return c; } -redisContext *redisConnectWithTimeout(const char *ip, int port, struct timeval tv) { - redisContext *c = redisContextInit(); +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectTcp(c,ip,port,&tv); return c; } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags &= ~REDIS_BLOCK; redisContextConnectTcp(c,ip,port,NULL); return c; } -redisContext *redisConnectUnix(const char *path) { +redisContext *redisConnectBindNonBlock(const char *ip, int port, + const char *source_addr) { redisContext *c = redisContextInit(); + c->flags &= ~REDIS_BLOCK; + redisContextConnectBindTcp(c,ip,port,NULL,source_addr); + return c; +} + +redisContext *redisConnectUnix(const char *path) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectUnix(c,path,NULL); return c; } -redisContext *redisConnectUnixWithTimeout(const char *path, struct timeval tv) { - redisContext *c = redisContextInit(); +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags |= REDIS_BLOCK; redisContextConnectUnix(c,path,&tv); return c; } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c = redisContextInit(); + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + c->flags &= ~REDIS_BLOCK; redisContextConnectUnix(c,path,NULL); return c; } +redisContext *redisConnectFd(int fd) { + redisContext *c; + + c = redisContextInit(); + if (c == NULL) + return NULL; + + c->fd = fd; + c->flags |= REDIS_BLOCK | REDIS_CONNECTED; + return c; +} + /* Set read/write timeout on a blocking socket. */ -int redisSetTimeout(redisContext *c, struct timeval tv) { +int redisSetTimeout(redisContext *c, const struct timeval tv) { if (c->flags & REDIS_BLOCK) return redisContextSetTimeout(c,tv); return REDIS_ERR; } +/* Enable connection KeepAlive. */ +int redisEnableKeepAlive(redisContext *c) { + if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK) + return REDIS_ERR; + return REDIS_OK; +} + /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * @@ -1078,7 +1141,7 @@ int redisBufferRead(redisContext *c) { nread = read(c->fd,buf,sizeof(buf)); if (nread == -1) { - if (errno == EAGAIN && !(c->flags & REDIS_BLOCK)) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ } else { __redisSetError(c,REDIS_ERR_IO,NULL); @@ -1115,7 +1178,7 @@ int redisBufferWrite(redisContext *c, int *done) { if (sdslen(c->obuf) > 0) { nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); if (nwritten == -1) { - if (errno == EAGAIN && !(c->flags & REDIS_BLOCK)) { + if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { /* Try again later */ } else { __redisSetError(c,REDIS_ERR_IO,NULL); @@ -1126,7 +1189,7 @@ int redisBufferWrite(redisContext *c, int *done) { sdsfree(c->obuf); c->obuf = sdsempty(); } else { - c->obuf = sdsrange(c->obuf,nwritten,-1); + sdsrange(c->obuf,nwritten,-1); } } } @@ -1181,7 +1244,7 @@ int redisGetReply(redisContext *c, void **reply) { * is used, you need to call redisGetReply yourself to retrieve * the reply (or replies in pub/sub). */ -int __redisAppendCommand(redisContext *c, char *cmd, size_t len) { +int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { sds newbuf; newbuf = sdscatlen(c->obuf,cmd,len); @@ -1194,6 +1257,15 @@ int __redisAppendCommand(redisContext *c, char *cmd, size_t len) { return REDIS_OK; } +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) { + + if (__redisAppendCommand(c, cmd, len) != REDIS_OK) { + return REDIS_ERR; + } + + return REDIS_OK; +} + int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { char *cmd; int len; diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index a73f50e957b..7700f4b8905 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -36,8 +36,8 @@ #include /* for struct timeval */ #define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 10 -#define HIREDIS_PATCH 1 +#define HIREDIS_MINOR 11 +#define HIREDIS_PATCH 0 #define REDIS_ERR -1 #define REDIS_OK 0 @@ -76,6 +76,9 @@ /* Flag that is set when the async context has one or more subscriptions. */ #define REDIS_SUBSCRIBED 0x20 +/* Flag that is set when monitor mode is active */ +#define REDIS_MONITORING 0x40 + #define REDIS_REPLY_STRING 1 #define REDIS_REPLY_ARRAY 2 #define REDIS_REPLY_INTEGER 3 @@ -83,6 +86,10 @@ #define REDIS_REPLY_STATUS 5 #define REDIS_REPLY_ERROR 6 +#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ + +#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ + #ifdef __cplusplus extern "C" { #endif @@ -122,6 +129,7 @@ typedef struct redisReader { char *buf; /* Read buffer */ size_t pos; /* Buffer cursor */ size_t len; /* Buffer length */ + size_t maxbuf; /* Max length of unused buffer */ redisReadTask rstack[9]; int ridx; /* Index of current read task */ @@ -165,13 +173,17 @@ typedef struct redisContext { } redisContext; redisContext *redisConnect(const char *ip, int port); -redisContext *redisConnectWithTimeout(const char *ip, int port, struct timeval tv); +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); +redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr); redisContext *redisConnectUnix(const char *path); -redisContext *redisConnectUnixWithTimeout(const char *path, struct timeval tv); +redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -int redisSetTimeout(redisContext *c, struct timeval tv); +redisContext *redisConnectFd(int fd); +int redisSetTimeout(redisContext *c, const struct timeval tv); +int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); +int redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); @@ -182,6 +194,10 @@ int redisBufferWrite(redisContext *c, int *done); int redisGetReply(redisContext *c, void **reply); int redisGetReplyFromReader(redisContext *c, void **reply); +/* Write a formatted command to the output buffer. Use these functions in blocking mode + * to get a pipeline of commands. */ +int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len); + /* Write a command to the output buffer. Use these functions in blocking mode * to get a pipeline of commands. */ int redisvAppendCommand(redisContext *c, const char *format, va_list ap); diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index 158e1dd8af8..bdb84ceedf7 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -45,6 +45,8 @@ #include #include #include +#include +#include #include "net.h" #include "sds.h" @@ -52,8 +54,15 @@ /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); +static void redisContextCloseFd(redisContext *c) { + if (c && c->fd >= 0) { + close(c->fd); + c->fd = -1; + } +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { - char buf[128]; + char buf[128] = { 0 }; size_t len = 0; if (prefix != NULL) @@ -62,11 +71,11 @@ static void __redisSetErrorFromErrno(redisContext *c, int type, const char *pref __redisSetError(c,type,buf); } -static int redisSetReuseAddr(redisContext *c, int fd) { +static int redisSetReuseAddr(redisContext *c) { int on = 1; - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { + if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } return REDIS_OK; @@ -78,23 +87,24 @@ static int redisCreateSocket(redisContext *c, int type) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } + c->fd = s; if (type == AF_INET) { - if (redisSetReuseAddr(c,s) == REDIS_ERR) { + if (redisSetReuseAddr(c) == REDIS_ERR) { return REDIS_ERR; } } - return s; + return REDIS_OK; } -static int redisSetBlocking(redisContext *c, int fd, int blocking) { +static int redisSetBlocking(redisContext *c, int blocking) { int flags; /* Set the socket nonblocking. * Note that fcntl(2) for F_GETFL and F_SETFL can't be * interrupted by a signal. */ - if ((flags = fcntl(fd, F_GETFL)) == -1) { + if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } @@ -103,84 +113,135 @@ static int redisSetBlocking(redisContext *c, int fd, int blocking) { else flags |= O_NONBLOCK; - if (fcntl(fd, F_SETFL, flags) == -1) { + if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - close(fd); + redisContextCloseFd(c); + return REDIS_ERR; + } + return REDIS_OK; +} + +int redisKeepAlive(redisContext *c, int interval) { + int val = 1; + int fd = c->fd; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval; + +#ifdef _OSX + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } +#else +#ifndef __sun + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = interval/3; + if (val == 0) val = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); + return REDIS_ERR; + } + + val = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); return REDIS_ERR; } +#endif +#endif + return REDIS_OK; } -static int redisSetTcpNoDelay(redisContext *c, int fd) { +static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; - if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { + if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } return REDIS_OK; } -static int redisContextWaitReady(redisContext *c, int fd, const struct timeval *timeout) { - struct timeval to; - struct timeval *toptr = NULL; - fd_set wfd; +#define __MAX_MSEC (((LONG_MAX) - 999) / 1000) + +static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) { + struct pollfd wfd[1]; + long msec; + + msec = -1; + wfd[0].fd = c->fd; + wfd[0].events = POLLOUT; /* Only use timeout when not NULL. */ if (timeout != NULL) { - to = *timeout; - toptr = &to; + if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL); + redisContextCloseFd(c); + return REDIS_ERR; + } + + msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000); + + if (msec < 0 || msec > INT_MAX) { + msec = INT_MAX; + } } if (errno == EINPROGRESS) { - FD_ZERO(&wfd); - FD_SET(fd, &wfd); + int res; - if (select(FD_SETSIZE, NULL, &wfd, NULL, toptr) == -1) { - __redisSetErrorFromErrno(c,REDIS_ERR_IO,"select(2)"); - close(fd); + if ((res = poll(wfd, 1, msec)) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); + redisContextCloseFd(c); return REDIS_ERR; - } - - if (!FD_ISSET(fd, &wfd)) { + } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } - if (redisCheckSocketError(c, fd) != REDIS_OK) + if (redisCheckSocketError(c) != REDIS_OK) return REDIS_ERR; return REDIS_OK; } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); + redisContextCloseFd(c); return REDIS_ERR; } -int redisCheckSocketError(redisContext *c, int fd) { +int redisCheckSocketError(redisContext *c) { int err = 0; socklen_t errlen = sizeof(err); - if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { + if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)"); - close(fd); return REDIS_ERR; } if (err) { errno = err; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - close(fd); return REDIS_ERR; } return REDIS_OK; } -int redisContextSetTimeout(redisContext *c, struct timeval tv) { +int redisContextSetTimeout(redisContext *c, const struct timeval tv) { if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; @@ -192,10 +253,12 @@ int redisContextSetTimeout(redisContext *c, struct timeval tv) { return REDIS_OK; } -int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct timeval *timeout) { +static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { int s, rv; char _port[6]; /* strlen("65535"); */ - struct addrinfo hints, *servinfo, *p; + struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); snprintf(_port, 6, "%d", port); @@ -203,33 +266,64 @@ int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct t hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; + /* Try with IPv6 if no IPv4 address was found. We do it in this order since + * in a Redis client you can't afford to test if you have IPv6 connectivity + * as this would add latency to every connect. Otherwise a more sensible + * route could be: Use IPv6 if both addresses are available and there is IPv6 + * connectivity. */ if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { - __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); - return REDIS_ERR; + hints.ai_family = AF_INET6; + if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); + return REDIS_ERR; + } } for (p = servinfo; p != NULL; p = p->ai_next) { if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; - if (redisSetBlocking(c,s,0) != REDIS_OK) + c->fd = s; + if (redisSetBlocking(c,0) != REDIS_OK) goto error; + if (source_addr) { + int bound = 0; + /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */ + if ((rv = getaddrinfo(source_addr, NULL, &hints, &bservinfo)) != 0) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + for (b = bservinfo; b != NULL; b = b->ai_next) { + if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { + bound = 1; + break; + } + } + freeaddrinfo(bservinfo); + if (!bound) { + char buf[128]; + snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno)); + __redisSetError(c,REDIS_ERR_OTHER,buf); + goto error; + } + } if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - close(s); + redisContextCloseFd(c); continue; } else if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { - if (redisContextWaitReady(c,s,timeout) != REDIS_OK) + if (redisContextWaitReady(c,timeout) != REDIS_OK) goto error; } } - if (blocking && redisSetBlocking(c,s,1) != REDIS_OK) + if (blocking && redisSetBlocking(c,1) != REDIS_OK) goto error; - if (redisSetTcpNoDelay(c,s) != REDIS_OK) + if (redisSetTcpNoDelay(c) != REDIS_OK) goto error; - c->fd = s; c->flags |= REDIS_CONNECTED; rv = REDIS_OK; goto end; @@ -248,32 +342,41 @@ int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct t return rv; // Need to return REDIS_OK if alright } -int redisContextConnectUnix(redisContext *c, const char *path, struct timeval *timeout) { - int s; +int redisContextConnectTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout) { + return _redisContextConnectTcp(c, addr, port, timeout, NULL); +} + +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr) { + return _redisContextConnectTcp(c, addr, port, timeout, source_addr); +} + +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { int blocking = (c->flags & REDIS_BLOCK); struct sockaddr_un sa; - if ((s = redisCreateSocket(c,AF_LOCAL)) < 0) + if (redisCreateSocket(c,AF_LOCAL) < 0) return REDIS_ERR; - if (redisSetBlocking(c,s,0) != REDIS_OK) + if (redisSetBlocking(c,0) != REDIS_OK) return REDIS_ERR; sa.sun_family = AF_LOCAL; strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { - if (redisContextWaitReady(c,s,timeout) != REDIS_OK) + if (redisContextWaitReady(c,timeout) != REDIS_OK) return REDIS_ERR; } } /* Reset socket to be blocking after connect(2). */ - if (blocking && redisSetBlocking(c,s,1) != REDIS_OK) + if (blocking && redisSetBlocking(c,1) != REDIS_OK) return REDIS_ERR; - c->fd = s; c->flags |= REDIS_CONNECTED; return REDIS_OK; } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index eb8a0a1cfbb..3763ab089da 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -35,13 +35,17 @@ #include "hiredis.h" -#if defined(__sun) +#if defined(__sun) || defined(_AIX) #define AF_LOCAL AF_UNIX #endif -int redisCheckSocketError(redisContext *c, int fd); -int redisContextSetTimeout(redisContext *c, struct timeval tv); -int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct timeval *timeout); -int redisContextConnectUnix(redisContext *c, const char *path, struct timeval *timeout); +int redisCheckSocketError(redisContext *c); +int redisContextSetTimeout(redisContext *c, const struct timeval tv); +int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); +int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, + const struct timeval *timeout, + const char *source_addr); +int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); +int redisKeepAlive(redisContext *c, int interval); #endif diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 0af9c672093..95454e997e9 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1,6 +1,6 @@ /* SDSLib, A C dynamic strings library * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,52 +32,76 @@ #include #include #include +#include #include "sds.h" +#include "zmalloc.h" -#ifdef SDS_ABORT_ON_OOM -static void sdsOomAbort(void) { - fprintf(stderr,"SDS: Out Of Memory (SDS_ABORT_ON_OOM defined)\n"); - abort(); -} -#endif - +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3"); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ sds sdsnewlen(const void *init, size_t initlen) { struct sdshdr *sh; - sh = malloc(sizeof(struct sdshdr)+initlen+1); -#ifdef SDS_ABORT_ON_OOM - if (sh == NULL) sdsOomAbort(); -#else + if (init) { + sh = zmalloc(sizeof(struct sdshdr)+initlen+1); + } else { + sh = zcalloc(sizeof(struct sdshdr)+initlen+1); + } if (sh == NULL) return NULL; -#endif sh->len = initlen; sh->free = 0; - if (initlen) { - if (init) memcpy(sh->buf, init, initlen); - else memset(sh->buf,0,initlen); - } + if (initlen && init) + memcpy(sh->buf, init, initlen); sh->buf[initlen] = '\0'; return (char*)sh->buf; } +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ sds sdsempty(void) { return sdsnewlen("",0); } +/* Create a new sds string starting from a null termined C string. */ sds sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); return sdsnewlen(init, initlen); } +/* Duplicate an sds string. */ sds sdsdup(const sds s) { return sdsnewlen(s, sdslen(s)); } +/* Free an sds string. No operation is performed if 's' is NULL. */ void sdsfree(sds s) { if (s == NULL) return; - free(s-sizeof(struct sdshdr)); + zfree(s-sizeof(struct sdshdr)); } +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ void sdsupdatelen(sds s) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); int reallen = strlen(s); @@ -85,7 +109,24 @@ void sdsupdatelen(sds s) { sh->len = reallen; } -static sds sdsMakeRoomFor(sds s, size_t addlen) { +/* Modify an sds string on-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ +void sdsclear(sds s) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + sh->free += sh->len; + sh->len = 0; + sh->buf[0] = '\0'; +} + +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *length* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { struct sdshdr *sh, *newsh; size_t free = sdsavail(s); size_t len, newlen; @@ -93,20 +134,86 @@ static sds sdsMakeRoomFor(sds s, size_t addlen) { if (free >= addlen) return s; len = sdslen(s); sh = (void*) (s-(sizeof(struct sdshdr))); - newlen = (len+addlen)*2; - newsh = realloc(sh, sizeof(struct sdshdr)+newlen+1); -#ifdef SDS_ABORT_ON_OOM - if (newsh == NULL) sdsOomAbort(); -#else + newlen = (len+addlen); + if (newlen < SDS_MAX_PREALLOC) + newlen *= 2; + else + newlen += SDS_MAX_PREALLOC; + newsh = zrealloc(sh, sizeof(struct sdshdr)+newlen+1); if (newsh == NULL) return NULL; -#endif newsh->free = newlen - len; return newsh->buf; } +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdsRemoveFreeSpace(sds s) { + struct sdshdr *sh; + + sh = (void*) (s-(sizeof(struct sdshdr))); + sh = zrealloc(sh, sizeof(struct sdshdr)+sh->len+1); + sh->free = 0; + return sh->buf; +} + +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ +size_t sdsAllocSize(sds s) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + return sizeof(*sh)+sh->len+sh->free+1; +} + +/* Increment the sds length and decrements the left free space at the + * end of the string according to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Usage example: + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nread); + */ +void sdsIncrLen(sds s, int incr) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + if (incr >= 0) + assert(sh->free >= (unsigned int)incr); + else + assert(sh->len >= (unsigned int)(-incr)); + sh->len += incr; + sh->free -= incr; + s[sh->len] = '\0'; +} + /* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. */ + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ sds sdsgrowzero(sds s, size_t len) { struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); size_t totlen, curlen = sh->len; @@ -124,6 +231,11 @@ sds sdsgrowzero(sds s, size_t len) { return s; } +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatlen(sds s, const void *t, size_t len) { struct sdshdr *sh; size_t curlen = sdslen(s); @@ -138,11 +250,25 @@ sds sdscatlen(sds s, const void *t, size_t len) { return s; } +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscat(sds s, const char *t) { return sdscatlen(s, t, strlen(t)); } -sds sdscpylen(sds s, char *t, size_t len) { +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatsds(sds s, const sds t) { + return sdscatlen(s, t, sdslen(t)); +} + +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ +sds sdscpylen(sds s, const char *t, size_t len) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t totlen = sh->free+sh->len; @@ -159,37 +285,144 @@ sds sdscpylen(sds s, char *t, size_t len) { return s; } -sds sdscpy(sds s, char *t) { +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ +sds sdscpy(sds s, const char *t) { return sdscpylen(s, t, strlen(t)); } +/* Helper for sdscatlonglong() doing the actual number -> string + * conversion. 's' must point to a string with room for at least + * SDS_LLSTR_SIZE bytes. + * + * The function returns the lenght of the null-terminated string + * representation stored at 's'. */ +#define SDS_LLSTR_SIZE 21 +int sdsll2str(char *s, long long value) { + char *p, aux; + unsigned long long v; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + v = (value < 0) ? -value : value; + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p++ = '-'; + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Identical sdsll2str(), but for unsigned long long type. */ +int sdsull2str(char *s, unsigned long long v) { + char *p, aux; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ +sds sdsfromlonglong(long long value) { + char buf[SDS_LLSTR_SIZE]; + int len = sdsll2str(buf,value); + + return sdsnewlen(buf,len); +} + +/* Like sdscatpritf() but gets va_list instead of being variadic. */ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { va_list cpy; - char *buf, *t; - size_t buflen = 16; + char staticbuf[1024], *buf = staticbuf, *t; + size_t buflen = strlen(fmt)*2; - while(1) { - buf = malloc(buflen); -#ifdef SDS_ABORT_ON_OOM - if (buf == NULL) sdsOomAbort(); -#else + /* We try to start using a static buffer for speed. + * If not possible we revert to heap allocation. */ + if (buflen > sizeof(staticbuf)) { + buf = zmalloc(buflen); if (buf == NULL) return NULL; -#endif + } else { + buflen = sizeof(staticbuf); + } + + /* Try with buffers two times bigger every time we fail to + * fit the string in the current buffer size. */ + while(1) { buf[buflen-2] = '\0'; va_copy(cpy,ap); vsnprintf(buf, buflen, fmt, cpy); + va_end(ap); if (buf[buflen-2] != '\0') { - free(buf); + if (buf != staticbuf) zfree(buf); buflen *= 2; + buf = zmalloc(buflen); + if (buf == NULL) return NULL; continue; } break; } + + /* Finally concat the obtained string to the SDS string and return it. */ t = sdscat(s, buf); - free(buf); + if (buf != staticbuf) zfree(buf); return t; } +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsempty("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ sds sdscatprintf(sds s, const char *fmt, ...) { va_list ap; char *t; @@ -199,6 +432,136 @@ sds sdscatprintf(sds s, const char *fmt, ...) { return t; } +/* This function is similar to sdscatprintf, but much faster as it does + * not rely on sprintf() family functions implemented by the libc that + * are often very slow. Moreover directly handling the sds string as + * new data is concatenated provides a performance improvement. + * + * However this function only handles an incompatible subset of printf-alike + * format specifiers: + * + * %s - C String + * %S - SDS string + * %i - signed int + * %I - 64 bit signed integer (long long, int64_t) + * %u - unsigned int + * %U - 64 bit unsigned integer (unsigned long long, uint64_t) + * %% - Verbatim "%" character. + */ +sds sdscatfmt(sds s, char const *fmt, ...) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + size_t initlen = sdslen(s); + const char *f = fmt; + int i; + va_list ap; + + va_start(ap,fmt); + f = fmt; /* Next format specifier byte to process. */ + i = initlen; /* Position of the next byte to write to dest str. */ + while(*f) { + char next, *str; + unsigned int l; + long long num; + unsigned long long unum; + + /* Make sure there is always space for at least 1 char. */ + if (sh->free == 0) { + s = sdsMakeRoomFor(s,1); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + + switch(*f) { + case '%': + next = *(f+1); + f++; + switch(next) { + case 's': + case 'S': + str = va_arg(ap,char*); + l = (next == 's') ? strlen(str) : sdslen(str); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,str,l); + sh->len += l; + sh->free -= l; + i += l; + break; + case 'i': + case 'I': + if (next == 'i') + num = va_arg(ap,int); + else + num = va_arg(ap,long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsll2str(buf,num); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + case 'u': + case 'U': + if (next == 'u') + unum = va_arg(ap,unsigned int); + else + unum = va_arg(ap,unsigned long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsull2str(buf,unum); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + default: /* Handle %% and generally %. */ + s[i++] = next; + sh->len += 1; + sh->free -= 1; + break; + } + break; + default: + s[i++] = *f; + sh->len += 1; + sh->free -= 1; + break; + } + f++; + } + va_end(ap); + + /* Add null-term */ + s[i] = '\0'; + return s; +} + +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"A. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ sds sdstrim(sds s, const char *cset) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); char *start, *end, *sp, *ep; @@ -216,11 +579,27 @@ sds sdstrim(sds s, const char *cset) { return s; } -sds sdsrange(sds s, int start, int end) { +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdsrange(s,1,-1); => "ello World" + */ +void sdsrange(sds s, int start, int end) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t newlen, len = sdslen(s); - if (len == 0) return s; + if (len == 0) return; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -244,22 +623,34 @@ sds sdsrange(sds s, int start, int end) { sh->buf[newlen] = 0; sh->free = sh->free+(sh->len-newlen); sh->len = newlen; - return s; } +/* Apply tolower() to every character of the sds string 's'. */ void sdstolower(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } +/* Apply toupper() to every character of the sds string 's'. */ void sdstoupper(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -int sdscmp(sds s1, sds s2) { +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * 1 if s1 > s2. + * -1 if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ +int sdscmp(const sds s1, const sds s2) { size_t l1, l2, minlen; int cmp; @@ -287,14 +678,15 @@ int sdscmp(sds s1, sds s2) { * requires length arguments. sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; + sds *tokens; + + if (seplen < 1 || len < 0) return NULL; + + tokens = zmalloc(sizeof(sds)*slots); + if (tokens == NULL) return NULL; - sds *tokens = malloc(sizeof(sds)*slots); -#ifdef SDS_ABORT_ON_OOM - if (tokens == NULL) sdsOomAbort(); -#endif - if (seplen < 1 || len < 0 || tokens == NULL) return NULL; if (len == 0) { *count = 0; return tokens; @@ -305,26 +697,14 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { sds *newtokens; slots *= 2; - newtokens = realloc(tokens,sizeof(sds)*slots); - if (newtokens == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + newtokens = zrealloc(tokens,sizeof(sds)*slots); + if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { tokens[elements] = sdsnewlen(s+start,j-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; j = j+seplen-1; /* skip the separator */ @@ -332,54 +712,37 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { } /* Add the final element. We are sure there is room in the tokens array. */ tokens[elements] = sdsnewlen(s+start,len-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; return tokens; -#ifndef SDS_ABORT_ON_OOM cleanup: { int i; for (i = 0; i < elements; i++) sdsfree(tokens[i]); - free(tokens); + zfree(tokens); + *count = 0; return NULL; } -#endif } +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ void sdsfreesplitres(sds *tokens, int count) { if (!tokens) return; while(count--) sdsfree(tokens[count]); - free(tokens); -} - -sds sdsfromlonglong(long long value) { - char buf[32], *p; - unsigned long long v; - - v = (value < 0) ? -value : value; - p = buf+31; /* point to the last character */ - do { - *p-- = '0'+(v%10); - v /= 10; - } while(v); - if (value < 0) *p-- = '-'; - p++; - return sdsnewlen(p,32-(p-buf)); + zfree(tokens); } -sds sdscatrepr(sds s, char *p, size_t len) { +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatrepr(sds s, const char *p, size_t len) { s = sdscatlen(s,"\"",1); - if (s == NULL) return NULL; - while(len--) { switch(*p) { case '\\': @@ -399,27 +762,64 @@ sds sdscatrepr(sds s, char *p, size_t len) { break; } p++; - if (s == NULL) return NULL; } return sdscatlen(s,"\"",1); } +/* Helper function for sdssplitargs() that returns non zero if 'c' + * is a valid hex digit. */ +int is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +/* Helper function for sdssplitargs() that converts a hex digit into an + * integer from 0 to 15 */ +int hex_digit_to_int(char c) { + switch(c) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 10; + case 'b': case 'B': return 11; + case 'c': case 'C': return 12; + case 'd': case 'D': return 13; + case 'e': case 'E': return 14; + case 'f': case 'F': return 15; + default: return 0; + } +} + /* Split a line into arguments, where every argument can be in the * following programming-language REPL-alike form: * * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. The caller should sdsfree() all the returned - * strings and finally free() the array itself. + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). * * Note that sdscatrepr() is able to convert back a string into * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' */ -sds *sdssplitargs(char *line, int *argc) { - char *p = line; +sds *sdssplitargs(const char *line, int *argc) { + const char *p = line; char *current = NULL; - char **vector = NULL, **_vector = NULL; + char **vector = NULL; *argc = 0; while(1) { @@ -427,17 +827,24 @@ sds *sdssplitargs(char *line, int *argc) { while(*p && isspace(*p)) p++; if (*p) { /* get a token */ - int inq=0; /* set to 1 if we are in "quotes" */ + int inq=0; /* set to 1 if we are in "quotes" */ + int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) { - current = sdsempty(); - if (current == NULL) goto err; - } - + if (current == NULL) current = sdsempty(); while(!done) { if (inq) { - if (*p == '\\' && *(p+1)) { + if (*p == '\\' && *(p+1) == 'x' && + is_hex_digit(*(p+2)) && + is_hex_digit(*(p+3))) + { + unsigned char byte; + + byte = (hex_digit_to_int(*(p+2))*16)+ + hex_digit_to_int(*(p+3)); + current = sdscatlen(current,(char*)&byte,1); + p += 3; + } else if (*p == '\\' && *(p+1)) { char c; p++; @@ -451,7 +858,23 @@ sds *sdssplitargs(char *line, int *argc) { } current = sdscatlen(current,&c,1); } else if (*p == '"') { - /* closing quote must be followed by a space */ + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else if (insq) { + if (*p == '\\' && *(p+1) == '\'') { + p++; + current = sdscatlen(current,"'",1); + } else if (*p == '\'') { + /* closing quote must be followed by a space or + * nothing at all. */ if (*(p+1) && !isspace(*(p+1))) goto err; done=1; } else if (!*p) { @@ -472,23 +895,24 @@ sds *sdssplitargs(char *line, int *argc) { case '"': inq=1; break; + case '\'': + insq=1; + break; default: current = sdscatlen(current,p,1); break; } } if (*p) p++; - if (current == NULL) goto err; } /* add the token to the vector */ - _vector = realloc(vector,((*argc)+1)*sizeof(char*)); - if (_vector == NULL) goto err; - - vector = _vector; + vector = zrealloc(vector,((*argc)+1)*sizeof(char*)); vector[*argc] = current; (*argc)++; current = NULL; } else { + /* Even on empty input string return something not NULL. */ + if (vector == NULL) vector = zmalloc(sizeof(void*)); return vector; } } @@ -496,30 +920,56 @@ sds *sdssplitargs(char *line, int *argc) { err: while((*argc)--) sdsfree(vector[*argc]); - if (vector != NULL) free(vector); - if (current != NULL) sdsfree(current); + zfree(vector); + if (current) sdsfree(current); + *argc = 0; return NULL; } +/* Modify the string substituting all the occurrences of the set of + * characters specified in the 'from' string to the corresponding character + * in the 'to' array. + * + * For instance: sdsmapchars(mystring, "ho", "01", 2) + * will have the effect of turning the string "hello" into "0ell1". + * + * The function returns the sds string pointer, that is always the same + * as the input pointer since no resize is needed. */ +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = sdslen(s); + + for (j = 0; j < l; j++) { + for (i = 0; i < setlen; i++) { + if (s[j] == from[i]) { + s[j] = to[i]; + break; + } + } + } + return s; +} + +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscat(join,sep); + } + return join; +} + #ifdef SDS_TEST_MAIN #include - -int __failed_tests = 0; -int __test_num = 0; -#define test_cond(descr,_c) do { \ - __test_num++; printf("%d - %s: ", __test_num, descr); \ - if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \ -} while(0); -#define test_report() do { \ - printf("%d tests, %d passed, %d failed\n", __test_num, \ - __test_num-__failed_tests, __failed_tests); \ - if (__failed_tests) { \ - printf("=== WARNING === We have failed tests here...\n"); \ - } \ -} while(0); +#include "testhelp.h" +#include "limits.h" int main(void) { { + struct sdshdr *sh; sds x = sdsnew("foo"), y; test_cond("Create a string and obtain the length", @@ -546,39 +996,61 @@ int main(void) { sdsfree(x); x = sdscatprintf(sdsempty(),"%d",123); test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) + sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) sdsfree(x); - x = sdstrim(sdsnew("xxciaoyyy"),"xy"); + x = sdsnew("--"); + x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("sdscatfmt() seems working in the base case", + sdslen(x) == 60 && + memcmp(x,"--Hello Hi! World -9223372036854775808," + "9223372036854775807--",60) == 0) + + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("sdscatfmt() seems working with unsigned numbers", + sdslen(x) == 35 && + memcmp(x,"--4294967295,18446744073709551615--",35) == 0) + + sdsfree(x); + x = sdsnew("xxciaoyyy"); + sdstrim(x,"xy"); test_cond("sdstrim() correctly trims characters", sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - y = sdsrange(sdsdup(x),1,1); + y = sdsdup(x); + sdsrange(y,1,1); test_cond("sdsrange(...,1,1)", sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,-1); + y = sdsdup(x); + sdsrange(y,1,-1); test_cond("sdsrange(...,1,-1)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),-2,-1); + y = sdsdup(x); + sdsrange(y,-2,-1); test_cond("sdsrange(...,-2,-1)", sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),2,1); + y = sdsdup(x); + sdsrange(y,2,1); test_cond("sdsrange(...,2,1)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,100); + y = sdsdup(x); + sdsrange(y,1,100); test_cond("sdsrange(...,1,100)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),100,100); + y = sdsdup(x); + sdsrange(y,100,100); test_cond("sdsrange(...,100,100)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) @@ -599,7 +1071,33 @@ int main(void) { x = sdsnew("aar"); y = sdsnew("bar"); test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + sdsfree(y); + sdsfree(x); + x = sdsnewlen("\a\n\0foo\r",7); + y = sdscatrepr(sdsempty(),x,sdslen(x)); + test_cond("sdscatrepr(...data...)", + memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) + + { + int oldfree; + + sdsfree(x); + x = sdsnew("0"); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0); + x = sdsMakeRoomFor(x,1); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0); + oldfree = sh->free; + x[1] = '1'; + sdsIncrLen(x,1); + test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1'); + test_cond("sdsIncrLen() -- len", sh->len == 2); + test_cond("sdsIncrLen() -- free", sh->free == oldfree-1); + } } test_report() + return 0; } #endif diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 94f5871f54f..37aaf7a28a4 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -31,14 +31,16 @@ #ifndef __SDS_H #define __SDS_H +#define SDS_MAX_PREALLOC (1024*1024) + #include #include typedef char *sds; struct sdshdr { - int len; - int free; + unsigned int len; + unsigned int free; char buf[]; }; @@ -58,12 +60,13 @@ sds sdsempty(void); size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); -size_t sdsavail(sds s); +size_t sdsavail(const sds s); sds sdsgrowzero(sds s, size_t len); sds sdscatlen(sds s, const void *t, size_t len); sds sdscat(sds s, const char *t); -sds sdscpylen(sds s, char *t, size_t len); -sds sdscpy(sds s, char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); sds sdscatvprintf(sds s, const char *fmt, va_list ap); #ifdef __GNUC__ @@ -73,16 +76,26 @@ sds sdscatprintf(sds s, const char *fmt, ...) sds sdscatprintf(sds s, const char *fmt, ...); #endif +sds sdscatfmt(sds s, char const *fmt, ...); sds sdstrim(sds s, const char *cset); -sds sdsrange(sds s, int start, int end); +void sdsrange(sds s, int start, int end); void sdsupdatelen(sds s); -int sdscmp(sds s1, sds s2); -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); void sdsfreesplitres(sds *tokens, int count); void sdstolower(sds s); void sdstoupper(sds s); sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, char *p, size_t len); -sds *sdssplitargs(char *line, int *argc); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); #endif diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 5945b655209..2cc35a46f52 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -8,12 +8,14 @@ #include #include #include +#include #include "hiredis.h" enum connection_type { CONN_TCP, - CONN_UNIX + CONN_UNIX, + CONN_FD }; struct config { @@ -22,6 +24,7 @@ struct config { struct { const char *host; int port; + struct timeval timeout; } tcp; struct { @@ -48,7 +51,7 @@ static redisContext *select_database(redisContext *c) { assert(reply != NULL); freeReplyObject(reply); - /* Make sure the DB is emtpy */ + /* Make sure the DB is empty */ reply = redisCommand(c,"DBSIZE"); assert(reply != NULL); if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) { @@ -62,7 +65,7 @@ static redisContext *select_database(redisContext *c) { return c; } -static void disconnect(redisContext *c) { +static int disconnect(redisContext *c, int keep_fd) { redisReply *reply; /* Make sure we're on DB 9. */ @@ -73,8 +76,11 @@ static void disconnect(redisContext *c) { assert(reply != NULL); freeReplyObject(reply); - /* Free the context as well. */ + /* Free the context as well, but keep the fd if requested. */ + if (keep_fd) + return redisFreeKeepFd(c); redisFree(c); + return -1; } static redisContext *connect(struct config config) { @@ -84,11 +90,22 @@ static redisContext *connect(struct config config) { c = redisConnect(config.tcp.host, config.tcp.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix.path); + } else if (config.type == CONN_FD) { + /* Create a dummy connection just to get an fd to inherit */ + redisContext *dummy_ctx = redisConnectUnix(config.unix.path); + if (dummy_ctx) { + int fd = disconnect(dummy_ctx, 1); + printf("Connecting to inherited fd %d\n", fd); + c = redisConnectFd(fd); + } } else { assert(NULL); } - if (c->err) { + if (c == NULL) { + printf("Connection error: can't allocate redis context\n"); + exit(1); + } else if (c->err) { printf("Connection error: %s\n", c->errstr); exit(1); } @@ -125,13 +142,13 @@ static void test_format_commands(void) { free(cmd); test("Format command with %%b string interpolation: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"b\0r",3); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); free(cmd); test("Format command with %%b and an empty string: "); - len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"",0); + len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); free(cmd); @@ -177,7 +194,7 @@ static void test_format_commands(void) { FLOAT_WIDTH_TEST(double); test("Format command with invalid printf format: "); - len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",3); + len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3); test_cond(len == -1); const char *argv[3]; @@ -200,10 +217,33 @@ static void test_format_commands(void) { free(cmd); } +static void test_append_formatted_commands(struct config config) { + redisContext *c; + redisReply *reply; + char *cmd; + int len; + + c = connect(config); + + test("Append format command: "); + + len = redisFormatCommand(&cmd, "SET foo bar"); + + test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK); + + assert(redisGetReply(c, (void*)&reply) == REDIS_OK); + + free(cmd); + freeReplyObject(reply); + + disconnect(c, 0); +} + static void test_reply_reader(void) { redisReader *reader; void *reply; int ret; + int i; test("Error handling in reply parser: "); reader = redisReaderCreate(); @@ -225,12 +265,13 @@ static void test_reply_reader(void) { strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); redisReaderFree(reader); - test("Set error on nested multi bulks with depth > 2: "); + test("Set error on nested multi bulks with depth > 7: "); reader = redisReaderCreate(); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); - redisReaderFeed(reader,(char*)"*1\r\n",4); + + for (i = 0; i < 9; i++) { + redisReaderFeed(reader,(char*)"*1\r\n",4); + } + ret = redisReaderGetReply(reader,NULL); test_cond(ret == REDIS_ERR && strncasecmp(reader->errstr,"No support for",14) == 0); @@ -284,7 +325,10 @@ static void test_blocking_connection_errors(void) { c = redisConnect((char*)"idontexist.local", 6379); test_cond(c->err == REDIS_ERR_OTHER && (strcmp(c->errstr,"Name or service not known") == 0 || - strcmp(c->errstr,"Can't resolve: idontexist.local") == 0)); + strcmp(c->errstr,"Can't resolve: idontexist.local") == 0 || + strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr,"No address associated with hostname") == 0 || + strcmp(c->errstr,"no address associated with name") == 0)); redisFree(c); test("Returns error when the port is not open: "); @@ -326,7 +370,7 @@ static void test_blocking_connection(struct config config) { freeReplyObject(reply); test("%%b String interpolation works: "); - reply = redisCommand(c,"SET %b %b","foo",3,"hello\x00world",11); + reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11); freeReplyObject(reply); reply = redisCommand(c,"GET foo"); test_cond(reply->type == REDIS_REPLY_STRING && @@ -374,7 +418,7 @@ static void test_blocking_connection(struct config config) { strcasecmp(reply->element[1]->str,"pong") == 0); freeReplyObject(reply); - disconnect(c); + disconnect(c, 0); } static void test_blocking_io_errors(struct config config) { @@ -428,6 +472,30 @@ static void test_blocking_io_errors(struct config config) { redisFree(c); } +static void test_invalid_timeout_errors(struct config config) { + redisContext *c; + + test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = 0; + config.tcp.timeout.tv_usec = 10000001; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO); + + test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: "); + + config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1; + config.tcp.timeout.tv_usec = 0; + + c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout); + + test_cond(c->err == REDIS_ERR_IO); + + redisFree(c); +} + static void test_throughput(struct config config) { redisContext *c = connect(config); redisReply **replies; @@ -490,7 +558,7 @@ static void test_throughput(struct config config) { free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - disconnect(c); + disconnect(c, 0); } // static long __test_callback_flags = 0; @@ -603,6 +671,7 @@ int main(int argc, char **argv) { } }; int throughput = 1; + int test_inherit_fd = 1; /* Ignore broken pipe signal (for I/O error tests). */ signal(SIGPIPE, SIG_IGN); @@ -621,6 +690,8 @@ int main(int argc, char **argv) { cfg.unix.path = argv[0]; } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) { throughput = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { + test_inherit_fd = 0; } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -636,6 +707,8 @@ int main(int argc, char **argv) { cfg.type = CONN_TCP; test_blocking_connection(cfg); test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path); @@ -644,6 +717,12 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); + if (test_inherit_fd) { + printf("\nTesting against inherited fd (%s):\n", cfg.unix.path); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } + if (fails) { printf("*** %d TESTS FAILED ***\n", fails); return 1; diff --git a/deps/hiredis/zmalloc.h b/deps/hiredis/zmalloc.h new file mode 100644 index 00000000000..99b87ace9ff --- /dev/null +++ b/deps/hiredis/zmalloc.h @@ -0,0 +1,13 @@ +/* Drop in replacement for zmalloc.h in order to just use libc malloc without + * any wrappering. */ + +#ifndef ZMALLOC_H +#define ZMALLOC_H + +#define zmalloc malloc +#define zrealloc realloc +#define zcalloc(x) calloc(x,1) +#define zfree free +#define zstrdup strdup + +#endif diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 32b4c424bde..4c408ec2c28 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -1,23 +1,72 @@ +/*.gcov.* + /autom4te.cache/ + +/bin/jemalloc.sh + /config.stamp /config.log /config.status /configure + /doc/html.xsl /doc/manpages.xsl /doc/jemalloc.xml /doc/jemalloc.html /doc/jemalloc.3 + /lib/ + /Makefile -/include/jemalloc/internal/jemalloc_internal\.h -/include/jemalloc/jemalloc\.h -/include/jemalloc/jemalloc_defs\.h -/test/jemalloc_test\.h + +/include/jemalloc/internal/jemalloc_internal.h +/include/jemalloc/internal/jemalloc_internal_defs.h +/include/jemalloc/internal/private_namespace.h +/include/jemalloc/internal/private_unnamespace.h +/include/jemalloc/internal/public_namespace.h +/include/jemalloc/internal/public_symbols.txt +/include/jemalloc/internal/public_unnamespace.h +/include/jemalloc/internal/size_classes.h +/include/jemalloc/jemalloc.h +/include/jemalloc/jemalloc_defs.h +/include/jemalloc/jemalloc_macros.h +/include/jemalloc/jemalloc_mangle.h +/include/jemalloc/jemalloc_mangle_jet.h +/include/jemalloc/jemalloc_protos.h +/include/jemalloc/jemalloc_protos_jet.h +/include/jemalloc/jemalloc_rename.h + /src/*.[od] -/test/*.[od] -/test/*.out -/test/[a-z]* -!test/*.c -!test/*.exp +/src/*.gcda +/src/*.gcno + +/test/test.sh +test/include/test/jemalloc_test.h +test/include/test/jemalloc_test_defs.h + +/test/integration/[A-Za-z]* +!/test/integration/[A-Za-z]*.* +/test/integration/*.[od] +/test/integration/*.gcda +/test/integration/*.gcno +/test/integration/*.out + +/test/src/*.[od] +/test/src/*.gcda +/test/src/*.gcno + +/test/stress/[A-Za-z]* +!/test/stress/[A-Za-z]*.* +/test/stress/*.[od] +/test/stress/*.gcda +/test/stress/*.gcno +/test/stress/*.out + +/test/unit/[A-Za-z]* +!/test/unit/[A-Za-z]*.* +/test/unit/*.[od] +/test/unit/*.gcda +/test/unit/*.gcno +/test/unit/*.out + /VERSION diff --git a/deps/jemalloc/COPYING b/deps/jemalloc/COPYING index 10ade120049..bdda0feb9e5 100644 --- a/deps/jemalloc/COPYING +++ b/deps/jemalloc/COPYING @@ -1,9 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are -subject to the following licenses: +subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2010 Jason Evans . +Copyright (C) 2002-2014 Jason Evans . All rights reserved. -Copyright (C) 2007-2010 Mozilla Foundation. All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2014 Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -24,28 +25,3 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Copyright (C) 2009-2010 Facebook, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Facebook, Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- diff --git a/deps/jemalloc/ChangeLog b/deps/jemalloc/ChangeLog index 326ee7a97e1..d56ee999e69 100644 --- a/deps/jemalloc/ChangeLog +++ b/deps/jemalloc/ChangeLog @@ -3,8 +3,306 @@ bug fixes are all mentioned, but internal enhancements are omitted here for brevity (even though they are more fun to write about). Much more detail can be found in the git revision history: - http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git - git://canonware.com/jemalloc.git + https://github.com/jemalloc/jemalloc + +* 3.6.0 (March 31, 2014) + + This version contains a critical bug fix for a regression present in 3.5.0 and + 3.5.1. + + Bug fixes: + - Fix a regression in arena_chunk_alloc() that caused crashes during + small/large allocation if chunk allocation failed. In the absence of this + bug, chunk allocation failure would result in allocation failure, e.g. NULL + return from malloc(). This regression was introduced in 3.5.0. + - Fix backtracing for gcc intrinsics-based backtracing by specifying + -fno-omit-frame-pointer to gcc. Note that the application (and all the + libraries it links to) must also be compiled with this option for + backtracing to be reliable. + - Use dss allocation precedence for huge allocations as well as small/large + allocations. + - Fix test assertion failure message formatting. This bug did not manifect on + x86_64 systems because of implementation subtleties in va_list. + - Fix inconsequential test failures for hash and SFMT code. + + New features: + - Support heap profiling on FreeBSD. This feature depends on the proc + filesystem being mounted during heap profile dumping. + +* 3.5.1 (February 25, 2014) + + This version primarily addresses minor bugs in test code. + + Bug fixes: + - Configure Solaris/Illumos to use MADV_FREE. + - Fix junk filling for mremap(2)-based huge reallocation. This is only + relevant if configuring with the --enable-mremap option specified. + - Avoid compilation failure if 'restrict' C99 keyword is not supported by the + compiler. + - Add a configure test for SSE2 rather than assuming it is usable on i686 + systems. This fixes test compilation errors, especially on 32-bit Linux + systems. + - Fix mallctl argument size mismatches (size_t vs. uint64_t) in the stats unit + test. + - Fix/remove flawed alignment-related overflow tests. + - Prevent compiler optimizations that could change backtraces in the + prof_accum unit test. + +* 3.5.0 (January 22, 2014) + + This version focuses on refactoring and automated testing, though it also + includes some non-trivial heap profiling optimizations not mentioned below. + + New features: + - Add the *allocx() API, which is a successor to the experimental *allocm() + API. The *allocx() functions are slightly simpler to use because they have + fewer parameters, they directly return the results of primary interest, and + mallocx()/rallocx() avoid the strict aliasing pitfall that + allocm()/rallocm() share with posix_memalign(). Note that *allocm() is + slated for removal in the next non-bugfix release. + - Add support for LinuxThreads. + + Bug fixes: + - Unless heap profiling is enabled, disable floating point code and don't link + with libm. This, in combination with e.g. EXTRA_CFLAGS=-mno-sse on x64 + systems, makes it possible to completely disable floating point register + use. Some versions of glibc neglect to save/restore caller-saved floating + point registers during dynamic lazy symbol loading, and the symbol loading + code uses whatever malloc the application happens to have linked/loaded + with, the result being potential floating point register corruption. + - Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling + backtrace creation in imemalign(). This bug impacted posix_memalign() and + aligned_alloc(). + - Fix a file descriptor leak in a prof_dump_maps() error path. + - Fix prof_dump() to close the dump file descriptor for all relevant error + paths. + - Fix rallocm() to use the arena specified by the ALLOCM_ARENA(s) flag for + allocation, not just deallocation. + - Fix a data race for large allocation stats counters. + - Fix a potential infinite loop during thread exit. This bug occurred on + Solaris, and could affect other platforms with similar pthreads TSD + implementations. + - Don't junk-fill reallocations unless usable size changes. This fixes a + violation of the *allocx()/*allocm() semantics. + - Fix growing large reallocation to junk fill new space. + - Fix huge deallocation to junk fill when munmap is disabled. + - Change the default private namespace prefix from empty to je_, and change + --with-private-namespace-prefix so that it prepends an additional prefix + rather than replacing je_. This reduces the likelihood of applications + which statically link jemalloc experiencing symbol name collisions. + - Add missing private namespace mangling (relevant when + --with-private-namespace is specified). + - Add and use JEMALLOC_INLINE_C so that static inline functions are marked as + static even for debug builds. + - Add a missing mutex unlock in a malloc_init_hard() error path. In practice + this error path is never executed. + - Fix numerous bugs in malloc_strotumax() error handling/reporting. These + bugs had no impact except for malformed inputs. + - Fix numerous bugs in malloc_snprintf(). These bugs were not exercised by + existing calls, so they had no impact. + +* 3.4.1 (October 20, 2013) + + Bug fixes: + - Fix a race in the "arenas.extend" mallctl that could cause memory corruption + of internal data structures and subsequent crashes. + - Fix Valgrind integration flaws that caused Valgrind warnings about reads of + uninitialized memory in: + + arena chunk headers + + internal zero-initialized data structures (relevant to tcache and prof + code) + - Preserve errno during the first allocation. A readlink(2) call during + initialization fails unless /etc/malloc.conf exists, so errno was typically + set during the first allocation prior to this fix. + - Fix compilation warnings reported by gcc 4.8.1. + +* 3.4.0 (June 2, 2013) + + This version is essentially a small bugfix release, but the addition of + aarch64 support requires that the minor version be incremented. + + Bug fixes: + - Fix race-triggered deadlocks in chunk_record(). These deadlocks were + typically triggered by multiple threads concurrently deallocating huge + objects. + + New features: + - Add support for the aarch64 architecture. + +* 3.3.1 (March 6, 2013) + + This version fixes bugs that are typically encountered only when utilizing + custom run-time options. + + Bug fixes: + - Fix a locking order bug that could cause deadlock during fork if heap + profiling were enabled. + - Fix a chunk recycling bug that could cause the allocator to lose track of + whether a chunk was zeroed. On FreeBSD, NetBSD, and OS X, it could cause + corruption if allocating via sbrk(2) (unlikely unless running with the + "dss:primary" option specified). This was completely harmless on Linux + unless using mlockall(2) (and unlikely even then, unless the + --disable-munmap configure option or the "dss:primary" option was + specified). This regression was introduced in 3.1.0 by the + mlockall(2)/madvise(2) interaction fix. + - Fix TLS-related memory corruption that could occur during thread exit if the + thread never allocated memory. Only the quarantine and prof facilities were + susceptible. + - Fix two quarantine bugs: + + Internal reallocation of the quarantined object array leaked the old + array. + + Reallocation failure for internal reallocation of the quarantined object + array (very unlikely) resulted in memory corruption. + - Fix Valgrind integration to annotate all internally allocated memory in a + way that keeps Valgrind happy about internal data structure access. + - Fix building for s390 systems. + +* 3.3.0 (January 23, 2013) + + This version includes a few minor performance improvements in addition to the + listed new features and bug fixes. + + New features: + - Add clipping support to lg_chunk option processing. + - Add the --enable-ivsalloc option. + - Add the --without-export option. + - Add the --disable-zone-allocator option. + + Bug fixes: + - Fix "arenas.extend" mallctl to output the number of arenas. + - Fix chunk_recycle() to unconditionally inform Valgrind that returned memory + is undefined. + - Fix build break on FreeBSD related to alloca.h. + +* 3.2.0 (November 9, 2012) + + In addition to a couple of bug fixes, this version modifies page run + allocation and dirty page purging algorithms in order to better control + page-level virtual memory fragmentation. + + Incompatible changes: + - Change the "opt.lg_dirty_mult" default from 5 to 3 (32:1 to 8:1). + + Bug fixes: + - Fix dss/mmap allocation precedence code to use recyclable mmap memory only + after primary dss allocation fails. + - Fix deadlock in the "arenas.purge" mallctl. This regression was introduced + in 3.1.0 by the addition of the "arena..purge" mallctl. + +* 3.1.0 (October 16, 2012) + + New features: + - Auto-detect whether running inside Valgrind, thus removing the need to + manually specify MALLOC_CONF=valgrind:true. + - Add the "arenas.extend" mallctl, which allows applications to create + manually managed arenas. + - Add the ALLOCM_ARENA() flag for {,r,d}allocm(). + - Add the "opt.dss", "arena..dss", and "stats.arenas..dss" mallctls, + which provide control over dss/mmap precedence. + - Add the "arena..purge" mallctl, which obsoletes "arenas.purge". + - Define LG_QUANTUM for hppa. + + Incompatible changes: + - Disable tcache by default if running inside Valgrind, in order to avoid + making unallocated objects appear reachable to Valgrind. + - Drop const from malloc_usable_size() argument on Linux. + + Bug fixes: + - Fix heap profiling crash if sampled object is freed via realloc(p, 0). + - Remove const from __*_hook variable declarations, so that glibc can modify + them during process forking. + - Fix mlockall(2)/madvise(2) interaction. + - Fix fork(2)-related deadlocks. + - Fix error return value for "thread.tcache.enabled" mallctl. + +* 3.0.0 (May 11, 2012) + + Although this version adds some major new features, the primary focus is on + internal code cleanup that facilitates maintainability and portability, most + of which is not reflected in the ChangeLog. This is the first release to + incorporate substantial contributions from numerous other developers, and the + result is a more broadly useful allocator (see the git revision history for + contribution details). Note that the license has been unified, thanks to + Facebook granting a license under the same terms as the other copyright + holders (see COPYING). + + New features: + - Implement Valgrind support, redzones, and quarantine. + - Add support for additional platforms: + + FreeBSD + + Mac OS X Lion + + MinGW + + Windows (no support yet for replacing the system malloc) + - Add support for additional architectures: + + MIPS + + SH4 + + Tilera + - Add support for cross compiling. + - Add nallocm(), which rounds a request size up to the nearest size class + without actually allocating. + - Implement aligned_alloc() (blame C11). + - Add the "thread.tcache.enabled" mallctl. + - Add the "opt.prof_final" mallctl. + - Update pprof (from gperftools 2.0). + - Add the --with-mangling option. + - Add the --disable-experimental option. + - Add the --disable-munmap option, and make it the default on Linux. + - Add the --enable-mremap option, which disables use of mremap(2) by default. + + Incompatible changes: + - Enable stats by default. + - Enable fill by default. + - Disable lazy locking by default. + - Rename the "tcache.flush" mallctl to "thread.tcache.flush". + - Rename the "arenas.pagesize" mallctl to "arenas.page". + - Change the "opt.lg_prof_sample" default from 0 to 19 (1 B to 512 KiB). + - Change the "opt.prof_accum" default from true to false. + + Removed features: + - Remove the swap feature, including the "config.swap", "swap.avail", + "swap.prezeroed", "swap.nfds", and "swap.fds" mallctls. + - Remove highruns statistics, including the + "stats.arenas..bins..highruns" and + "stats.arenas..lruns..highruns" mallctls. + - As part of small size class refactoring, remove the "opt.lg_[qc]space_max", + "arenas.cacheline", "arenas.subpage", "arenas.[tqcs]space_{min,max}", and + "arenas.[tqcs]bins" mallctls. + - Remove the "arenas.chunksize" mallctl. + - Remove the "opt.lg_prof_tcmax" option. + - Remove the "opt.lg_prof_bt_max" option. + - Remove the "opt.lg_tcache_gc_sweep" option. + - Remove the --disable-tiny option, including the "config.tiny" mallctl. + - Remove the --enable-dynamic-page-shift configure option. + - Remove the --enable-sysv configure option. + + Bug fixes: + - Fix a statistics-related bug in the "thread.arena" mallctl that could cause + invalid statistics and crashes. + - Work around TLS deallocation via free() on Linux. This bug could cause + write-after-free memory corruption. + - Fix a potential deadlock that could occur during interval- and + growth-triggered heap profile dumps. + - Fix large calloc() zeroing bugs due to dropping chunk map unzeroed flags. + - Fix chunk_alloc_dss() to stop claiming memory is zeroed. This bug could + cause memory corruption and crashes with --enable-dss specified. + - Fix fork-related bugs that could cause deadlock in children between fork + and exec. + - Fix malloc_stats_print() to honor 'b' and 'l' in the opts parameter. + - Fix realloc(p, 0) to act like free(p). + - Do not enforce minimum alignment in memalign(). + - Check for NULL pointer in malloc_usable_size(). + - Fix an off-by-one heap profile statistics bug that could be observed in + interval- and growth-triggered heap profiles. + - Fix the "epoch" mallctl to update cached stats even if the passed in epoch + is 0. + - Fix bin->runcur management to fix a layout policy bug. This bug did not + affect correctness. + - Fix a bug in choose_arena_hard() that potentially caused more arenas to be + initialized than necessary. + - Add missing "opt.lg_tcache_max" mallctl implementation. + - Use glibc allocator hooks to make mixed allocator usage less likely. + - Fix build issues for --disable-tcache. + - Don't mangle pthread_create() when --with-private-namespace is specified. * 2.2.5 (November 14, 2011) diff --git a/deps/jemalloc/INSTALL b/deps/jemalloc/INSTALL index 2a1e469cb8b..841704d2a08 100644 --- a/deps/jemalloc/INSTALL +++ b/deps/jemalloc/INSTALL @@ -26,6 +26,19 @@ any of the following arguments (not a definitive list) to 'configure': Embed one or more library paths, so that libjemalloc can find the libraries it is linked to. This works only on ELF-based systems. +--with-mangling= + Mangle public symbols specified in which is a comma-separated list of + name:mangled pairs. + + For example, to use ld's --wrap option as an alternative method for + overriding libc's malloc implementation, specify something like: + + --with-mangling=malloc:__wrap_malloc,free:__wrap_free[...] + + Note that mangling happens prior to application of the prefix specified by + --with-jemalloc-prefix, and mangled symbols are then ignored when applying + the prefix. + --with-jemalloc-prefix= Prefix all public APIs with . For example, if is "prefix_", API changes like the following occur: @@ -42,11 +55,16 @@ any of the following arguments (not a definitive list) to 'configure': jemalloc overlays the default malloc zone, but makes no attempt to actually replace the "malloc", "calloc", etc. symbols. +--without-export + Don't export public APIs. This can be useful when building jemalloc as a + static library, or to avoid exporting public APIs when using the zone + allocator on OSX. + --with-private-namespace= - Prefix all library-private APIs with . For shared libraries, + Prefix all library-private APIs with je_. For shared libraries, symbol visibility mechanisms prevent these symbols from being exported, but for static libraries, naming collisions are a real possibility. By - default, the prefix is "" (empty string). + default, is empty, which results in a symbol prefix of je_ . --with-install-suffix= Append to the base name of all installed files, such that multiple @@ -61,9 +79,28 @@ any of the following arguments (not a definitive list) to 'configure': --enable-debug Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. + Implies --enable-ivsalloc. ---enable-stats - Enable statistics gathering functionality. See the "opt.stats_print" +--enable-code-coverage + Enable code coverage support, for use during jemalloc test development. + Additional testing targets are available if this option is enabled: + + coverage + coverage_unit + coverage_integration + coverage_stress + + These targets do not clear code coverage results from previous runs, and + there are interactions between the various coverage targets, so it is + usually advisable to run 'make clean' between repeated code coverage runs. + +--enable-ivsalloc + Enable validation code, which verifies that pointers reside within + jemalloc-owned chunks before dereferencing them. This incurs a substantial + performance hit. + +--disable-stats + Disable statistics gathering functionality. See the "opt.stats_print" option documentation for usage details. --enable-prof @@ -90,51 +127,54 @@ any of the following arguments (not a definitive list) to 'configure': Statically link against the specified libunwind.a rather than dynamically linking with -lunwind. ---disable-tiny - Disable tiny (sub-quantum-sized) object support. Technically it is not - legal for a malloc implementation to allocate objects with less than - quantum alignment (8 or 16 bytes, depending on architecture), but in - practice it never causes any problems if, for example, 4-byte allocations - are 4-byte-aligned. - --disable-tcache Disable thread-specific caches for small objects. Objects are cached and released in bulk, thus reducing the total number of mutex operations. See the "opt.tcache" option for usage details. ---enable-swap - Enable mmap()ed swap file support. When this feature is built in, it is - possible to specify one or more files that act as backing store. This - effectively allows for per application swap files. +--enable-mremap + Enable huge realloc() via mremap(2). mremap() is disabled by default + because the flavor used is specific to Linux, which has a quirk in its + virtual memory allocation algorithm that causes semi-permanent VM map holes + under normal jemalloc operation. + +--disable-munmap + Disable virtual memory deallocation via munmap(2); instead keep track of + the virtual memory for later use. munmap() is disabled by default (i.e. + --disable-munmap is implied) on Linux, which has a quirk in its virtual + memory allocation algorithm that causes semi-permanent VM map holes under + normal jemalloc operation. --enable-dss Enable support for page allocation/deallocation via sbrk(2), in addition to mmap(2). ---enable-fill - Enable support for junk/zero filling of memory. See the "opt.junk"/ - "opt.zero" option documentation for usage details. +--disable-fill + Disable support for junk/zero filling of memory, quarantine, and redzones. + See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option + documentation for usage details. + +--disable-valgrind + Disable support for Valgrind. + +--disable-experimental + Disable support for the experimental API (*allocm()). + +--disable-zone-allocator + Disable zone allocator for Darwin. This means jemalloc won't be hooked as + the default allocator on OSX/iOS. + +--enable-utrace + Enable utrace(2)-based allocation tracing. This feature is not broadly + portable (FreeBSD has it, but Linux and OS X do not). --enable-xmalloc Enable support for optional immediate termination due to out-of-memory errors, as is commonly implemented by "xmalloc" wrapper function for malloc. See the "opt.xmalloc" option documentation for usage details. ---enable-sysv - Enable support for System V semantics, wherein malloc(0) returns NULL - rather than a minimal allocation. See the "opt.sysv" option documentation - for usage details. - ---enable-dynamic-page-shift - Under most conditions, the system page size never changes (usually 4KiB or - 8KiB, depending on architecture and configuration), and unless this option - is enabled, jemalloc assumes that page size can safely be determined during - configuration and hard-coded. Enabling dynamic page size determination has - a measurable impact on performance, since the compiler is forced to load - the page size from memory rather than embedding immediate values. - ---disable-lazy-lock - Disable code that wraps pthread_create() to detect when an application +--enable-lazy-lock + Enable code that wraps pthread_create() to detect when an application switches from single-threaded to multi-threaded mode, so that it can avoid mutex locking/unlocking operations while in single-threaded mode. In practice, this feature usually has little impact on performance unless @@ -181,11 +221,24 @@ PATH="?" === Advanced compilation ======================================================= +To build only parts of jemalloc, use the following targets: + + build_lib_shared + build_lib_static + build_lib + build_doc_html + build_doc_man + build_doc + To install only parts of jemalloc, use the following targets: install_bin install_include + install_lib_shared + install_lib_static install_lib + install_doc_html + install_doc_man install_doc To clean up build results to varying degrees, use the following make targets: @@ -248,10 +301,6 @@ directory, issue configuration and build commands: The manual page is generated in both html and roff formats. Any web browser can be used to view the html manual. The roff manual page can be formatted -prior to installation via any of the following commands: +prior to installation via the following command: nroff -man -t doc/jemalloc.3 - - groff -man -t -Tps doc/jemalloc.3 | ps2pdf - doc/jemalloc.3.pdf - - (cd doc; groff -man -man-ext -t -Thtml jemalloc.3 > jemalloc.3.html) diff --git a/deps/jemalloc/Makefile.in b/deps/jemalloc/Makefile.in index de7492f951a..d6b7d6ea3b4 100644 --- a/deps/jemalloc/Makefile.in +++ b/deps/jemalloc/Makefile.in @@ -17,129 +17,261 @@ INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@ LIBDIR := $(DESTDIR)@LIBDIR@ DATADIR := $(DESTDIR)@DATADIR@ MANDIR := $(DESTDIR)@MANDIR@ +srcroot := @srcroot@ +objroot := @objroot@ +abs_srcroot := @abs_srcroot@ +abs_objroot := @abs_objroot@ # Build parameters. -CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include +CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include CFLAGS := @CFLAGS@ -ifeq (macho, @abi@) -CFLAGS += -dynamic -endif LDFLAGS := @LDFLAGS@ +EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ LIBS := @LIBS@ RPATH_EXTRA := @RPATH_EXTRA@ -ifeq (macho, @abi@) -SO := dylib -WL_SONAME := dylib_install_name +SO := @so@ +IMPORTLIB := @importlib@ +O := @o@ +A := @a@ +EXE := @exe@ +LIBPREFIX := @libprefix@ +REV := @rev@ +install_suffix := @install_suffix@ +ABI := @abi@ +XSLTPROC := @XSLTPROC@ +AUTOCONF := @AUTOCONF@ +_RPATH = @RPATH@ +RPATH = $(if $(1),$(call _RPATH,$(1))) +cfghdrs_in := @cfghdrs_in@ +cfghdrs_out := @cfghdrs_out@ +cfgoutputs_in := @cfgoutputs_in@ +cfgoutputs_out := @cfgoutputs_out@ +enable_autogen := @enable_autogen@ +enable_code_coverage := @enable_code_coverage@ +enable_experimental := @enable_experimental@ +enable_zone_allocator := @enable_zone_allocator@ +DSO_LDFLAGS = @DSO_LDFLAGS@ +SOREV = @SOREV@ +PIC_CFLAGS = @PIC_CFLAGS@ +CTARGET = @CTARGET@ +LDTARGET = @LDTARGET@ +MKLIB = @MKLIB@ +AR = @AR@ +ARFLAGS = @ARFLAGS@ +CC_MM = @CC_MM@ + +ifeq (macho, $(ABI)) +TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib" else -SO := so -WL_SONAME := soname -endif -REV := 1 -ifeq (macho, @abi@) -TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH=@objroot@lib +ifeq (pecoff, $(ABI)) +TEST_LIBRARY_PATH := PATH="$(PATH):$(objroot)lib" else TEST_LIBRARY_PATH := endif +endif + +LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) # Lists of files. -BINS := @srcroot@bin/pprof -CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \ - @objroot@include/jemalloc/jemalloc_defs@install_suffix@.h -CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \ - @srcroot@src/base.c @srcroot@src/bitmap.c @srcroot@src/chunk.c \ - @srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \ - @srcroot@src/chunk_swap.c @srcroot@src/ckh.c @srcroot@src/ctl.c \ - @srcroot@src/extent.c @srcroot@src/hash.c @srcroot@src/huge.c \ - @srcroot@src/mb.c @srcroot@src/mutex.c @srcroot@src/prof.c \ - @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c -ifeq (macho, @abi@) -CSRCS += @srcroot@src/zone.c +BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh +C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h +C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \ + $(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \ + $(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \ + $(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \ + $(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \ + $(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \ + $(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \ + $(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c +ifeq ($(enable_zone_allocator), 1) +C_SRCS += $(srcroot)src/zone.c +endif +ifeq ($(IMPORTLIB),$(SO)) +STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A) +endif +ifdef PIC_CFLAGS +STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_pic.$(A) +else +STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_s.$(A) endif -STATIC_LIBS := @objroot@lib/libjemalloc@install_suffix@.a -DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) \ - @objroot@lib/libjemalloc@install_suffix@_pic.a -MAN3 := @objroot@doc/jemalloc@install_suffix@.3 -DOCS_XML := @objroot@doc/jemalloc@install_suffix@.xml -DOCS_HTML := $(DOCS_XML:@objroot@%.xml=@srcroot@%.html) -DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3) +DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) +ifneq ($(SOREV),$(SO)) +DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) +endif +MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 +DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml +DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) +DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) -CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \ - @srcroot@test/bitmap.c @srcroot@test/mremap.c \ - @srcroot@test/posix_memalign.c @srcroot@test/rallocm.c \ - @srcroot@test/thread_arena.c - -.PHONY: all dist doc_html doc_man doc +C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \ + $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ + $(srcroot)test/src/thd.c +C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c +TESTS_UNIT := $(srcroot)test/unit/bitmap.c \ + $(srcroot)test/unit/ckh.c \ + $(srcroot)test/unit/hash.c \ + $(srcroot)test/unit/junk.c \ + $(srcroot)test/unit/mallctl.c \ + $(srcroot)test/unit/math.c \ + $(srcroot)test/unit/mq.c \ + $(srcroot)test/unit/mtx.c \ + $(srcroot)test/unit/prof_accum.c \ + $(srcroot)test/unit/prof_gdump.c \ + $(srcroot)test/unit/prof_idump.c \ + $(srcroot)test/unit/ql.c \ + $(srcroot)test/unit/qr.c \ + $(srcroot)test/unit/quarantine.c \ + $(srcroot)test/unit/rb.c \ + $(srcroot)test/unit/rtree.c \ + $(srcroot)test/unit/SFMT.c \ + $(srcroot)test/unit/stats.c \ + $(srcroot)test/unit/tsd.c \ + $(srcroot)test/unit/util.c \ + $(srcroot)test/unit/zero.c +TESTS_UNIT_AUX := $(srcroot)test/unit/prof_accum_a.c \ + $(srcroot)test/unit/prof_accum_b.c +TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ + $(srcroot)test/integration/allocated.c \ + $(srcroot)test/integration/mallocx.c \ + $(srcroot)test/integration/mremap.c \ + $(srcroot)test/integration/posix_memalign.c \ + $(srcroot)test/integration/rallocx.c \ + $(srcroot)test/integration/thread_arena.c \ + $(srcroot)test/integration/thread_tcache_enabled.c \ + $(srcroot)test/integration/xallocx.c +ifeq ($(enable_experimental), 1) +TESTS_INTEGRATION += $(srcroot)test/integration/allocm.c \ + $(srcroot)test/integration/MALLOCX_ARENA.c \ + $(srcroot)test/integration/rallocm.c +endif +TESTS_STRESS := +TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS) + +C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O)) +C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O)) +C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O)) +C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O)) +C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) +C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) +C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O)) +C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS) + +TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_UNIT_AUX_OBJS := $(TESTS_UNIT_AUX:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O)) +TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS) + +.PHONY: all dist build_doc_html build_doc_man build_doc .PHONY: install_bin install_include install_lib -.PHONY: install_html install_man install_doc install +.PHONY: install_doc_html install_doc_man install_doc install .PHONY: tests check clean distclean relclean -.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o) +.SECONDARY : $(TESTS_OBJS) # Default target. -all: $(DSOS) $(STATIC_LIBS) +all: build_lib -dist: doc +dist: build_doc -@srcroot@doc/%.html : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/html.xsl - @XSLTPROC@ -o $@ @objroot@doc/html.xsl $< +$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl + $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< -@srcroot@doc/%.3 : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/manpages.xsl - @XSLTPROC@ -o $@ @objroot@doc/manpages.xsl $< +$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl + $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< -doc_html: $(DOCS_HTML) -doc_man: $(DOCS_MAN3) -doc: $(DOCS) +build_doc_html: $(DOCS_HTML) +build_doc_man: $(DOCS_MAN3) +build_doc: $(DOCS) # # Include generated dependency files. # --include $(CSRCS:@srcroot@%.c=@objroot@%.d) --include $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) --include $(CTESTS:@srcroot@%.c=@objroot@%.d) +ifdef CC_MM +-include $(C_OBJS:%.$(O)=%.d) +-include $(C_PIC_OBJS:%.$(O)=%.d) +-include $(C_JET_OBJS:%.$(O)=%.d) +-include $(C_TESTLIB_OBJS:%.$(O)=%.d) +-include $(TESTS_OBJS:%.$(O)=%.d) +endif -@objroot@src/%.o: @srcroot@src/%.c - @mkdir -p $(@D) - $(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)" +$(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c +$(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c +$(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS) +$(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c +$(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET +$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST +$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST +$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c +$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c +$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB +$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include +$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST +$(TESTS_UNIT_AUX_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST +define make-unit-link-dep +$(1): TESTS_UNIT_LINK_OBJS += $(2) +$(1): $(2) +endef +$(foreach test, $(TESTS_UNIT:$(srcroot)test/unit/%.c=$(objroot)test/unit/%$(EXE)), $(eval $(call make-unit-link-dep,$(test),$(filter $(test:%=%_a.$(O)) $(test:%=%_b.$(O)),$(TESTS_UNIT_AUX_OBJS))))) +$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST +$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST +$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c +$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include +ifneq ($(IMPORTLIB),$(SO)) +$(C_OBJS): CPPFLAGS += -DDLLEXPORT +endif + +ifndef CC_MM +# Dependencies. +HEADER_DIRS = $(srcroot)include/jemalloc/internal \ + $(objroot)include/jemalloc $(objroot)include/jemalloc/internal +HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)) +$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS) +$(TESTS_OBJS): $(objroot)test/unit/jemalloc_test.h +endif -@objroot@src/%.pic.o: @srcroot@src/%.c +$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O): @mkdir -p $(@D) - $(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $(basename $@))))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.pic.o \2/g\" > $(@:%.o=%.d)" + $(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $< +ifdef CC_MM + @$(CC) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $< +endif -%.$(SO) : %.$(SO).$(REV) +ifneq ($(SOREV),$(SO)) +%.$(SO) : %.$(SOREV) @mkdir -p $(@D) ln -sf $( $(@:%.o=%.d)" + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) -# Automatic dependency generation misses #include "*.c". -@objroot@test/bitmap.o : @objroot@src/bitmap.o +$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) + @mkdir -p $(@D) + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(EXTRA_LDFLAGS) -@objroot@test/%: @objroot@test/%.o \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) +$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) -ifneq (@RPATH@, ) - $(CC) -o $@ $< @RPATH@@objroot@lib -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -else - $(CC) -o $@ $< -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -endif + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) + +build_lib_shared: $(DSOS) +build_lib_static: $(STATIC_LIBS) +build_lib: build_lib_shared build_lib_static install_bin: install -d $(BINDIR) @@ -150,110 +282,157 @@ done install_include: install -d $(INCLUDEDIR)/jemalloc - @for h in $(CHDRS); do \ + @for h in $(C_HDRS); do \ echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \ install -m 644 $$h $(INCLUDEDIR)/jemalloc; \ done -install_lib: $(DSOS) $(STATIC_LIBS) +install_lib_shared: $(DSOS) install -d $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR) - ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO) - install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR) + install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR) +ifneq ($(SOREV),$(SO)) + ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO) +endif -install_html: - install -d $(DATADIR)/doc/jemalloc@install_suffix@ +install_lib_static: $(STATIC_LIBS) + install -d $(LIBDIR) + @for l in $(STATIC_LIBS); do \ + echo "install -m 755 $$l $(LIBDIR)"; \ + install -m 755 $$l $(LIBDIR); \ +done + +install_lib: install_lib_shared install_lib_static + +install_doc_html: + install -d $(DATADIR)/doc/jemalloc$(install_suffix) @for d in $(DOCS_HTML); do \ - echo "install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@"; \ - install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@; \ + echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \ + install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \ done -install_man: +install_doc_man: install -d $(MANDIR)/man3 @for d in $(DOCS_MAN3); do \ echo "install -m 644 $$d $(MANDIR)/man3"; \ install -m 644 $$d $(MANDIR)/man3; \ done -install_doc: install_html install_man +install_doc: install_doc_html install_doc_man install: install_bin install_include install_lib install_doc -tests: $(CTESTS:@srcroot@%.c=@objroot@%) - -check: tests - @mkdir -p @objroot@test - @$(SHELL) -c 'total=0; \ - failures=0; \ - echo "========================================="; \ - for t in $(CTESTS:@srcroot@%.c=@objroot@%); do \ - total=`expr $$total + 1`; \ - /bin/echo -n "$${t} ... "; \ - $(TEST_LIBRARY_PATH) $${t} @abs_srcroot@ @abs_objroot@ \ - > @objroot@$${t}.out 2>&1; \ - if test -e "@srcroot@$${t}.exp"; then \ - diff -u @srcroot@$${t}.exp \ - @objroot@$${t}.out >/dev/null 2>&1; \ - fail=$$?; \ - if test "$${fail}" -eq "1" ; then \ - failures=`expr $${failures} + 1`; \ - echo "*** FAIL ***"; \ - else \ - echo "pass"; \ - fi; \ - else \ - echo "*** FAIL *** (.exp file is missing)"; \ - failures=`expr $${failures} + 1`; \ - fi; \ - done; \ - echo "========================================="; \ - echo "Failures: $${failures}/$${total}"' +tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE)) +tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) +tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE)) +tests: tests_unit tests_integration tests_stress + +check_unit_dir: + @mkdir -p $(objroot)test/unit +check_integration_dir: + @mkdir -p $(objroot)test/integration +check_stress_dir: + @mkdir -p $(objroot)test/stress +check_dir: check_unit_dir check_integration_dir check_stress_dir + +check_unit: tests_unit check_unit_dir + $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) +check_integration: tests_integration check_integration_dir + $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) +check_stress: tests_stress check_stress_dir + $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%) +check: tests check_dir + $(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%) + +ifeq ($(enable_code_coverage), 1) +coverage_unit: check_unit + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) + +coverage_integration: check_integration + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS) + +coverage_stress: check_stress + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS) + +coverage: check + $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS) + $(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS) +endif clean: - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out) + rm -f $(C_OBJS) + rm -f $(C_PIC_OBJS) + rm -f $(C_JET_OBJS) + rm -f $(C_TESTLIB_OBJS) + rm -f $(C_OBJS:%.$(O)=%.d) + rm -f $(C_OBJS:%.$(O)=%.gcda) + rm -f $(C_OBJS:%.$(O)=%.gcno) + rm -f $(C_PIC_OBJS:%.$(O)=%.d) + rm -f $(C_PIC_OBJS:%.$(O)=%.gcda) + rm -f $(C_PIC_OBJS:%.$(O)=%.gcno) + rm -f $(C_JET_OBJS:%.$(O)=%.d) + rm -f $(C_JET_OBJS:%.$(O)=%.gcda) + rm -f $(C_JET_OBJS:%.$(O)=%.gcno) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda) + rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno) + rm -f $(TESTS_OBJS:%.$(O)=%$(EXE)) + rm -f $(TESTS_OBJS) + rm -f $(TESTS_OBJS:%.$(O)=%.d) + rm -f $(TESTS_OBJS:%.$(O)=%.gcda) + rm -f $(TESTS_OBJS:%.$(O)=%.gcno) + rm -f $(TESTS_OBJS:%.$(O)=%.out) rm -f $(DSOS) $(STATIC_LIBS) + rm -f $(objroot)*.gcov.* distclean: clean - rm -rf @objroot@autom4te.cache - rm -f @objroot@config.log - rm -f @objroot@config.status - rm -f @objroot@config.stamp - rm -f @cfghdrs_out@ - rm -f @cfgoutputs_out@ + rm -rf $(objroot)autom4te.cache + rm -f $(objroot)bin/jemalloc.sh + rm -f $(objroot)config.log + rm -f $(objroot)config.status + rm -f $(objroot)config.stamp + rm -f $(cfghdrs_out) + rm -f $(cfgoutputs_out) relclean: distclean - rm -f @objroot@configure - rm -f @srcroot@VERSION + rm -f $(objroot)configure + rm -f $(srcroot)VERSION rm -f $(DOCS_HTML) rm -f $(DOCS_MAN3) #=============================================================================== # Re-configuration rules. -ifeq (@enable_autogen@, 1) -@srcroot@configure : @srcroot@configure.ac - cd ./@srcroot@ && @AUTOCONF@ +ifeq ($(enable_autogen), 1) +$(srcroot)configure : $(srcroot)configure.ac + cd ./$(srcroot) && $(AUTOCONF) -@objroot@config.status : @srcroot@configure - ./@objroot@config.status --recheck +$(objroot)config.status : $(srcroot)configure + ./$(objroot)config.status --recheck -@srcroot@config.stamp.in : @srcroot@configure.ac - echo stamp > @srcroot@config.stamp.in +$(srcroot)config.stamp.in : $(srcroot)configure.ac + echo stamp > $(srcroot)config.stamp.in -@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure - ./@objroot@config.status +$(objroot)config.stamp : $(cfgoutputs_in) $(cfghdrs_in) $(srcroot)configure + ./$(objroot)config.status @touch $@ # There must be some action in order for make to re-read Makefile when it is # out of date. -@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp +$(cfgoutputs_out) $(cfghdrs_out) : $(objroot)config.stamp @true endif diff --git a/deps/jemalloc/README b/deps/jemalloc/README index 4d7b552bf3d..9b268f42288 100644 --- a/deps/jemalloc/README +++ b/deps/jemalloc/README @@ -1,10 +1,14 @@ -jemalloc is a general-purpose scalable concurrent malloc(3) implementation. -This distribution is a stand-alone "portable" implementation that currently -targets Linux and Apple OS X. jemalloc is included as the default allocator in -the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox -web browser on Microsoft Windows-related platforms. Depending on your needs, -one of the other divergent versions may suit your needs better than this -distribution. +jemalloc is a general purpose malloc(3) implementation that emphasizes +fragmentation avoidance and scalable concurrency support. jemalloc first came +into use as the FreeBSD libc allocator in 2005, and since then it has found its +way into numerous applications that rely on its predictable behavior. In 2010 +jemalloc development efforts broadened to include developer support features +such as heap profiling, Valgrind integration, and extensive monitoring/tuning +hooks. Modern jemalloc releases continue to be integrated back into FreeBSD, +and therefore versatility remains critical. Ongoing development efforts trend +toward making jemalloc among the best allocators for a broad range of demanding +applications, and eliminating/mitigating weaknesses that have practical +repercussions for real world applications. The COPYING file contains copyright and licensing information. diff --git a/deps/jemalloc/VERSION b/deps/jemalloc/VERSION index aa85f5a2acf..dace31ba7b6 100644 --- a/deps/jemalloc/VERSION +++ b/deps/jemalloc/VERSION @@ -1 +1 @@ -2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760 +3.6.0-0-g46c0af68bd248b04df75e4f92d5fb804c3d75340 diff --git a/deps/jemalloc/bin/jemalloc.sh.in b/deps/jemalloc/bin/jemalloc.sh.in new file mode 100644 index 00000000000..cdf36737591 --- /dev/null +++ b/deps/jemalloc/bin/jemalloc.sh.in @@ -0,0 +1,9 @@ +#!/bin/sh + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ + +@LD_PRELOAD_VAR@=${libdir}/libjemalloc.@SOREV@ +export @LD_PRELOAD_VAR@ +exec "$@" diff --git a/deps/jemalloc/bin/pprof b/deps/jemalloc/bin/pprof index 280ddcc8329..a309943c1cb 100755 --- a/deps/jemalloc/bin/pprof +++ b/deps/jemalloc/bin/pprof @@ -72,7 +72,7 @@ use strict; use warnings; use Getopt::Long; -my $PPROF_VERSION = "1.7"; +my $PPROF_VERSION = "2.0"; # These are the object tools we use which can come from a # user-specified location using --tools, from the PPROF_TOOLS @@ -87,13 +87,14 @@ my %obj_tool_map = ( #"addr2line_pdb" => "addr2line-pdb", # ditto #"otool" => "otool", # equivalent of objdump on OS X ); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $EVINCE = "evince"; # could also be xpdf or perhaps acroread -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; +# NOTE: these are lists, so you can put in commandline flags if you want. +my @DOT = ("dot"); # leave non-absolute, since it may be in /usr/local +my @GV = ("gv"); +my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread +my @KCACHEGRIND = ("kcachegrind"); +my @PS2PDF = ("ps2pdf"); # These are used for dynamic profiles -my $URL_FETCHER = "curl -s"; +my @URL_FETCHER = ("curl", "-s"); # These are the web pages that servers need to support for dynamic profiles my $HEAP_PAGE = "/pprof/heap"; @@ -104,7 +105,10 @@ my $GROWTH_PAGE = "/pprof/growth"; my $CONTENTION_PAGE = "/pprof/contention"; my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $CENSUSPROFILE_PAGE = "/pprof/censusprofile"; # must support "?seconds=#" +my $CENSUSPROFILE_PAGE = "/pprof/censusprofile(?:\\?.*)?"; # must support cgi-param + # "?seconds=#", + # "?tags_regexp=#" and + # "?type=#". my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; @@ -122,6 +126,11 @@ my $UNKNOWN_BINARY = "(unknown)"; # 64-bit profiles. To err on the safe size, default to 64-bit here: my $address_length = 16; +my $dev_null = "/dev/null"; +if (! -e $dev_null && $^O =~ /MSWin/) { # $^O is the OS perl was built for + $dev_null = "nul"; +} + # A list of paths to search for shared object files my @prefix_list = (); @@ -151,7 +160,8 @@ pprof [options] The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. - For instance: "pprof http://myserver.com:80$HEAP_PAGE". + For instance: + pprof http://myserver.com:80$HEAP_PAGE If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). pprof --symbols Maps addresses to symbol names. In this mode, stdin should be a @@ -162,7 +172,7 @@ pprof --symbols For more help with querying remote servers, including how to add the necessary server-side support code, see this filename (or one like it): - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html + /usr/doc/gperftools-$PPROF_VERSION/pprof_remote_servers.html Options: --cum Sort by cumulative data @@ -260,7 +270,7 @@ EOF sub version_string { return < 0) { + if (IsProfileURL($ARGV[0])) { + $main::use_symbol_page = 1; + } elsif (IsSymbolizedProfileFile($ARGV[0])) { + $main::use_symbolized_profile = 1; + $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file + } } if ($main::use_symbol_page || $main::use_symbolized_profile) { @@ -540,7 +552,7 @@ sub Init() { ConfigureObjTools($main::prog) } - # Break the opt_list_prefix into the prefix_list array + # Break the opt_lib_prefix into the prefix_list array @prefix_list = split (',', $main::opt_lib_prefix); # Remove trailing / from the prefixes, in the list to prevent @@ -636,9 +648,9 @@ sub Main() { # Print if (!$main::opt_interactive) { if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); + PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); } elsif ($main::opt_list) { - PrintListing($libs, $flat, $cumulative, $main::opt_list); + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); } elsif ($main::opt_text) { # Make sure the output is empty when have nothing to report # (only matters when --heapcheck is given but we must be @@ -646,7 +658,7 @@ sub Main() { if ($total != 0) { printf("Total: %s %s\n", Unparse($total), Units()); } - PrintText($symbols, $flat, $cumulative, $total, -1); + PrintText($symbols, $flat, $cumulative, -1); } elsif ($main::opt_raw) { PrintSymbolizedProfile($symbols, $profile, $main::prog); } elsif ($main::opt_callgrind) { @@ -656,7 +668,7 @@ sub Main() { if ($main::opt_gv) { RunGV(TempName($main::next_tmpfile, "ps"), ""); } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); } elsif ($main::opt_web) { my $tmp = TempName($main::next_tmpfile, "svg"); RunWeb($tmp); @@ -705,24 +717,25 @@ sub ReadlineMightFail { sub RunGV { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >/dev/null 2>&1")) { + if (!system(ShellEscape(@GV, "--version") . " >$dev_null 2>&1")) { # Options using double dash are supported by this gv version. # Also, turn on noantialias to better handle bug in gv for # postscript files with large dimensions. # TODO: Maybe we should not pass the --noantialias flag # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); + system(ShellEscape(@GV, "--scale=$main::opt_scale", "--noantialias", $fname) + . $bg); } else { # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); + print STDERR ShellEscape(@GV, "-scale", $main::opt_scale) . "\n"; + system(ShellEscape(@GV, "-scale", "$main::opt_scale", $fname) . $bg); } } sub RunEvince { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - system("$EVINCE " . $fname . $bg); + system(ShellEscape(@EVINCE, $fname) . $bg); } sub RunWeb { @@ -756,8 +769,8 @@ sub RunWeb { sub RunKcachegrind { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); + print STDERR "Starting '@KCACHEGRIND " . $fname . $bg . "'\n"; + system(ShellEscape(@KCACHEGRIND, $fname) . $bg); } @@ -834,14 +847,14 @@ sub InteractiveCommand { my $ignore; ($routine, $ignore) = ParseInteractiveArgs($3); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintText($symbols, $flat, $cumulative, $total, $line_limit); + PrintText($symbols, $flat, $cumulative, $line_limit); return 1; } if (m/^\s*callgrind\s*([^ \n]*)/) { @@ -861,21 +874,22 @@ sub InteractiveCommand { return 1; } - if (m/^\s*list\s*(.+)/) { + if (m/^\s*(web)?list\s*(.+)/) { + my $html = (defined($1) && ($1 eq "web")); $main::opt_list = 1; my $routine; my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); + ($routine, $ignore) = ParseInteractiveArgs($2); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintListing($libs, $flat, $cumulative, $routine); + PrintListing($total, $libs, $flat, $cumulative, $routine, $html); return 1; } if (m/^\s*disasm\s*(.+)/) { @@ -886,14 +900,14 @@ sub InteractiveCommand { ($routine, $ignore) = ParseInteractiveArgs($1); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintDisassembly($libs, $flat, $cumulative, $routine, $total); + PrintDisassembly($libs, $flat, $cumulative, $routine); return 1; } if (m/^\s*(gv|web|evince)\s*(.*)/) { @@ -913,7 +927,8 @@ sub InteractiveCommand { ($focus, $ignore) = ParseInteractiveArgs($2); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, + $focus, $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -941,6 +956,7 @@ sub InteractiveCommand { sub ProcessProfile { + my $total_count = shift; my $orig_profile = shift; my $symbols = shift; my $focus = shift; @@ -948,7 +964,6 @@ sub ProcessProfile { # Process current profile to account for various settings my $profile = $orig_profile; - my $total_count = TotalProfile($profile); printf("Total: %s %s\n", Unparse($total_count), Units()); if ($focus ne '') { $profile = FocusProfile($symbols, $profile, $focus); @@ -995,6 +1010,11 @@ Commands: list [routine_regexp] [-ignore1] [-ignore2] Show source listing of routines whose names match "routine_regexp" + weblist [routine_regexp] [-ignore1] [-ignore2] + Displays a source listing of routines whose names match "routine_regexp" + in a web browser. You can click on source lines to view the + corresponding disassembly. + top [--cum] [-ignore1] [-ignore2] top20 [--cum] [-ignore1] [-ignore2] top37 [--cum] [-ignore1] [-ignore2] @@ -1019,8 +1039,8 @@ parameters will be ignored. Further pprof details are available at this location (or one similar): - /usr/doc/google-perftools-$PPROF_VERSION/cpu_profiler.html - /usr/doc/google-perftools-$PPROF_VERSION/heap_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/cpu_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/heap_profiler.html ENDOFHELP } @@ -1137,9 +1157,10 @@ sub PrintText { my $symbols = shift; my $flat = shift; my $cumulative = shift; - my $total = shift; my $line_limit = shift; + my $total = TotalProfile($flat); + # Which profile to sort by? my $s = $main::opt_cum ? $cumulative : $flat; @@ -1169,7 +1190,29 @@ sub PrintText { $sym); } $lines++; - last if ($line_limit >= 0 && $lines > $line_limit); + last if ($line_limit >= 0 && $lines >= $line_limit); + } +} + +# Callgrind format has a compression for repeated function and file +# names. You show the name the first time, and just use its number +# subsequently. This can cut down the file to about a third or a +# quarter of its uncompressed size. $key and $val are the key/value +# pair that would normally be printed by callgrind; $map is a map from +# value to number. +sub CompressedCGName { + my($key, $val, $map) = @_; + my $idx = $map->{$val}; + # For very short keys, providing an index hurts rather than helps. + if (length($val) <= 3) { + return "$key=$val\n"; + } elsif (defined($idx)) { + return "$key=($idx)\n"; + } else { + # scalar(keys $map) gives the number of items in the map. + $idx = scalar(keys(%{$map})) + 1; + $map->{$val} = $idx; + return "$key=($idx) $val\n"; } } @@ -1177,13 +1220,16 @@ sub PrintText { sub PrintCallgrind { my $calls = shift; my $filename; + my %filename_to_index_map; + my %fnname_to_index_map; + if ($main::opt_interactive) { $filename = shift; print STDERR "Writing callgrind file to '$filename'.\n" } else { $filename = "&STDOUT"; } - open(CG, ">".$filename ); + open(CG, ">$filename"); printf CG ("events: Hits\n\n"); foreach my $call ( map { $_->[0] } sort { $a->[1] cmp $b ->[1] || @@ -1197,11 +1243,14 @@ sub PrintCallgrind { $callee_file, $callee_line, $callee_function ) = ( $1, $2, $3, $5, $6, $7 ); - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); + # TODO(csilvers): for better compression, collect all the + # caller/callee_files and functions first, before printing + # anything, and only compress those referenced more than once. + printf CG CompressedCGName("fl", $caller_file, \%filename_to_index_map); + printf CG CompressedCGName("fn", $caller_function, \%fnname_to_index_map); if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); + printf CG CompressedCGName("cfl", $callee_file, \%filename_to_index_map); + printf CG CompressedCGName("cfn", $callee_function, \%fnname_to_index_map); printf CG ("calls=$count $callee_line\n"); } printf CG ("$caller_line $count\n\n"); @@ -1214,7 +1263,8 @@ sub PrintDisassembly { my $flat = shift; my $cumulative = shift; my $disasm_opts = shift; - my $total = shift; + + my $total = TotalProfile($flat); foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); @@ -1249,10 +1299,10 @@ sub Disassemble { my $end_addr = shift; my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); + my $cmd = ShellEscape($objdump, "-C", "-d", "-l", "--no-show-raw-insn", + "--start-address=0x$start_addr", + "--stop-address=0x$end_addr", $prog); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); my @result = (); my $filename = ""; my $linenumber = -1; @@ -1315,13 +1365,33 @@ sub ByName { return ShortFunctionName($a) cmp ShortFunctionName($b); } -# Print source-listing for all all routines that match $main::opt_list +# Print source-listing for all all routines that match $list_opts sub PrintListing { + my $total = shift; my $libs = shift; my $flat = shift; my $cumulative = shift; my $list_opts = shift; + my $html = shift; + my $output = \*STDOUT; + my $fname = ""; + + if ($html) { + # Arrange to write the output to a temporary file + $fname = TempName($main::next_tmpfile, "html"); + $main::next_tmpfile++; + if (!open(TEMP, ">$fname")) { + print STDERR "$fname: $!\n"; + return; + } + $output = \*TEMP; + print $output HtmlListingHeader(); + printf $output ("
%s
Total: %s %s
\n", + $main::prog, Unparse($total), Units()); + } + + my $listed = 0; foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); my $offset = AddressSub($lib->[1], $lib->[3]); @@ -1333,15 +1403,113 @@ sub PrintListing { my $addr = AddressAdd($start_addr, $offset); for (my $i = 0; $i < $length; $i++) { if (defined($cumulative->{$addr})) { - PrintSource($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr); + $listed += PrintSource( + $lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr, + $html, + $output); last; } $addr = AddressInc($addr); } } } + + if ($html) { + if ($listed > 0) { + print $output HtmlListingFooter(); + close($output); + RunWeb($fname); + } else { + close($output); + unlink($fname); + } + } +} + +sub HtmlListingHeader { + return <<'EOF'; + + + +Pprof listing + + + + +EOF +} + +sub HtmlListingFooter { + return <<'EOF'; + + +EOF +} + +sub HtmlEscape { + my $text = shift; + $text =~ s/&/&/g; + $text =~ s//>/g; + return $text; } # Returns the indentation of the line, if it has any non-whitespace @@ -1355,6 +1523,45 @@ sub Indentation { } } +# If the symbol table contains inlining info, Disassemble() may tag an +# instruction with a location inside an inlined function. But for +# source listings, we prefer to use the location in the function we +# are listing. So use MapToSymbols() to fetch full location +# information for each instruction and then pick out the first +# location from a location list (location list contains callers before +# callees in case of inlining). +# +# After this routine has run, each entry in $instructions contains: +# [0] start address +# [1] filename for function we are listing +# [2] line number for function we are listing +# [3] disassembly +# [4] limit address +# [5] most specific filename (may be different from [1] due to inlining) +# [6] most specific line number (may be different from [2] due to inlining) +sub GetTopLevelLineNumbers { + my ($lib, $offset, $instructions) = @_; + my $pcs = []; + for (my $i = 0; $i <= $#{$instructions}; $i++) { + push(@{$pcs}, $instructions->[$i]->[0]); + } + my $symbols = {}; + MapToSymbols($lib, $offset, $pcs, $symbols); + for (my $i = 0; $i <= $#{$instructions}; $i++) { + my $e = $instructions->[$i]; + push(@{$e}, $e->[1]); + push(@{$e}, $e->[2]); + my $addr = $e->[0]; + my $sym = $symbols->{$addr}; + if (defined($sym)) { + if ($#{$sym} >= 2 && $sym->[1] =~ m/^(.*):(\d+)$/) { + $e->[1] = $1; # File name + $e->[2] = $2; # Line number + } + } + } +} + # Print source-listing for one routine sub PrintSource { my $prog = shift; @@ -1364,9 +1571,12 @@ sub PrintSource { my $cumulative = shift; my $start_addr = shift; my $end_addr = shift; + my $html = shift; + my $output = shift; # Disassemble all instructions (just to get line numbers) my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + GetTopLevelLineNumbers($prog, $offset, \@instructions); # Hack 1: assume that the first source file encountered in the # disassembly contains the routine @@ -1379,7 +1589,7 @@ sub PrintSource { } if (!defined($filename)) { print STDERR "no filename found in $routine\n"; - return; + return 0; } # Hack 2: assume that the largest line number from $filename is the @@ -1412,7 +1622,7 @@ sub PrintSource { { if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; my $first_indentation = -1; @@ -1440,12 +1650,24 @@ sub PrintSource { # Assign all samples to the range $firstline,$lastline, # Hack 4: If an instruction does not occur in the range, its samples # are moved to the next instruction that occurs in the range. - my $samples1 = {}; - my $samples2 = {}; - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts + my $samples1 = {}; # Map from line number to flat count + my $samples2 = {}; # Map from line number to cumulative count + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + my %disasm = (); # Map from line number to disassembly + my $running_disasm = ""; # Unassigned disassembly + my $skip_marker = "---\n"; + if ($html) { + $skip_marker = ""; + for (my $l = $firstline; $l <= $lastline; $l++) { + $disasm{$l} = ""; + } + } + my $last_dis_filename = ''; + my $last_dis_linenum = -1; + my $last_touched_line = -1; # To detect gaps in disassembly for a line foreach my $e (@instructions) { # Add up counts for all address that fall inside this instruction my $c1 = 0; @@ -1454,6 +1676,38 @@ sub PrintSource { $c1 += GetEntry($flat, $a); $c2 += GetEntry($cumulative, $a); } + + if ($html) { + my $dis = sprintf(" %6s %6s \t\t%8s: %s ", + HtmlPrintNumber($c1), + HtmlPrintNumber($c2), + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); + + # Append the most specific source line associated with this instruction + if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) }; + $dis = HtmlEscape($dis); + my $f = $e->[5]; + my $l = $e->[6]; + if ($f ne $last_dis_filename) { + $dis .= sprintf("%s:%d", + HtmlEscape(CleanFileName($f)), $l); + } elsif ($l ne $last_dis_linenum) { + # De-emphasize the unchanged file name portion + $dis .= sprintf("%s" . + ":%d", + HtmlEscape(CleanFileName($f)), $l); + } else { + # De-emphasize the entire location + $dis .= sprintf("%s:%d", + HtmlEscape(CleanFileName($f)), $l); + } + $last_dis_filename = $f; + $last_dis_linenum = $l; + $running_disasm .= $dis; + $running_disasm .= "\n"; + } + $running1 += $c1; $running2 += $c2; $total1 += $c1; @@ -1468,23 +1722,49 @@ sub PrintSource { AddEntry($samples2, $line, $running2); $running1 = 0; $running2 = 0; + if ($html) { + if ($line != $last_touched_line && $disasm{$line} ne '') { + $disasm{$line} .= "\n"; + } + $disasm{$line} .= $running_disasm; + $running_disasm = ''; + $last_touched_line = $line; + } } } # Assign any leftover samples to $lastline AddEntry($samples1, $lastline, $running1); AddEntry($samples2, $lastline, $running2); - - printf("ROUTINE ====================== %s in %s\n" . - "%6s %6s Total %s (flat / cumulative)\n", - ShortFunctionName($routine), - $filename, - Units(), - Unparse($total1), - Unparse($total2)); + if ($html) { + if ($lastline != $last_touched_line && $disasm{$lastline} ne '') { + $disasm{$lastline} .= "\n"; + } + $disasm{$lastline} .= $running_disasm; + } + + if ($html) { + printf $output ( + "

%s

%s\n
\n" .
+      "Total:%6s %6s (flat / cumulative %s)\n",
+      HtmlEscape(ShortFunctionName($routine)),
+      HtmlEscape(CleanFileName($filename)),
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  } else {
+    printf $output (
+      "ROUTINE ====================== %s in %s\n" .
+      "%6s %6s Total %s (flat / cumulative)\n",
+      ShortFunctionName($routine),
+      CleanFileName($filename),
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  }
   if (!open(FILE, "<$filename")) {
     print STDERR "$filename: $!\n";
-    return;
+    return 0;
   }
   my $l = 0;
   while () {
@@ -1494,16 +1774,47 @@ sub PrintSource {
         (($l <= $oldlastline + 5) || ($l <= $lastline))) {
       chop;
       my $text = $_;
-      if ($l == $firstline) { printf("---\n"); }
-      printf("%6s %6s %4d: %s\n",
-             UnparseAlt(GetEntry($samples1, $l)),
-             UnparseAlt(GetEntry($samples2, $l)),
-             $l,
-             $text);
-      if ($l == $lastline)  { printf("---\n"); }
+      if ($l == $firstline) { print $output $skip_marker; }
+      my $n1 = GetEntry($samples1, $l);
+      my $n2 = GetEntry($samples2, $l);
+      if ($html) {
+        # Emit a span that has one of the following classes:
+        #    livesrc -- has samples
+        #    deadsrc -- has disassembly, but with no samples
+        #    nop     -- has no matching disasembly
+        # Also emit an optional span containing disassembly.
+        my $dis = $disasm{$l};
+        my $asm = "";
+        if (defined($dis) && $dis ne '') {
+          $asm = "" . $dis . "";
+        }
+        my $source_class = (($n1 + $n2 > 0) 
+                            ? "livesrc" 
+                            : (($asm ne "") ? "deadsrc" : "nop"));
+        printf $output (
+          "%5d " .
+          "%6s %6s %s%s\n",
+          $l, $source_class,
+          HtmlPrintNumber($n1),
+          HtmlPrintNumber($n2),
+          HtmlEscape($text),
+          $asm);
+      } else {
+        printf $output(
+          "%6s %6s %4d: %s\n",
+          UnparseAlt($n1),
+          UnparseAlt($n2),
+          $l,
+          $text);
+      }
+      if ($l == $lastline)  { print $output $skip_marker; }
     };
   }
   close(FILE);
+  if ($html) {
+    print $output "
\n"; + } + return 1; } # Return the source line for the specified file/linenumber. @@ -1646,21 +1957,11 @@ sub PrintDisassembledFunction { # Print disassembly for (my $x = $first_inst; $x <= $last_inst; $x++) { my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - # Trim symbols - my $d = $e->[3]; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) - while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments - printf("%6s %6s %8s: %6s\n", UnparseAlt($flat_count[$x]), UnparseAlt($cum_count[$x]), - $address, - $d); + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); } } } @@ -1706,19 +2007,24 @@ sub PrintDot { # Open DOT output file my $output; + my $escaped_dot = ShellEscape(@DOT); + my $escaped_ps2pdf = ShellEscape(@PS2PDF); if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "ps")); + $output = "| $escaped_dot -Tps2 >$escaped_outfile"; } elsif ($main::opt_evince) { - $output = "| $DOT -Tps2 | $PS2PDF - " . TempName($main::next_tmpfile, "pdf"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "pdf")); + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - $escaped_outfile"; } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; + $output = "| $escaped_dot -Tps2"; } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - -"; } elsif ($main::opt_web || $main::opt_svg) { # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "svg")); + $output = "| $escaped_dot -Tsvg >$escaped_outfile"; } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; + $output = "| $escaped_dot -Tgif"; } else { $output = ">&STDOUT"; } @@ -1770,7 +2076,7 @@ sub PrintDot { if ($f != $c) { $extra = sprintf("\\rof %s (%s)", Unparse($c), - Percent($c, $overall_total)); + Percent($c, $local_total)); } my $style = ""; if ($main::opt_heapcheck) { @@ -1789,7 +2095,7 @@ sub PrintDot { $node{$a}, $sym, Unparse($f), - Percent($f, $overall_total), + Percent($f, $local_total), $extra, $fs, $style, @@ -1799,10 +2105,12 @@ sub PrintDot { # Get edges and counts per edge my %edge = (); my $n; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$raw})) { # TODO: omit low %age edges $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); for (my $i = 1; $i <= $#translated; $i++) { my $src = $translated[$i]; my $dst = $translated[$i-1]; @@ -2186,6 +2494,50 @@ function handleMouseUp(evt) { EOF } +# Provides a map from fullname to shortname for cases where the +# shortname is ambiguous. The symlist has both the fullname and +# shortname for all symbols, which is usually fine, but sometimes -- +# such as overloaded functions -- two different fullnames can map to +# the same shortname. In that case, we use the address of the +# function to disambiguate the two. This function fills in a map that +# maps fullnames to modified shortnames in such cases. If a fullname +# is not present in the map, the 'normal' shortname provided by the +# symlist is the appropriate one to use. +sub FillFullnameToShortnameMap { + my $symbols = shift; + my $fullname_to_shortname_map = shift; + my $shortnames_seen_once = {}; + my $shortnames_seen_more_than_once = {}; + + foreach my $symlist (values(%{$symbols})) { + # TODO(csilvers): deal with inlined symbols too. + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + if ($fullname !~ /<[0-9a-fA-F]+>$/) { # fullname doesn't end in an address + next; # the only collisions we care about are when addresses differ + } + if (defined($shortnames_seen_once->{$shortname}) && + $shortnames_seen_once->{$shortname} ne $fullname) { + $shortnames_seen_more_than_once->{$shortname} = 1; + } else { + $shortnames_seen_once->{$shortname} = $fullname; + } + } + + foreach my $symlist (values(%{$symbols})) { + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + # TODO(csilvers): take in a list of addresses we care about, and only + # store in the map if $symlist->[1] is in that list. Saves space. + next if defined($fullname_to_shortname_map->{$fullname}); + if (defined($shortnames_seen_more_than_once->{$shortname})) { + if ($fullname =~ /<0*([^>]*)>$/) { # fullname has address at end of it + $fullname_to_shortname_map->{$fullname} = "$shortname\@$1"; + } + } + } +} + # Return a small number that identifies the argument. # Multiple calls with the same argument will return the same number. # Calls with different arguments will return different numbers. @@ -2202,6 +2554,7 @@ sub ShortIdFor { # Translate a stack of addresses into a stack of symbols sub TranslateStack { my $symbols = shift; + my $fullname_to_shortname_map = shift; my $k = shift; my @addrs = split(/\n/, $k); @@ -2233,6 +2586,9 @@ sub TranslateStack { my $func = $symlist->[$j-2]; my $fileline = $symlist->[$j-1]; my $fullfunc = $symlist->[$j]; + if (defined($fullname_to_shortname_map->{$fullfunc})) { + $func = $fullname_to_shortname_map->{$fullfunc}; + } if ($j > 2) { $func = "$func (inline)"; } @@ -2319,6 +2675,16 @@ sub UnparseAlt { } } +# Alternate pretty-printed form: 0 maps to "" +sub HtmlPrintNumber { + my $num = shift; + if ($num == 0) { + return ""; + } else { + return Unparse($num); + } +} + # Return output units sub Units { if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { @@ -2475,6 +2841,13 @@ sub RemoveUninterestingFrames { '__builtin_vec_new', 'operator new', 'operator new[]', + # The entry to our memory-allocation routines on OS X + 'malloc_zone_malloc', + 'malloc_zone_calloc', + 'malloc_zone_valloc', + 'malloc_zone_realloc', + 'malloc_zone_memalign', + 'malloc_zone_free', # These mark the beginning/end of our custom sections '__start_google_malloc', '__stop_google_malloc', @@ -2566,9 +2939,11 @@ sub ReduceProfile { my $symbols = shift; my $profile = shift; my $result = {}; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$profile})) { my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); my @path = (); my %seen = (); $seen{''} = 1; # So that empty keys are skipped @@ -2775,7 +3150,8 @@ sub AddEntries { sub CheckSymbolPage { my $url = SymbolPageURL(); - open(SYMBOL, "$URL_FETCHER '$url' |"); + my $command = ShellEscape(@URL_FETCHER, $url); + open(SYMBOL, "$command |") or error($command); my $line = ; $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines close(SYMBOL); @@ -2832,7 +3208,7 @@ sub SymbolPageURL { sub FetchProgramName() { my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); my $url = "$baseURL$PROGRAM_NAME_PAGE"; - my $command_line = "$URL_FETCHER '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, $url); open(CMDLINE, "$command_line |") or error($command_line); my $cmdline = ; $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2849,7 +3225,7 @@ sub FetchProgramName() { # curl. Redirection happens on borg hosts. sub ResolveRedirectionForCurl { my $url = shift; - my $command_line = "$URL_FETCHER --head '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, "--head", $url); open(CMDLINE, "$command_line |") or error($command_line); while () { s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2861,18 +3237,18 @@ sub ResolveRedirectionForCurl { return $url; } -# Add a timeout flat to URL_FETCHER +# Add a timeout flat to URL_FETCHER. Returns a new list. sub AddFetchTimeout { - my $fetcher = shift; my $timeout = shift; + my @fetcher = shift; if (defined($timeout)) { - if ($fetcher =~ m/\bcurl -s/) { - $fetcher .= sprintf(" --max-time %d", $timeout); - } elsif ($fetcher =~ m/\brpcget\b/) { - $fetcher .= sprintf(" --deadline=%d", $timeout); + if (join(" ", @fetcher) =~ m/\bcurl -s/) { + push(@fetcher, "--max-time", sprintf("%d", $timeout)); + } elsif (join(" ", @fetcher) =~ m/\brpcget\b/) { + push(@fetcher, sprintf("--deadline=%d", $timeout)); } } - return $fetcher; + return @fetcher; } # Reads a symbol map from the file handle name given as $1, returning @@ -2932,15 +3308,17 @@ sub FetchSymbols { my $url = SymbolPageURL(); my $command_line; - if ($URL_FETCHER =~ m/\bcurl -s/) { + if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) { $url = ResolveRedirectionForCurl($url); - $command_line = "$URL_FETCHER -d '\@$main::tmpfile_sym' '$url'"; + $command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym", + $url); } else { - $command_line = "$URL_FETCHER --post '$url' < '$main::tmpfile_sym'"; + $command_line = (ShellEscape(@URL_FETCHER, "--post", $url) + . " < " . ShellEscape($main::tmpfile_sym)); } # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. - my $cppfilt = $obj_tool_map{"c++filt"}; - open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); + my $escaped_cppfilt = ShellEscape($obj_tool_map{"c++filt"}); + open(SYMBOL, "$command_line | $escaped_cppfilt |") or error($command_line); $symbol_map = ReadSymbols(*SYMBOL{IO}); close(SYMBOL); } @@ -2956,8 +3334,8 @@ sub FetchSymbols { my $shortpc = $pc; $shortpc =~ s/^0*//; # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. + # and also other functions it has inlined. They are separated (in + # PrintSymbolizedProfile), by --, which is illegal in function names. my $fullnames; if (defined($symbol_map->{$shortpc})) { $fullnames = $symbol_map->{$shortpc}; @@ -3035,8 +3413,8 @@ sub FetchDynamicProfile { return $real_profile; } - my $fetcher = AddFetchTimeout($URL_FETCHER, $fetch_timeout); - my $cmd = "$fetcher '$url' > '$tmp_profile'"; + my @fetcher = AddFetchTimeout($fetch_timeout, @URL_FETCHER); + my $cmd = ShellEscape(@fetcher, $url) . " > " . ShellEscape($tmp_profile); if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE|$CENSUSPROFILE_PAGE/){ print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; if ($encourage_patience) { @@ -3047,7 +3425,7 @@ sub FetchDynamicProfile { } (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); - (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); + (system("mv", $tmp_profile, $real_profile) == 0) || error("Unable to rename profile\n"); print STDERR "Wrote profile to $real_profile\n"; $main::collected_profile = $real_profile; return $main::collected_profile; @@ -3161,7 +3539,7 @@ BEGIN { my $has_q = 0; eval { $has_q = pack("Q", "1") ? 1 : 1; }; if (!$has_q) { - $self->{perl_is_64bit} = 0; + $self->{perl_is_64bit} = 0; } read($self->{file}, $str, 8); if (substr($str, 4, 4) eq chr(0)x4) { @@ -3197,17 +3575,17 @@ BEGIN { # TODO(csilvers): if this is a 32-bit perl, the math below # could end up in a too-large int, which perl will promote # to a double, losing necessary precision. Deal with that. - # Right now, we just die. - my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); + # Right now, we just die. + my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); if ($self->{unpack_code} eq 'N') { # big-endian - ($lo, $hi) = ($hi, $lo); - } - my $value = $lo + $hi * (2**32); - if (!$self->{perl_is_64bit} && # check value is exactly represented - (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { - ::error("Need a 64-bit perl to process this 64-bit profile.\n"); - } - push(@b64_values, $value); + ($lo, $hi) = ($hi, $lo); + } + my $value = $lo + $hi * (2**32); + if (!$self->{perl_is_64bit} && # check value is exactly represented + (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { + ::error("Need a 64-bit perl to process this 64-bit profile.\n"); + } + push(@b64_values, $value); } @$slots = @b64_values; } @@ -3335,7 +3713,7 @@ sub ReadProfile { if (!$main::use_symbolized_profile) { # we have both a binary and symbolized profiles, abort error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " . - "a binary arg. Try again without passing\n $prog\n"); + "a binary arg. Try again without passing\n $prog\n"); } # Read the symbol section of the symbolized profile file. $symbols = ReadSymbols(*PROFILE{IO}); @@ -3636,18 +4014,18 @@ sub ReadHeapProfile { # The sampling frequency is the rate of a Poisson process. # This means that the probability of sampling an allocation of # size X with sampling rate Y is 1 - exp(-X/Y) - if ($n1 != 0) { - my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - } - if ($n2 != 0) { - my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } + if ($n1 != 0) { + my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + } + if ($n2 != 0) { + my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } } else { # Remote-heap version 1 my $ratio; @@ -3771,19 +4149,19 @@ sub ReadSynchProfile { return $r; } -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... +# Given a hex value in the form "0x1abcd" or "1abcd", return either +# "0001abcd" or "000000000001abcd", depending on the current (global) +# address length. sub HexExtend { my $addr = shift; - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + $addr =~ s/^(0x)?0*//; + my $zeros_needed = $address_length - length($addr); + if ($zeros_needed < 0) { + printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + return $addr; } - - return substr("000000000000000".$addr, -$address_length); + return ("0" x $zeros_needed) . $addr; } ##### Symbol extraction ##### @@ -3819,8 +4197,12 @@ sub FindLibrary { # For libc libraries, the copy in /usr/lib/debug contains debugging symbols sub DebuggingLibrary { my $file = shift; - if ($file =~ m|^/| && -f "/usr/lib/debug$file") { - return "/usr/lib/debug$file"; + if ($file =~ m|^/|) { + if (-f "/usr/lib/debug$file") { + return "/usr/lib/debug$file"; + } elsif (-f "/usr/lib/debug$file.debug") { + return "/usr/lib/debug$file.debug"; + } } return undef; } @@ -3834,9 +4216,8 @@ sub ParseTextSectionHeaderFromObjdump { my $file_offset; # Get objdump output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); + my $cmd = ShellEscape($obj_tool_map{"objdump"}, "-h", $lib); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); while () { s/\r//g; # turn windows-looking lines into unix-looking lines # Idx Name Size VMA LMA File off Algn @@ -3874,9 +4255,8 @@ sub ParseTextSectionHeaderFromOtool { my $file_offset = undef; # Get otool output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); + my $command = ShellEscape($obj_tool_map{"otool"}, "-l", $lib); + open(OTOOL, "$command |") || error("$command: $!\n"); my $cmd = ""; my $sectname = ""; my $segname = ""; @@ -3984,6 +4364,19 @@ sub ParseLibraries { $finish = HexExtend($2); $offset = $zero_offset; $lib = $3; + } + # FreeBSD 10.0 virtual memory map /proc/curproc/map as defined in + # function procfs_doprocmap (sys/fs/procfs/procfs_map.c) + # + # Example: + # 0x800600000 0x80061a000 26 0 0xfffff800035a0000 r-x 75 33 0x1004 COW NC vnode /libexec/ld-elf.s + # o.1 NCH -1 + elsif ($l =~ /^(0x$h)\s(0x$h)\s\d+\s\d+\s0x$h\sr-x\s\d+\s\d+\s0x\d+\s(COW|NCO)\s(NC|NNC)\svnode\s(\S+\.so(\.\d+)*)/) { + $start = HexExtend($1); + $finish = HexExtend($2); + $offset = $zero_offset; + $lib = FindLibrary($5); + } else { next; } @@ -4006,6 +4399,7 @@ sub ParseLibraries { } } + if($main::opt_debug) { printf STDERR "$start:$finish ($offset) $lib\n"; } push(@{$result}, [$lib, $start, $finish, $offset]); } @@ -4213,23 +4607,29 @@ sub ExtractSymbols { my $finish = $lib->[2]; my $offset = $lib->[3]; + # Use debug library if it exists + my $debug_libname = DebuggingLibrary($libname); + if ($debug_libname) { + $libname = $debug_libname; + } + # Get list of pcs that belong in this library. my $contained = []; my ($start_pc_index, $finish_pc_index); # Find smallest finish_pc_index such that $finish < $pc[$finish_pc_index]. for ($finish_pc_index = $#pcs + 1; $finish_pc_index > 0; - $finish_pc_index--) { + $finish_pc_index--) { last if $pcs[$finish_pc_index - 1] le $finish; } # Find smallest start_pc_index such that $start <= $pc[$start_pc_index]. for ($start_pc_index = $finish_pc_index; $start_pc_index > 0; - $start_pc_index--) { + $start_pc_index--) { last if $pcs[$start_pc_index - 1] lt $start; } # This keeps PC values higher than $pc[$finish_pc_index] in @pcs, # in case there are overlaps in libraries and the main binary. @{$contained} = splice(@pcs, $start_pc_index, - $finish_pc_index - $start_pc_index); + $finish_pc_index - $start_pc_index); # Map to symbols MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); } @@ -4251,15 +4651,15 @@ sub MapToSymbols { # Figure out the addr2line command to use my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; + my $cmd = ShellEscape($addr2line, "-f", "-C", "-e", $image); if (exists $obj_tool_map{"addr2line_pdb"}) { $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; + $cmd = ShellEscape($addr2line, "--demangle", "-f", "-C", "-e", $image); } # If "addr2line" isn't installed on the system at all, just use # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >/dev/null 2>&1") != 0) { + if (system(ShellEscape($addr2line, "--help") . " >$dev_null 2>&1") != 0) { MapSymbolsWithNM($image, $offset, $pclist, $symbols); return; } @@ -4273,11 +4673,10 @@ sub MapToSymbols { $sep_address = undef; # May be filled in by MapSymbolsWithNM() my $nm_symbols = {}; MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. if (defined($sep_address)) { # Only add " -i" to addr2line if the binary supports it. # addr2line --help returns 0, but not if it sees an unknown flag first. - if (system("$cmd -i --help >/dev/null 2>&1") == 0) { + if (system("$cmd -i --help >$dev_null 2>&1") == 0) { $cmd .= " -i"; } else { $sep_address = undef; # no need for sep_address if we don't support -i @@ -4299,13 +4698,14 @@ sub MapToSymbols { close(ADDRESSES); if ($debug) { print("----\n"); - system("cat $main::tmpfile_sym"); + system("cat", $main::tmpfile_sym); print("----\n"); - system("$cmd <$main::tmpfile_sym"); + system("$cmd < " . ShellEscape($main::tmpfile_sym)); print("----\n"); } - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); + open(SYMBOLS, "$cmd <" . ShellEscape($main::tmpfile_sym) . " |") + || error("$cmd: $!\n"); my $count = 0; # Index in pclist while () { # Read fullfunction and filelineinfo from next pair of lines @@ -4325,15 +4725,29 @@ sub MapToSymbols { my $pcstr = $pclist->[$count]; my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { + my $nms = $nm_symbols->{$pcstr}; + if (defined($nms)) { + if ($fullfunction eq '??') { + # nm found a symbol for us. $function = $nms->[0]; $fullfunction = $nms->[2]; + } else { + # MapSymbolsWithNM tags each routine with its starting address, + # useful in case the image has multiple occurrences of this + # routine. (It uses a syntax that resembles template paramters, + # that are automatically stripped out by ShortFunctionName().) + # addr2line does not provide the same information. So we check + # if nm disambiguated our symbol, and if so take the annotated + # (nm) version of the routine-name. TODO(csilvers): this won't + # catch overloaded, inlined symbols, which nm doesn't see. + # Better would be to do a check similar to nm's, in this fn. + if ($nms->[2] =~ m/^\Q$function\E/) { # sanity check it's the right fn + $function = $nms->[0]; + $fullfunction = $nms->[2]; + } } } - + # Prepend to accumulated symbols for pcstr # (so that caller comes before callee) my $sym = $symbols->{$pcstr}; @@ -4344,7 +4758,7 @@ sub MapToSymbols { unshift(@{$sym}, $function, $filelinenum, $fullfunction); if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately + # Inlining is off, so this entry ends immediately $count++; } } @@ -4407,6 +4821,31 @@ sub ShortFunctionName { return $function; } +# Trim overly long symbols found in disassembler output +sub CleanDisassembly { + my $d = shift; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + return $d; +} + +# Clean file name for display +sub CleanFileName { + my ($f) = @_; + $f =~ s|^/proc/self/cwd/||; + $f =~ s|^\./||; + return $f; +} + +# Make address relative to section and clean up for display +sub UnparseAddress { + my ($offset, $address) = @_; + $address = AddressSub($address, $offset); + $address =~ s/^0x//; + $address =~ s/^0*//; + return $address; +} + ##### Miscellaneous ##### # Find the right versions of the above object tools to use. The @@ -4423,8 +4862,18 @@ sub ConfigureObjTools { # predictably return error status in prod. (-e $prog_file) || error("$prog_file does not exist.\n"); - # Follow symlinks (at least for systems where "file" supports that) - my $file_type = `/usr/bin/file -L $prog_file 2>/dev/null || /usr/bin/file $prog_file`; + my $file_type = undef; + if (-e "/usr/bin/file") { + # Follow symlinks (at least for systems where "file" supports that). + my $escaped_prog_file = ShellEscape($prog_file); + $file_type = `/usr/bin/file -L $escaped_prog_file 2>$dev_null || + /usr/bin/file $escaped_prog_file`; + } elsif ($^O == "MSWin32") { + $file_type = "MS Windows"; + } else { + print STDERR "WARNING: Can't determine the file type of $prog_file"; + } + if ($file_type =~ /64-bit/) { # Change $address_length to 16 if the program file is ELF 64-bit. # We can't detect this from many (most?) heap or lock contention @@ -4500,6 +4949,19 @@ sub ConfigureTool { return $path; } +sub ShellEscape { + my @escaped_words = (); + foreach my $word (@_) { + my $escaped_word = $word; + if ($word =~ m![^a-zA-Z0-9/.,_=-]!) { # check for anything not in whitelist + $escaped_word =~ s/'/'\\''/; + $escaped_word = "'$escaped_word'"; + } + push(@escaped_words, $escaped_word); + } + return join(" ", @escaped_words); +} + sub cleanup { unlink($main::tmpfile_sym); unlink(keys %main::tempnames); @@ -4537,11 +4999,11 @@ sub error { # names match "$regexp" and returns them in a hashtable mapping from # procedure name to a two-element vector of [start address, end address] sub GetProcedureBoundariesViaNm { - my $nm_command = shift; + my $escaped_nm_command = shift; # shell-escaped my $regexp = shift; my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); + open(NM, "$escaped_nm_command |") || error("$escaped_nm_command: $!\n"); my $last_start = "0"; my $routine = ""; while () { @@ -4581,7 +5043,7 @@ sub GetProcedureBoundariesViaNm { # Tag this routine with the starting address in case the image # has multiple occurrences of this routine. We use a syntax - # that resembles template paramters that are automatically + # that resembles template parameters that are automatically # stripped out by ShortFunctionName() $this_routine .= "<$start_val>"; @@ -4619,6 +5081,21 @@ sub GetProcedureBoundaries { my $image = shift; my $regexp = shift; + # If $image doesn't start with /, then put ./ in front of it. This works + # around an obnoxious bug in our probing of nm -f behavior. + # "nm -f $image" is supposed to fail on GNU nm, but if: + # + # a. $image starts with [BbSsPp] (for example, bin/foo/bar), AND + # b. you have a.out in your current directory (a not uncommon occurence) + # + # then "nm -f $image" succeeds because -f only looks at the first letter of + # the argument, which looks valid because it's [BbSsPp], and then since + # there's no image provided, it looks for a.out and finds it. + # + # This regex makes sure that $image starts with . or /, forcing the -f + # parsing to fail since . and / are not valid formats. + $image =~ s#^[^/]#./$&#; + # For libc libraries, the copy in /usr/lib/debug contains debugging symbols my $debugging = DebuggingLibrary($image); if ($debugging) { @@ -4636,28 +5113,29 @@ sub GetProcedureBoundaries { # --demangle and -f. my $demangle_flag = ""; my $cppfilt_flag = ""; - if (system("$nm --demangle $image >/dev/null 2>&1") == 0) { + my $to_devnull = ">$dev_null 2>&1"; + if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) { # In this mode, we do "nm --demangle " $demangle_flag = "--demangle"; $cppfilt_flag = ""; - } elsif (system("$cppfilt $image >/dev/null 2>&1") == 0) { + } elsif (system(ShellEscape($cppfilt, $image) . $to_devnull) == 0) { # In this mode, we do "nm | c++filt" - $cppfilt_flag = " | $cppfilt"; + $cppfilt_flag = " | " . ShellEscape($cppfilt); }; my $flatten_flag = ""; - if (system("$nm -f $image >/dev/null 2>&1") == 0) { + if (system(ShellEscape($nm, "-f", $image) . $to_devnull) == 0) { $flatten_flag = "-f"; } # Finally, in the case $imagie isn't a debug library, we try again with # -D to at least get *exported* symbols. If we can't use --demangle, # we use c++filt instead, if it exists on this system. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", + my @nm_commands = (ShellEscape($nm, "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", + ShellEscape($nm, "-D", "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", # 6nm is for Go binaries - "6nm $image 2>/dev/null | sort", + ShellEscape("6nm", "$image") . " 2>$dev_null | sort", ); # If the executable is an MS Windows PDB-format executable, we'll @@ -4665,8 +5143,9 @@ sub GetProcedureBoundaries { # want to use both unix nm and windows-specific nm_pdb, since # PDB-format executables can apparently include dwarf .o files. if (exists $obj_tool_map{"nm_pdb"}) { - my $nm_pdb = $obj_tool_map{"nm_pdb"}; - push(@nm_commands, "$nm_pdb --demangle $image 2>/dev/null"); + push(@nm_commands, + ShellEscape($obj_tool_map{"nm_pdb"}, "--demangle", $image) + . " 2>$dev_null"); } foreach my $nm_command (@nm_commands) { diff --git a/deps/jemalloc/config.guess b/deps/jemalloc/config.guess index 0773d0f6312..b79252d6b10 100755 --- a/deps/jemalloc/config.guess +++ b/deps/jemalloc/config.guess @@ -1,13 +1,12 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2004-03-03' +timestamp='2013-06-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -16,24 +15,22 @@ timestamp='2004-03-03' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# Originally written by Per Bothner . -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). +# +# Originally written by Per Bothner. # -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD # -# The plan is that this can be called by configure scripts if you -# don't specify an explicit build system type. +# Please send patches with a ChangeLog entry to config-patches@gnu.org. + me=`echo "$0" | sed -e 's,.*/,,'` @@ -53,8 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -66,11 +62,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -104,7 +100,7 @@ set_cc_for_build=' trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; : ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; @@ -123,7 +119,7 @@ case $CC_FOR_BUILD,$HOST_CC,$CC in ;; ,,*) CC_FOR_BUILD=$CC ;; ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ;' +esac ; set_cc_for_build= ;' # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) @@ -136,12 +132,33 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "${UNAME_SYSTEM}" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + eval $set_cc_for_build + cat <<-EOF > $dummy.c + #include + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + ;; +esac + # Note: order is significant - the case branches are not exclusive. case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward @@ -158,6 +175,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched @@ -166,7 +184,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep __ELF__ >/dev/null + | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? @@ -176,7 +194,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in fi ;; *) - os=netbsd + os=netbsd ;; esac # The OS release @@ -196,71 +214,34 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "${machine}-${os}${release}" - exit 0 ;; - amd64:OpenBSD:*:*) - echo x86_64-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - amiga:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - arc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - cats:OpenBSD:*:*) - echo arm-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - hp300:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mac68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - macppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme88k:OpenBSD:*:*) - echo m88k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvmeppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pegasos:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pmax:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sgi:OpenBSD:*:*) - echo mipseb-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sun3:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - wgrisc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; + *:Bitrig:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + exit ;; *:OpenBSD:*:*) - echo ${UNAME_MACHINE}-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} - exit 0 ;; + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; macppc:MirBSD:*:*) - echo powerppc-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on @@ -306,40 +287,46 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - exit 0 ;; - Alpha*:OpenVMS:*:*) - echo alpha-hp-vms - exit 0 ;; + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; Alpha\ *:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # Should we change UNAME_MACHINE based on the output of uname instead # of the specific Alpha model? echo alpha-pc-interix - exit 0 ;; + exit ;; 21064:Windows_NT:50:3) echo alpha-dec-winnt3.5 - exit 0 ;; + exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 - exit 0;; + exit ;; *:[Aa]miga[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-amigaos - exit 0 ;; + exit ;; *:[Mm]orph[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-morphos - exit 0 ;; + exit ;; *:OS/390:*:*) echo i370-ibm-openedition - exit 0 ;; + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; *:OS400:*:*) - echo powerpc-ibm-os400 - exit 0 ;; + echo powerpc-ibm-os400 + exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} - exit 0;; + exit ;; + arm*:riscos:*:*|arm*:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) echo hppa1.1-hitachi-hiuxmpp - exit 0;; + exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. if test "`(/bin/universe) 2>/dev/null`" = att ; then @@ -347,32 +334,51 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in else echo pyramid-pyramid-bsd fi - exit 0 ;; + exit ;; NILE*:*:*:dcosx) echo pyramid-pyramid-svr4 - exit 0 ;; + exit ;; DRS?6000:unix:4.0:6*) echo sparc-icl-nx6 - exit 0 ;; - DRS?6000:UNIX_SV:4.2*:7*) + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in - sparc) echo sparc-icl-nx7 && exit 0 ;; + sparc) echo sparc-icl-nx7; exit ;; esac ;; + s390x:SunOS:*:*) + echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4H:SunOS:5.*:*) echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - i86pc:SunOS:5.*:*) - echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + eval $set_cc_for_build + SUN_ARCH="i386" + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH="x86_64" + fi + fi + echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in Series*|S4*) @@ -381,10 +387,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in esac # Japanese Language versions have a version number like `4.1.3-JL'. echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` - exit 0 ;; + exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 @@ -396,10 +402,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo sparc-sun-sunos${UNAME_RELEASE} ;; esac - exit 0 ;; + exit ;; aushp:SunOS:*:*) echo sparc-auspex-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor @@ -409,41 +415,41 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; m68k:machten:*:*) echo m68k-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; powerpc:machten:*:*) echo powerpc-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 - exit 0 ;; + exit ;; RISC*:ULTRIX:*:*) echo mips-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; VAX*:ULTRIX*:*:*) echo vax-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) echo clipper-intergraph-clix${UNAME_RELEASE} - exit 0 ;; + exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -467,35 +473,36 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c \ - && $dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \ - && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos${UNAME_RELEASE} - exit 0 ;; + exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax - exit 0 ;; + exit ;; Motorola:*:4.3:PL8-*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:Power_UNIX:*:*) echo powerpc-harris-powerunix - exit 0 ;; + exit ;; m88k:CX/UX:7*:*) echo m88k-harris-cxux7 - exit 0 ;; + exit ;; m88k:*:4*:R4*) echo m88k-motorola-sysv4 - exit 0 ;; + exit ;; m88k:*:3*:R3*) echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] then if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ @@ -508,29 +515,29 @@ EOF else echo i586-dg-dgux${UNAME_RELEASE} fi - exit 0 ;; + exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 - exit 0 ;; + exit ;; M88*:*:R3*:*) # Delta 88k system running SVR3 echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) echo m88k-tektronix-sysv3 - exit 0 ;; + exit ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) echo m68k-tektronix-bsd - exit 0 ;; + exit ;; *:IRIX*:*:*) echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` - exit 0 ;; + exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit 0 ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix - exit 0 ;; + exit ;; ia64:AIX:*:*) if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` @@ -538,7 +545,7 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then eval $set_cc_for_build @@ -553,15 +560,19 @@ EOF exit(0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 - echo rs6000-ibm-aix3.2.5 + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then echo rs6000-ibm-aix3.2.4 else echo rs6000-ibm-aix3.2 fi - exit 0 ;; - *:AIX:*:[45]) + exit ;; + *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 @@ -574,28 +585,28 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${IBM_ARCH}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:*:*) echo rs6000-ibm-aix - exit 0 ;; + exit ;; ibmrt:4.4BSD:*|romp-ibm:BSD:*) echo romp-ibm-bsd4.4 - exit 0 ;; + exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to - exit 0 ;; # report: romp-ibm BSD 4.3 + exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx - exit 0 ;; + exit ;; DPX/2?00:B.O.S.:*:*) echo m68k-bull-sysv3 - exit 0 ;; + exit ;; 9000/[34]??:4.3bsd:1.*:*) echo m68k-hp-bsd - exit 0 ;; + exit ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) echo m68k-hp-bsd4.4 - exit 0 ;; + exit ;; 9000/[34678]??:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` case "${UNAME_MACHINE}" in @@ -604,52 +615,52 @@ EOF 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 - esac ;; - esac + esac ;; + esac fi if [ "${HP_ARCH}" = "" ]; then eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + sed 's/^ //' << EOF >$dummy.c - #define _HPUX_SOURCE - #include - #include + #define _HPUX_SOURCE + #include + #include - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } EOF (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa @@ -657,9 +668,19 @@ EOF esac if [ ${HP_ARCH} = "hppa2.0w" ] then - # avoid double evaluation of $set_cc_for_build - test -n "$CC_FOR_BUILD" || eval $set_cc_for_build - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E -) | grep __LP64__ >/dev/null + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ then HP_ARCH="hppa2.0w" else @@ -667,11 +688,11 @@ EOF fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; ia64:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` echo ia64-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; 3050*:HI-UX:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -699,337 +720,345 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 - exit 0 ;; + exit ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) echo hppa1.1-hp-bsd - exit 0 ;; + exit ;; 9000/8??:4.3bsd:*:*) echo hppa1.0-hp-bsd - exit 0 ;; + exit ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix - exit 0 ;; + exit ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) echo hppa1.1-hp-osf - exit 0 ;; + exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf - exit 0 ;; + exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then echo ${UNAME_MACHINE}-unknown-osf1mk else echo ${UNAME_MACHINE}-unknown-osf1 fi - exit 0 ;; + exit ;; parisc*:Lites*:*:*) echo hppa1.1-hp-lites - exit 0 ;; + exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd - exit 0 ;; + exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd - exit 0 ;; + exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd - exit 0 ;; + exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd - exit 0 ;; + exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*[A-Z]90:*:*:*) echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*TS:*:*:*) echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*T3E:*:*:*) echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*SV1:*:*:*) echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; *:UNICOS/mp:*:*) - echo nv1-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; sparc*:BSD/OS:*:*) echo sparc-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:BSD/OS:*:*) echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:FreeBSD:*:*) - # Determine whether the default compiler uses glibc. - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #if __GLIBC__ >= 2 - LIBC=gnu - #else - LIBC= - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - # GNU/KFreeBSD systems have a "k" prefix to indicate we are using - # FreeBSD's kernel, but not the complete OS. - case ${LIBC} in gnu) kernel_only='k' ;; esac - echo ${UNAME_MACHINE}-unknown-${kernel_only}freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`${LIBC:+-$LIBC} - exit 0 ;; + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in + amd64) + echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + *) + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + esac + exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin - exit 0 ;; - i*:MINGW*:*) + exit ;; + *:MINGW64*:*) + echo ${UNAME_MACHINE}-pc-mingw64 + exit ;; + *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 - exit 0 ;; + exit ;; + i*:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; i*:PW*:*) echo ${UNAME_MACHINE}-pc-pw32 - exit 0 ;; - x86:Interix*:[34]*) - echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//' - exit 0 ;; + exit ;; + *:Interix*:*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) echo i${UNAME_MACHINE}-pc-mks - exit 0 ;; + exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; i*:Windows_NT*:* | Pentium*:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we # UNAME_MACHINE based on the output of uname instead of i386? echo i586-pc-interix - exit 0 ;; + exit ;; i*:UWIN*:*) echo ${UNAME_MACHINE}-pc-uwin - exit 0 ;; + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; p*:CYGWIN*:*) echo powerpcle-unknown-cygwin - exit 0 ;; + exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` - exit 0 ;; + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix - exit 0 ;; + exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; arm*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + else + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + fi + fi + exit ;; + avr32*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; cris:Linux:*:*) - echo cris-axis-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + crisv32:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + frv:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + hexagon:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + i*86:Linux:*:*) + echo ${UNAME_MACHINE}-pc-linux-${LIBC} + exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - mips:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #undef CPU - #undef mips - #undef mipsel + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mipsel + CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips + CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; - mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips64 - #undef mips64el - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mips64el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips64 - #else - CPU= - #endif - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 - ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu - exit 0 ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu - exit 0 ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} - exit 0 ;; + or1k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + or32:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-${LIBC} + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-${LIBC} + exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; + PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; + PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; + *) echo hppa-unknown-linux-${LIBC} ;; esac - exit 0 ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu - exit 0 ;; + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-${LIBC} + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-${LIBC} + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-${LIBC} + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-${LIBC} + exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux - exit 0 ;; + echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-${LIBC} + exit ;; x86_64:Linux:*:*) - echo x86_64-unknown-linux-gnu - exit 0 ;; - i*86:Linux:*:*) - # The BFD linker knows what the default object file format is, so - # first see if it will tell us. cd to the root directory to prevent - # problems with other programs or directories called `ld' in the path. - # Set LC_ALL=C to ensure ld outputs messages in English. - ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ - | sed -ne '/supported targets:/!d - s/[ ][ ]*/ /g - s/.*supported targets: *// - s/ .*// - p'` - case "$ld_supported_targets" in - elf32-i386) - TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" - ;; - a.out-i386-linux) - echo "${UNAME_MACHINE}-pc-linux-gnuaout" - exit 0 ;; - coff-i386) - echo "${UNAME_MACHINE}-pc-linux-gnucoff" - exit 0 ;; - "") - # Either a pre-BFD a.out linker (linux-gnuoldld) or - # one that does not give us useful --help. - echo "${UNAME_MACHINE}-pc-linux-gnuoldld" - exit 0 ;; - esac - # Determine whether the default compiler is a.out or elf - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #ifdef __ELF__ - # ifdef __GLIBC__ - # if __GLIBC__ >= 2 - LIBC=gnu - # else - LIBC=gnulibc1 - # endif - # else - LIBC=gnulibc1 - # endif - #else - #ifdef __INTEL_COMPILER - LIBC=gnu - #else - LIBC=gnuaout - #endif - #endif - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - test x"${LIBC}" != x && echo "${UNAME_MACHINE}-pc-linux-${LIBC}" && exit 0 - test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0 - ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. echo i386-sequent-sysv4 - exit 0 ;; + exit ;; i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. + # Use sysv4.2uw... so that sysv4* matches it. echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. echo ${UNAME_MACHINE}-pc-os2-emx - exit 0 ;; + exit ;; i*86:XTS-300:*:STOP) echo ${UNAME_MACHINE}-unknown-stop - exit 0 ;; + exit ;; i*86:atheos:*:*) echo ${UNAME_MACHINE}-unknown-atheos - exit 0 ;; - i*86:syllable:*:*) + exit ;; + i*86:syllable:*:*) echo ${UNAME_MACHINE}-pc-syllable - exit 0 ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; i*86:*DOS:*:*) echo ${UNAME_MACHINE}-pc-msdosdjgpp - exit 0 ;; + exit ;; i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then @@ -1037,15 +1066,16 @@ EOF else echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} fi - exit 0 ;; - i*86:*:5:[78]*) + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null 2>&1 ; then echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 fi - exit 0 ;; + exit ;; mini*:CTIX:SYS*5:*) # "miniframe" echo m68010-convergent-sysv - exit 0 ;; + exit ;; mc68k:UNIX:SYSTEM5:3.51m) echo m68k-convergent-sysv - exit 0 ;; + exit ;; M680?0:D-NIX:5.3:*) echo m68k-diab-dnix - exit 0 ;; - M68*:*:R3V[567]*:*) - test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0) + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4.3${OS_REL} && exit 0 + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;; + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4 && exit 0 ;; + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) echo m68k-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 - exit 0 ;; + exit ;; TSUNAMI:LynxOS:2.*:*) echo sparc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; SM[BE]S:UNIX_SV:*:*) echo mips-dde-sysv${UNAME_RELEASE} - exit 0 ;; + exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; RM*:SINIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` @@ -1137,68 +1180,99 @@ EOF else echo ns32k-sni-sysv fi - exit 0 ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit 0 ;; + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm echo hppa1.1-stratus-sysv4 - exit 0 ;; + exit ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. echo i860-stratus-sysv4 - exit 0 ;; + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos - exit 0 ;; + exit ;; mc68*:A/UX:*:*) echo m68k-apple-aux${UNAME_RELEASE} - exit 0 ;; + exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 - exit 0 ;; + exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv${UNAME_RELEASE} else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv${UNAME_RELEASE} fi - exit 0 ;; + exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos - exit 0 ;; + exit ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. echo powerpc-apple-beos - exit 0 ;; + exit ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. echo i586-pc-beos - exit 0 ;; + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; + x86_64:Haiku:*:*) + echo x86_64-unknown-haiku + exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-5:SUPER-UX:*:*) echo sx5-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-6:SUPER-UX:*:*) echo sx6-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Rhapsody:*:*) echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Darwin:*:*) - case `uname -p` in - *86) UNAME_PROCESSOR=i686 ;; - powerpc) UNAME_PROCESSOR=powerpc ;; - esac + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + eval $set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} - exit 0 ;; + exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = "x86"; then @@ -1206,22 +1280,28 @@ EOF UNAME_MACHINE=pc fi echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} - exit 0 ;; + exit ;; *:QNX:*:4*) echo i386-pc-qnx - exit 0 ;; + exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; + NSE-*:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; NSR-?:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} - exit 0 ;; + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux - exit 0 ;; + exit ;; BS2000:POSIX*:*:*) echo bs2000-siemens-sysv - exit 0 ;; + exit ;; DS/*:UNIX_System_V:*:*) echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 @@ -1232,36 +1312,55 @@ EOF UNAME_MACHINE="$cputype" fi echo ${UNAME_MACHINE}-unknown-plan9 - exit 0 ;; + exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 - exit 0 ;; + exit ;; *:TENEX:*:*) echo pdp10-unknown-tenex - exit 0 ;; + exit ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) echo pdp10-dec-tops20 - exit 0 ;; + exit ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) echo pdp10-xkl-tops20 - exit 0 ;; + exit ;; *:TOPS-20:*:*) echo pdp10-unknown-tops20 - exit 0 ;; + exit ;; *:ITS:*:*) echo pdp10-unknown-its - exit 0 ;; + exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} - exit 0 ;; + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; *:DragonFly:*:*) echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` - exit 0 ;; + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; + i*86:AROS:*:*) + echo ${UNAME_MACHINE}-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; esac -#echo '(No uname command or uname output not recognized.)' 1>&2 -#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 - eval $set_cc_for_build cat >$dummy.c < printf ("m68k-sony-newsos%s\n", #ifdef NEWSOS4 - "4" + "4" #else - "" + "" #endif - ); exit (0); + ); exit (0); #endif #endif #if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix"); exit (0); + printf ("arm-acorn-riscix\n"); exit (0); #endif #if defined (hp300) && !defined (hpux) @@ -1377,11 +1476,12 @@ main () } EOF -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && $dummy && exit 0 +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } # Apollos put the system type in the environment. -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; } +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } # Convex versions that predate uname can use getsysinfo(1) @@ -1390,22 +1490,22 @@ then case `getsysinfo -f cpu_type` in c1*) echo c1-convex-bsd - exit 0 ;; + exit ;; c2*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; c34*) echo c34-convex-bsd - exit 0 ;; + exit ;; c38*) echo c38-convex-bsd - exit 0 ;; + exit ;; c4*) echo c4-convex-bsd - exit 0 ;; + exit ;; esac fi @@ -1416,7 +1516,9 @@ This script, last modified $timestamp, has failed to recognize the operating system you are using. It is advised that you download the most up to date version of the config scripts from - ftp://ftp.gnu.org/pub/gnu/config/ + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +and + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD If the version you run ($0) is already up to date, please send the following data and any information you think might be diff --git a/deps/jemalloc/config.sub b/deps/jemalloc/config.sub index 264f820aa55..61cb4bc22db 100755 --- a/deps/jemalloc/config.sub +++ b/deps/jemalloc/config.sub @@ -1,42 +1,40 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2004-02-23' +timestamp='2013-10-01' -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - +# along with this program; if not, see . +# # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. + +# Please send patches with a ChangeLog entry to config-patches@gnu.org. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. # If it is invalid, we print an error message on stderr and exit with code 1. # Otherwise, we print the canonical config type on stdout and succeed. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases # that are meaningful with *any* GNU software. @@ -70,8 +68,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -83,11 +80,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -99,7 +96,7 @@ while test $# -gt 0 ; do *local*) # First pass through any local machine types. echo $1 - exit 0;; + exit ;; * ) break ;; @@ -118,11 +115,18 @@ esac # Here we must recognize all the valid KERNEL-OS combinations. maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in - nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \ - kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*) + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ + linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | \ + kopensolaris*-gnu* | \ + storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; + android-linux) + os=-linux-android + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + ;; *) basic_machine=`echo $1 | sed 's/-[^-]*$//'` if [ $basic_machine != $1 ] @@ -145,10 +149,13 @@ case $os in -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis) + -apple | -axis | -knuth | -cray | -microblaze*) os= basic_machine=$1 ;; + -bluegene*) + os=-cnk + ;; -sim | -cisco | -oki | -wec | -winbond) os= basic_machine=$1 @@ -163,13 +170,17 @@ case $os in os=-chorusos basic_machine=$1 ;; - -chorusrdb) - os=-chorusrdb + -chorusrdb) + os=-chorusrdb basic_machine=$1 - ;; + ;; -hiux*) os=-hiuxwe2 ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco5) os=-sco3.2v5 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -186,6 +197,10 @@ case $os in # Don't forget version if it is 3.2v4 or newer. basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco*) os=-sco3.2v2 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -203,6 +218,12 @@ case $os in -isc*) basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` ;; + -lynx*178) + os=-lynxos178 + ;; + -lynx*5) + os=-lynxos5 + ;; -lynx*) os=-lynxos ;; @@ -227,57 +248,107 @@ case $basic_machine in # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ + | aarch64 | aarch64_be \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \ - | c4x | clipper \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ + | avr | avr32 \ + | be32 | be64 \ + | bfin \ + | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | fr30 | frv \ + | epiphany \ + | fido | fr30 | frv \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ | i370 | i860 | i960 | ia64 \ | ip2k | iq2000 \ - | m32r | m68000 | m68k | m88k | mcore \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ - | mips64vr | mips64vrel \ + | mips64octeon | mips64octeonel \ | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ | mips64vr4100 | mips64vr4100el \ | mips64vr4300 | mips64vr4300el \ | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ + | moxie \ + | mt \ | msp430 \ + | nds32 | nds32le | nds32be \ + | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | openrisc | or32 \ + | open8 \ + | or1k | or32 \ | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ - | sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ - | sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \ - | strongarm \ - | tahoe | thumb | tic4x | tic80 | tron \ - | v850 | v850e \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | we32k \ - | x86 | xscale | xstormy16 | xtensa \ - | z8k) + | x86 | xc16x | xstormy16 | xtensa \ + | z8k | z80) basic_machine=$basic_machine-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12) - # Motorola 68HC11/12. + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) basic_machine=$basic_machine-unknown os=-none ;; m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) ;; + ms1) + basic_machine=mt-unknown + ;; + + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xgate) + basic_machine=$basic_machine-unknown + os=-none + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; # We use `pc' rather than `unknown' # because (1) that's what they normally are, and @@ -293,59 +364,83 @@ case $basic_machine in # Recognize the basic CPU types with company name. 580-* \ | a29k-* \ + | aarch64-* | aarch64_be-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* \ - | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ - | clipper-* | cydra-* \ + | avr-* | avr32-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ | elxsi-* \ - | f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ | i*86-* | i860-* | i960-* | ia64-* \ | ip2k-* | iq2000-* \ - | m32r-* \ + | k1om-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | mcore-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ + | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ - | mips64vr-* | mips64vrel-* \ + | mips64octeon-* | mips64octeonel-* \ | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ | mips64vr4100-* | mips64vr4100el-* \ | mips64vr4300-* | mips64vr4300el-* \ | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ | msp430-* \ - | none-* | np1-* | nv1-* | ns16k-* | ns32k-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ - | romp-* | rs6000-* \ - | sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \ - | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \ - | tahoe-* | thumb-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tile*-* \ | tron-* \ - | v850-* | v850e-* | vax-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ | we32k-* \ - | x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \ - | xtensa-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* \ + | xstormy16-* | xtensa*-* \ | ymp-* \ - | z8k-*) + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. @@ -363,7 +458,7 @@ case $basic_machine in basic_machine=a29k-amd os=-udi ;; - abacus) + abacus) basic_machine=abacus-unknown ;; adobe68k) @@ -409,6 +504,10 @@ case $basic_machine in basic_machine=m68k-apollo os=-bsd ;; + aros) + basic_machine=i386-pc + os=-aros + ;; aux) basic_machine=m68k-apple os=-aux @@ -417,10 +516,35 @@ case $basic_machine in basic_machine=ns32k-sequent os=-dynix ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=-cnk + ;; + c54x-*) + basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; c90) basic_machine=c90-cray os=-unicos ;; + cegcc) + basic_machine=arm-unknown + os=-cegcc + ;; convex-c1) basic_machine=c1-convex os=-bsd @@ -445,13 +569,20 @@ case $basic_machine in basic_machine=j90-cray os=-unicos ;; - cr16c) - basic_machine=cr16c-unknown + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown os=-elf ;; crds | unos) basic_machine=m68k-crds ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; cris | cris-* | etrax*) basic_machine=cris-axis ;; @@ -481,6 +612,14 @@ case $basic_machine in basic_machine=m88k-motorola os=-sysv3 ;; + dicos) + basic_machine=i686-pc + os=-dicos + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; dpx20 | dpx20-*) basic_machine=rs6000-bull os=-bosx @@ -592,7 +731,6 @@ case $basic_machine in i370-ibm* | ibm*) basic_machine=i370-ibm ;; -# I'm not sure what "Sysv32" means. Should this be sysv3.2? i*86v32) basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` os=-sysv32 @@ -631,6 +769,14 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; m88k-omron*) basic_machine=m88k-omron ;; @@ -642,10 +788,21 @@ case $basic_machine in basic_machine=ns32k-utek os=-sysv ;; + microblaze*) + basic_machine=microblaze-xilinx + ;; + mingw64) + basic_machine=x86_64-pc + os=-mingw64 + ;; mingw32) - basic_machine=i386-pc + basic_machine=i686-pc os=-mingw32 ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; miniframe) basic_machine=m68000-convergent ;; @@ -659,10 +816,6 @@ case $basic_machine in mips3*) basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown ;; - mmix*) - basic_machine=mmix-knuth - os=-mmixware - ;; monitor) basic_machine=m68k-rom68k os=-coff @@ -675,10 +828,21 @@ case $basic_machine in basic_machine=i386-pc os=-msdos ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + msys) + basic_machine=i686-pc + os=-msys + ;; mvs) basic_machine=i370-ibm os=-mvs ;; + nacl) + basic_machine=le32-unknown + os=-nacl + ;; ncr3000) basic_machine=i486-ncr os=-sysv4 @@ -743,9 +907,11 @@ case $basic_machine in np1) basic_machine=np1-gould ;; - nv1) - basic_machine=nv1-cray - os=-unicosmp + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem ;; nsr-tandem) basic_machine=nsr-tandem @@ -754,9 +920,8 @@ case $basic_machine in basic_machine=hppa1.1-oki os=-proelf ;; - or32 | or32-*) + openrisc | openrisc-*) basic_machine=or32-unknown - os=-coff ;; os400) basic_machine=powerpc-ibm @@ -778,6 +943,14 @@ case $basic_machine in basic_machine=i860-intel os=-osf ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; pbd) basic_machine=sparc-tti ;; @@ -787,6 +960,12 @@ case $basic_machine in pc532 | pc532-*) basic_machine=ns32k-pc532 ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc ;; @@ -816,9 +995,10 @@ case $basic_machine in ;; power) basic_machine=power-ibm ;; - ppc) basic_machine=powerpc-unknown + ppc | ppcbe) basic_machine=powerpc-unknown ;; - ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; ppcle | powerpclittle | ppc-le | powerpc-little) basic_machine=powerpcle-unknown @@ -843,6 +1023,14 @@ case $basic_machine in basic_machine=i586-unknown os=-pw32 ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=-rdos + ;; + rdos32) + basic_machine=i386-pc + os=-rdos + ;; rom68k) basic_machine=m68k-rom68k os=-coff @@ -869,6 +1057,10 @@ case $basic_machine in sb1el) basic_machine=mipsisa64sb1el-unknown ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; sei) basic_machine=mips-sei os=-seiux @@ -880,6 +1072,9 @@ case $basic_machine in basic_machine=sh-hitachi os=-hms ;; + sh5el) + basic_machine=sh5le-unknown + ;; sh64) basic_machine=sh64-unknown ;; @@ -901,6 +1096,9 @@ case $basic_machine in basic_machine=i860-stratus os=-sysv4 ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; sun2) basic_machine=m68000-sun ;; @@ -957,17 +1155,9 @@ case $basic_machine in basic_machine=t90-cray os=-unicos ;; - tic54x | c54x*) - basic_machine=tic54x-unknown - os=-coff - ;; - tic55x | c55x*) - basic_machine=tic55x-unknown - os=-coff - ;; - tic6x | c6x*) - basic_machine=tic6x-unknown - os=-coff + tile*) + basic_machine=$basic_machine-unknown + os=-linux-gnu ;; tx39) basic_machine=mipstx39-unknown @@ -1029,9 +1219,16 @@ case $basic_machine in basic_machine=hppa1.1-winbond os=-proelf ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; xps | xps100) basic_machine=xps100-honeywell ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ;; ymp) basic_machine=ymp-cray os=-unicos @@ -1040,6 +1237,10 @@ case $basic_machine in basic_machine=z8k-unknown os=-sim ;; + z80-*-coff) + basic_machine=z80-unknown + os=-sim + ;; none) basic_machine=none-none os=-none @@ -1059,6 +1260,9 @@ case $basic_machine in romp) basic_machine=romp-ibm ;; + mmix) + basic_machine=mmix-knuth + ;; rs6000) basic_machine=rs6000-ibm ;; @@ -1075,13 +1279,10 @@ case $basic_machine in we32k) basic_machine=we32k-att ;; - sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele) + sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) basic_machine=sh-unknown ;; - sh64) - basic_machine=sh64-unknown - ;; - sparc | sparcv9 | sparcv9b) + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) basic_machine=sparc-sun ;; cydra) @@ -1125,9 +1326,12 @@ esac if [ x"$os" != x"" ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. + # First match some system type aliases + # that might get confused with valid system types. # -solaris* is a basic system type, with this one exception. + -auroraux) + os=-auroraux + ;; -solaris1 | -solaris1.*) os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; @@ -1148,26 +1352,31 @@ case $os in # Each alternative MUST END IN A *, to match a version number. # -sysv* is not here because it comes later, after sysvr4. -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ + | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* \ + | -aos* | -aros* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -bitrig* | -openbsd* | -solidbsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* \ - | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \ + | -chorusos* | -chorusrdb* | -cegcc* \ + | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-musl* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*) + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1185,7 +1394,7 @@ case $os in os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) ;; -mac*) @@ -1206,7 +1415,7 @@ case $os in -opened*) os=-openedition ;; - -os400*) + -os400*) os=-os400 ;; -wince*) @@ -1255,7 +1464,7 @@ case $os in -sinix*) os=-sysv4 ;; - -tpf*) + -tpf*) os=-tpf ;; -triton*) @@ -1291,8 +1500,13 @@ case $os in -aros*) os=-aros ;; - -kaos*) - os=-kaos + -zvmoe) + os=-zvmoe + ;; + -dicos*) + os=-dicos + ;; + -nacl*) ;; -none) ;; @@ -1316,6 +1530,12 @@ else # system, and we'll never get to this point. case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; *-acorn) os=-riscix1.2 ;; @@ -1325,9 +1545,24 @@ case $basic_machine in arm*-semi) os=-aout ;; - c4x-* | tic4x-*) - os=-coff - ;; + c4x-* | tic4x-*) + os=-coff + ;; + c8051-*) + os=-elf + ;; + hexagon-*) + os=-elf + ;; + tic54x-*) + os=-coff + ;; + tic55x-*) + os=-coff + ;; + tic6x-*) + os=-coff + ;; # This must come before the *-dec entry. pdp10-*) os=-tops20 @@ -1346,19 +1581,22 @@ case $basic_machine in ;; m68000-sun) os=-sunos3 - # This also exists in the configure program, but was not the - # default. - # os=-sunos4 ;; m68*-cisco) os=-aout ;; + mep-*) + os=-elf + ;; mips*-cisco) os=-elf ;; mips*-*) os=-elf ;; + or1k-*) + os=-elf + ;; or32-*) os=-coff ;; @@ -1371,9 +1609,15 @@ case $basic_machine in *-be) os=-beos ;; + *-haiku) + os=-haiku + ;; *-ibm) os=-aix ;; + *-knuth) + os=-mmixware + ;; *-wec) os=-proelf ;; @@ -1476,7 +1720,7 @@ case $basic_machine in -sunos*) vendor=sun ;; - -aix*) + -cnk*|-aix*) vendor=ibm ;; -beos*) @@ -1539,7 +1783,7 @@ case $basic_machine in esac echo $basic_machine$os -exit 0 +exit # Local variables: # eval: (add-hook 'write-file-hooks 'time-stamp) diff --git a/deps/jemalloc/configure b/deps/jemalloc/configure index 610884fb323..2e5496bfb84 100755 --- a/deps/jemalloc/configure +++ b/deps/jemalloc/configure @@ -1,11 +1,9 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65. +# Generated by GNU Autoconf 2.69. # # -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, -# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, -# Inc. +# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # # This configure script is free software; the Free Software Foundation @@ -89,6 +87,7 @@ fi IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. +as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -133,6 +132,31 @@ export LANGUAGE # CDPATH. (unset CDPATH) >/dev/null 2>&1 && unset CDPATH +# Use a proper internal environment variable to ensure we don't fall + # into an infinite loop, continuously re-executing ourselves. + if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then + _as_can_reexec=no; export _as_can_reexec; + # We cannot yet assume a decent shell, so we have to provide a +# neutralization value for shells without unset; and this also +# works around shells that cannot unset nonexistent variables. +# Preserve -v and -x to the replacement shell. +BASH_ENV=/dev/null +ENV=/dev/null +(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV +case $- in # (((( + *v*x* | *x*v* ) as_opts=-vx ;; + *v* ) as_opts=-v ;; + *x* ) as_opts=-x ;; + * ) as_opts= ;; +esac +exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} +# Admittedly, this is quite paranoid, since all the known shells bail +# out after a failed `exec'. +$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 +as_fn_exit 255 + fi + # We don't want this to propagate to other subprocesses. + { _as_can_reexec=; unset _as_can_reexec;} if test "x$CONFIG_SHELL" = x; then as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : emulate sh @@ -166,7 +190,8 @@ if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : else exitcode=1; echo positional parameters were not saved. fi -test x\$exitcode = x0 || exit 1" +test x\$exitcode = x0 || exit 1 +test -x / || exit 1" as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && @@ -211,14 +236,25 @@ IFS=$as_save_IFS if test "x$CONFIG_SHELL" != x; then : - # We cannot yet assume a decent shell, so we have to provide a - # neutralization value for shells without unset; and this also - # works around shells that cannot unset nonexistent variables. - BASH_ENV=/dev/null - ENV=/dev/null - (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV - export CONFIG_SHELL - exec "$CONFIG_SHELL" "$as_myself" ${1+"$@"} + export CONFIG_SHELL + # We cannot yet assume a decent shell, so we have to provide a +# neutralization value for shells without unset; and this also +# works around shells that cannot unset nonexistent variables. +# Preserve -v and -x to the replacement shell. +BASH_ENV=/dev/null +ENV=/dev/null +(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV +case $- in # (((( + *v*x* | *x*v* ) as_opts=-vx ;; + *v* ) as_opts=-v ;; + *x* ) as_opts=-x ;; + * ) as_opts= ;; +esac +exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} +# Admittedly, this is quite paranoid, since all the known shells bail +# out after a failed `exec'. +$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 +exit 255 fi if test x$as_have_required = xno; then : @@ -316,10 +352,18 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p + +# as_fn_executable_p FILE +# ----------------------- +# Test if FILE is an executable regular file. +as_fn_executable_p () +{ + test -f "$1" && test -x "$1" +} # as_fn_executable_p # as_fn_append VAR VALUE # ---------------------- # Append the text in VALUE to the end of the definition contained in VAR. Take @@ -356,19 +400,19 @@ else fi # as_fn_arith -# as_fn_error ERROR [LINENO LOG_FD] -# --------------------------------- +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with status $?, using 1 if that was 0. +# script with STATUS, using 1 if that was 0. as_fn_error () { - as_status=$?; test $as_status -eq 0 && as_status=1 - if test "$3"; then - as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - $as_echo "$as_me: error: $1" >&2 + $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -441,6 +485,10 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits chmod +x "$as_me.lineno" || { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } + # If we had to re-execute with $CONFIG_SHELL, we're ensured to have + # already done that, so ensure we don't try to do so again and fall + # in an infinite loop. This has already happened in practice. + _as_can_reexec=no; export _as_can_reexec # Don't try to exec as it changes $[0], causing all sort of problems # (the dirname of $[0] is not the place where we might find the # original and so on. Autoconf is especially sensitive to this). @@ -475,16 +523,16 @@ if (echo >conf$$.file) 2>/dev/null; then # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -p'. + # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || - as_ln_s='cp -p' + as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else - as_ln_s='cp -p' + as_ln_s='cp -pR' fi else - as_ln_s='cp -p' + as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null @@ -496,28 +544,8 @@ else as_mkdir_p=false fi -if test -x / >/dev/null 2>&1; then - as_test_x='test -x' -else - if ls -dL / >/dev/null 2>&1; then - as_ls_L_option=L - else - as_ls_L_option= - fi - as_test_x=' - eval sh -c '\'' - if test -d "$1"; then - test -d "$1/."; - else - case $1 in #( - -*)set "./$1";; - esac; - case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( - ???[sx]*):;;*)false;;esac;fi - '\'' sh - ' -fi -as_executable_p=$as_test_x +as_test_x='test -x' +as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" @@ -530,7 +558,7 @@ test -n "$DJDIR" || exec 7<&0 &1 # Name of the host. -# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, +# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, # so uname gets run too. ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` @@ -597,6 +625,7 @@ cfgoutputs_out cfgoutputs_in cfghdrs_out cfghdrs_in +enable_zone_allocator enable_tls enable_lazy_lock jemalloc_version_gid @@ -605,29 +634,50 @@ jemalloc_version_bugfix jemalloc_version_minor jemalloc_version_major jemalloc_version -enable_dynamic_page_shift -enable_sysv enable_xmalloc +enable_valgrind +enable_utrace enable_fill enable_dss -enable_swap +enable_munmap +enable_mremap enable_tcache -enable_tiny enable_prof enable_stats enable_debug +je_ install_suffix +private_namespace +enable_code_coverage +enable_experimental AUTOCONF LD -AR RANLIB INSTALL_DATA INSTALL_SCRIPT INSTALL_PROGRAM enable_autogen RPATH_EXTRA +CC_MM +AROUT +ARFLAGS +MKLIB +LDTARGET +CTARGET +PIC_CFLAGS +SOREV +EXTRA_LDFLAGS +DSO_LDFLAGS +libprefix +exe +a +o +importlib +so +LD_PRELOAD_VAR RPATH abi +AR host_os host_vendor host_cpu @@ -658,6 +708,7 @@ abs_objroot objroot abs_srcroot srcroot +rev target_alias host_alias build_alias @@ -702,27 +753,33 @@ enable_option_checking with_xslroot with_rpath enable_autogen +enable_experimental +enable_code_coverage +with_mangling with_jemalloc_prefix +with_export with_private_namespace with_install_suffix enable_cc_silence enable_debug +enable_ivsalloc enable_stats enable_prof enable_prof_libunwind with_static_libunwind enable_prof_libgcc enable_prof_gcc -enable_tiny enable_tcache -enable_swap +enable_mremap +enable_munmap enable_dss enable_fill +enable_utrace +enable_valgrind enable_xmalloc -enable_sysv -enable_dynamic_page_shift enable_lazy_lock enable_tls +enable_zone_allocator ' ac_precious_vars='build_alias host_alias @@ -795,8 +852,9 @@ do fi case $ac_option in - *=*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; - *) ac_optarg=yes ;; + *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *=) ac_optarg= ;; + *) ac_optarg=yes ;; esac # Accept the important Cygnus configure options, so we can diagnose typos. @@ -841,7 +899,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid feature name: $ac_useropt" + as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -867,7 +925,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid feature name: $ac_useropt" + as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1071,7 +1129,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid package name: $ac_useropt" + as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1087,7 +1145,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid package name: $ac_useropt" + as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1117,8 +1175,8 @@ do | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; - -*) as_fn_error "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information." + -*) as_fn_error $? "unrecognized option: \`$ac_option' +Try \`$0 --help' for more information" ;; *=*) @@ -1126,7 +1184,7 @@ Try \`$0 --help' for more information." # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error "invalid variable name: \`$ac_envvar'" ;; + as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; @@ -1136,7 +1194,7 @@ Try \`$0 --help' for more information." $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 - : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option} + : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" ;; esac @@ -1144,13 +1202,13 @@ done if test -n "$ac_prev"; then ac_option=--`echo $ac_prev | sed 's/_/-/g'` - as_fn_error "missing argument to $ac_option" + as_fn_error $? "missing argument to $ac_option" fi if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; - fatal) as_fn_error "unrecognized options: $ac_unrecognized_opts" ;; + fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi @@ -1173,7 +1231,7 @@ do [\\/$]* | ?:[\\/]* ) continue;; NONE | '' ) case $ac_var in *prefix ) continue;; esac;; esac - as_fn_error "expected an absolute directory name for --$ac_var: $ac_val" + as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done # There might be people who depend on the old broken behavior: `$host' @@ -1187,8 +1245,6 @@ target=$target_alias if test "x$host_alias" != x; then if test "x$build_alias" = x; then cross_compiling=maybe - $as_echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. - If a cross compiler is detected then cross compile mode will be used." >&2 elif test "x$build_alias" != "x$host_alias"; then cross_compiling=yes fi @@ -1203,9 +1259,9 @@ test "$silent" = yes && exec 6>/dev/null ac_pwd=`pwd` && test -n "$ac_pwd" && ac_ls_di=`ls -di .` && ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || - as_fn_error "working directory cannot be determined" + as_fn_error $? "working directory cannot be determined" test "X$ac_ls_di" = "X$ac_pwd_ls_di" || - as_fn_error "pwd does not report name of working directory" + as_fn_error $? "pwd does not report name of working directory" # Find the source files, if location was not specified. @@ -1244,11 +1300,11 @@ else fi if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." - as_fn_error "cannot find sources ($ac_unique_file) in $srcdir" + as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" ac_abs_confdir=`( - cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error "$ac_msg" + cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` # When building in place, set srcdir=. if test "$ac_abs_confdir" = "$ac_pwd"; then @@ -1288,7 +1344,7 @@ Configuration: --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking...' messages + -q, --quiet, --silent do not print \`checking ...' messages --cache-file=FILE cache test results in FILE [disabled] -C, --config-cache alias for \`--cache-file=config.cache' -n, --no-create do not create output files @@ -1346,34 +1402,39 @@ Optional Features: --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-autogen Automatically regenerate configure output + --disable-experimental Disable support for the experimental API + --enable-code-coverage Enable code coverage --enable-cc-silence Silence irrelevant compiler warnings - --enable-debug Build debugging code - --enable-stats Enable statistics calculation/reporting + --enable-debug Build debugging code (implies --enable-ivsalloc) + --enable-ivsalloc Validate pointers passed through the public API + --disable-stats Disable statistics calculation/reporting --enable-prof Enable allocation profiling --enable-prof-libunwind Use libunwind for backtracing --disable-prof-libgcc Do not use libgcc for backtracing --disable-prof-gcc Do not use gcc intrinsics for backtracing - --disable-tiny Disable tiny (sub-quantum) allocations --disable-tcache Disable per thread caches - --enable-swap Enable mmap()ped swap files + --enable-mremap Enable mremap(2) for huge realloc() + --disable-munmap Disable VM deallocation via munmap(2) --enable-dss Enable allocation from DSS - --enable-fill Support junk/zero filling option + --disable-fill Disable support for junk/zero filling, quarantine, + and redzones + --enable-utrace Enable utrace(2)-based tracing + --disable-valgrind Disable support for Valgrind --enable-xmalloc Support xmalloc option - --enable-sysv Support SYSV semantics option - --enable-dynamic-page-shift - Determine page size at run time (don't trust - configure result) - --disable-lazy-lock Disable lazy locking (always lock, even when - single-threaded) + --enable-lazy-lock Enable lazy locking (only lock when multi-threaded) --disable-tls Disable thread-local storage (__thread keyword) + --disable-zone-allocator + Disable zone allocator for Darwin Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-xslroot= XSL stylesheet root path --with-rpath= Colon-separated rpath (ELF systems only) + --with-mangling= Mangle symbols in --with-jemalloc-prefix= Prefix to prepend to all public APIs + --without-export disable exporting jemalloc public APIs --with-private-namespace= Prefix to prepend to all library-private APIs --with-install-suffix= @@ -1459,9 +1520,9 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF configure -generated by GNU Autoconf 2.65 +generated by GNU Autoconf 2.69 -Copyright (C) 2009 Free Software Foundation, Inc. +Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -1505,11 +1566,48 @@ sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_compile +# ac_fn_c_try_cpp LINENO +# ---------------------- +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_cpp + # ac_fn_c_try_run LINENO # ---------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. Assumes @@ -1547,47 +1645,41 @@ sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=$ac_status fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_run -# ac_fn_c_try_cpp LINENO -# ---------------------- -# Try to preprocess conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_cpp () +# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists and can be compiled using the include files in +# INCLUDES, setting the cache variable VAR accordingly. +ac_fn_c_check_header_compile () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_cpp conftest.$ac_ext" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } >/dev/null && { - test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || - test ! -s conftest.err - }; then : - ac_retval=0 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} - as_fn_set_status $ac_retval +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno -} # ac_fn_c_try_cpp +} # ac_fn_c_check_header_compile # ac_fn_c_compute_int LINENO EXPR VAR INCLUDES # -------------------------------------------- @@ -1606,7 +1698,8 @@ int main () { static int test_array [1 - 2 * !(($2) >= 0)]; -test_array [0] = 0 +test_array [0] = 0; +return test_array [0]; ; return 0; @@ -1622,7 +1715,8 @@ int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; -test_array [0] = 0 +test_array [0] = 0; +return test_array [0]; ; return 0; @@ -1648,7 +1742,8 @@ int main () { static int test_array [1 - 2 * !(($2) < 0)]; -test_array [0] = 0 +test_array [0] = 0; +return test_array [0]; ; return 0; @@ -1664,7 +1759,8 @@ int main () { static int test_array [1 - 2 * !(($2) >= $ac_mid)]; -test_array [0] = 0 +test_array [0] = 0; +return test_array [0]; ; return 0; @@ -1698,7 +1794,8 @@ int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; -test_array [0] = 0 +test_array [0] = 0; +return test_array [0]; ; return 0; @@ -1762,129 +1859,11 @@ rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ rm -f conftest.val fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_compute_int -# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES -# ------------------------------------------------------- -# Tests whether HEADER exists and can be compiled using the include files in -# INCLUDES, setting the cache variable VAR accordingly. -ac_fn_c_check_header_compile () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -#include <$2> -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - eval "$3=yes" -else - eval "$3=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} - -} # ac_fn_c_check_header_compile - -# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES -# ------------------------------------------------------- -# Tests whether HEADER exists, giving a warning if it cannot be compiled using -# the include files in INCLUDES and setting the cache variable VAR -# accordingly. -ac_fn_c_check_header_mongrel () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - $as_echo_n "(cached) " >&6 -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -else - # Is the header compilable? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 -$as_echo_n "checking $2 usability... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -#include <$2> -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_header_compiler=yes -else - ac_header_compiler=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 -$as_echo "$ac_header_compiler" >&6; } - -# Is the header present? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 -$as_echo_n "checking $2 presence... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include <$2> -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - ac_header_preproc=yes -else - ac_header_preproc=no -fi -rm -f conftest.err conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 -$as_echo "$ac_header_preproc" >&6; } - -# So? What about this header? -case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( - yes:no: ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 -$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; - no:yes:* ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 -$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 -$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 -$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 -$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; -esac - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - $as_echo_n "(cached) " >&6 -else - eval "$3=\$ac_header_compiler" -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} - -} # ac_fn_c_check_header_mongrel - # ac_fn_c_try_link LINENO # ----------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. @@ -1912,7 +1891,7 @@ $as_echo "$ac_try_echo"; } >&5 test ! -s conftest.err } && test -s conftest$ac_exeext && { test "$cross_compiling" = yes || - $as_test_x conftest$ac_exeext + test -x conftest$ac_exeext }; then : ac_retval=0 else @@ -1926,7 +1905,7 @@ fi # interfere with the next link command; also delete a directory that is # left behind by Apple's compiler. We do this before executing the actions. rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_link @@ -1939,7 +1918,7 @@ ac_fn_c_check_func () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : +if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -1994,39 +1973,126 @@ fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_func -# ac_fn_c_check_type LINENO TYPE VAR INCLUDES -# ------------------------------------------- -# Tests whether TYPE exists after having included INCLUDES, setting cache -# variable VAR accordingly. -ac_fn_c_check_type () +# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists, giving a warning if it cannot be compiled using +# the include files in INCLUDES and setting the cache variable VAR +# accordingly. +ac_fn_c_check_header_mongrel () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if eval \${$3+:} false; then : { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : +if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } else - eval "$3=no" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 -int -main () -{ -if (sizeof ($2)) - return 0; - ; - return 0; -} +#include <$2> _ACEOF if ac_fn_c_try_compile "$LINENO"; then : - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -$4 +#include <$2> +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_mongrel + +# ac_fn_c_check_type LINENO TYPE VAR INCLUDES +# ------------------------------------------- +# Tests whether TYPE exists after having included INCLUDES, setting cache +# variable VAR accordingly. +ac_fn_c_check_type () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=no" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main () +{ +if (sizeof ($2)) + return 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 int main () { @@ -2048,7 +2114,7 @@ fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_type cat >config.log <<_ACEOF @@ -2056,7 +2122,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by $as_me, which was -generated by GNU Autoconf 2.65. Invocation command line was +generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2166,11 +2232,9 @@ trap 'exit_status=$? { echo - cat <<\_ASBOX -## ---------------- ## + $as_echo "## ---------------- ## ## Cache variables. ## -## ---------------- ## -_ASBOX +## ---------------- ##" echo # The following way of writing the cache mishandles newlines in values, ( @@ -2204,11 +2268,9 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; ) echo - cat <<\_ASBOX -## ----------------- ## + $as_echo "## ----------------- ## ## Output variables. ## -## ----------------- ## -_ASBOX +## ----------------- ##" echo for ac_var in $ac_subst_vars do @@ -2221,11 +2283,9 @@ _ASBOX echo if test -n "$ac_subst_files"; then - cat <<\_ASBOX -## ------------------- ## + $as_echo "## ------------------- ## ## File substitutions. ## -## ------------------- ## -_ASBOX +## ------------------- ##" echo for ac_var in $ac_subst_files do @@ -2239,11 +2299,9 @@ _ASBOX fi if test -s confdefs.h; then - cat <<\_ASBOX -## ----------- ## + $as_echo "## ----------- ## ## confdefs.h. ## -## ----------- ## -_ASBOX +## ----------- ##" echo cat confdefs.h echo @@ -2298,7 +2356,12 @@ _ACEOF ac_site_file1=NONE ac_site_file2=NONE if test -n "$CONFIG_SITE"; then - ac_site_file1=$CONFIG_SITE + # We do not want a PATH search for config.site. + case $CONFIG_SITE in #(( + -*) ac_site_file1=./$CONFIG_SITE;; + */*) ac_site_file1=$CONFIG_SITE;; + *) ac_site_file1=./$CONFIG_SITE;; + esac elif test "x$prefix" != xNONE; then ac_site_file1=$prefix/share/config.site ac_site_file2=$prefix/etc/config.site @@ -2313,7 +2376,11 @@ do { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 $as_echo "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 - . "$ac_site_file" + . "$ac_site_file" \ + || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "failed to load site script $ac_site_file +See \`config.log' for more details" "$LINENO" 5; } fi done @@ -2389,7 +2456,7 @@ if $ac_cache_corrupted; then $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 $as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 + as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## ## Main body of script. ## @@ -2409,6 +2476,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu +rev=1 + + srcroot=$srcdir if test "x${srcroot}" = "x." ; then srcroot="" @@ -2452,7 +2522,7 @@ MANDIR=`eval echo $MANDIR` set dummy xsltproc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_XSLTPROC+set}" = set; then : +if ${ac_cv_path_XSLTPROC+:} false; then : $as_echo_n "(cached) " >&6 else case $XSLTPROC in @@ -2466,7 +2536,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_XSLTPROC="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2475,6 +2545,7 @@ done done IFS=$as_save_IFS + test -z "$ac_cv_path_XSLTPROC" && ac_cv_path_XSLTPROC="false" ;; esac fi @@ -2488,16 +2559,25 @@ $as_echo "no" >&6; } fi +if test -d "/usr/share/xml/docbook/stylesheet/docbook-xsl" ; then + DEFAULT_XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +elif test -d "/usr/share/sgml/docbook/xsl-stylesheets" ; then + DEFAULT_XSLROOT="/usr/share/sgml/docbook/xsl-stylesheets" +else + DEFAULT_XSLROOT="" +fi # Check whether --with-xslroot was given. if test "${with_xslroot+set}" = set; then : - withval=$with_xslroot; if test "x$with_xslroot" = "xno" ; then - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + withval=$with_xslroot; +if test "x$with_xslroot" = "xno" ; then + XSLROOT="${DEFAULT_XSLROOT}" else XSLROOT="${with_xslroot}" fi + else - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + XSLROOT="${DEFAULT_XSLROOT}" fi @@ -2514,7 +2594,7 @@ if test -n "$ac_tool_prefix"; then set dummy ${ac_tool_prefix}gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2526,7 +2606,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="${ac_tool_prefix}gcc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2554,7 +2634,7 @@ if test -z "$ac_cv_prog_CC"; then set dummy gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then : +if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then @@ -2566,7 +2646,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CC="gcc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2607,7 +2687,7 @@ if test -z "$CC"; then set dummy ${ac_tool_prefix}cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2619,7 +2699,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="${ac_tool_prefix}cc" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2647,7 +2727,7 @@ if test -z "$CC"; then set dummy cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2660,7 +2740,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then ac_prog_rejected=yes continue @@ -2706,7 +2786,7 @@ if test -z "$CC"; then set dummy $ac_tool_prefix$ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2718,7 +2798,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="$ac_tool_prefix$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2750,7 +2830,7 @@ do set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then : +if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then @@ -2762,7 +2842,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CC="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -2804,8 +2884,8 @@ fi test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "no acceptable C compiler found in \$PATH -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "no acceptable C compiler found in \$PATH +See \`config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 @@ -2919,9 +2999,8 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "C compiler cannot create executables -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "C compiler cannot create executables +See \`config.log' for more details" "$LINENO" 5; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } @@ -2963,8 +3042,8 @@ done else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest conftest$ac_cv_exeext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 @@ -3021,9 +3100,9 @@ $as_echo "$ac_try_echo"; } >&5 else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run C compiled programs. +as_fn_error $? "cannot run C compiled programs. If you meant to cross compile, use \`--host'. -See \`config.log' for more details." "$LINENO" 5; } +See \`config.log' for more details" "$LINENO" 5; } fi fi fi @@ -3034,7 +3113,7 @@ rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 $as_echo_n "checking for suffix of object files... " >&6; } -if test "${ac_cv_objext+set}" = set; then : +if ${ac_cv_objext+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3074,8 +3153,8 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot compute suffix of object files: cannot compile -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot compute suffix of object files: cannot compile +See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest.$ac_cv_objext conftest.$ac_ext fi @@ -3085,7 +3164,7 @@ OBJEXT=$ac_cv_objext ac_objext=$OBJEXT { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 $as_echo_n "checking whether we are using the GNU C compiler... " >&6; } -if test "${ac_cv_c_compiler_gnu+set}" = set; then : +if ${ac_cv_c_compiler_gnu+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3122,7 +3201,7 @@ ac_test_CFLAGS=${CFLAGS+set} ac_save_CFLAGS=$CFLAGS { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 $as_echo_n "checking whether $CC accepts -g... " >&6; } -if test "${ac_cv_prog_cc_g+set}" = set; then : +if ${ac_cv_prog_cc_g+:} false; then : $as_echo_n "(cached) " >&6 else ac_save_c_werror_flag=$ac_c_werror_flag @@ -3200,7 +3279,7 @@ else fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 $as_echo_n "checking for $CC option to accept ISO C89... " >&6; } -if test "${ac_cv_prog_cc_c89+set}" = set; then : +if ${ac_cv_prog_cc_c89+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_prog_cc_c89=no @@ -3209,8 +3288,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include -#include -#include +struct stat; /* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ struct buf { int x; }; FILE * (*rcsopen) (struct buf *, struct stat *, int); @@ -3295,11 +3373,43 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu +if test "x$GCC" != "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler is MSVC" >&5 +$as_echo_n "checking whether compiler is MSVC... " >&6; } +if ${je_cv_msvc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + +#ifndef _MSC_VER + int fail-1; +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_msvc=yes +else + je_cv_msvc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_msvc" >&5 +$as_echo "$je_cv_msvc" >&6; } +fi + if test "x$CFLAGS" = "x" ; then no_CFLAGS="yes" if test "x$GCC" = "xyes" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -std=gnu99" >&5 $as_echo_n "checking whether compiler supports -std=gnu99... " >&6; } TCFLAGS="${CFLAGS}" @@ -3308,13 +3418,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -std=gnu99" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3328,20 +3432,25 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-std=gnu99 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test "x$je_cv_cflags_appended" = "x-std=gnu99" ; then + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_HAS_RESTRICT 1 +_ACEOF + fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Wall" >&5 $as_echo_n "checking whether compiler supports -Wall... " >&6; } @@ -3351,13 +3460,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -Wall" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3371,19 +3474,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-Wall + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -pipe" >&5 @@ -3394,13 +3496,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -pipe" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3414,19 +3510,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-pipe + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -g3" >&5 @@ -3437,13 +3532,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -g3" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3457,39 +3546,31 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-g3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - fi -fi -if test "x$EXTRA_CFLAGS" != "x" ; then + elif test "x$je_cv_msvc" = "xyes" ; then + CC="$CC -nologo" -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports $EXTRA_CFLAGS" >&5 -$as_echo_n "checking whether compiler supports $EXTRA_CFLAGS... " >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Zi" >&5 +$as_echo_n "checking whether compiler supports -Zi... " >&6; } TCFLAGS="${CFLAGS}" if test "x${CFLAGS}" = "x" ; then - CFLAGS="$EXTRA_CFLAGS" + CFLAGS="-Zi" else - CFLAGS="${CFLAGS} $EXTRA_CFLAGS" + CFLAGS="${CFLAGS} -Zi" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3503,84 +3584,195 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-Zi + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 -$as_echo_n "checking how to run the C preprocessor... " >&6; } -# On Suns, sometimes $CPP names a directory. -if test -n "$CPP" && test -d "$CPP"; then - CPP= -fi -if test -z "$CPP"; then - if test "${ac_cv_prog_CPP+set}" = set; then : - $as_echo_n "(cached) " >&6 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -MT" >&5 +$as_echo_n "checking whether compiler supports -MT... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-MT" else - # Double quotes because CPP needs to be expanded - for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" - do - ac_preproc_ok=false -for ac_c_preproc_warn_flag in '' yes -do - # Use a header file that comes with gcc, so configuring glibc - # with a fresh cross-compiler works. - # Prefer to if __STDC__ is defined, since - # exists even on freestanding compilers. - # On the NeXT, cc -E runs the code through the compiler's parser, - # not just through cpp. "Syntax error" is here to catch this case. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + CFLAGS="${CFLAGS} -MT" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#ifdef __STDC__ -# include -#else -# include -#endif - Syntax error -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : -else - # Broken: fails on valid input. -continue -fi -rm -f conftest.err conftest.$ac_ext - # OK, works on sane cases. Now check whether nonexistent headers - # can be detected and how. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include +int +main () +{ + + return 0; + + ; + return 0; +} _ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - # Broken: success on invalid input. -continue +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-MT + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } else - # Passes both tests. -ac_preproc_ok=: -break + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + fi -rm -f conftest.err conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -W3" >&5 +$as_echo_n "checking whether compiler supports -W3... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-W3" +else + CFLAGS="${CFLAGS} -W3" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-W3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" + fi +fi +if test "x$EXTRA_CFLAGS" != "x" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports $EXTRA_CFLAGS" >&5 +$as_echo_n "checking whether compiler supports $EXTRA_CFLAGS... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="$EXTRA_CFLAGS" +else + CFLAGS="${CFLAGS} $EXTRA_CFLAGS" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=$EXTRA_CFLAGS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 +$as_echo_n "checking how to run the C preprocessor... " >&6; } +# On Suns, sometimes $CPP names a directory. +if test -n "$CPP" && test -d "$CPP"; then + CPP= +fi +if test -z "$CPP"; then + if ${ac_cv_prog_CPP+:} false; then : + $as_echo_n "(cached) " >&6 +else + # Double quotes because CPP needs to be expanded + for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" + do + ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.err conftest.$ac_ext +rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : break fi @@ -3619,7 +3811,7 @@ else # Broken: fails on valid input. continue fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. @@ -3635,18 +3827,18 @@ else ac_preproc_ok=: break fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.err conftest.$ac_ext +rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } fi ac_ext=c @@ -3659,7 +3851,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 $as_echo_n "checking for grep that handles long lines and -e... " >&6; } -if test "${ac_cv_path_GREP+set}" = set; then : +if ${ac_cv_path_GREP+:} false; then : $as_echo_n "(cached) " >&6 else if test -z "$GREP"; then @@ -3673,7 +3865,7 @@ do for ac_prog in grep ggrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" - { test -f "$ac_path_GREP" && $as_test_x "$ac_path_GREP"; } || continue + as_fn_executable_p "$ac_path_GREP" || continue # Check for GNU ac_path_GREP and select it if it is found. # Check for GNU $ac_path_GREP case `"$ac_path_GREP" --version 2>&1` in @@ -3708,7 +3900,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_GREP"; then - as_fn_error "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_GREP=$GREP @@ -3722,7 +3914,7 @@ $as_echo "$ac_cv_path_GREP" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 $as_echo_n "checking for egrep... " >&6; } -if test "${ac_cv_path_EGREP+set}" = set; then : +if ${ac_cv_path_EGREP+:} false; then : $as_echo_n "(cached) " >&6 else if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 @@ -3739,7 +3931,7 @@ do for ac_prog in egrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" - { test -f "$ac_path_EGREP" && $as_test_x "$ac_path_EGREP"; } || continue + as_fn_executable_p "$ac_path_EGREP" || continue # Check for GNU ac_path_EGREP and select it if it is found. # Check for GNU $ac_path_EGREP case `"$ac_path_EGREP" --version 2>&1` in @@ -3774,7 +3966,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_EGREP"; then - as_fn_error "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_EGREP=$EGREP @@ -3789,7 +3981,7 @@ $as_echo "$ac_cv_path_EGREP" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 $as_echo_n "checking for ANSI C header files... " >&6; } -if test "${ac_cv_header_stdc+set}" = set; then : +if ${ac_cv_header_stdc+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3906,8 +4098,7 @@ do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default " -eval as_val=\$$as_ac_Header - if test "x$as_val" = x""yes; then : +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF @@ -3917,13 +4108,243 @@ fi done + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether byte ordering is bigendian" >&5 +$as_echo_n "checking whether byte ordering is bigendian... " >&6; } +if ${ac_cv_c_bigendian+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_c_bigendian=unknown + # See if we're dealing with a universal compiler. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifndef __APPLE_CC__ + not a universal capable compiler + #endif + typedef int dummy; + +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + + # Check for potential -arch flags. It is not universal unless + # there are at least two -arch flags with different values. + ac_arch= + ac_prev= + for ac_word in $CC $CFLAGS $CPPFLAGS $LDFLAGS; do + if test -n "$ac_prev"; then + case $ac_word in + i?86 | x86_64 | ppc | ppc64) + if test -z "$ac_arch" || test "$ac_arch" = "$ac_word"; then + ac_arch=$ac_word + else + ac_cv_c_bigendian=universal + break + fi + ;; + esac + ac_prev= + elif test "x$ac_word" = "x-arch"; then + ac_prev=arch + fi + done +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test $ac_cv_c_bigendian = unknown; then + # See if sys/param.h defines the BYTE_ORDER macro. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + #include + +int +main () +{ +#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \ + && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \ + && LITTLE_ENDIAN) + bogus endian macros + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + # It does; now see whether it defined to BIG_ENDIAN or not. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + #include + +int +main () +{ +#if BYTE_ORDER != BIG_ENDIAN + not big endian + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_c_bigendian=yes +else + ac_cv_c_bigendian=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi + if test $ac_cv_c_bigendian = unknown; then + # See if defines _LITTLE_ENDIAN or _BIG_ENDIAN (e.g., Solaris). + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +int +main () +{ +#if ! (defined _LITTLE_ENDIAN || defined _BIG_ENDIAN) + bogus endian macros + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + # It does; now see whether it defined to _BIG_ENDIAN or not. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +int +main () +{ +#ifndef _BIG_ENDIAN + not big endian + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_c_bigendian=yes +else + ac_cv_c_bigendian=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi + if test $ac_cv_c_bigendian = unknown; then + # Compile a test program. + if test "$cross_compiling" = yes; then : + # Try to guess by grepping values from an object file. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +short int ascii_mm[] = + { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; + short int ascii_ii[] = + { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; + int use_ascii (int i) { + return ascii_mm[i] + ascii_ii[i]; + } + short int ebcdic_ii[] = + { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; + short int ebcdic_mm[] = + { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 }; + int use_ebcdic (int i) { + return ebcdic_mm[i] + ebcdic_ii[i]; + } + extern int foo; + +int +main () +{ +return use_ascii (foo) == use_ebcdic (foo); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + if grep BIGenDianSyS conftest.$ac_objext >/dev/null; then + ac_cv_c_bigendian=yes + fi + if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then + if test "$ac_cv_c_bigendian" = unknown; then + ac_cv_c_bigendian=no + else + # finding both strings is unlikely to happen, but who knows? + ac_cv_c_bigendian=unknown + fi + fi +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default +int +main () +{ + + /* Are we little or big endian? From Harbison&Steele. */ + union + { + long int l; + char c[sizeof (long int)]; + } u; + u.l = 1; + return u.c[sizeof (long int) - 1] == 1; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + ac_cv_c_bigendian=no +else + ac_cv_c_bigendian=yes +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_bigendian" >&5 +$as_echo "$ac_cv_c_bigendian" >&6; } + case $ac_cv_c_bigendian in #( + yes) + ac_cv_big_endian=1;; #( + no) + ac_cv_big_endian=0 ;; #( + universal) + +$as_echo "#define AC_APPLE_UNIVERSAL_BUILD 1" >>confdefs.h + + ;; #( + *) + as_fn_error $? "unknown endianness + presetting ac_cv_c_bigendian=no (or yes) will help" "$LINENO" 5 ;; + esac + +if test "x${ac_cv_big_endian}" = "x1" ; then + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_BIG_ENDIAN +_ACEOF + +fi + # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 $as_echo_n "checking size of void *... " >&6; } -if test "${ac_cv_sizeof_void_p+set}" = set; then : +if ${ac_cv_sizeof_void_p+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default"; then : @@ -3932,9 +4353,8 @@ else if test "$ac_cv_type_void_p" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (void *) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (void *) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 fi @@ -3956,7 +4376,7 @@ if test "x${ac_cv_sizeof_void_p}" = "x8" ; then elif test "x${ac_cv_sizeof_void_p}" = "x4" ; then LG_SIZEOF_PTR=2 else - as_fn_error "Unsupported pointer size: ${ac_cv_sizeof_void_p}" "$LINENO" 5 + as_fn_error $? "Unsupported pointer size: ${ac_cv_sizeof_void_p}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_PTR $LG_SIZEOF_PTR @@ -3969,7 +4389,7 @@ _ACEOF # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of int" >&5 $as_echo_n "checking size of int... " >&6; } -if test "${ac_cv_sizeof_int+set}" = set; then : +if ${ac_cv_sizeof_int+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default"; then : @@ -3978,9 +4398,8 @@ else if test "$ac_cv_type_int" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (int) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (int) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_int=0 fi @@ -4002,7 +4421,7 @@ if test "x${ac_cv_sizeof_int}" = "x8" ; then elif test "x${ac_cv_sizeof_int}" = "x4" ; then LG_SIZEOF_INT=2 else - as_fn_error "Unsupported int size: ${ac_cv_sizeof_int}" "$LINENO" 5 + as_fn_error $? "Unsupported int size: ${ac_cv_sizeof_int}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_INT $LG_SIZEOF_INT @@ -4015,7 +4434,7 @@ _ACEOF # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of long" >&5 $as_echo_n "checking size of long... " >&6; } -if test "${ac_cv_sizeof_long+set}" = set; then : +if ${ac_cv_sizeof_long+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default"; then : @@ -4024,9 +4443,8 @@ else if test "$ac_cv_type_long" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (long) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (long) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long=0 fi @@ -4048,28 +4466,81 @@ if test "x${ac_cv_sizeof_long}" = "x8" ; then elif test "x${ac_cv_sizeof_long}" = "x4" ; then LG_SIZEOF_LONG=2 else - as_fn_error "Unsupported long size: ${ac_cv_sizeof_long}" "$LINENO" 5 + as_fn_error $? "Unsupported long size: ${ac_cv_sizeof_long}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_LONG $LG_SIZEOF_LONG _ACEOF -ac_aux_dir= -for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do - for ac_t in install-sh install.sh shtool; do - if test -f "$ac_dir/$ac_t"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/$ac_t -c" - break 2 - fi - done -done -if test -z "$ac_aux_dir"; then - as_fn_error "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 +# The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of intmax_t" >&5 +$as_echo_n "checking size of intmax_t... " >&6; } +if ${ac_cv_sizeof_intmax_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (intmax_t))" "ac_cv_sizeof_intmax_t" "$ac_includes_default"; then : + +else + if test "$ac_cv_type_intmax_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (intmax_t) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_intmax_t=0 + fi fi -# These three variables are undocumented and unsupported, +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_intmax_t" >&5 +$as_echo "$ac_cv_sizeof_intmax_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_INTMAX_T $ac_cv_sizeof_intmax_t +_ACEOF + + +if test "x${ac_cv_sizeof_intmax_t}" = "x16" ; then + LG_SIZEOF_INTMAX_T=4 +elif test "x${ac_cv_sizeof_intmax_t}" = "x8" ; then + LG_SIZEOF_INTMAX_T=3 +elif test "x${ac_cv_sizeof_intmax_t}" = "x4" ; then + LG_SIZEOF_INTMAX_T=2 +else + as_fn_error $? "Unsupported intmax_t size: ${ac_cv_sizeof_intmax_t}" "$LINENO" 5 +fi +cat >>confdefs.h <<_ACEOF +#define LG_SIZEOF_INTMAX_T $LG_SIZEOF_INTMAX_T +_ACEOF + + +ac_aux_dir= +for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do + if test -f "$ac_dir/install-sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install-sh -c" + break + elif test -f "$ac_dir/install.sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install.sh -c" + break + elif test -f "$ac_dir/shtool"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/shtool install -c" + break + fi +done +if test -z "$ac_aux_dir"; then + as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 +fi + +# These three variables are undocumented and unsupported, # and are intended to be withdrawn in a future Autoconf release. # They can cause serious problems if a builder's source tree is in a directory # whose full name contains unusual characters. @@ -4080,27 +4551,27 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. # Make sure we can run config.sub. $SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 || - as_fn_error "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 + as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5 $as_echo_n "checking build system type... " >&6; } -if test "${ac_cv_build+set}" = set; then : +if ${ac_cv_build+:} false; then : $as_echo_n "(cached) " >&6 else ac_build_alias=$build_alias test "x$ac_build_alias" = x && ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"` test "x$ac_build_alias" = x && - as_fn_error "cannot guess build type; you must specify one" "$LINENO" 5 + as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` || - as_fn_error "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 $as_echo "$ac_cv_build" >&6; } case $ac_cv_build in *-*-*) ;; -*) as_fn_error "invalid value of canonical build" "$LINENO" 5;; +*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;; esac build=$ac_cv_build ac_save_IFS=$IFS; IFS='-' @@ -4118,14 +4589,14 @@ case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5 $as_echo_n "checking host system type... " >&6; } -if test "${ac_cv_host+set}" = set; then : +if ${ac_cv_host+:} false; then : $as_echo_n "(cached) " >&6 else if test "x$host_alias" = x; then ac_cv_host=$ac_cv_build else ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` || - as_fn_error "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 fi fi @@ -4133,7 +4604,7 @@ fi $as_echo "$ac_cv_host" >&6; } case $ac_cv_host in *-*-*) ;; -*) as_fn_error "invalid value of canonical host" "$LINENO" 5;; +*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;; esac host=$ac_cv_host ac_save_IFS=$IFS; IFS='-' @@ -4153,15 +4624,12 @@ CPU_SPINWAIT="" case "${host_cpu}" in i[345]86) ;; - i686) + i686|x86_64) -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __asm__ is compilable" >&5 -$as_echo_n "checking whether __asm__ is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pause instruction is compilable" >&5 +$as_echo_n "checking whether pause instruction is compilable... " >&6; } +if ${je_cv_pause+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4174,64 +4642,62 @@ __asm__ volatile("pause"); return 0; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - asm="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_pause=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - asm="no" - + je_cv_pause=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_pause" >&5 +$as_echo "$je_cv_pause" >&6; } - - if test "x${asm}" = "xyes" ; then + if test "x${je_cv_pause}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi - ;; - x86_64) -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __asm__ syntax is compilable" >&5 -$as_echo_n "checking whether __asm__ syntax is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether SSE2 intrinsics is compilable" >&5 +$as_echo_n "checking whether SSE2 intrinsics is compilable... " >&6; } +if ${je_cv_sse2+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include + int main () { -__asm__ volatile("pause"); return 0; + ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - asm="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_sse2=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - asm="no" - + je_cv_sse2=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sse2" >&5 +$as_echo "$je_cv_sse2" >&6; } + if test "x${je_cv_sse2}" = "xyes" ; then + cat >>confdefs.h <<_ACEOF +#define HAVE_SSE2 +_ACEOF - if test "x${asm}" = "xyes" ; then - CPU_SPINWAIT='__asm__ volatile("pause")' fi + ;; + powerpc) + cat >>confdefs.h <<_ACEOF +#define HAVE_ALTIVEC +_ACEOF + ;; *) ;; @@ -4241,28 +4707,156 @@ cat >>confdefs.h <<_ACEOF _ACEOF +LD_PRELOAD_VAR="LD_PRELOAD" +so="so" +importlib="${so}" +o="$ac_objext" +a="a" +exe="$ac_exeext" +libprefix="lib" +DSO_LDFLAGS='-shared -Wl,-soname,$(@F)' +RPATH='-Wl,-rpath,$(1)' +SOREV="${so}.${rev}" +PIC_CFLAGS='-fPIC -DPIC' +CTARGET='-o $@' +LDTARGET='-o $@' +EXTRA_LDFLAGS= +ARFLAGS='crus' +AROUT=' $@' +CC_MM=1 + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}ar", so it can be a program name with args. +set dummy ${ac_tool_prefix}ar; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$AR"; then + ac_cv_prog_AR="$AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_AR="${ac_tool_prefix}ar" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AR=$ac_cv_prog_AR +if test -n "$AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5 +$as_echo "$AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_AR"; then + ac_ct_AR=$AR + # Extract the first word of "ar", so it can be a program name with args. +set dummy ar; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_AR+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_AR"; then + ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_AR="ar" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_AR=$ac_cv_prog_ac_ct_AR +if test -n "$ac_ct_AR"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5 +$as_echo "$ac_ct_AR" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_AR" = x; then + AR=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + AR=$ac_ct_AR + fi +else + AR="$ac_cv_prog_AR" +fi + + +default_munmap="1" +JEMALLOC_USABLE_SIZE_CONST="const" case "${host}" in *-*-darwin*) - CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" + CFLAGS="$CFLAGS" abi="macho" - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h RPATH="" + LD_PRELOAD_VAR="DYLD_INSERT_LIBRARIES" + so="dylib" + importlib="${so}" + force_tls="0" + DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)' + SOREV="${rev}.${so}" + sbrk_deprecated="1" ;; *-*-freebsd*) CFLAGS="$CFLAGS" abi="elf" - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h - RPATH="-Wl,-rpath," + force_lazy_lock="1" ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" abi="elf" - $as_echo "#define JEMALLOC_PURGE_MADVISE_DONTNEED 1" >>confdefs.h + $as_echo "#define JEMALLOC_HAS_ALLOCA_H 1" >>confdefs.h + + $as_echo "#define JEMALLOC_PURGE_MADVISE_DONTNEED " >>confdefs.h - RPATH="-Wl,-rpath," + $as_echo "#define JEMALLOC_THREADED_INIT " >>confdefs.h + + JEMALLOC_USABLE_SIZE_CONST="" + default_munmap="0" ;; *-*-netbsd*) { $as_echo "$as_me:${as_lineno-$LINENO}: checking ABI" >&5 @@ -4291,35 +4885,85 @@ fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $abi" >&5 $as_echo "$abi" >&6; } - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h - RPATH="-Wl,-rpath," ;; *-*-solaris2*) CFLAGS="$CFLAGS" abi="elf" - RPATH="-Wl,-R," + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h + + RPATH='-Wl,-R,$(1)' CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" LIBS="$LIBS -lposix4 -lsocket -lnsl" ;; + *-ibm-aix*) + if "$LG_SIZEOF_PTR" = "8"; then + LD_PRELOAD_VAR="LDR_PRELOAD64" + else + LD_PRELOAD_VAR="LDR_PRELOAD" + fi + abi="xcoff" + ;; + *-*-mingw*) + abi="pecoff" + force_tls="0" + RPATH="" + so="dll" + if test "x$je_cv_msvc" = "xyes" ; then + importlib="lib" + DSO_LDFLAGS="-LD" + EXTRA_LDFLAGS="-link -DEBUG" + CTARGET='-Fo$@' + LDTARGET='-Fe$@' + AR='lib' + ARFLAGS='-nologo -out:' + AROUT='$@' + CC_MM= + else + importlib="${so}" + DSO_LDFLAGS="-shared" + fi + a="lib" + libprefix="" + SOREV="${so}" + PIC_CFLAGS="" + ;; *) { $as_echo "$as_me:${as_lineno-$LINENO}: result: Unsupported operating system: ${host}" >&5 $as_echo "Unsupported operating system: ${host}" >&6; } abi="elf" - RPATH="-Wl,-rpath," ;; esac +cat >>confdefs.h <<_ACEOF +#define JEMALLOC_USABLE_SIZE_CONST $JEMALLOC_USABLE_SIZE_CONST +_ACEOF + + + + + + + + + + + + + + + + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __attribute__ syntax is compilable" >&5 $as_echo_n "checking whether __attribute__ syntax is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_attribute+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4332,22 +4976,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - attribute="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_attribute=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - attribute="no" - + je_cv_attribute=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_attribute" >&5 +$as_echo "$je_cv_attribute" >&6; } - -if test "x${attribute}" = "xyes" ; then +if test "x${je_cv_attribute}" = "xyes" ; then $as_echo "#define JEMALLOC_HAVE_ATTR " >>confdefs.h if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then @@ -4360,13 +5000,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -fvisibility=hidden" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4380,65 +5014,94 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-fvisibility=hidden + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi fi +SAVED_CFLAGS="${CFLAGS}" - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mremap(...MREMAP_FIXED...) is compilable" >&5 -$as_echo_n "checking whether mremap(...MREMAP_FIXED...) is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Werror" >&5 +$as_echo_n "checking whether compiler supports -Werror... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-Werror" else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + CFLAGS="${CFLAGS} -Werror" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#define _GNU_SOURCE -#include int main () { -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); + return 0; ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-Werror + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } - mremap_fixed="yes" else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } - mremap_fixed="no" + CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether tls_model attribute is compilable" >&5 +$as_echo_n "checking whether tls_model attribute is compilable... " >&6; } +if ${je_cv_tls_model+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +static __thread int + __attribute__((tls_model("initial-exec"))) foo; + foo = 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_tls_model=yes +else + je_cv_tls_model=no fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_tls_model" >&5 +$as_echo "$je_cv_tls_model" >&6; } +CFLAGS="${SAVED_CFLAGS}" +if test "x${je_cv_tls_model}" = "xyes" ; then + $as_echo "#define JEMALLOC_TLS_MODEL __attribute__((tls_model(\"initial-exec\")))" >>confdefs.h -if test "x${mremap_fixed}" = "xyes" ; then - $as_echo "#define JEMALLOC_MREMAP_FIXED 1" >>confdefs.h +else + $as_echo "#define JEMALLOC_TLS_MODEL " >>confdefs.h fi @@ -4489,7 +5152,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 $as_echo_n "checking for a BSD-compatible install... " >&6; } if test -z "$INSTALL"; then -if test "${ac_cv_path_install+set}" = set; then : +if ${ac_cv_path_install+:} false; then : $as_echo_n "(cached) " >&6 else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4509,7 +5172,7 @@ case $as_dir/ in #(( # by default. for ac_prog in ginstall scoinst install; do for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then if test $ac_prog = install && grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then # AIX install. It has an incompatible calling convention. @@ -4570,7 +5233,7 @@ if test -n "$ac_tool_prefix"; then set dummy ${ac_tool_prefix}ranlib; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_RANLIB+set}" = set; then : +if ${ac_cv_prog_RANLIB+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$RANLIB"; then @@ -4582,7 +5245,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -4610,7 +5273,7 @@ if test -z "$ac_cv_prog_RANLIB"; then set dummy ranlib; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then : +if ${ac_cv_prog_ac_ct_RANLIB+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_RANLIB"; then @@ -4622,7 +5285,7 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_RANLIB="ranlib" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 @@ -4657,16 +5320,16 @@ else RANLIB="$ac_cv_prog_RANLIB" fi -# Extract the first word of "ar", so it can be a program name with args. -set dummy ar; ac_word=$2 +# Extract the first word of "ld", so it can be a program name with args. +set dummy ld; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_AR+set}" = set; then : +if ${ac_cv_path_LD+:} false; then : $as_echo_n "(cached) " >&6 else - case $AR in + case $LD in [\\/]* | ?:[\\/]*) - ac_cv_path_AR="$AR" # Let the user override the test with a path. + ac_cv_path_LD="$LD" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4675,8 +5338,8 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then - ac_cv_path_AR="$as_dir/$ac_word$ac_exec_ext" + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_LD="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -4684,29 +5347,30 @@ done done IFS=$as_save_IFS + test -z "$ac_cv_path_LD" && ac_cv_path_LD="false" ;; esac fi -AR=$ac_cv_path_AR -if test -n "$AR"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5 -$as_echo "$AR" >&6; } +LD=$ac_cv_path_LD +if test -n "$LD"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LD" >&5 +$as_echo "$LD" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi -# Extract the first word of "ld", so it can be a program name with args. -set dummy ld; ac_word=$2 +# Extract the first word of "autoconf", so it can be a program name with args. +set dummy autoconf; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_LD+set}" = set; then : +if ${ac_cv_path_AUTOCONF+:} false; then : $as_echo_n "(cached) " >&6 else - case $LD in + case $AUTOCONF in [\\/]* | ?:[\\/]*) - ac_cv_path_LD="$LD" # Let the user override the test with a path. + ac_cv_path_AUTOCONF="$AUTOCONF" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4715,8 +5379,8 @@ do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then - ac_cv_path_LD="$as_dir/$ac_word$ac_exec_ext" + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_AUTOCONF="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -4724,66 +5388,170 @@ done done IFS=$as_save_IFS + test -z "$ac_cv_path_AUTOCONF" && ac_cv_path_AUTOCONF="false" ;; esac fi -LD=$ac_cv_path_LD -if test -n "$LD"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LD" >&5 -$as_echo "$LD" >&6; } +AUTOCONF=$ac_cv_path_AUTOCONF +if test -n "$AUTOCONF"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AUTOCONF" >&5 +$as_echo "$AUTOCONF" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi -# Extract the first word of "autoconf", so it can be a program name with args. -set dummy autoconf; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_AUTOCONF+set}" = set; then : - $as_echo_n "(cached) " >&6 + +public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free mallocx rallocx xallocx sallocx dallocx nallocx mallctl mallctlnametomib mallctlbymib malloc_stats_print malloc_usable_size" + +ac_fn_c_check_func "$LINENO" "memalign" "ac_cv_func_memalign" +if test "x$ac_cv_func_memalign" = xyes; then : + $as_echo "#define JEMALLOC_OVERRIDE_MEMALIGN " >>confdefs.h + + public_syms="${public_syms} memalign" +fi + +ac_fn_c_check_func "$LINENO" "valloc" "ac_cv_func_valloc" +if test "x$ac_cv_func_valloc" = xyes; then : + $as_echo "#define JEMALLOC_OVERRIDE_VALLOC " >>confdefs.h + + public_syms="${public_syms} valloc" +fi + + +# Check whether --enable-experimental was given. +if test "${enable_experimental+set}" = set; then : + enableval=$enable_experimental; if test "x$enable_experimental" = "xno" ; then + enable_experimental="0" else - case $AUTOCONF in - [\\/]* | ?:[\\/]*) - ac_cv_path_AUTOCONF="$AUTOCONF" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then - ac_cv_path_AUTOCONF="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 + enable_experimental="1" +fi + +else + enable_experimental="1" + +fi + +if test "x$enable_experimental" = "x1" ; then + $as_echo "#define JEMALLOC_EXPERIMENTAL " >>confdefs.h + + public_syms="${public_syms} allocm dallocm nallocm rallocm sallocm" +fi + + +GCOV_FLAGS= +# Check whether --enable-code-coverage was given. +if test "${enable_code_coverage+set}" = set; then : + enableval=$enable_code_coverage; if test "x$enable_code_coverage" = "xno" ; then + enable_code_coverage="0" +else + enable_code_coverage="1" +fi + +else + enable_code_coverage="0" + +fi + +if test "x$enable_code_coverage" = "x1" ; then + deoptimize="no" + echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || deoptimize="yes" + if test "x${deoptimize}" = "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -O0" >&5 +$as_echo_n "checking whether compiler supports -O0... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-O0" +else + CFLAGS="${CFLAGS} -O0" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-O0 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + fi -done - done -IFS=$as_save_IFS - ;; -esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -fprofile-arcs -ftest-coverage" >&5 +$as_echo_n "checking whether compiler supports -fprofile-arcs -ftest-coverage... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-fprofile-arcs -ftest-coverage" +else + CFLAGS="${CFLAGS} -fprofile-arcs -ftest-coverage" fi -AUTOCONF=$ac_cv_path_AUTOCONF -if test -n "$AUTOCONF"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AUTOCONF" >&5 -$as_echo "$AUTOCONF" >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-fprofile-arcs -ftest-coverage + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + EXTRA_LDFLAGS="$EXTRA_LDFLAGS -fprofile-arcs -ftest-coverage" + $as_echo "#define JEMALLOC_CODE_COVERAGE " >>confdefs.h + fi +# Check whether --with-mangling was given. +if test "${with_mangling+set}" = set; then : + withval=$with_mangling; mangling_map="$with_mangling" +else + mangling_map="" +fi + + # Check whether --with-jemalloc_prefix was given. if test "${with_jemalloc_prefix+set}" = set; then : withval=$with_jemalloc_prefix; JEMALLOC_PREFIX="$with_jemalloc_prefix" else - if test "x$abi" != "xmacho" ; then + if test "x$abi" != "xmacho" -a "x$abi" != "xpecoff"; then JEMALLOC_PREFIX="" else JEMALLOC_PREFIX="je_" @@ -4801,36 +5569,34 @@ _ACEOF #define JEMALLOC_CPREFIX "$JEMALLOC_CPREFIX" _ACEOF - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix) ${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix -_ACEOF +fi + + +# Check whether --with-export was given. +if test "${with_export+set}" = set; then : + withval=$with_export; if test "x$with_export" = "xno"; then + $as_echo "#define JEMALLOC_EXPORT /**/" >>confdefs.h + +fi fi + # Check whether --with-private_namespace was given. if test "${with_private_namespace+set}" = set; then : - withval=$with_private_namespace; JEMALLOC_PRIVATE_NAMESPACE="$with_private_namespace" + withval=$with_private_namespace; JEMALLOC_PRIVATE_NAMESPACE="${with_private_namespace}je_" else - JEMALLOC_PRIVATE_NAMESPACE="" + JEMALLOC_PRIVATE_NAMESPACE="je_" fi cat >>confdefs.h <<_ACEOF -#define JEMALLOC_PRIVATE_NAMESPACE "$JEMALLOC_PRIVATE_NAMESPACE" -_ACEOF - -if test "x$JEMALLOC_PRIVATE_NAMESPACE" != "x" ; then - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix) ${JEMALLOC_PRIVATE_NAMESPACE}##string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix +#define JEMALLOC_PRIVATE_NAMESPACE $JEMALLOC_PRIVATE_NAMESPACE _ACEOF -else - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix) string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix -_ACEOF +private_namespace="$JEMALLOC_PRIVATE_NAMESPACE" -fi # Check whether --with-install_suffix was given. @@ -4844,35 +5610,70 @@ fi install_suffix="$INSTALL_SUFFIX" +je_="je_" + + cfgoutputs_in="${srcroot}Makefile.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_macros.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_protos.h.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/jemalloc_test.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/test.sh.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/include/test/jemalloc_test.h.in" cfgoutputs_out="Makefile" cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" -cfgoutputs_out="${cfgoutputs_out} doc/jemalloc${install_suffix}.xml" -cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h" +cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_out="${cfgoutputs_out} test/jemalloc_test.h" +cfgoutputs_out="${cfgoutputs_out} test/test.sh" +cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" cfgoutputs_tup="Makefile" cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in" cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" -cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc${install_suffix}.xml:doc/jemalloc.xml.in" -cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in" +cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jemalloc/jemalloc_macros.h.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" +cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" +cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" - -cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" - -cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_namespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_symbols.txt" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_namespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_rename.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_mangle.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}test/include/test/jemalloc_test_defs.h.in" + +cfghdrs_out="include/jemalloc/jemalloc_defs.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc${install_suffix}.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_namespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_unnamespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle_jet.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/jemalloc_internal_defs.h" +cfghdrs_out="${cfghdrs_out} test/include/test/jemalloc_test_defs.h" + +cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:${srcroot}test/include/test/jemalloc_test_defs.h.in" # Check whether --enable-cc-silence was given. if test "${enable_cc_silence+set}" = set; then : @@ -4888,7 +5689,7 @@ else fi if test "x$enable_cc_silence" = "x1" ; then - $as_echo "#define JEMALLOC_CC_SILENCE 1" >>confdefs.h + $as_echo "#define JEMALLOC_CC_SILENCE " >>confdefs.h fi @@ -4908,14 +5709,31 @@ fi if test "x$enable_debug" = "x1" ; then $as_echo "#define JEMALLOC_DEBUG " >>confdefs.h - $as_echo "#define JEMALLOC_IVSALLOC " >>confdefs.h + enable_ivsalloc="1" +fi + + +# Check whether --enable-ivsalloc was given. +if test "${enable_ivsalloc+set}" = set; then : + enableval=$enable_ivsalloc; if test "x$enable_ivsalloc" = "xno" ; then + enable_ivsalloc="0" +else + enable_ivsalloc="1" +fi + +else + enable_ivsalloc="0" fi +if test "x$enable_ivsalloc" = "x1" ; then + $as_echo "#define JEMALLOC_IVSALLOC " >>confdefs.h + +fi if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then optimize="no" - echo "$EXTRA_CFLAGS" | grep "\-O" >/dev/null || optimize="yes" + echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || optimize="yes" if test "x${optimize}" = "xyes" ; then if test "x$GCC" = "xyes" ; then @@ -4927,13 +5745,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -O3" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4947,19 +5759,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-O3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -funroll-loops" >&5 @@ -4970,13 +5781,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -funroll-loops" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4990,19 +5795,55 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-funroll-loops + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + elif test "x$je_cv_msvc" = "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -O2" >&5 +$as_echo_n "checking whether compiler supports -O2... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-O2" +else + CFLAGS="${CFLAGS} -O2" fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-O2 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext else @@ -5014,13 +5855,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -O" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5034,19 +5869,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-O + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi fi @@ -5061,7 +5895,7 @@ else fi else - enable_stats="0" + enable_stats="1" fi @@ -5110,7 +5944,7 @@ if test "${with_static_libunwind+set}" = set; then : LUNWIND="-lunwind" else if test ! -f "$with_static_libunwind" ; then - as_fn_error "Static libunwind not found: $with_static_libunwind" "$LINENO" 5 + as_fn_error $? "Static libunwind not found: $with_static_libunwind" "$LINENO" 5 fi LUNWIND="$with_static_libunwind" fi @@ -5123,7 +5957,7 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then for ac_header in libunwind.h do : ac_fn_c_check_header_mongrel "$LINENO" "libunwind.h" "ac_cv_header_libunwind_h" "$ac_includes_default" -if test "x$ac_cv_header_libunwind_h" = x""yes; then : +if test "x$ac_cv_header_libunwind_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_LIBUNWIND_H 1 _ACEOF @@ -5137,7 +5971,7 @@ done if test "x$LUNWIND" = "x-lunwind" ; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for backtrace in -lunwind" >&5 $as_echo_n "checking for backtrace in -lunwind... " >&6; } -if test "${ac_cv_lib_unwind_backtrace+set}" = set; then : +if ${ac_cv_lib_unwind_backtrace+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5171,7 +6005,7 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_unwind_backtrace" >&5 $as_echo "$ac_cv_lib_unwind_backtrace" >&6; } -if test "x$ac_cv_lib_unwind_backtrace" = x""yes; then : +if test "x$ac_cv_lib_unwind_backtrace" = xyes; then : LIBS="$LIBS $LUNWIND" else enable_prof_libunwind="0" @@ -5205,7 +6039,7 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ for ac_header in unwind.h do : ac_fn_c_check_header_mongrel "$LINENO" "unwind.h" "ac_cv_header_unwind_h" "$ac_includes_default" -if test "x$ac_cv_header_unwind_h" = x""yes; then : +if test "x$ac_cv_header_unwind_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_UNWIND_H 1 _ACEOF @@ -5218,7 +6052,7 @@ done { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _Unwind_Backtrace in -lgcc" >&5 $as_echo_n "checking for _Unwind_Backtrace in -lgcc... " >&6; } -if test "${ac_cv_lib_gcc__Unwind_Backtrace+set}" = set; then : +if ${ac_cv_lib_gcc__Unwind_Backtrace+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5252,30 +6086,12 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gcc__Unwind_Backtrace" >&5 $as_echo "$ac_cv_lib_gcc__Unwind_Backtrace" >&6; } -if test "x$ac_cv_lib_gcc__Unwind_Backtrace" = x""yes; then : +if test "x$ac_cv_lib_gcc__Unwind_Backtrace" = xyes; then : LIBS="$LIBS -lgcc" else enable_prof_libgcc="0" fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking libgcc-based backtracing reliability on ${host_cpu}" >&5 -$as_echo_n "checking libgcc-based backtracing reliability on ${host_cpu}... " >&6; } - case "${host_cpu}" in - i[3456]86) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: unreliable" >&5 -$as_echo "unreliable" >&6; } - enable_prof_libgcc="0"; - ;; - x86_64) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: reliable" >&5 -$as_echo "reliable" >&6; } - ;; - *) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: unreliable" >&5 -$as_echo "unreliable" >&6; } - enable_prof_libgcc="0"; - ;; - esac if test "x${enable_prof_libgcc}" = "x1" ; then backtrace_method="libgcc" $as_echo "#define JEMALLOC_PROF_LIBGCC " >>confdefs.h @@ -5300,6 +6116,42 @@ fi if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \ -a "x$GCC" = "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -fno-omit-frame-pointer" >&5 +$as_echo_n "checking whether compiler supports -fno-omit-frame-pointer... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-fno-omit-frame-pointer" +else + CFLAGS="${CFLAGS} -fno-omit-frame-pointer" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_cflags_appended=-fno-omit-frame-pointer + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + je_cv_cflags_appended= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + backtrace_method="gcc intrinsics" $as_echo "#define JEMALLOC_PROF_GCC " >>confdefs.h @@ -5316,65 +6168,111 @@ $as_echo_n "checking configured backtracing method... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $backtrace_method" >&5 $as_echo "$backtrace_method" >&6; } if test "x$enable_prof" = "x1" ; then - LIBS="$LIBS -lm" + if test "x${force_tls}" = "x0" ; then + as_fn_error $? "Heap profiling requires TLS" "$LINENO" 5; + fi + force_tls="1" + + if test "x$abi" != "xpecoff"; then + LIBS="$LIBS -lm" + fi + $as_echo "#define JEMALLOC_PROF " >>confdefs.h fi -# Check whether --enable-tiny was given. -if test "${enable_tiny+set}" = set; then : - enableval=$enable_tiny; if test "x$enable_tiny" = "xno" ; then - enable_tiny="0" +# Check whether --enable-tcache was given. +if test "${enable_tcache+set}" = set; then : + enableval=$enable_tcache; if test "x$enable_tcache" = "xno" ; then + enable_tcache="0" else - enable_tiny="1" + enable_tcache="1" fi else - enable_tiny="1" + enable_tcache="1" fi -if test "x$enable_tiny" = "x1" ; then - $as_echo "#define JEMALLOC_TINY " >>confdefs.h +if test "x$enable_tcache" = "x1" ; then + $as_echo "#define JEMALLOC_TCACHE " >>confdefs.h fi -# Check whether --enable-tcache was given. -if test "${enable_tcache+set}" = set; then : - enableval=$enable_tcache; if test "x$enable_tcache" = "xno" ; then - enable_tcache="0" +# Check whether --enable-mremap was given. +if test "${enable_mremap+set}" = set; then : + enableval=$enable_mremap; if test "x$enable_mremap" = "xno" ; then + enable_mremap="0" else - enable_tcache="1" + enable_mremap="1" fi else - enable_tcache="1" + enable_mremap="0" fi -if test "x$enable_tcache" = "x1" ; then - $as_echo "#define JEMALLOC_TCACHE " >>confdefs.h +if test "x$enable_mremap" = "x1" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mremap(...MREMAP_FIXED...) is compilable" >&5 +$as_echo_n "checking whether mremap(...MREMAP_FIXED...) is compilable... " >&6; } +if ${je_cv_mremap_fixed+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define _GNU_SOURCE +#include + +int +main () +{ + +void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_mremap_fixed=yes +else + je_cv_mremap_fixed=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_mremap_fixed" >&5 +$as_echo "$je_cv_mremap_fixed" >&6; } + if test "x${je_cv_mremap_fixed}" = "xno" ; then + enable_mremap="0" + fi fi +if test "x$enable_mremap" = "x1" ; then + $as_echo "#define JEMALLOC_MREMAP " >>confdefs.h +fi -# Check whether --enable-swap was given. -if test "${enable_swap+set}" = set; then : - enableval=$enable_swap; if test "x$enable_swap" = "xno" ; then - enable_swap="0" + +# Check whether --enable-munmap was given. +if test "${enable_munmap+set}" = set; then : + enableval=$enable_munmap; if test "x$enable_munmap" = "xno" ; then + enable_munmap="0" else - enable_swap="1" + enable_munmap="1" fi else - enable_swap="0" + enable_munmap="${default_munmap}" fi -if test "x$enable_swap" = "x1" ; then - $as_echo "#define JEMALLOC_SWAP " >>confdefs.h +if test "x$enable_munmap" = "x1" ; then + $as_echo "#define JEMALLOC_MUNMAP " >>confdefs.h fi @@ -5392,6 +6290,26 @@ else fi +ac_fn_c_check_func "$LINENO" "sbrk" "ac_cv_func_sbrk" +if test "x$ac_cv_func_sbrk" = xyes; then : + have_sbrk="1" +else + have_sbrk="0" +fi + +if test "x$have_sbrk" = "x1" ; then + if test "x$sbrk_deprecated" == "x1" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Disabling dss allocation because sbrk is deprecated" >&5 +$as_echo "Disabling dss allocation because sbrk is deprecated" >&6; } + enable_dss="0" + else + $as_echo "#define JEMALLOC_HAVE_SBRK " >>confdefs.h + + fi +else + enable_dss="0" +fi + if test "x$enable_dss" = "x1" ; then $as_echo "#define JEMALLOC_DSS " >>confdefs.h @@ -5407,7 +6325,7 @@ else fi else - enable_fill="0" + enable_fill="1" fi @@ -5417,94 +6335,189 @@ if test "x$enable_fill" = "x1" ; then fi -# Check whether --enable-xmalloc was given. -if test "${enable_xmalloc+set}" = set; then : - enableval=$enable_xmalloc; if test "x$enable_xmalloc" = "xno" ; then - enable_xmalloc="0" +# Check whether --enable-utrace was given. +if test "${enable_utrace+set}" = set; then : + enableval=$enable_utrace; if test "x$enable_utrace" = "xno" ; then + enable_utrace="0" else - enable_xmalloc="1" + enable_utrace="1" fi else - enable_xmalloc="0" + enable_utrace="0" fi -if test "x$enable_xmalloc" = "x1" ; then - $as_echo "#define JEMALLOC_XMALLOC " >>confdefs.h +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether utrace(2) is compilable" >&5 +$as_echo_n "checking whether utrace(2) is compilable... " >&6; } +if ${je_cv_utrace+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include +#include +#include + +int +main () +{ + + utrace((void *)0, 0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_utrace=yes +else + je_cv_utrace=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_utrace" >&5 +$as_echo "$je_cv_utrace" >&6; } + +if test "x${je_cv_utrace}" = "xno" ; then + enable_utrace="0" fi +if test "x$enable_utrace" = "x1" ; then + $as_echo "#define JEMALLOC_UTRACE " >>confdefs.h +fi -# Check whether --enable-sysv was given. -if test "${enable_sysv+set}" = set; then : - enableval=$enable_sysv; if test "x$enable_sysv" = "xno" ; then - enable_sysv="0" + +# Check whether --enable-valgrind was given. +if test "${enable_valgrind+set}" = set; then : + enableval=$enable_valgrind; if test "x$enable_valgrind" = "xno" ; then + enable_valgrind="0" else - enable_sysv="1" + enable_valgrind="1" fi else - enable_sysv="0" + enable_valgrind="1" + +fi + +if test "x$enable_valgrind" = "x1" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether valgrind is compilable" >&5 +$as_echo_n "checking whether valgrind is compilable... " >&6; } +if ${je_cv_valgrind+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include + +#if !defined(VALGRIND_RESIZEINPLACE_BLOCK) +# error "Incompatible Valgrind version" +#endif +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_valgrind=yes +else + je_cv_valgrind=no fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_valgrind" >&5 +$as_echo "$je_cv_valgrind" >&6; } -if test "x$enable_sysv" = "x1" ; then - $as_echo "#define JEMALLOC_SYSV " >>confdefs.h + if test "x${je_cv_valgrind}" = "xno" ; then + enable_valgrind="0" + fi + if test "x$enable_valgrind" = "x1" ; then + $as_echo "#define JEMALLOC_VALGRIND " >>confdefs.h + fi fi -# Check whether --enable-dynamic_page_shift was given. -if test "${enable_dynamic_page_shift+set}" = set; then : - enableval=$enable_dynamic_page_shift; if test "x$enable_dynamic_page_shift" = "xno" ; then - enable_dynamic_page_shift="0" +# Check whether --enable-xmalloc was given. +if test "${enable_xmalloc+set}" = set; then : + enableval=$enable_xmalloc; if test "x$enable_xmalloc" = "xno" ; then + enable_xmalloc="0" else - enable_dynamic_page_shift="1" + enable_xmalloc="1" fi else - enable_dynamic_page_shift="0" + enable_xmalloc="0" fi -if test "x$enable_dynamic_page_shift" = "x1" ; then - $as_echo "#define DYNAMIC_PAGE_SHIFT " >>confdefs.h +if test "x$enable_xmalloc" = "x1" ; then + $as_echo "#define JEMALLOC_XMALLOC " >>confdefs.h fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking STATIC_PAGE_SHIFT" >&5 $as_echo_n "checking STATIC_PAGE_SHIFT... " >&6; } -if test "$cross_compiling" = yes; then : +if ${je_cv_static_page_shift+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include -#include + #include +#ifdef _WIN32 +#include +#else +#include +#endif +#include int main () { - long result; + int result; FILE *f; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwPageSize; +#else result = sysconf(_SC_PAGESIZE); +#endif if (result == -1) { return 1; } + result = ffsl(result) - 1; + f = fopen("conftest.out", "w"); if (f == NULL) { return 1; } - fprintf(f, "%u\n", ffs((int)result) - 1); - close(f); + fprintf(f, "%d\n", result); + fclose(f); return 0; @@ -5513,21 +6526,26 @@ main () } _ACEOF if ac_fn_c_try_run "$LINENO"; then : - STATIC_PAGE_SHIFT=`cat conftest.out` - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STATIC_PAGE_SHIFT" >&5 -$as_echo "$STATIC_PAGE_SHIFT" >&6; } - cat >>confdefs.h <<_ACEOF -#define STATIC_PAGE_SHIFT $STATIC_PAGE_SHIFT -_ACEOF - + je_cv_static_page_shift=`cat conftest.out` else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: error" >&5 -$as_echo "error" >&6; } + je_cv_static_page_shift=undefined fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_static_page_shift" >&5 +$as_echo "$je_cv_static_page_shift" >&6; } + +if test "x$je_cv_static_page_shift" != "xundefined"; then + cat >>confdefs.h <<_ACEOF +#define STATIC_PAGE_SHIFT $je_cv_static_page_shift +_ACEOF + +else + as_fn_error $? "cannot determine value for STATIC_PAGE_SHIFT" "$LINENO" 5 +fi if test -d "${srcroot}.git" ; then @@ -5547,23 +6565,24 @@ jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $5}' -for ac_header in pthread.h +if test "x$abi" != "xpecoff" ; then + for ac_header in pthread.h do : ac_fn_c_check_header_mongrel "$LINENO" "pthread.h" "ac_cv_header_pthread_h" "$ac_includes_default" -if test "x$ac_cv_header_pthread_h" = x""yes; then : +if test "x$ac_cv_header_pthread_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_PTHREAD_H 1 _ACEOF else - as_fn_error "pthread.h is missing" "$LINENO" 5 + as_fn_error $? "pthread.h is missing" "$LINENO" 5 fi done -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthread" >&5 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthread" >&5 $as_echo_n "checking for pthread_create in -lpthread... " >&6; } -if test "${ac_cv_lib_pthread_pthread_create+set}" = set; then : +if ${ac_cv_lib_pthread_pthread_create+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5597,15 +6616,100 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread_pthread_create" >&5 $as_echo "$ac_cv_lib_pthread_pthread_create" >&6; } -if test "x$ac_cv_lib_pthread_pthread_create" = x""yes; then : +if test "x$ac_cv_lib_pthread_pthread_create" = xyes; then : LIBS="$LIBS -lpthread" else - as_fn_error "libpthread is missing" "$LINENO" 5 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 +$as_echo_n "checking for library containing pthread_create... " >&6; } +if ${ac_cv_search_pthread_create+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (); +int +main () +{ +return pthread_create (); + ; + return 0; +} +_ACEOF +for ac_lib in '' ; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_pthread_create=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_pthread_create+:} false; then : + break fi +done +if ${ac_cv_search_pthread_create+:} false; then : +else + ac_cv_search_pthread_create=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 +$as_echo "$ac_cv_search_pthread_create" >&6; } +ac_res=$ac_cv_search_pthread_create +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +else + as_fn_error $? "libpthread is missing" "$LINENO" 5 +fi + +fi + +fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" +ac_fn_c_check_func "$LINENO" "_malloc_thread_cleanup" "ac_cv_func__malloc_thread_cleanup" +if test "x$ac_cv_func__malloc_thread_cleanup" = xyes; then : + have__malloc_thread_cleanup="1" +else + have__malloc_thread_cleanup="0" + +fi + +if test "x$have__malloc_thread_cleanup" = "x1" ; then + $as_echo "#define JEMALLOC_MALLOC_THREAD_CLEANUP " >>confdefs.h + + force_tls="1" +fi + +ac_fn_c_check_func "$LINENO" "_pthread_mutex_init_calloc_cb" "ac_cv_func__pthread_mutex_init_calloc_cb" +if test "x$ac_cv_func__pthread_mutex_init_calloc_cb" = xyes; then : + have__pthread_mutex_init_calloc_cb="1" +else + have__pthread_mutex_init_calloc_cb="0" + +fi + +if test "x$have__pthread_mutex_init_calloc_cb" = "x1" ; then + $as_echo "#define JEMALLOC_MUTEX_INIT_CB 1" >>confdefs.h + +fi + # Check whether --enable-lazy_lock was given. if test "${enable_lazy_lock+set}" = set; then : enableval=$enable_lazy_lock; if test "x$enable_lazy_lock" = "xno" ; then @@ -5615,28 +6719,38 @@ else fi else - enable_lazy_lock="1" + enable_lazy_lock="0" fi +if test "x$enable_lazy_lock" = "x0" -a "x${force_lazy_lock}" = "x1" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing lazy-lock to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing lazy-lock to avoid allocator/threading bootstrap issues" >&6; } + enable_lazy_lock="1" +fi if test "x$enable_lazy_lock" = "x1" ; then - for ac_header in dlfcn.h + if test "x$abi" != "xpecoff" ; then + for ac_header in dlfcn.h do : ac_fn_c_check_header_mongrel "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default" -if test "x$ac_cv_header_dlfcn_h" = x""yes; then : +if test "x$ac_cv_header_dlfcn_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_DLFCN_H 1 _ACEOF else - as_fn_error "dlfcn.h is missing" "$LINENO" 5 + as_fn_error $? "dlfcn.h is missing" "$LINENO" 5 fi done - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 -$as_echo_n "checking for dlopen in -ldl... " >&6; } -if test "${ac_cv_lib_dl_dlopen+set}" = set; then : + ac_fn_c_check_func "$LINENO" "dlsym" "ac_cv_func_dlsym" +if test "x$ac_cv_func_dlsym" = xyes; then : + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlsym in -ldl" >&5 +$as_echo_n "checking for dlsym in -ldl... " >&6; } +if ${ac_cv_lib_dl_dlsym+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5650,32 +6764,36 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char dlopen (); +char dlsym (); int main () { -return dlopen (); +return dlsym (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dl_dlopen=yes + ac_cv_lib_dl_dlsym=yes else - ac_cv_lib_dl_dlopen=no + ac_cv_lib_dl_dlsym=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 -$as_echo "$ac_cv_lib_dl_dlopen" >&6; } -if test "x$ac_cv_lib_dl_dlopen" = x""yes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlsym" >&5 +$as_echo "$ac_cv_lib_dl_dlsym" >&6; } +if test "x$ac_cv_lib_dl_dlsym" = xyes; then : LIBS="$LIBS -ldl" else - as_fn_error "libdl is missing" "$LINENO" 5 + as_fn_error $? "libdl is missing" "$LINENO" 5 fi + +fi + + fi $as_echo "#define JEMALLOC_LAZY_LOCK " >>confdefs.h fi @@ -5694,16 +6812,20 @@ else fi +if test "x${enable_tls}" = "x0" -a "x${force_tls}" = "x1" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing TLS to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing TLS to avoid allocator/threading bootstrap issues" >&6; } + enable_tls="1" +fi +if test "x${enable_tls}" = "x1" -a "x${force_tls}" = "x0" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing no TLS to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing no TLS to avoid allocator/threading bootstrap issues" >&6; } + enable_tls="0" +fi if test "x${enable_tls}" = "x1" ; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for TLS" >&5 $as_echo_n "checking for TLS... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ __thread int x; @@ -5720,7 +6842,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -5728,80 +6850,249 @@ else $as_echo "no" >&6; } enable_tls="0" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi +if test "x${enable_tls}" = "x1" ; then + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_TLS +_ACEOF + +elif test "x${force_tls}" = "x1" ; then + as_fn_error $? "Failed to configure TLS, which is mandatory for correct function" "$LINENO" 5 fi -if test "x${enable_tls}" = "x0" ; then - cat >>confdefs.h <<_ACEOF -#define NO_TLS + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program using ffsl is compilable" >&5 +$as_echo_n "checking whether a program using ffsl is compilable... " >&6; } +if ${je_cv_function_ffsl+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +int +main () +{ + + { + int rv = ffsl(0x08); + printf("%d\n", rv); + } + + ; + return 0; +} _ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_function_ffsl=yes +else + je_cv_function_ffsl=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_function_ffsl" >&5 +$as_echo "$je_cv_function_ffsl" >&6; } + +if test "x${je_cv_function_ffsl}" != "xyes" ; then + as_fn_error $? "Cannot build without ffsl(3)" "$LINENO" 5 +fi + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether atomic(9) is compilable" >&5 +$as_echo_n "checking whether atomic(9) is compilable... " >&6; } +if ${je_cv_atomic9+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +int +main () +{ + + { + uint32_t x32 = 0; + volatile uint32_t *x32p = &x32; + atomic_fetchadd_32(x32p, 1); + } + { + unsigned long xlong = 0; + volatile unsigned long *xlongp = &xlong; + atomic_fetchadd_long(xlongp, 1); + } + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_atomic9=yes +else + je_cv_atomic9=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_atomic9" >&5 +$as_echo "$je_cv_atomic9" >&6; } + +if test "x${je_cv_atomic9}" = "xyes" ; then + $as_echo "#define JEMALLOC_ATOMIC9 1" >>confdefs.h + +fi + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSAtomic*() is compilable" >&5 +$as_echo_n "checking whether Darwin OSAtomic*() is compilable... " >&6; } +if ${je_cv_osatomic+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include + +int +main () +{ + + { + int32_t x32 = 0; + volatile int32_t *x32p = &x32; + OSAtomicAdd32(1, x32p); + } + { + int64_t x64 = 0; + volatile int64_t *x64p = &x64; + OSAtomicAdd64(1, x64p); + } + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_osatomic=yes +else + je_cv_osatomic=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_osatomic" >&5 +$as_echo "$je_cv_osatomic" >&6; } + +if test "x${je_cv_osatomic}" = "xyes" ; then + $as_echo "#define JEMALLOC_OSATOMIC " >>confdefs.h + +fi + + + + +if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 32-bit __sync_{add,sub}_and_fetch()" >&5 +$as_echo_n "checking whether to force 32-bit __sync_{add,sub}_and_fetch()... " >&6; } +if ${je_cv_sync_compare_and_swap_4+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -fi + #include +int +main () +{ -ac_fn_c_check_func "$LINENO" "ffsl" "ac_cv_func_ffsl" -if test "x$ac_cv_func_ffsl" = x""yes; then : + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 + { + uint32_t x32 = 0; + __sync_add_and_fetch(&x32, 42); + __sync_sub_and_fetch(&x32, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 is defined, no need to force + #endif + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_sync_compare_and_swap_4=yes else - as_fn_error "Cannot build without ffsl(3)" "$LINENO" 5 + je_cv_sync_compare_and_swap_4=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_4" >&5 +$as_echo "$je_cv_sync_compare_and_swap_4" >&6; } + if test "x${je_cv_sync_compare_and_swap_4}" = "xyes" ; then + $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_4 " >>confdefs.h + fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSAtomic*() is compilable" >&5 -$as_echo_n "checking whether Darwin OSAtomic*() is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 64-bit __sync_{add,sub}_and_fetch()" >&5 +$as_echo_n "checking whether to force 64-bit __sync_{add,sub}_and_fetch()... " >&6; } +if ${je_cv_sync_compare_and_swap_8+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include -#include + #include int main () { - { - int32_t x32 = 0; - volatile int32_t *x32p = &x32; - OSAtomicAdd32(1, x32p); - } - { - int64_t x64 = 0; - volatile int64_t *x64p = &x64; - OSAtomicAdd64(1, x64p); - } + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 + { + uint64_t x64 = 0; + __sync_add_and_fetch(&x64, 42); + __sync_sub_and_fetch(&x64, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 is defined, no need to force + #endif ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - osatomic="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_sync_compare_and_swap_8=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - osatomic="no" - + je_cv_sync_compare_and_swap_8=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_8" >&5 +$as_echo "$je_cv_sync_compare_and_swap_8" >&6; } + if test "x${je_cv_sync_compare_and_swap_8}" = "xyes" ; then + $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_8 " >>confdefs.h -if test "x${osatomic}" = "xyes" ; then - $as_echo "#define JEMALLOC_OSATOMIC 1" >>confdefs.h + fi fi @@ -5809,11 +7100,8 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSSpin*() is compilable" >&5 $as_echo_n "checking whether Darwin OSSpin*() is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_osspin+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5833,187 +7121,263 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - osspin="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_osspin=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - osspin="no" - + je_cv_osspin=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_osspin" >&5 +$as_echo "$je_cv_osspin" >&6; } - -if test "x${osspin}" = "xyes" ; then - $as_echo "#define JEMALLOC_OSSPIN 1" >>confdefs.h +if test "x${je_cv_osspin}" = "xyes" ; then + $as_echo "#define JEMALLOC_OSSPIN " >>confdefs.h fi -ac_fn_c_check_func "$LINENO" "memalign" "ac_cv_func_memalign" -if test "x$ac_cv_func_memalign" = x""yes; then : - $as_echo "#define JEMALLOC_OVERRIDE_MEMALIGN 1" >>confdefs.h +# Check whether --enable-zone-allocator was given. +if test "${enable_zone_allocator+set}" = set; then : + enableval=$enable_zone_allocator; if test "x$enable_zone_allocator" = "xno" ; then + enable_zone_allocator="0" +else + enable_zone_allocator="1" +fi +else + if test "x${abi}" = "xmacho"; then + enable_zone_allocator="1" fi -ac_fn_c_check_func "$LINENO" "valloc" "ac_cv_func_valloc" -if test "x$ac_cv_func_valloc" = x""yes; then : - $as_echo "#define JEMALLOC_OVERRIDE_VALLOC 1" >>confdefs.h fi -if test "x${abi}" = "xmacho" ; then - $as_echo "#define JEMALLOC_IVSALLOC 1" >>confdefs.h +if test "x${enable_zone_allocator}" = "x1" ; then + if test "x${abi}" != "xmacho"; then + as_fn_error $? "--enable-zone-allocator is only supported on Darwin" "$LINENO" 5 + fi + $as_echo "#define JEMALLOC_IVSALLOC " >>confdefs.h - $as_echo "#define JEMALLOC_ZONE 1" >>confdefs.h + $as_echo "#define JEMALLOC_ZONE " >>confdefs.h { $as_echo "$as_me:${as_lineno-$LINENO}: checking malloc zone version" >&5 $as_echo_n "checking malloc zone version... " >&6; } + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include #include int main () { +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 14 ? 1 : -1] + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=3 +else - static malloc_zone_t zone; - static struct malloc_introspection_t zone_introspect; - - zone.size = NULL; - zone.malloc = NULL; - zone.calloc = NULL; - zone.valloc = NULL; - zone.free = NULL; - zone.realloc = NULL; - zone.destroy = NULL; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = 6; - zone.memalign = NULL; - zone.free_definite_size = NULL; - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = NULL; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = NULL; - zone_introspect.force_unlock = NULL; - zone_introspect.statistics = NULL; - zone_introspect.zone_locked = NULL; + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 15 ? 1 : -1] ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_ZONE_VERSION 6 + JEMALLOC_ZONE_VERSION=5 +else + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 16 ? 1 : -1] + + ; + return 0; +} _ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_introspection_t) == sizeof(void *) * 9 ? 1 : -1] - { $as_echo "$as_me:${as_lineno-$LINENO}: result: 6" >&5 -$as_echo "6" >&6; } + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=6 else - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_ZONE_VERSION 3 + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_introspection_t) == sizeof(void *) * 13 ? 1 : -1] + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=7 +else + JEMALLOC_ZONE_VERSION= + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 17 ? 1 : -1] + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=8 +else + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) > sizeof(void *) * 17 ? 1 : -1] + + ; + return 0; +} _ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=9 +else + JEMALLOC_ZONE_VERSION= - { $as_echo "$as_me:${as_lineno-$LINENO}: result: 3" >&5 -$as_echo "3" >&6; } fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +$as_echo "unsupported" >&6; } + as_fn_error $? "Unsupported malloc zone version" "$LINENO" 5 + fi + if test "${JEMALLOC_ZONE_VERSION}" = 9; then + JEMALLOC_ZONE_VERSION=8 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: > 8" >&5 +$as_echo "> 8" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JEMALLOC_ZONE_VERSION" >&5 +$as_echo "$JEMALLOC_ZONE_VERSION" >&6; } + fi + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_ZONE_VERSION $JEMALLOC_ZONE_VERSION +_ACEOF + fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdbool.h that conforms to C99" >&5 $as_echo_n "checking for stdbool.h that conforms to C99... " >&6; } -if test "${ac_cv_header_stdbool_h+set}" = set; then : +if ${ac_cv_header_stdbool_h+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include -#ifndef bool - "error: bool is not defined" -#endif -#ifndef false - "error: false is not defined" -#endif -#if false - "error: false is not 0" -#endif -#ifndef true - "error: true is not defined" -#endif -#if true != 1 - "error: true is not 1" -#endif -#ifndef __bool_true_false_are_defined - "error: __bool_true_false_are_defined is not defined" -#endif - - struct s { _Bool s: 1; _Bool t; } s; - - char a[true == 1 ? 1 : -1]; - char b[false == 0 ? 1 : -1]; - char c[__bool_true_false_are_defined == 1 ? 1 : -1]; - char d[(bool) 0.5 == true ? 1 : -1]; - bool e = &s; - char f[(_Bool) 0.0 == false ? 1 : -1]; - char g[true]; - char h[sizeof (_Bool)]; - char i[sizeof s.t]; - enum { j = false, k = true, l = false * true, m = true * 256 }; - /* The following fails for - HP aC++/ANSI C B3910B A.05.55 [Dec 04 2003]. */ - _Bool n[m]; - char o[sizeof n == m * sizeof n[0] ? 1 : -1]; - char p[-1 - (_Bool) 0 < 0 && -1 - (bool) 0 < 0 ? 1 : -1]; -# if defined __xlc__ || defined __GNUC__ - /* Catch a bug in IBM AIX xlc compiler version 6.0.0.0 - reported by James Lemley on 2005-10-05; see - http://lists.gnu.org/archive/html/bug-coreutils/2005-10/msg00086.html - This test is not quite right, since xlc is allowed to - reject this program, as the initializer for xlcbug is - not one of the forms that C requires support for. - However, doing the test right would require a runtime - test, and that would make cross-compilation harder. - Let us hope that IBM fixes the xlc bug, and also adds - support for this kind of constant expression. In the - meantime, this test will reject xlc, which is OK, since - our stdbool.h substitute should suffice. We also test - this with GCC, where it should work, to detect more - quickly whether someone messes up the test in the - future. */ - char digs[] = "0123456789"; - int xlcbug = 1 / (&(digs + 5)[-2 + (bool) 1] == &digs[4] ? 1 : -1); -# endif - /* Catch a bug in an HP-UX C compiler. See - http://gcc.gnu.org/ml/gcc-patches/2003-12/msg02303.html - http://lists.gnu.org/archive/html/bug-coreutils/2005-11/msg00161.html - */ - _Bool q = true; - _Bool *pq = &q; + #include + #ifndef bool + "error: bool is not defined" + #endif + #ifndef false + "error: false is not defined" + #endif + #if false + "error: false is not 0" + #endif + #ifndef true + "error: true is not defined" + #endif + #if true != 1 + "error: true is not 1" + #endif + #ifndef __bool_true_false_are_defined + "error: __bool_true_false_are_defined is not defined" + #endif + + struct s { _Bool s: 1; _Bool t; } s; + + char a[true == 1 ? 1 : -1]; + char b[false == 0 ? 1 : -1]; + char c[__bool_true_false_are_defined == 1 ? 1 : -1]; + char d[(bool) 0.5 == true ? 1 : -1]; + /* See body of main program for 'e'. */ + char f[(_Bool) 0.0 == false ? 1 : -1]; + char g[true]; + char h[sizeof (_Bool)]; + char i[sizeof s.t]; + enum { j = false, k = true, l = false * true, m = true * 256 }; + /* The following fails for + HP aC++/ANSI C B3910B A.05.55 [Dec 04 2003]. */ + _Bool n[m]; + char o[sizeof n == m * sizeof n[0] ? 1 : -1]; + char p[-1 - (_Bool) 0 < 0 && -1 - (bool) 0 < 0 ? 1 : -1]; + /* Catch a bug in an HP-UX C compiler. See + http://gcc.gnu.org/ml/gcc-patches/2003-12/msg02303.html + http://lists.gnu.org/archive/html/bug-coreutils/2005-11/msg00161.html + */ + _Bool q = true; + _Bool *pq = &q; int main () { - *pq |= q; - *pq |= ! q; - /* Refer to every declared value, to avoid compiler optimizations. */ - return (!a + !b + !c + !d + !e + !f + !g + !h + !i + !!j + !k + !!l - + !m + !n + !o + !p + !q + !pq); + bool e = &s; + *pq |= q; + *pq |= ! q; + /* Refer to every declared value, to avoid compiler optimizations. */ + return (!a + !b + !c + !d + !e + !f + !g + !h + !i + !!j + !k + !!l + + !m + !n + !o + !p + !q + !pq); ; return 0; @@ -6028,8 +7392,8 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdbool_h" >&5 $as_echo "$ac_cv_header_stdbool_h" >&6; } -ac_fn_c_check_type "$LINENO" "_Bool" "ac_cv_type__Bool" "$ac_includes_default" -if test "x$ac_cv_type__Bool" = x""yes; then : + ac_fn_c_check_type "$LINENO" "_Bool" "ac_cv_type__Bool" "$ac_includes_default" +if test "x$ac_cv_type__Bool" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE__BOOL 1 @@ -6038,6 +7402,7 @@ _ACEOF fi + if test $ac_cv_header_stdbool_h = yes; then $as_echo "#define HAVE_STDBOOL_H 1" >>confdefs.h @@ -6046,11 +7411,36 @@ fi +ac_config_commands="$ac_config_commands include/jemalloc/internal/private_namespace.h" + +ac_config_commands="$ac_config_commands include/jemalloc/internal/private_unnamespace.h" + +ac_config_commands="$ac_config_commands include/jemalloc/internal/public_symbols.txt" + +ac_config_commands="$ac_config_commands include/jemalloc/internal/public_namespace.h" + +ac_config_commands="$ac_config_commands include/jemalloc/internal/public_unnamespace.h" + +ac_config_commands="$ac_config_commands include/jemalloc/internal/size_classes.h" + +ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_protos_jet.h" + +ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_rename.h" + +ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_mangle.h" + +ac_config_commands="$ac_config_commands include/jemalloc/jemalloc_mangle_jet.h" + +ac_config_commands="$ac_config_commands include/jemalloc/jemalloc.h" + + + ac_config_headers="$ac_config_headers $cfghdrs_tup" -ac_config_files="$ac_config_files $cfgoutputs_tup config.stamp" + +ac_config_files="$ac_config_files $cfgoutputs_tup config.stamp bin/jemalloc.sh" @@ -6118,10 +7508,21 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; :end' >>confcache if diff "$cache_file" confcache >/dev/null 2>&1; then :; else if test -w "$cache_file"; then - test "x$cache_file" != "x/dev/null" && + if test "x$cache_file" != "x/dev/null"; then { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 $as_echo "$as_me: updating cache $cache_file" >&6;} - cat confcache >$cache_file + if test ! -f "$cache_file" || test -h "$cache_file"; then + cat confcache >"$cache_file" + else + case $cache_file in #( + */* | ?:*) + mv -f confcache "$cache_file"$$ && + mv -f "$cache_file"$$ "$cache_file" ;; #( + *) + mv -f confcache "$cache_file" ;; + esac + fi + fi else { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 $as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} @@ -6137,6 +7538,7 @@ DEFS=-DHAVE_CONFIG_H ac_libobjs= ac_ltlibobjs= +U= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' @@ -6152,7 +7554,8 @@ LTLIBOBJS=$ac_ltlibobjs -: ${CONFIG_STATUS=./config.status} + +: "${CONFIG_STATUS=./config.status}" ac_write_fail=0 ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files $CONFIG_STATUS" @@ -6253,6 +7656,7 @@ fi IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. +as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6298,19 +7702,19 @@ export LANGUAGE (unset CDPATH) >/dev/null 2>&1 && unset CDPATH -# as_fn_error ERROR [LINENO LOG_FD] -# --------------------------------- +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with status $?, using 1 if that was 0. +# script with STATUS, using 1 if that was 0. as_fn_error () { - as_status=$?; test $as_status -eq 0 && as_status=1 - if test "$3"; then - as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - $as_echo "$as_me: error: $1" >&2 + $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -6448,16 +7852,16 @@ if (echo >conf$$.file) 2>/dev/null; then # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -p'. + # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || - as_ln_s='cp -p' + as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else - as_ln_s='cp -p' + as_ln_s='cp -pR' fi else - as_ln_s='cp -p' + as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null @@ -6506,7 +7910,7 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p @@ -6517,28 +7921,16 @@ else as_mkdir_p=false fi -if test -x / >/dev/null 2>&1; then - as_test_x='test -x' -else - if ls -dL / >/dev/null 2>&1; then - as_ls_L_option=L - else - as_ls_L_option= - fi - as_test_x=' - eval sh -c '\'' - if test -d "$1"; then - test -d "$1/."; - else - case $1 in #( - -*)set "./$1";; - esac; - case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #(( - ???[sx]*):;;*)false;;esac;fi - '\'' sh - ' -fi -as_executable_p=$as_test_x + +# as_fn_executable_p FILE +# ----------------------- +# Test if FILE is an executable regular file. +as_fn_executable_p () +{ + test -f "$1" && test -x "$1" +} # as_fn_executable_p +as_test_x='test -x' +as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" @@ -6560,7 +7952,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # values after options handling. ac_log=" This file was extended by $as_me, which was -generated by GNU Autoconf 2.65. Invocation command line was +generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -6586,6 +7978,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 # Files that config.status was made for. config_files="$ac_config_files" config_headers="$ac_config_headers" +config_commands="$ac_config_commands" _ACEOF @@ -6615,6 +8008,9 @@ $config_files Configuration headers: $config_headers +Configuration commands: +$config_commands + Report bugs to the package provider." _ACEOF @@ -6622,10 +8018,10 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ config.status -configured by $0, generated by GNU Autoconf 2.65, +configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" -Copyright (C) 2009 Free Software Foundation, Inc. +Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." @@ -6641,11 +8037,16 @@ ac_need_defaults=: while test $# != 0 do case $1 in - --*=*) + --*=?*) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` ac_shift=: ;; + --*=) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg= + ac_shift=: + ;; *) ac_option=$1 ac_optarg=$2 @@ -6667,6 +8068,7 @@ do $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; @@ -6679,7 +8081,7 @@ do ac_need_defaults=false;; --he | --h) # Conflict between --help and --header - as_fn_error "ambiguous option: \`$1' + as_fn_error $? "ambiguous option: \`$1' Try \`$0 --help' for more information.";; --help | --hel | -h ) $as_echo "$ac_cs_usage"; exit ;; @@ -6688,7 +8090,7 @@ Try \`$0 --help' for more information.";; ac_cs_silent=: ;; # This is an error. - -*) as_fn_error "unrecognized option: \`$1' + -*) as_fn_error $? "unrecognized option: \`$1' Try \`$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" @@ -6708,7 +8110,7 @@ fi _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 if \$ac_cs_recheck; then - set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion + set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion shift \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 CONFIG_SHELL='$SHELL' @@ -6729,6 +8131,58 @@ _ASBOX _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +# +# INIT-COMMANDS +# + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + mangling_map="${mangling_map}" + public_syms="${public_syms}" + JEMALLOC_PREFIX="${JEMALLOC_PREFIX}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + + + srcdir="${srcdir}" + objroot="${objroot}" + install_suffix="${install_suffix}" + + _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 @@ -6737,11 +8191,23 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 for ac_config_target in $ac_config_targets do case $ac_config_target in + "include/jemalloc/internal/private_namespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/private_namespace.h" ;; + "include/jemalloc/internal/private_unnamespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/private_unnamespace.h" ;; + "include/jemalloc/internal/public_symbols.txt") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/public_symbols.txt" ;; + "include/jemalloc/internal/public_namespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/public_namespace.h" ;; + "include/jemalloc/internal/public_unnamespace.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/public_unnamespace.h" ;; + "include/jemalloc/internal/size_classes.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/size_classes.h" ;; + "include/jemalloc/jemalloc_protos_jet.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_protos_jet.h" ;; + "include/jemalloc/jemalloc_rename.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_rename.h" ;; + "include/jemalloc/jemalloc_mangle.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_mangle.h" ;; + "include/jemalloc/jemalloc_mangle_jet.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc_mangle_jet.h" ;; + "include/jemalloc/jemalloc.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/jemalloc.h" ;; "$cfghdrs_tup") CONFIG_HEADERS="$CONFIG_HEADERS $cfghdrs_tup" ;; "$cfgoutputs_tup") CONFIG_FILES="$CONFIG_FILES $cfgoutputs_tup" ;; "config.stamp") CONFIG_FILES="$CONFIG_FILES config.stamp" ;; + "bin/jemalloc.sh") CONFIG_FILES="$CONFIG_FILES bin/jemalloc.sh" ;; - *) as_fn_error "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac done @@ -6753,6 +8219,7 @@ done if $ac_need_defaults; then test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands fi # Have a temporary directory for convenience. Make it in the build tree @@ -6763,9 +8230,10 @@ fi # after its creation but before its name has been assigned to `$tmp'. $debug || { - tmp= + tmp= ac_tmp= trap 'exit_status=$? - { test -z "$tmp" || test ! -d "$tmp" || rm -fr "$tmp"; } && exit $exit_status + : "${ac_tmp:=$tmp}" + { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status ' 0 trap 'as_fn_exit 1' 1 2 13 15 } @@ -6773,12 +8241,13 @@ $debug || { tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && - test -n "$tmp" && test -d "$tmp" + test -d "$tmp" } || { tmp=./conf$$-$RANDOM (umask 077 && mkdir "$tmp") -} || as_fn_error "cannot create a temporary directory in ." "$LINENO" 5 +} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 +ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. @@ -6795,12 +8264,12 @@ if test "x$ac_cr" = x; then fi ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then - ac_cs_awk_cr='\r' + ac_cs_awk_cr='\\r' else ac_cs_awk_cr=$ac_cr fi -echo 'BEGIN {' >"$tmp/subs1.awk" && +echo 'BEGIN {' >"$ac_tmp/subs1.awk" && _ACEOF @@ -6809,18 +8278,18 @@ _ACEOF echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && echo "_ACEOF" } >conf$$subs.sh || - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 -ac_delim_num=`echo "$ac_subst_vars" | grep -c '$'` + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 +ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -6828,7 +8297,7 @@ done rm -f conf$$subs.sh cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -cat >>"\$tmp/subs1.awk" <<\\_ACAWK && +cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && _ACEOF sed -n ' h @@ -6876,7 +8345,7 @@ t delim rm -f conf$$subs.awk cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK -cat >>"\$tmp/subs1.awk" <<_ACAWK && +cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && for (key in S) S_is_set[key] = 1 FS = "" @@ -6908,21 +8377,29 @@ if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" else cat -fi < "$tmp/subs1.awk" > "$tmp/subs.awk" \ - || as_fn_error "could not setup config files machinery" "$LINENO" 5 +fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ + || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 _ACEOF -# VPATH may cause trouble with some makes, so we remove $(srcdir), -# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and +# VPATH may cause trouble with some makes, so we remove sole $(srcdir), +# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and # trailing colons and then remove the whole line if VPATH becomes empty # (actually we leave an empty line to preserve line numbers). if test "x$srcdir" = x.; then - ac_vpsub='/^[ ]*VPATH[ ]*=/{ -s/:*\$(srcdir):*/:/ -s/:*\${srcdir}:*/:/ -s/:*@srcdir@:*/:/ -s/^\([^=]*=[ ]*\):*/\1/ + ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ +h +s/// +s/^/:/ +s/[ ]*$/:/ +s/:\$(srcdir):/:/g +s/:\${srcdir}:/:/g +s/:@srcdir@:/:/g +s/^:*// s/:*$// +x +s/\(=[ ]*\).*/\1/ +G +s/\n// s/^[^=]*=[ ]*$// }' fi @@ -6934,7 +8411,7 @@ fi # test -n "$CONFIG_FILES" # No need to generate them if there are no CONFIG_HEADERS. # This happens for instance with `./config.status Makefile'. if test -n "$CONFIG_HEADERS"; then -cat >"$tmp/defines.awk" <<\_ACAWK || +cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF @@ -6946,11 +8423,11 @@ _ACEOF # handling of long lines. ac_delim='%!_!# ' for ac_last_try in false false :; do - ac_t=`sed -n "/$ac_delim/p" confdefs.h` - if test -z "$ac_t"; then + ac_tt=`sed -n "/$ac_delim/p" confdefs.h` + if test -z "$ac_tt"; then break elif $ac_last_try; then - as_fn_error "could not make $CONFIG_HEADERS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -7035,11 +8512,11 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 - as_fn_error "could not setup config headers machinery" "$LINENO" 5 + as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 fi # test -n "$CONFIG_HEADERS" -eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS " +eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" shift for ac_tag do @@ -7048,7 +8525,7 @@ do esac case $ac_mode$ac_tag in :[FHL]*:*);; - :L* | :C*:*) as_fn_error "invalid tag \`$ac_tag'" "$LINENO" 5;; + :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac @@ -7067,7 +8544,7 @@ do for ac_f do case $ac_f in - -) ac_f="$tmp/stdin";; + -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, # because $ac_f cannot contain `:'. @@ -7076,7 +8553,7 @@ do [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || - as_fn_error "cannot find input file: \`$ac_f'" "$LINENO" 5;; + as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" @@ -7102,8 +8579,8 @@ $as_echo "$as_me: creating $ac_file" >&6;} esac case $ac_tag in - *:-:* | *:-) cat >"$tmp/stdin" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 ;; + *:-:* | *:-) cat >"$ac_tmp/stdin" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; esac ;; esac @@ -7233,23 +8710,24 @@ s&@abs_top_builddir@&$ac_abs_top_builddir&;t t s&@INSTALL@&$ac_INSTALL&;t t $ac_datarootdir_hack " -eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$tmp/subs.awk" >$tmp/out \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 +eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ + >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && - { ac_out=`sed -n '/\${datarootdir}/p' "$tmp/out"`; test -n "$ac_out"; } && - { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' "$tmp/out"`; test -z "$ac_out"; } && + { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && + { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ + "$ac_tmp/out"`; test -z "$ac_out"; } && { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined." >&5 +which seems to be undefined. Please make sure it is defined" >&5 $as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined." >&2;} +which seems to be undefined. Please make sure it is defined" >&2;} - rm -f "$tmp/stdin" + rm -f "$ac_tmp/stdin" case $ac_file in - -) cat "$tmp/out" && rm -f "$tmp/out";; - *) rm -f "$ac_file" && mv "$tmp/out" "$ac_file";; + -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; + *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; esac \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; :H) # @@ -7258,27 +8736,89 @@ which seems to be undefined. Please make sure it is defined." >&2;} if test x"$ac_file" != x-; then { $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" - } >"$tmp/config.h" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 - if diff "$ac_file" "$tmp/config.h" >/dev/null 2>&1; then + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" + } >"$ac_tmp/config.h" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 $as_echo "$as_me: $ac_file is unchanged" >&6;} else rm -f "$ac_file" - mv "$tmp/config.h" "$ac_file" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 + mv "$ac_tmp/config.h" "$ac_file" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 fi else $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" \ - || as_fn_error "could not create -" "$LINENO" 5 + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ + || as_fn_error $? "could not create -" "$LINENO" 5 fi ;; - + :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5 +$as_echo "$as_me: executing $ac_file commands" >&6;} + ;; esac + + case $ac_file$ac_mode in + "include/jemalloc/internal/private_namespace.h":C) + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/private_namespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_namespace.h" + ;; + "include/jemalloc/internal/private_unnamespace.h":C) + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/private_unnamespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_unnamespace.h" + ;; + "include/jemalloc/internal/public_symbols.txt":C) + f="${objroot}include/jemalloc/internal/public_symbols.txt" + mkdir -p "${objroot}include/jemalloc/internal" + cp /dev/null "${f}" + for nm in `echo ${mangling_map} |tr ',' ' '` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'` + m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'` + echo "${n}:${m}" >> "${f}" + public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${n}\$" |tr '\n' ' '` + done + for sym in ${public_syms} ; do + n="${sym}" + m="${JEMALLOC_PREFIX}${sym}" + echo "${n}:${m}" >> "${f}" + done + ;; + "include/jemalloc/internal/public_namespace.h":C) + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/public_namespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_namespace.h" + ;; + "include/jemalloc/internal/public_unnamespace.h":C) + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/public_unnamespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_unnamespace.h" + ;; + "include/jemalloc/internal/size_classes.h":C) + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h" + ;; + "include/jemalloc/jemalloc_protos_jet.h":C) + mkdir -p "${objroot}include/jemalloc" + cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h" + ;; + "include/jemalloc/jemalloc_rename.h":C) + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_rename.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/jemalloc_rename.h" + ;; + "include/jemalloc/jemalloc_mangle.h":C) + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" je_ > "${objroot}include/jemalloc/jemalloc_mangle.h" + ;; + "include/jemalloc/jemalloc_mangle_jet.h":C) + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" jet_ > "${objroot}include/jemalloc/jemalloc_mangle_jet.h" + ;; + "include/jemalloc/jemalloc.h":C) + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc.sh" "${objroot}" > "${objroot}include/jemalloc/jemalloc${install_suffix}.h" + ;; + + esac done # for ac_tag @@ -7287,7 +8827,7 @@ _ACEOF ac_clean_files=$ac_clean_files_save test $ac_write_fail = 0 || - as_fn_error "write failure creating $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 # configure is writing to config.log, and then calls config.status. @@ -7308,7 +8848,7 @@ if test "$no_create" != yes; then exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. - $ac_cs_success || as_fn_exit $? + $ac_cs_success || as_fn_exit 1 fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 @@ -7318,8 +8858,10 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: ===============================================================================" >&5 $as_echo "===============================================================================" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: jemalloc version : $jemalloc_version" >&5 -$as_echo "jemalloc version : $jemalloc_version" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: jemalloc version : ${jemalloc_version}" >&5 +$as_echo "jemalloc version : ${jemalloc_version}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: library revision : ${rev}" >&5 +$as_echo "library revision : ${rev}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 $as_echo "" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: CC : ${CC}" >&5 @@ -7330,6 +8872,8 @@ $as_echo "CPPFLAGS : ${CPPFLAGS}" >&6; } $as_echo "CFLAGS : ${CFLAGS}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: LDFLAGS : ${LDFLAGS}" >&5 $as_echo "LDFLAGS : ${LDFLAGS}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: EXTRA_LDFLAGS : ${EXTRA_LDFLAGS}" >&5 +$as_echo "EXTRA_LDFLAGS : ${EXTRA_LDFLAGS}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: LIBS : ${LIBS}" >&5 $as_echo "LIBS : ${LIBS}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: RPATH_EXTRA : ${RPATH_EXTRA}" >&5 @@ -7376,10 +8920,14 @@ $as_echo " : ${JEMALLOC_PRIVATE_NAMESPACE}" >&6; } $as_echo "install_suffix : ${install_suffix}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: autogen : ${enable_autogen}" >&5 $as_echo "autogen : ${enable_autogen}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: experimental : ${enable_experimental}" >&5 +$as_echo "experimental : ${enable_experimental}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: cc-silence : ${enable_cc_silence}" >&5 $as_echo "cc-silence : ${enable_cc_silence}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug : ${enable_debug}" >&5 $as_echo "debug : ${enable_debug}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: code-coverage : ${enable_code_coverage}" >&5 +$as_echo "code-coverage : ${enable_code_coverage}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: stats : ${enable_stats}" >&5 $as_echo "stats : ${enable_stats}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: prof : ${enable_prof}" >&5 @@ -7390,22 +8938,22 @@ $as_echo "prof-libunwind : ${enable_prof_libunwind}" >&6; } $as_echo "prof-libgcc : ${enable_prof_libgcc}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: prof-gcc : ${enable_prof_gcc}" >&5 $as_echo "prof-gcc : ${enable_prof_gcc}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: tiny : ${enable_tiny}" >&5 -$as_echo "tiny : ${enable_tiny}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: tcache : ${enable_tcache}" >&5 $as_echo "tcache : ${enable_tcache}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: fill : ${enable_fill}" >&5 $as_echo "fill : ${enable_fill}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: utrace : ${enable_utrace}" >&5 +$as_echo "utrace : ${enable_utrace}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: valgrind : ${enable_valgrind}" >&5 +$as_echo "valgrind : ${enable_valgrind}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: xmalloc : ${enable_xmalloc}" >&5 $as_echo "xmalloc : ${enable_xmalloc}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: sysv : ${enable_sysv}" >&5 -$as_echo "sysv : ${enable_sysv}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: swap : ${enable_swap}" >&5 -$as_echo "swap : ${enable_swap}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: mremap : ${enable_mremap}" >&5 +$as_echo "mremap : ${enable_mremap}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: munmap : ${enable_munmap}" >&5 +$as_echo "munmap : ${enable_munmap}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: dss : ${enable_dss}" >&5 $as_echo "dss : ${enable_dss}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: dynamic_page_shift : ${enable_dynamic_page_shift}" >&5 -$as_echo "dynamic_page_shift : ${enable_dynamic_page_shift}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: lazy_lock : ${enable_lazy_lock}" >&5 $as_echo "lazy_lock : ${enable_lazy_lock}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: tls : ${enable_tls}" >&5 diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac index b58aa520998..4de81dc1d9f 100644 --- a/deps/jemalloc/configure.ac +++ b/deps/jemalloc/configure.ac @@ -14,32 +14,39 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} $1" fi -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ ]], [[ return 0; ]])], + [je_cv_cflags_appended=$1] AC_MSG_RESULT([yes]), + [je_cv_cflags_appended=] AC_MSG_RESULT([no]) [CFLAGS="${TCFLAGS}"] ) ]) dnl JE_COMPILABLE(label, hcode, mcode, rvar) +dnl +dnl Use AC_LINK_IFELSE() rather than AC_COMPILE_IFELSE() so that linker errors +dnl cause failure. AC_DEFUN([JE_COMPILABLE], [ -AC_MSG_CHECKING([whether $1 is compilable]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[$2], [$3])], - AC_MSG_RESULT([yes]) - [$4="yes"], - AC_MSG_RESULT([no]) - [$4="no"] -) +AC_CACHE_CHECK([whether $1 is compilable], + [$4], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([$2], + [$3])], + [$4=yes], + [$4=no])]) ]) dnl ============================================================================ +dnl Library revision. +rev=1 +AC_SUBST([rev]) + srcroot=$srcdir if test "x${srcroot}" = "x." ; then srcroot="" @@ -81,15 +88,24 @@ MANDIR=`eval echo $MANDIR` AC_SUBST([MANDIR]) dnl Support for building documentation. -AC_PATH_PROG([XSLTPROC], [xsltproc], , [$PATH]) +AC_PATH_PROG([XSLTPROC], [xsltproc], [false], [$PATH]) +if test -d "/usr/share/xml/docbook/stylesheet/docbook-xsl" ; then + DEFAULT_XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +elif test -d "/usr/share/sgml/docbook/xsl-stylesheets" ; then + DEFAULT_XSLROOT="/usr/share/sgml/docbook/xsl-stylesheets" +else + dnl Documentation building will fail if this default gets used. + DEFAULT_XSLROOT="" +fi AC_ARG_WITH([xslroot], - [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], + [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], [ if test "x$with_xslroot" = "xno" ; then - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + XSLROOT="${DEFAULT_XSLROOT}" else XSLROOT="${with_xslroot}" -fi, - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +fi +], + XSLROOT="${DEFAULT_XSLROOT}" ) AC_SUBST([XSLROOT]) @@ -97,13 +113,35 @@ dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, dnl just prevent autoconf from molesting CFLAGS. CFLAGS=$CFLAGS AC_PROG_CC +if test "x$GCC" != "xyes" ; then + AC_CACHE_CHECK([whether compiler is MSVC], + [je_cv_msvc], + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], + [ +#ifndef _MSC_VER + int fail[-1]; +#endif +])], + [je_cv_msvc=yes], + [je_cv_msvc=no])]) +fi + if test "x$CFLAGS" = "x" ; then no_CFLAGS="yes" if test "x$GCC" = "xyes" ; then JE_CFLAGS_APPEND([-std=gnu99]) + if test "x$je_cv_cflags_appended" = "x-std=gnu99" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT]) + fi JE_CFLAGS_APPEND([-Wall]) JE_CFLAGS_APPEND([-pipe]) JE_CFLAGS_APPEND([-g3]) + elif test "x$je_cv_msvc" = "xyes" ; then + CC="$CC -nologo" + JE_CFLAGS_APPEND([-Zi]) + JE_CFLAGS_APPEND([-MT]) + JE_CFLAGS_APPEND([-W3]) + CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" fi fi dnl Append EXTRA_CFLAGS to CFLAGS, if defined. @@ -112,6 +150,11 @@ if test "x$EXTRA_CFLAGS" != "x" ; then fi AC_PROG_CPP +AC_C_BIGENDIAN([ac_cv_big_endian=1], [ac_cv_big_endian=0]) +if test "x${ac_cv_big_endian}" = "x1" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_BIG_ENDIAN], [ ]) +fi + AC_CHECK_SIZEOF([void *]) if test "x${ac_cv_sizeof_void_p}" = "x8" ; then LG_SIZEOF_PTR=3 @@ -142,31 +185,71 @@ else fi AC_DEFINE_UNQUOTED([LG_SIZEOF_LONG], [$LG_SIZEOF_LONG]) +AC_CHECK_SIZEOF([intmax_t]) +if test "x${ac_cv_sizeof_intmax_t}" = "x16" ; then + LG_SIZEOF_INTMAX_T=4 +elif test "x${ac_cv_sizeof_intmax_t}" = "x8" ; then + LG_SIZEOF_INTMAX_T=3 +elif test "x${ac_cv_sizeof_intmax_t}" = "x4" ; then + LG_SIZEOF_INTMAX_T=2 +else + AC_MSG_ERROR([Unsupported intmax_t size: ${ac_cv_sizeof_intmax_t}]) +fi +AC_DEFINE_UNQUOTED([LG_SIZEOF_INTMAX_T], [$LG_SIZEOF_INTMAX_T]) + AC_CANONICAL_HOST dnl CPU-specific settings. CPU_SPINWAIT="" case "${host_cpu}" in i[[345]]86) ;; - i686) - JE_COMPILABLE([__asm__], [], [[__asm__ volatile("pause"); return 0;]], - [asm]) - if test "x${asm}" = "xyes" ; then + i686|x86_64) + JE_COMPILABLE([pause instruction], [], + [[__asm__ volatile("pause"); return 0;]], + [je_cv_pause]) + if test "x${je_cv_pause}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi - ;; - x86_64) - JE_COMPILABLE([__asm__ syntax], [], - [[__asm__ volatile("pause"); return 0;]], [asm]) - if test "x${asm}" = "xyes" ; then - CPU_SPINWAIT='__asm__ volatile("pause")' + dnl emmintrin.h fails to compile unless MMX, SSE, and SSE2 are + dnl supported. + JE_COMPILABLE([SSE2 intrinsics], [ +#include +], [], [je_cv_sse2]) + if test "x${je_cv_sse2}" = "xyes" ; then + AC_DEFINE_UNQUOTED([HAVE_SSE2], [ ]) fi ;; + powerpc) + AC_DEFINE_UNQUOTED([HAVE_ALTIVEC], [ ]) + ;; *) ;; esac AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT]) +LD_PRELOAD_VAR="LD_PRELOAD" +so="so" +importlib="${so}" +o="$ac_objext" +a="a" +exe="$ac_exeext" +libprefix="lib" +DSO_LDFLAGS='-shared -Wl,-soname,$(@F)' +RPATH='-Wl,-rpath,$(1)' +SOREV="${so}.${rev}" +PIC_CFLAGS='-fPIC -DPIC' +CTARGET='-o $@' +LDTARGET='-o $@' +EXTRA_LDFLAGS= +ARFLAGS='crus' +AROUT=' $@' +CC_MM=1 + +AN_MAKEVAR([AR], [AC_PROG_AR]) +AN_PROGRAM([ar], [AC_PROG_AR]) +AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL(AR, ar, :)]) +AC_PROG_AR + dnl Platform-specific settings. abi and RPATH can probably be determined dnl programmatically, but doing so is error-prone, which makes it generally dnl not worth the trouble. @@ -174,25 +257,37 @@ dnl dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. +default_munmap="1" +JEMALLOC_USABLE_SIZE_CONST="const" case "${host}" in *-*-darwin*) - CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" + CFLAGS="$CFLAGS" abi="macho" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) RPATH="" + LD_PRELOAD_VAR="DYLD_INSERT_LIBRARIES" + so="dylib" + importlib="${so}" + force_tls="0" + DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)' + SOREV="${rev}.${so}" + sbrk_deprecated="1" ;; *-*-freebsd*) CFLAGS="$CFLAGS" abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + force_lazy_lock="1" ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) + AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) + JEMALLOC_USABLE_SIZE_CONST="" + default_munmap="0" ;; *-*-netbsd*) AC_MSG_CHECKING([ABI]) @@ -206,45 +301,101 @@ case "${host}" in [CFLAGS="$CFLAGS"; abi="elf"], [abi="aout"]) AC_MSG_RESULT([$abi]) - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) ;; *-*-solaris2*) CFLAGS="$CFLAGS" abi="elf" - RPATH="-Wl,-R," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + RPATH='-Wl,-R,$(1)' dnl Solaris needs this for sigwait(). CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" LIBS="$LIBS -lposix4 -lsocket -lnsl" ;; + *-ibm-aix*) + if "$LG_SIZEOF_PTR" = "8"; then + dnl 64bit AIX + LD_PRELOAD_VAR="LDR_PRELOAD64" + else + dnl 32bit AIX + LD_PRELOAD_VAR="LDR_PRELOAD" + fi + abi="xcoff" + ;; + *-*-mingw*) + abi="pecoff" + force_tls="0" + RPATH="" + so="dll" + if test "x$je_cv_msvc" = "xyes" ; then + importlib="lib" + DSO_LDFLAGS="-LD" + EXTRA_LDFLAGS="-link -DEBUG" + CTARGET='-Fo$@' + LDTARGET='-Fe$@' + AR='lib' + ARFLAGS='-nologo -out:' + AROUT='$@' + CC_MM= + else + importlib="${so}" + DSO_LDFLAGS="-shared" + fi + a="lib" + libprefix="" + SOREV="${so}" + PIC_CFLAGS="" + ;; *) AC_MSG_RESULT([Unsupported operating system: ${host}]) abi="elf" - RPATH="-Wl,-rpath," ;; esac +AC_DEFINE_UNQUOTED([JEMALLOC_USABLE_SIZE_CONST], [$JEMALLOC_USABLE_SIZE_CONST]) AC_SUBST([abi]) AC_SUBST([RPATH]) +AC_SUBST([LD_PRELOAD_VAR]) +AC_SUBST([so]) +AC_SUBST([importlib]) +AC_SUBST([o]) +AC_SUBST([a]) +AC_SUBST([exe]) +AC_SUBST([libprefix]) +AC_SUBST([DSO_LDFLAGS]) +AC_SUBST([EXTRA_LDFLAGS]) +AC_SUBST([SOREV]) +AC_SUBST([PIC_CFLAGS]) +AC_SUBST([CTARGET]) +AC_SUBST([LDTARGET]) +AC_SUBST([MKLIB]) +AC_SUBST([ARFLAGS]) +AC_SUBST([AROUT]) +AC_SUBST([CC_MM]) JE_COMPILABLE([__attribute__ syntax], [static __attribute__((unused)) void foo(void){}], [], - [attribute]) -if test "x${attribute}" = "xyes" ; then + [je_cv_attribute]) +if test "x${je_cv_attribute}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ]) if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then JE_CFLAGS_APPEND([-fvisibility=hidden]) fi fi - -JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ -#define _GNU_SOURCE -#include -], [ -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); -], [mremap_fixed]) -if test "x${mremap_fixed}" = "xyes" ; then - AC_DEFINE([JEMALLOC_MREMAP_FIXED]) +dnl Check for tls_model attribute support (clang 3.0 still lacks support). +SAVED_CFLAGS="${CFLAGS}" +JE_CFLAGS_APPEND([-Werror]) +JE_COMPILABLE([tls_model attribute], [], + [static __thread int + __attribute__((tls_model("initial-exec"))) foo; + foo = 0;], + [je_cv_tls_model]) +CFLAGS="${SAVED_CFLAGS}" +if test "x${je_cv_tls_model}" = "xyes" ; then + AC_DEFINE([JEMALLOC_TLS_MODEL], + [__attribute__((tls_model("initial-exec")))]) +else + AC_DEFINE([JEMALLOC_TLS_MODEL], [ ]) fi dnl Support optional additions to rpath. @@ -274,15 +425,72 @@ AC_SUBST([enable_autogen]) AC_PROG_INSTALL AC_PROG_RANLIB -AC_PATH_PROG([AR], [ar], , [$PATH]) -AC_PATH_PROG([LD], [ld], , [$PATH]) -AC_PATH_PROG([AUTOCONF], [autoconf], , [$PATH]) +AC_PATH_PROG([LD], [ld], [false], [$PATH]) +AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH]) + +public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free mallocx rallocx xallocx sallocx dallocx nallocx mallctl mallctlnametomib mallctlbymib malloc_stats_print malloc_usable_size" + +dnl Check for allocator-related functions that should be wrapped. +AC_CHECK_FUNC([memalign], + [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ]) + public_syms="${public_syms} memalign"]) +AC_CHECK_FUNC([valloc], + [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC], [ ]) + public_syms="${public_syms} valloc"]) + +dnl Support the experimental API by default. +AC_ARG_ENABLE([experimental], + [AS_HELP_STRING([--disable-experimental], + [Disable support for the experimental API])], +[if test "x$enable_experimental" = "xno" ; then + enable_experimental="0" +else + enable_experimental="1" +fi +], +[enable_experimental="1"] +) +if test "x$enable_experimental" = "x1" ; then + AC_DEFINE([JEMALLOC_EXPERIMENTAL], [ ]) + public_syms="${public_syms} allocm dallocm nallocm rallocm sallocm" +fi +AC_SUBST([enable_experimental]) + +dnl Do not compute test code coverage by default. +GCOV_FLAGS= +AC_ARG_ENABLE([code-coverage], + [AS_HELP_STRING([--enable-code-coverage], + [Enable code coverage])], +[if test "x$enable_code_coverage" = "xno" ; then + enable_code_coverage="0" +else + enable_code_coverage="1" +fi +], +[enable_code_coverage="0"] +) +if test "x$enable_code_coverage" = "x1" ; then + deoptimize="no" + echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || deoptimize="yes" + if test "x${deoptimize}" = "xyes" ; then + JE_CFLAGS_APPEND([-O0]) + fi + JE_CFLAGS_APPEND([-fprofile-arcs -ftest-coverage]) + EXTRA_LDFLAGS="$EXTRA_LDFLAGS -fprofile-arcs -ftest-coverage" + AC_DEFINE([JEMALLOC_CODE_COVERAGE], [ ]) +fi +AC_SUBST([enable_code_coverage]) + +dnl Perform no name mangling by default. +AC_ARG_WITH([mangling], + [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], + [mangling_map="$with_mangling"], [mangling_map=""]) dnl Do not prefix public APIs by default. AC_ARG_WITH([jemalloc_prefix], [AS_HELP_STRING([--with-jemalloc-prefix=], [Prefix to prepend to all public APIs])], [JEMALLOC_PREFIX="$with_jemalloc_prefix"], - [if test "x$abi" != "xmacho" ; then + [if test "x$abi" != "xmacho" -a "x$abi" != "xpecoff"; then JEMALLOC_PREFIX="" else JEMALLOC_PREFIX="je_" @@ -292,21 +500,24 @@ if test "x$JEMALLOC_PREFIX" != "x" ; then JEMALLOC_CPREFIX=`echo ${JEMALLOC_PREFIX} | tr "a-z" "A-Z"` AC_DEFINE_UNQUOTED([JEMALLOC_PREFIX], ["$JEMALLOC_PREFIX"]) AC_DEFINE_UNQUOTED([JEMALLOC_CPREFIX], ["$JEMALLOC_CPREFIX"]) - AC_DEFINE_UNQUOTED([JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix)], [${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix]) fi -dnl Do not mangle library-private APIs by default. +AC_ARG_WITH([export], + [AS_HELP_STRING([--without-export], [disable exporting jemalloc public APIs])], + [if test "x$with_export" = "xno"; then + AC_DEFINE([JEMALLOC_EXPORT],[]) +fi] +) + +dnl Mangle library-private APIs. AC_ARG_WITH([private_namespace], [AS_HELP_STRING([--with-private-namespace=], [Prefix to prepend to all library-private APIs])], - [JEMALLOC_PRIVATE_NAMESPACE="$with_private_namespace"], - [JEMALLOC_PRIVATE_NAMESPACE=""] + [JEMALLOC_PRIVATE_NAMESPACE="${with_private_namespace}je_"], + [JEMALLOC_PRIVATE_NAMESPACE="je_"] ) -AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], ["$JEMALLOC_PRIVATE_NAMESPACE"]) -if test "x$JEMALLOC_PRIVATE_NAMESPACE" != "x" ; then - AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [${JEMALLOC_PRIVATE_NAMESPACE}##string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) -else - AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) -fi +AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], [$JEMALLOC_PRIVATE_NAMESPACE]) +private_namespace="$JEMALLOC_PRIVATE_NAMESPACE" +AC_SUBST([private_namespace]) dnl Do not add suffix to installed files by default. AC_ARG_WITH([install_suffix], @@ -317,35 +528,72 @@ AC_ARG_WITH([install_suffix], install_suffix="$INSTALL_SUFFIX" AC_SUBST([install_suffix]) +dnl Substitute @je_@ in jemalloc_protos.h.in, primarily to make generation of +dnl jemalloc_protos_jet.h easy. +je_="je_" +AC_SUBST([je_]) + cfgoutputs_in="${srcroot}Makefile.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_macros.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_protos.h.in" cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/jemalloc_test.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/test.sh.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/include/test/jemalloc_test.h.in" cfgoutputs_out="Makefile" cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" -cfgoutputs_out="${cfgoutputs_out} doc/jemalloc${install_suffix}.xml" -cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h" +cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_out="${cfgoutputs_out} test/jemalloc_test.h" +cfgoutputs_out="${cfgoutputs_out} test/test.sh" +cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" cfgoutputs_tup="Makefile" cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in" cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" -cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc${install_suffix}.xml:doc/jemalloc.xml.in" -cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in" +cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jemalloc/jemalloc_macros.h.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" +cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" +cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" - -cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" - -cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_namespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_symbols.txt" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_namespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_rename.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_mangle.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc.sh" +cfghdrs_in="${cfghdrs_in} ${srcroot}test/include/test/jemalloc_test_defs.h.in" + +cfghdrs_out="include/jemalloc/jemalloc_defs.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc${install_suffix}.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_namespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_unnamespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle_jet.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/jemalloc_internal_defs.h" +cfghdrs_out="${cfghdrs_out} test/include/test/jemalloc_test_defs.h" + +cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:${srcroot}test/include/test/jemalloc_test_defs.h.in" dnl Do not silence irrelevant compiler warnings by default, since enabling this dnl option incurs a performance penalty. @@ -361,12 +609,12 @@ fi [enable_cc_silence="0"] ) if test "x$enable_cc_silence" = "x1" ; then - AC_DEFINE([JEMALLOC_CC_SILENCE]) + AC_DEFINE([JEMALLOC_CC_SILENCE], [ ]) fi dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], - [AS_HELP_STRING([--enable-debug], [Build debugging code])], + [AS_HELP_STRING([--enable-debug], [Build debugging code (implies --enable-ivsalloc)])], [if test "x$enable_debug" = "xno" ; then enable_debug="0" else @@ -377,35 +625,53 @@ fi ) if test "x$enable_debug" = "x1" ; then AC_DEFINE([JEMALLOC_DEBUG], [ ]) - AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) + enable_ivsalloc="1" fi AC_SUBST([enable_debug]) +dnl Do not validate pointers by default. +AC_ARG_ENABLE([ivsalloc], + [AS_HELP_STRING([--enable-ivsalloc], [Validate pointers passed through the public API])], +[if test "x$enable_ivsalloc" = "xno" ; then + enable_ivsalloc="0" +else + enable_ivsalloc="1" +fi +], +[enable_ivsalloc="0"] +) +if test "x$enable_ivsalloc" = "x1" ; then + AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) +fi + dnl Only optimize if not debugging. if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. optimize="no" - echo "$EXTRA_CFLAGS" | grep "\-O" >/dev/null || optimize="yes" + echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || optimize="yes" if test "x${optimize}" = "xyes" ; then if test "x$GCC" = "xyes" ; then JE_CFLAGS_APPEND([-O3]) JE_CFLAGS_APPEND([-funroll-loops]) + elif test "x$je_cv_msvc" = "xyes" ; then + JE_CFLAGS_APPEND([-O2]) else JE_CFLAGS_APPEND([-O]) fi fi fi -dnl Do not enable statistics calculation by default. +dnl Enable statistics calculation by default. AC_ARG_ENABLE([stats], - [AS_HELP_STRING([--enable-stats], [Enable statistics calculation/reporting])], + [AS_HELP_STRING([--disable-stats], + [Disable statistics calculation/reporting])], [if test "x$enable_stats" = "xno" ; then enable_stats="0" else enable_stats="1" fi ], -[enable_stats="0"] +[enable_stats="1"] ) if test "x$enable_stats" = "x1" ; then AC_DEFINE([JEMALLOC_STATS], [ ]) @@ -481,22 +747,6 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ -a "x$GCC" = "xyes" ; then AC_CHECK_HEADERS([unwind.h], , [enable_prof_libgcc="0"]) AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [LIBS="$LIBS -lgcc"], [enable_prof_libgcc="0"]) - dnl The following is conservative, in that it only has entries for CPUs on - dnl which jemalloc has been tested. - AC_MSG_CHECKING([libgcc-based backtracing reliability on ${host_cpu}]) - case "${host_cpu}" in - i[[3456]]86) - AC_MSG_RESULT([unreliable]) - enable_prof_libgcc="0"; - ;; - x86_64) - AC_MSG_RESULT([reliable]) - ;; - *) - AC_MSG_RESULT([unreliable]) - enable_prof_libgcc="0"; - ;; - esac if test "x${enable_prof_libgcc}" = "x1" ; then backtrace_method="libgcc" AC_DEFINE([JEMALLOC_PROF_LIBGCC], [ ]) @@ -518,6 +768,7 @@ fi ) if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \ -a "x$GCC" = "xyes" ; then + JE_CFLAGS_APPEND([-fno-omit-frame-pointer]) backtrace_method="gcc intrinsics" AC_DEFINE([JEMALLOC_PROF_GCC], [ ]) else @@ -531,27 +782,20 @@ fi AC_MSG_CHECKING([configured backtracing method]) AC_MSG_RESULT([$backtrace_method]) if test "x$enable_prof" = "x1" ; then - LIBS="$LIBS -lm" + if test "x${force_tls}" = "x0" ; then + AC_MSG_ERROR([Heap profiling requires TLS]); + fi + force_tls="1" + + if test "x$abi" != "xpecoff"; then + dnl Heap profiling uses the log(3) function. + LIBS="$LIBS -lm" + fi + AC_DEFINE([JEMALLOC_PROF], [ ]) fi AC_SUBST([enable_prof]) -dnl Enable tiny allocations by default. -AC_ARG_ENABLE([tiny], - [AS_HELP_STRING([--disable-tiny], [Disable tiny (sub-quantum) allocations])], -[if test "x$enable_tiny" = "xno" ; then - enable_tiny="0" -else - enable_tiny="1" -fi -], -[enable_tiny="1"] -) -if test "x$enable_tiny" = "x1" ; then - AC_DEFINE([JEMALLOC_TINY], [ ]) -fi -AC_SUBST([enable_tiny]) - dnl Enable thread-specific caching by default. AC_ARG_ENABLE([tcache], [AS_HELP_STRING([--disable-tcache], [Disable per thread caches])], @@ -568,21 +812,48 @@ if test "x$enable_tcache" = "x1" ; then fi AC_SUBST([enable_tcache]) -dnl Do not enable mmap()ped swap files by default. -AC_ARG_ENABLE([swap], - [AS_HELP_STRING([--enable-swap], [Enable mmap()ped swap files])], -[if test "x$enable_swap" = "xno" ; then - enable_swap="0" +dnl Disable mremap() for huge realloc() by default. +AC_ARG_ENABLE([mremap], + [AS_HELP_STRING([--enable-mremap], [Enable mremap(2) for huge realloc()])], +[if test "x$enable_mremap" = "xno" ; then + enable_mremap="0" +else + enable_mremap="1" +fi +], +[enable_mremap="0"] +) +if test "x$enable_mremap" = "x1" ; then + JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ +#define _GNU_SOURCE +#include +], [ +void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); +], [je_cv_mremap_fixed]) + if test "x${je_cv_mremap_fixed}" = "xno" ; then + enable_mremap="0" + fi +fi +if test "x$enable_mremap" = "x1" ; then + AC_DEFINE([JEMALLOC_MREMAP], [ ]) +fi +AC_SUBST([enable_mremap]) + +dnl Enable VM deallocation via munmap() by default. +AC_ARG_ENABLE([munmap], + [AS_HELP_STRING([--disable-munmap], [Disable VM deallocation via munmap(2)])], +[if test "x$enable_munmap" = "xno" ; then + enable_munmap="0" else - enable_swap="1" + enable_munmap="1" fi ], -[enable_swap="0"] +[enable_munmap="${default_munmap}"] ) -if test "x$enable_swap" = "x1" ; then - AC_DEFINE([JEMALLOC_SWAP], [ ]) +if test "x$enable_munmap" = "x1" ; then + AC_DEFINE([JEMALLOC_MUNMAP], [ ]) fi -AC_SUBST([enable_swap]) +AC_SUBST([enable_munmap]) dnl Do not enable allocation from DSS by default. AC_ARG_ENABLE([dss], @@ -595,102 +866,159 @@ fi ], [enable_dss="0"] ) +dnl Check whether the BSD/SUSv1 sbrk() exists. If not, disable DSS support. +AC_CHECK_FUNC([sbrk], [have_sbrk="1"], [have_sbrk="0"]) +if test "x$have_sbrk" = "x1" ; then + if test "x$sbrk_deprecated" == "x1" ; then + AC_MSG_RESULT([Disabling dss allocation because sbrk is deprecated]) + enable_dss="0" + else + AC_DEFINE([JEMALLOC_HAVE_SBRK], [ ]) + fi +else + enable_dss="0" +fi + if test "x$enable_dss" = "x1" ; then AC_DEFINE([JEMALLOC_DSS], [ ]) fi AC_SUBST([enable_dss]) -dnl Do not support the junk/zero filling option by default. +dnl Support the junk/zero filling option by default. AC_ARG_ENABLE([fill], - [AS_HELP_STRING([--enable-fill], [Support junk/zero filling option])], + [AS_HELP_STRING([--disable-fill], + [Disable support for junk/zero filling, quarantine, and redzones])], [if test "x$enable_fill" = "xno" ; then enable_fill="0" else enable_fill="1" fi ], -[enable_fill="0"] +[enable_fill="1"] ) if test "x$enable_fill" = "x1" ; then AC_DEFINE([JEMALLOC_FILL], [ ]) fi AC_SUBST([enable_fill]) -dnl Do not support the xmalloc option by default. -AC_ARG_ENABLE([xmalloc], - [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], -[if test "x$enable_xmalloc" = "xno" ; then - enable_xmalloc="0" +dnl Disable utrace(2)-based tracing by default. +AC_ARG_ENABLE([utrace], + [AS_HELP_STRING([--enable-utrace], [Enable utrace(2)-based tracing])], +[if test "x$enable_utrace" = "xno" ; then + enable_utrace="0" else - enable_xmalloc="1" + enable_utrace="1" fi ], -[enable_xmalloc="0"] +[enable_utrace="0"] ) -if test "x$enable_xmalloc" = "x1" ; then - AC_DEFINE([JEMALLOC_XMALLOC], [ ]) +JE_COMPILABLE([utrace(2)], [ +#include +#include +#include +#include +#include +], [ + utrace((void *)0, 0); +], [je_cv_utrace]) +if test "x${je_cv_utrace}" = "xno" ; then + enable_utrace="0" fi -AC_SUBST([enable_xmalloc]) +if test "x$enable_utrace" = "x1" ; then + AC_DEFINE([JEMALLOC_UTRACE], [ ]) +fi +AC_SUBST([enable_utrace]) -dnl Do not support the SYSV option by default. -AC_ARG_ENABLE([sysv], - [AS_HELP_STRING([--enable-sysv], [Support SYSV semantics option])], -[if test "x$enable_sysv" = "xno" ; then - enable_sysv="0" +dnl Support Valgrind by default. +AC_ARG_ENABLE([valgrind], + [AS_HELP_STRING([--disable-valgrind], [Disable support for Valgrind])], +[if test "x$enable_valgrind" = "xno" ; then + enable_valgrind="0" else - enable_sysv="1" + enable_valgrind="1" fi ], -[enable_sysv="0"] +[enable_valgrind="1"] ) -if test "x$enable_sysv" = "x1" ; then - AC_DEFINE([JEMALLOC_SYSV], [ ]) +if test "x$enable_valgrind" = "x1" ; then + JE_COMPILABLE([valgrind], [ +#include +#include + +#if !defined(VALGRIND_RESIZEINPLACE_BLOCK) +# error "Incompatible Valgrind version" +#endif +], [], [je_cv_valgrind]) + if test "x${je_cv_valgrind}" = "xno" ; then + enable_valgrind="0" + fi + if test "x$enable_valgrind" = "x1" ; then + AC_DEFINE([JEMALLOC_VALGRIND], [ ]) + fi fi -AC_SUBST([enable_sysv]) +AC_SUBST([enable_valgrind]) -dnl Do not determine page shift at run time by default. -AC_ARG_ENABLE([dynamic_page_shift], - [AS_HELP_STRING([--enable-dynamic-page-shift], - [Determine page size at run time (don't trust configure result)])], -[if test "x$enable_dynamic_page_shift" = "xno" ; then - enable_dynamic_page_shift="0" +dnl Do not support the xmalloc option by default. +AC_ARG_ENABLE([xmalloc], + [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], +[if test "x$enable_xmalloc" = "xno" ; then + enable_xmalloc="0" else - enable_dynamic_page_shift="1" + enable_xmalloc="1" fi ], -[enable_dynamic_page_shift="0"] +[enable_xmalloc="0"] ) -if test "x$enable_dynamic_page_shift" = "x1" ; then - AC_DEFINE([DYNAMIC_PAGE_SHIFT], [ ]) +if test "x$enable_xmalloc" = "x1" ; then + AC_DEFINE([JEMALLOC_XMALLOC], [ ]) fi -AC_SUBST([enable_dynamic_page_shift]) +AC_SUBST([enable_xmalloc]) -AC_MSG_CHECKING([STATIC_PAGE_SHIFT]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[[#include -#include +AC_CACHE_CHECK([STATIC_PAGE_SHIFT], + [je_cv_static_page_shift], + AC_RUN_IFELSE([AC_LANG_PROGRAM( +[[ #include -]], [[ - long result; +#ifdef _WIN32 +#include +#else +#include +#endif +#include +]], +[[ + int result; FILE *f; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwPageSize; +#else result = sysconf(_SC_PAGESIZE); +#endif if (result == -1) { return 1; } + result = ffsl(result) - 1; + f = fopen("conftest.out", "w"); if (f == NULL) { return 1; } - fprintf(f, "%u\n", ffs((int)result) - 1); - close(f); + fprintf(f, "%d\n", result); + fclose(f); return 0; ]])], - [STATIC_PAGE_SHIFT=`cat conftest.out`] - AC_MSG_RESULT([$STATIC_PAGE_SHIFT]) - AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$STATIC_PAGE_SHIFT]), - AC_MSG_RESULT([error])) + [je_cv_static_page_shift=`cat conftest.out`], + [je_cv_static_page_shift=undefined])) + +if test "x$je_cv_static_page_shift" != "xundefined"; then + AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$je_cv_static_page_shift]) +else + AC_MSG_ERROR([cannot determine value for STATIC_PAGE_SHIFT]) +fi dnl ============================================================================ dnl jemalloc configuration. @@ -716,28 +1044,65 @@ AC_SUBST([jemalloc_version_gid]) dnl ============================================================================ dnl Configure pthreads. -AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) -AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], - [AC_MSG_ERROR([libpthread is missing])]) +if test "x$abi" != "xpecoff" ; then + AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) + dnl Some systems may embed pthreads functionality in libc; check for libpthread + dnl first, but try libc too before failing. + AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], + [AC_SEARCH_LIBS([pthread_create], , , + AC_MSG_ERROR([libpthread is missing]))]) +fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" -dnl Enable lazy locking by default. +dnl Check whether the BSD-specific _malloc_thread_cleanup() exists. If so, use +dnl it rather than pthreads TSD cleanup functions to support cleanup during +dnl thread exit, in order to avoid pthreads library recursion during +dnl bootstrapping. +AC_CHECK_FUNC([_malloc_thread_cleanup], + [have__malloc_thread_cleanup="1"], + [have__malloc_thread_cleanup="0"] + ) +if test "x$have__malloc_thread_cleanup" = "x1" ; then + AC_DEFINE([JEMALLOC_MALLOC_THREAD_CLEANUP], [ ]) + force_tls="1" +fi + +dnl Check whether the BSD-specific _pthread_mutex_init_calloc_cb() exists. If +dnl so, mutex initialization causes allocation, and we need to implement this +dnl callback function in order to prevent recursive allocation. +AC_CHECK_FUNC([_pthread_mutex_init_calloc_cb], + [have__pthread_mutex_init_calloc_cb="1"], + [have__pthread_mutex_init_calloc_cb="0"] + ) +if test "x$have__pthread_mutex_init_calloc_cb" = "x1" ; then + AC_DEFINE([JEMALLOC_MUTEX_INIT_CB]) +fi + +dnl Disable lazy locking by default. AC_ARG_ENABLE([lazy_lock], - [AS_HELP_STRING([--disable-lazy-lock], - [Disable lazy locking (always lock, even when single-threaded)])], + [AS_HELP_STRING([--enable-lazy-lock], + [Enable lazy locking (only lock when multi-threaded)])], [if test "x$enable_lazy_lock" = "xno" ; then enable_lazy_lock="0" else enable_lazy_lock="1" fi ], -[enable_lazy_lock="1"] +[enable_lazy_lock="0"] ) +if test "x$enable_lazy_lock" = "x0" -a "x${force_lazy_lock}" = "x1" ; then + AC_MSG_RESULT([Forcing lazy-lock to avoid allocator/threading bootstrap issues]) + enable_lazy_lock="1" +fi if test "x$enable_lazy_lock" = "x1" ; then - AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) - AC_CHECK_LIB([dl], [dlopen], [LIBS="$LIBS -ldl"], - [AC_MSG_ERROR([libdl is missing])]) + if test "x$abi" != "xpecoff" ; then + AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) + AC_CHECK_FUNC([dlsym], [], + [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], + [AC_MSG_ERROR([libdl is missing])]) + ]) + fi AC_DEFINE([JEMALLOC_LAZY_LOCK], [ ]) fi AC_SUBST([enable_lazy_lock]) @@ -752,9 +1117,17 @@ fi , enable_tls="1" ) +if test "x${enable_tls}" = "x0" -a "x${force_tls}" = "x1" ; then + AC_MSG_RESULT([Forcing TLS to avoid allocator/threading bootstrap issues]) + enable_tls="1" +fi +if test "x${enable_tls}" = "x1" -a "x${force_tls}" = "x0" ; then + AC_MSG_RESULT([Forcing no TLS to avoid allocator/threading bootstrap issues]) + enable_tls="0" +fi if test "x${enable_tls}" = "x1" ; then AC_MSG_CHECKING([for TLS]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ __thread int x; ]], [[ @@ -767,17 +1140,52 @@ AC_RUN_IFELSE([AC_LANG_PROGRAM( enable_tls="0") fi AC_SUBST([enable_tls]) -if test "x${enable_tls}" = "x0" ; then - AC_DEFINE_UNQUOTED([NO_TLS], [ ]) +if test "x${enable_tls}" = "x1" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_TLS], [ ]) +elif test "x${force_tls}" = "x1" ; then + AC_MSG_ERROR([Failed to configure TLS, which is mandatory for correct function]) fi dnl ============================================================================ dnl Check for ffsl(3), and fail if not found. This function exists on all dnl platforms that jemalloc currently has a chance of functioning on without dnl modification. +JE_COMPILABLE([a program using ffsl], [ +#include +#include +#include +], [ + { + int rv = ffsl(0x08); + printf("%d\n", rv); + } +], [je_cv_function_ffsl]) +if test "x${je_cv_function_ffsl}" != "xyes" ; then + AC_MSG_ERROR([Cannot build without ffsl(3)]) +fi -AC_CHECK_FUNC([ffsl], [], - [AC_MSG_ERROR([Cannot build without ffsl(3)])]) +dnl ============================================================================ +dnl Check for atomic(9) operations as provided on FreeBSD. + +JE_COMPILABLE([atomic(9)], [ +#include +#include +#include +], [ + { + uint32_t x32 = 0; + volatile uint32_t *x32p = &x32; + atomic_fetchadd_32(x32p, 1); + } + { + unsigned long xlong = 0; + volatile unsigned long *xlongp = &xlong; + atomic_fetchadd_long(xlongp, 1); + } +], [je_cv_atomic9]) +if test "x${je_cv_atomic9}" = "xyes" ; then + AC_DEFINE([JEMALLOC_ATOMIC9]) +fi dnl ============================================================================ dnl Check for atomic(3) operations as provided on Darwin. @@ -796,9 +1204,43 @@ JE_COMPILABLE([Darwin OSAtomic*()], [ volatile int64_t *x64p = &x64; OSAtomicAdd64(1, x64p); } -], [osatomic]) -if test "x${osatomic}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSATOMIC]) +], [je_cv_osatomic]) +if test "x${je_cv_osatomic}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSATOMIC], [ ]) +fi + +dnl ============================================================================ +dnl Check whether __sync_{add,sub}_and_fetch() are available despite +dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. + +AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[ + AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()], + [je_cv_sync_compare_and_swap_$2], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([ + #include + ], + [ + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 + { + uint$1_t x$1 = 0; + __sync_add_and_fetch(&x$1, 42); + __sync_sub_and_fetch(&x$1, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force + #endif + ])], + [je_cv_sync_compare_and_swap_$2=yes], + [je_cv_sync_compare_and_swap_$2=no])]) + + if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then + AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ]) + fi +]) + +if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then + JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4) + JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) fi dnl ============================================================================ @@ -811,69 +1253,171 @@ JE_COMPILABLE([Darwin OSSpin*()], [ OSSpinLock lock = 0; OSSpinLockLock(&lock); OSSpinLockUnlock(&lock); -], [osspin]) -if test "x${osspin}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSSPIN]) +], [je_cv_osspin]) +if test "x${je_cv_osspin}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSSPIN], [ ]) fi -dnl ============================================================================ -dnl Check for allocator-related functions that should be wrapped. - -AC_CHECK_FUNC([memalign], - [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN])]) -AC_CHECK_FUNC([valloc], - [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC])]) - dnl ============================================================================ dnl Darwin-related configuration. -if test "x${abi}" = "xmacho" ; then - AC_DEFINE([JEMALLOC_IVSALLOC]) - AC_DEFINE([JEMALLOC_ZONE]) +AC_ARG_ENABLE([zone-allocator], + [AS_HELP_STRING([--disable-zone-allocator], + [Disable zone allocator for Darwin])], +[if test "x$enable_zone_allocator" = "xno" ; then + enable_zone_allocator="0" +else + enable_zone_allocator="1" +fi +], +[if test "x${abi}" = "xmacho"; then + enable_zone_allocator="1" +fi +] +) +AC_SUBST([enable_zone_allocator]) + +if test "x${enable_zone_allocator}" = "x1" ; then + if test "x${abi}" != "xmacho"; then + AC_MSG_ERROR([--enable-zone-allocator is only supported on Darwin]) + fi + AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) + AC_DEFINE([JEMALLOC_ZONE], [ ]) dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 dnl releases. malloc_zone_t and malloc_introspection_t have new fields in dnl 10.6, which is the only source-level indication of the change. AC_MSG_CHECKING([malloc zone version]) - AC_TRY_COMPILE([#include -#include ], [ - static malloc_zone_t zone; - static struct malloc_introspection_t zone_introspect; - - zone.size = NULL; - zone.malloc = NULL; - zone.calloc = NULL; - zone.valloc = NULL; - zone.free = NULL; - zone.realloc = NULL; - zone.destroy = NULL; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = 6; - zone.memalign = NULL; - zone.free_definite_size = NULL; - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = NULL; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = NULL; - zone_introspect.force_unlock = NULL; - zone_introspect.statistics = NULL; - zone_introspect.zone_locked = NULL; -], [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [6]) - AC_MSG_RESULT([6])], - [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [3]) - AC_MSG_RESULT([3])]) + AC_DEFUN([JE_ZONE_PROGRAM], + [AC_LANG_PROGRAM( + [#include ], + [static foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]] + )]) + + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,14)],[JEMALLOC_ZONE_VERSION=3],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,15)],[JEMALLOC_ZONE_VERSION=5],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,16)],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,9)],[JEMALLOC_ZONE_VERSION=6],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,13)],[JEMALLOC_ZONE_VERSION=7],[JEMALLOC_ZONE_VERSION=] + )])],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,17)],[JEMALLOC_ZONE_VERSION=8],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,>,17)],[JEMALLOC_ZONE_VERSION=9],[JEMALLOC_ZONE_VERSION=] + )])])])]) + if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then + AC_MSG_RESULT([unsupported]) + AC_MSG_ERROR([Unsupported malloc zone version]) + fi + if test "${JEMALLOC_ZONE_VERSION}" = 9; then + JEMALLOC_ZONE_VERSION=8 + AC_MSG_RESULT([> 8]) + else + AC_MSG_RESULT([$JEMALLOC_ZONE_VERSION]) + fi + AC_DEFINE_UNQUOTED(JEMALLOC_ZONE_VERSION, [$JEMALLOC_ZONE_VERSION]) fi dnl ============================================================================ dnl Check for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL +dnl ============================================================================ +dnl Define commands that generate output files. + +AC_CONFIG_COMMANDS([include/jemalloc/internal/private_namespace.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/private_namespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_namespace.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/private_unnamespace.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/private_unnamespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_unnamespace.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/public_symbols.txt], [ + f="${objroot}include/jemalloc/internal/public_symbols.txt" + mkdir -p "${objroot}include/jemalloc/internal" + cp /dev/null "${f}" + for nm in `echo ${mangling_map} |tr ',' ' '` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $[]1}'` + m=`echo ${nm} |tr ':' ' ' |awk '{print $[]2}'` + echo "${n}:${m}" >> "${f}" + dnl Remove name from public_syms so that it isn't redefined later. + public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${n}\$" |tr '\n' ' '` + done + for sym in ${public_syms} ; do + n="${sym}" + m="${JEMALLOC_PREFIX}${sym}" + echo "${n}:${m}" >> "${f}" + done +], [ + srcdir="${srcdir}" + objroot="${objroot}" + mangling_map="${mangling_map}" + public_syms="${public_syms}" + JEMALLOC_PREFIX="${JEMALLOC_PREFIX}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/public_namespace.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/public_namespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_namespace.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/public_unnamespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_unnamespace.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ + mkdir -p "${objroot}include/jemalloc" + cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_rename.h], [ + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_rename.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/jemalloc_rename.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_mangle.h], [ + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" je_ > "${objroot}include/jemalloc/jemalloc_mangle.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_mangle_jet.h], [ + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" jet_ > "${objroot}include/jemalloc/jemalloc_mangle_jet.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" +]) +AC_CONFIG_COMMANDS([include/jemalloc/jemalloc.h], [ + mkdir -p "${objroot}include/jemalloc" + "${srcdir}/include/jemalloc/jemalloc.sh" "${objroot}" > "${objroot}include/jemalloc/jemalloc${install_suffix}.h" +], [ + srcdir="${srcdir}" + objroot="${objroot}" + install_suffix="${install_suffix}" +]) + dnl Process .in files. AC_SUBST([cfghdrs_in]) AC_SUBST([cfghdrs_out]) @@ -881,7 +1425,8 @@ AC_CONFIG_HEADERS([$cfghdrs_tup]) dnl ============================================================================ dnl Generate outputs. -AC_CONFIG_FILES([$cfgoutputs_tup config.stamp]) + +AC_CONFIG_FILES([$cfgoutputs_tup config.stamp bin/jemalloc.sh]) AC_SUBST([cfgoutputs_in]) AC_SUBST([cfgoutputs_out]) AC_OUTPUT @@ -889,12 +1434,14 @@ AC_OUTPUT dnl ============================================================================ dnl Print out the results of configuration. AC_MSG_RESULT([===============================================================================]) -AC_MSG_RESULT([jemalloc version : $jemalloc_version]) +AC_MSG_RESULT([jemalloc version : ${jemalloc_version}]) +AC_MSG_RESULT([library revision : ${rev}]) AC_MSG_RESULT([]) AC_MSG_RESULT([CC : ${CC}]) AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) AC_MSG_RESULT([CFLAGS : ${CFLAGS}]) AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) +AC_MSG_RESULT([EXTRA_LDFLAGS : ${EXTRA_LDFLAGS}]) AC_MSG_RESULT([LIBS : ${LIBS}]) AC_MSG_RESULT([RPATH_EXTRA : ${RPATH_EXTRA}]) AC_MSG_RESULT([]) @@ -918,21 +1465,23 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) +AC_MSG_RESULT([experimental : ${enable_experimental}]) AC_MSG_RESULT([cc-silence : ${enable_cc_silence}]) AC_MSG_RESULT([debug : ${enable_debug}]) +AC_MSG_RESULT([code-coverage : ${enable_code_coverage}]) AC_MSG_RESULT([stats : ${enable_stats}]) AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) -AC_MSG_RESULT([tiny : ${enable_tiny}]) AC_MSG_RESULT([tcache : ${enable_tcache}]) AC_MSG_RESULT([fill : ${enable_fill}]) +AC_MSG_RESULT([utrace : ${enable_utrace}]) +AC_MSG_RESULT([valgrind : ${enable_valgrind}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) -AC_MSG_RESULT([sysv : ${enable_sysv}]) -AC_MSG_RESULT([swap : ${enable_swap}]) +AC_MSG_RESULT([mremap : ${enable_mremap}]) +AC_MSG_RESULT([munmap : ${enable_munmap}]) AC_MSG_RESULT([dss : ${enable_dss}]) -AC_MSG_RESULT([dynamic_page_shift : ${enable_dynamic_page_shift}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([tls : ${enable_tls}]) AC_MSG_RESULT([===============================================================================]) diff --git a/deps/jemalloc/coverage.sh b/deps/jemalloc/coverage.sh new file mode 100755 index 00000000000..6d1362a8c1c --- /dev/null +++ b/deps/jemalloc/coverage.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +objdir=$1 +suffix=$2 +shift 2 +objs=$@ + +gcov -b -p -f -o "${objdir}" ${objs} + +# Move gcov outputs so that subsequent gcov invocations won't clobber results +# for the same sources with different compilation flags. +for f in `find . -maxdepth 1 -type f -name '*.gcov'` ; do + mv "${f}" "${f}.${suffix}" +done diff --git a/deps/jemalloc/doc/jemalloc.3 b/deps/jemalloc/doc/jemalloc.3 index 0401cfe895e..d04fbb498f8 100644 --- a/deps/jemalloc/doc/jemalloc.3 +++ b/deps/jemalloc/doc/jemalloc.3 @@ -1,13 +1,13 @@ '\" t .\" Title: JEMALLOC .\" Author: Jason Evans -.\" Generator: DocBook XSL Stylesheets v1.75.2 -.\" Date: 11/14/2011 +.\" Generator: DocBook XSL Stylesheets v1.78.1 +.\" Date: 03/31/2014 .\" Manual: User Manual -.\" Source: jemalloc 2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760 +.\" Source: jemalloc 3.6.0-0-g46c0af68bd248b04df75e4f92d5fb804c3d75340 .\" Language: English .\" -.TH "JEMALLOC" "3" "11/14/2011" "jemalloc 2.2.5-0-gfc1bb70e5f0d" "User Manual" +.TH "JEMALLOC" "3" "03/31/2014" "jemalloc 3.6.0-0-g46c0af68bd24" "User Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -31,7 +31,7 @@ jemalloc \- general purpose memory allocation functions .SH "LIBRARY" .PP -This manual describes jemalloc 2\&.2\&.5\-0\-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760\&. More information can be found at the +This manual describes jemalloc 3\&.6\&.0\-0\-g46c0af68bd248b04df75e4f92d5fb804c3d75340\&. More information can be found at the \m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&. .SH "SYNOPSIS" .sp @@ -48,21 +48,35 @@ This manual describes jemalloc 2\&.2\&.5\-0\-gfc1bb70e5f0d9a58b39efa39cc549b5af5 .BI "void *calloc(size_t\ " "number" ", size_t\ " "size" ");" .HP \w'int\ posix_memalign('u .BI "int posix_memalign(void\ **" "ptr" ", size_t\ " "alignment" ", size_t\ " "size" ");" +.HP \w'void\ *aligned_alloc('u +.BI "void *aligned_alloc(size_t\ " "alignment" ", size_t\ " "size" ");" .HP \w'void\ *realloc('u .BI "void *realloc(void\ *" "ptr" ", size_t\ " "size" ");" .HP \w'void\ free('u .BI "void free(void\ *" "ptr" ");" .SS "Non\-standard API" -.HP \w'size_t\ malloc_usable_size('u -.BI "size_t malloc_usable_size(const\ void\ *" "ptr" ");" -.HP \w'void\ malloc_stats_print('u -.BI "void malloc_stats_print(void\ " "(*write_cb)" "\ (void\ *,\ const\ char\ *), void\ *" "cbopaque" ", const\ char\ *" "opts" ");" +.HP \w'void\ *mallocx('u +.BI "void *mallocx(size_t\ " "size" ", int\ " "flags" ");" +.HP \w'void\ *rallocx('u +.BI "void *rallocx(void\ *" "ptr" ", size_t\ " "size" ", int\ " "flags" ");" +.HP \w'size_t\ xallocx('u +.BI "size_t xallocx(void\ *" "ptr" ", size_t\ " "size" ", size_t\ " "extra" ", int\ " "flags" ");" +.HP \w'size_t\ sallocx('u +.BI "size_t sallocx(void\ *" "ptr" ", int\ " "flags" ");" +.HP \w'void\ dallocx('u +.BI "void dallocx(void\ *" "ptr" ", int\ " "flags" ");" +.HP \w'size_t\ nallocx('u +.BI "size_t nallocx(size_t\ " "size" ", int\ " "flags" ");" .HP \w'int\ mallctl('u .BI "int mallctl(const\ char\ *" "name" ", void\ *" "oldp" ", size_t\ *" "oldlenp" ", void\ *" "newp" ", size_t\ " "newlen" ");" .HP \w'int\ mallctlnametomib('u .BI "int mallctlnametomib(const\ char\ *" "name" ", size_t\ *" "mibp" ", size_t\ *" "miblenp" ");" .HP \w'int\ mallctlbymib('u .BI "int mallctlbymib(const\ size_t\ *" "mib" ", size_t\ " "miblen" ", void\ *" "oldp" ", size_t\ *" "oldlenp" ", void\ *" "newp" ", size_t\ " "newlen" ");" +.HP \w'void\ malloc_stats_print('u +.BI "void malloc_stats_print(void\ " "(*write_cb)" "\ (void\ *,\ const\ char\ *), void\ *" "cbopaque" ", const\ char\ *" "opts" ");" +.HP \w'size_t\ malloc_usable_size('u +.BI "size_t malloc_usable_size(const\ void\ *" "ptr" ");" .HP \w'void\ (*malloc_message)('u .BI "void (*malloc_message)(void\ *" "cbopaque" ", const\ char\ *" "s" ");" .PP @@ -76,6 +90,8 @@ const char *\fImalloc_conf\fR; .BI "int sallocm(const\ void\ *" "ptr" ", size_t\ *" "rsize" ", int\ " "flags" ");" .HP \w'int\ dallocm('u .BI "int dallocm(void\ *" "ptr" ", int\ " "flags" ");" +.HP \w'int\ nallocm('u +.BI "int nallocm(size_t\ *" "rsize" ", size_t\ " "size" ", int\ " "flags" ");" .SH "DESCRIPTION" .SS "Standard API" .PP @@ -110,6 +126,18 @@ must be a power of 2 at least as large as sizeof(\fBvoid *\fR)\&. .PP The +\fBaligned_alloc\fR\fB\fR +function allocates +\fIsize\fR +bytes of memory such that the allocation\*(Aqs base address is an even multiple of +\fIalignment\fR\&. The requested +\fIalignment\fR +must be a power of 2\&. Behavior is undefined if +\fIsize\fR +is not an integral multiple of +\fIalignment\fR\&. +.PP +The \fBrealloc\fR\fB\fR function changes the size of the previously allocated memory referenced by \fIptr\fR @@ -140,36 +168,105 @@ is .SS "Non\-standard API" .PP The -\fBmalloc_usable_size\fR\fB\fR -function returns the usable size of the allocation pointed to by -\fIptr\fR\&. The return value may be larger than the size that was requested during allocation\&. The -\fBmalloc_usable_size\fR\fB\fR -function is not a mechanism for in\-place -\fBrealloc\fR\fB\fR; rather it is provided solely as a tool for introspection purposes\&. Any discrepancy between the requested allocation size and the size reported by -\fBmalloc_usable_size\fR\fB\fR -should not be depended on, since such behavior is entirely implementation\-dependent\&. +\fBmallocx\fR\fB\fR, +\fBrallocx\fR\fB\fR, +\fBxallocx\fR\fB\fR, +\fBsallocx\fR\fB\fR, +\fBdallocx\fR\fB\fR, and +\fBnallocx\fR\fB\fR +functions all have a +\fIflags\fR +argument that can be used to specify options\&. The functions only check the options that are contextually relevant\&. Use bitwise or (|) operations to specify one or more of the following: +.PP +\fBMALLOCX_LG_ALIGN(\fR\fB\fIla\fR\fR\fB) \fR +.RS 4 +Align the memory allocation to start at an address that is a multiple of +(1 << \fIla\fR)\&. This macro does not validate that +\fIla\fR +is within the valid range\&. +.RE +.PP +\fBMALLOCX_ALIGN(\fR\fB\fIa\fR\fR\fB) \fR +.RS 4 +Align the memory allocation to start at an address that is a multiple of +\fIa\fR, where +\fIa\fR +is a power of two\&. This macro does not validate that +\fIa\fR +is a power of 2\&. +.RE +.PP +\fBMALLOCX_ZERO\fR +.RS 4 +Initialize newly allocated memory to contain zero bytes\&. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those that are initialized to contain zero bytes\&. If this macro is absent, newly allocated memory is uninitialized\&. +.RE +.PP +\fBMALLOCX_ARENA(\fR\fB\fIa\fR\fR\fB) \fR +.RS 4 +Use the arena specified by the index +\fIa\fR +(and by necessity bypass the thread cache)\&. This macro has no effect for huge regions, nor for regions that were allocated via an arena other than the one specified\&. This macro does not validate that +\fIa\fR +specifies an arena index in the valid range\&. +.RE .PP The -\fBmalloc_stats_print\fR\fB\fR -function writes human\-readable summary statistics via the -\fIwrite_cb\fR -callback function pointer and -\fIcbopaque\fR -data passed to -\fIwrite_cb\fR, or -\fBmalloc_message\fR\fB\fR -if -\fIwrite_cb\fR +\fBmallocx\fR\fB\fR +function allocates at least +\fIsize\fR +bytes of memory, and returns a pointer to the base address of the allocation\&. Behavior is undefined if +\fIsize\fR is -\fBNULL\fR\&. This function can be called repeatedly\&. General information that never changes during execution can be omitted by specifying "g" as a character within the -\fIopts\fR -string\&. Note that -\fBmalloc_message\fR\fB\fR -uses the -\fBmallctl*\fR\fB\fR -functions internally, so inconsistent statistics can be reported if multiple threads use these functions simultaneously\&. If -\fB\-\-enable\-stats\fR -is specified during configuration, \(lqm\(rq and \(lqa\(rq can be specified to omit merged arena and per arena statistics, respectively; \(lqb\(rq and \(lql\(rq can be specified to omit per size class statistics for bins and large objects, respectively\&. Unrecognized characters are silently ignored\&. Note that thread caching may prevent some statistics from being completely up to date, since extra locking would be required to merge counters that track thread cache operations\&. +\fB0\fR, or if request size overflows due to size class and/or alignment constraints\&. +.PP +The +\fBrallocx\fR\fB\fR +function resizes the allocation at +\fIptr\fR +to be at least +\fIsize\fR +bytes, and returns a pointer to the base address of the resulting allocation, which may or may not have moved from its original location\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if request size overflows due to size class and/or alignment constraints\&. +.PP +The +\fBxallocx\fR\fB\fR +function resizes the allocation at +\fIptr\fR +in place to be at least +\fIsize\fR +bytes, and returns the real size of the allocation\&. If +\fIextra\fR +is non\-zero, an attempt is made to resize the allocation to be at least +(\fIsize\fR + \fIextra\fR) +bytes, though inability to allocate the extra byte(s) will not by itself result in failure to resize\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if +(\fIsize\fR + \fIextra\fR > \fBSIZE_T_MAX\fR)\&. +.PP +The +\fBsallocx\fR\fB\fR +function returns the real size of the allocation at +\fIptr\fR\&. +.PP +The +\fBdallocx\fR\fB\fR +function causes the memory referenced by +\fIptr\fR +to be made available for future allocations\&. +.PP +The +\fBnallocx\fR\fB\fR +function allocates no memory, but it performs the same size computation as the +\fBmallocx\fR\fB\fR +function, and returns the real size of the allocation that would result from the equivalent +\fBmallocx\fR\fB\fR +function call\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if request size overflows due to size class and/or alignment constraints\&. .PP The \fBmallctl\fR\fB\fR @@ -213,15 +310,14 @@ that is smaller than the number of period\-separated name components, which resu .\} .nf unsigned nbins, i; - -int mib[4]; +size_t mib[4]; size_t len, miblen; len = sizeof(nbins); mallctl("arenas\&.nbins", &nbins, &len, NULL, 0); miblen = 4; -mallnametomib("arenas\&.bin\&.0\&.size", mib, &miblen); +mallctlnametomib("arenas\&.bin\&.0\&.size", mib, &miblen); for (i = 0; i < nbins; i++) { size_t bin_size; @@ -234,15 +330,50 @@ for (i = 0; i < nbins; i++) { .if n \{\ .RE .\} +.PP +The +\fBmalloc_stats_print\fR\fB\fR +function writes human\-readable summary statistics via the +\fIwrite_cb\fR +callback function pointer and +\fIcbopaque\fR +data passed to +\fIwrite_cb\fR, or +\fBmalloc_message\fR\fB\fR +if +\fIwrite_cb\fR +is +\fBNULL\fR\&. This function can be called repeatedly\&. General information that never changes during execution can be omitted by specifying "g" as a character within the +\fIopts\fR +string\&. Note that +\fBmalloc_message\fR\fB\fR +uses the +\fBmallctl*\fR\fB\fR +functions internally, so inconsistent statistics can be reported if multiple threads use these functions simultaneously\&. If +\fB\-\-enable\-stats\fR +is specified during configuration, \(lqm\(rq and \(lqa\(rq can be specified to omit merged arena and per arena statistics, respectively; \(lqb\(rq and \(lql\(rq can be specified to omit per size class statistics for bins and large objects, respectively\&. Unrecognized characters are silently ignored\&. Note that thread caching may prevent some statistics from being completely up to date, since extra locking would be required to merge counters that track thread cache operations\&. +.PP +The +\fBmalloc_usable_size\fR\fB\fR +function returns the usable size of the allocation pointed to by +\fIptr\fR\&. The return value may be larger than the size that was requested during allocation\&. The +\fBmalloc_usable_size\fR\fB\fR +function is not a mechanism for in\-place +\fBrealloc\fR\fB\fR; rather it is provided solely as a tool for introspection purposes\&. Any discrepancy between the requested allocation size and the size reported by +\fBmalloc_usable_size\fR\fB\fR +should not be depended on, since such behavior is entirely implementation\-dependent\&. .SS "Experimental API" .PP -The experimental API is subject to change or removal without regard for backward compatibility\&. +The experimental API is subject to change or removal without regard for backward compatibility\&. If +\fB\-\-disable\-experimental\fR +is specified during configuration, the experimental API is omitted\&. .PP The \fBallocm\fR\fB\fR, \fBrallocm\fR\fB\fR, -\fBsallocm\fR\fB\fR, and -\fBdallocm\fR\fB\fR +\fBsallocm\fR\fB\fR, +\fBdallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions all have a \fIflags\fR argument that can be used to specify options\&. The functions only check the options that are contextually relevant\&. Use bitwise or (|) operations to specify one or more of the following: @@ -267,7 +398,7 @@ is a power of 2\&. .PP \fBALLOCM_ZERO\fR .RS 4 -Initialize newly allocated memory to contain zero bytes\&. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those that are initialized to contain zero bytes\&. If this option is absent, newly allocated memory is uninitialized\&. +Initialize newly allocated memory to contain zero bytes\&. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those that are initialized to contain zero bytes\&. If this macro is absent, newly allocated memory is uninitialized\&. .RE .PP \fBALLOCM_NO_MOVE\fR @@ -275,6 +406,15 @@ Initialize newly allocated memory to contain zero bytes\&. In the growing reallo For reallocation, fail rather than moving the object\&. This constraint can apply to both growth and shrinkage\&. .RE .PP +\fBALLOCM_ARENA(\fR\fB\fIa\fR\fR\fB) \fR +.RS 4 +Use the arena specified by the index +\fIa\fR +(and by necessity bypass the thread cache)\&. This macro has no effect for huge regions, nor for regions that were allocated via an arena other than the one specified\&. This macro does not validate that +\fIa\fR +specifies an arena index in the valid range\&. +.RE +.PP The \fBallocm\fR\fB\fR function allocates at least @@ -286,7 +426,10 @@ to the base address of the allocation, and sets to the real size of the allocation if \fIrsize\fR is not -\fBNULL\fR\&. +\fBNULL\fR\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if request size overflows due to size class and/or alignment constraints\&. .PP The \fBrallocm\fR\fB\fR @@ -304,8 +447,11 @@ is not \fBNULL\fR\&. If \fIextra\fR is non\-zero, an attempt is made to resize the allocation to be at least -\fIsize\fR + \fIextra\fR) +(\fIsize\fR + \fIextra\fR) bytes, though inability to allocate the extra byte(s) will not by itself result in failure\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, if request size overflows due to size class and/or alignment constraints, or if (\fIsize\fR + \fIextra\fR > \fBSIZE_T_MAX\fR)\&. .PP The @@ -319,6 +465,23 @@ The function causes the memory referenced by \fIptr\fR to be made available for future allocations\&. +.PP +The +\fBnallocm\fR\fB\fR +function allocates no memory, but it performs the same size computation as the +\fBallocm\fR\fB\fR +function, and if +\fIrsize\fR +is not +\fBNULL\fR +it sets +\fI*rsize\fR +to the real size of the allocation that would result from the equivalent +\fBallocm\fR\fB\fR +function call\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if request size overflows due to size class and/or alignment constraints\&. .SH "TUNING" .PP Once, when the first call is made to one of the memory allocation routines, the allocator initializes its internals based in part on various options that can be specified at compile\- or run\-time\&. @@ -326,7 +489,19 @@ Once, when the first call is made to one of the memory allocation routines, the The string pointed to by the global variable \fImalloc_conf\fR, the \(lqname\(rq of the file referenced by the symbolic link named /etc/malloc\&.conf, and the value of the environment variable -\fBMALLOC_CONF\fR, will be interpreted, in that order, from left to right as options\&. +\fBMALLOC_CONF\fR, will be interpreted, in that order, from left to right as options\&. Note that +\fImalloc_conf\fR +may be read before +\fBmain\fR\fB\fR +is entered, so the declaration of +\fImalloc_conf\fR +should specify an initializer that contains the final value to be read by jemalloc\&. +\fImalloc_conf\fR +is a compile\-time setting, whereas +/etc/malloc\&.conf +and +\fBMALLOC_CONF\fR +can be safely set any time prior to program invocation\&. .PP An options string is a comma\-separated list of option:value pairs\&. There is one key corresponding to each "opt\&.*" @@ -346,9 +521,9 @@ Traditionally, allocators have used to obtain memory, which is suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory\&. If \fB\-\-enable\-dss\fR is specified during configuration, this allocator uses both -\fBsbrk\fR(2) +\fBmmap\fR(2) and -\fBmmap\fR(2), in that order of preference; otherwise only +\fBsbrk\fR(2), in that order of preference; otherwise only \fBmmap\fR(2) is used\&. .PP @@ -364,14 +539,8 @@ User objects are broken into three categories according to size: small, large, a .PP Each chunk that is managed by an arena tracks its contents as runs of contiguous pages (unused, backing a set of small objects, or backing one large object)\&. The combination of chunk alignment and chunk page maps makes it possible to determine all metadata regarding small and large allocations in constant time\&. .PP -Small objects are managed in groups by page runs\&. Each run maintains a frontier and free list to track which regions are in use\&. Unless -\fB\-\-disable\-tiny\fR -is specified during configuration, allocation requests that are no more than half the quantum (8 or 16, depending on architecture) are rounded up to the nearest power of two that is at least -sizeof(\fBvoid *\fR)\&. Allocation requests that are more than half the quantum, but no more than the minimum cacheline\-multiple size class (see the -"opt\&.lg_qspace_max" -option) are rounded up to the nearest multiple of the quantum\&. Allocation requests that are more than the minimum cacheline\-multiple size class, but no more than the minimum subpage\-multiple size class (see the -"opt\&.lg_cspace_max" -option) are rounded up to the nearest multiple of the cacheline size (64)\&. Allocation requests that are more than the minimum subpage\-multiple size class, but no more than the maximum subpage\-multiple size class are rounded up to the nearest multiple of the subpage size (256)\&. Allocation requests that are more than the maximum subpage\-multiple size class, but small enough to fit in an arena\-managed chunk (see the +Small objects are managed in groups by page runs\&. Each run maintains a frontier and free list to track which regions are in use\&. Allocation requests that are no more than half the quantum (8 or 16, depending on architecture) are rounded up to the nearest power of two that is at least +sizeof(\fBdouble\fR)\&. All other small object size classes are multiples of the quantum, spaced such that internal fragmentation is limited to approximately 25% for all but the smallest size classes\&. Allocation requests that are larger than the maximum small size class, but small enough to fit in an arena\-managed chunk (see the "opt\&.lg_chunk" option), are rounded up to the nearest run size\&. Allocation requests that are too large to fit in an arena\-managed chunk are rounded up to the nearest multiple of the chunk size\&. .PP @@ -387,51 +556,73 @@ Table 1\&. .B Table\ \&1.\ \&Size classes .TS allbox tab(:); -lB lB lB. +lB rB lB. T{ Category T}:T{ -Subcategory +Spacing T}:T{ Size T} .T& -l l l -^ l l -^ l l -^ l l -l s l -l s l. +l r l +^ r l +^ r l +^ r l +^ r l +^ r l +^ r l +l r l +l r l. T{ Small T}:T{ -Tiny +lg T}:T{ [8] T} :T{ -Quantum\-spaced +16 T}:T{ [16, 32, 48, \&.\&.\&., 128] T} :T{ -Cacheline\-spaced +32 +T}:T{ +[160, 192, 224, 256] +T} +:T{ +64 +T}:T{ +[320, 384, 448, 512] +T} +:T{ +128 T}:T{ -[192, 256, 320, \&.\&.\&., 512] +[640, 768, 896, 1024] T} :T{ -Subpage\-spaced +256 T}:T{ -[768, 1024, 1280, \&.\&.\&., 3840] +[1280, 1536, 1792, 2048] +T} +:T{ +512 +T}:T{ +[2560, 3072, 3584] T} T{ Large T}:T{ +4 KiB +T}:T{ [4 KiB, 8 KiB, 12 KiB, \&.\&.\&., 4072 KiB] T} T{ Huge T}:T{ +4 MiB +T}:T{ [4 MiB, 8 MiB, 12 MiB, \&.\&.\&.] T} .TE @@ -481,12 +672,6 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP -"config\&.dynamic_page_shift" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-dynamic\-page\-shift\fR -was specified during build configuration\&. -.RE -.PP "config\&.fill" (\fBbool\fR) r\- .RS 4 \fB\-\-enable\-fill\fR @@ -499,6 +684,18 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP +"config\&.mremap" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-mremap\fR +was specified during build configuration\&. +.RE +.PP +"config\&.munmap" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-munmap\fR +was specified during build configuration\&. +.RE +.PP "config\&.prof" (\fBbool\fR) r\- .RS 4 \fB\-\-enable\-prof\fR @@ -523,34 +720,28 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP -"config\&.swap" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-swap\fR -was specified during build configuration\&. -.RE -.PP -"config\&.sysv" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-sysv\fR -was specified during build configuration\&. -.RE -.PP "config\&.tcache" (\fBbool\fR) r\- .RS 4 \fB\-\-disable\-tcache\fR was not specified during build configuration\&. .RE .PP -"config\&.tiny" (\fBbool\fR) r\- +"config\&.tls" (\fBbool\fR) r\- .RS 4 -\fB\-\-disable\-tiny\fR +\fB\-\-disable\-tls\fR was not specified during build configuration\&. .RE .PP -"config\&.tls" (\fBbool\fR) r\- +"config\&.utrace" (\fBbool\fR) r\- .RS 4 -\fB\-\-disable\-tls\fR -was not specified during build configuration\&. +\fB\-\-enable\-utrace\fR +was specified during build configuration\&. +.RE +.PP +"config\&.valgrind" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-valgrind\fR +was specified during build configuration\&. .RE .PP "config\&.xmalloc" (\fBbool\fR) r\- @@ -568,31 +759,30 @@ in these cases\&. This option is disabled by default unless is specified during configuration, in which case it is enabled by default\&. .RE .PP -"opt\&.lg_qspace_max" (\fBsize_t\fR) r\- -.RS 4 -Size (log base 2) of the maximum size class that is a multiple of the quantum (8 or 16 bytes, depending on architecture)\&. Above this size, cacheline spacing is used for size classes\&. The default value is 128 bytes (2^7)\&. -.RE -.PP -"opt\&.lg_cspace_max" (\fBsize_t\fR) r\- +"opt\&.dss" (\fBconst char *\fR) r\- .RS 4 -Size (log base 2) of the maximum size class that is a multiple of the cacheline size (64)\&. Above this size, subpage spacing (256 bytes) is used for size classes\&. The default value is 512 bytes (2^9)\&. +dss (\fBsbrk\fR(2)) allocation precedence as related to +\fBmmap\fR(2) +allocation\&. The following settings are supported: \(lqdisabled\(rq, \(lqprimary\(rq, and \(lqsecondary\(rq\&. The default is \(lqsecondary\(rq if +"config\&.dss" +is true, \(lqdisabled\(rq otherwise\&. .RE .PP "opt\&.lg_chunk" (\fBsize_t\fR) r\- .RS 4 -Virtual memory chunk size (log base 2)\&. The default chunk size is 4 MiB (2^22)\&. +Virtual memory chunk size (log base 2)\&. If a chunk size outside the supported size range is specified, the size is silently clipped to the minimum/maximum supported size\&. The default chunk size is 4 MiB (2^22)\&. .RE .PP "opt\&.narenas" (\fBsize_t\fR) r\- .RS 4 -Maximum number of arenas to use\&. The default maximum number of arenas is four times the number of CPUs, or one if there is a single CPU\&. +Maximum number of arenas to use for automatic multiplexing of threads and arenas\&. The default is four times the number of CPUs, or one if there is a single CPU\&. .RE .PP "opt\&.lg_dirty_mult" (\fBssize_t\fR) r\- .RS 4 Per\-arena minimum ratio (log base 2) of active to dirty pages\&. Some dirty unused pages may be allowed to accumulate, within the limit set by the ratio (or one chunk worth of dirty pages, whichever is greater), before informing the kernel about some of those pages via \fBmadvise\fR(2) -or a similar system call\&. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused\&. The default minimum ratio is 32:1 (2^5:1); an option value of \-1 will disable dirty page purging\&. +or a similar system call\&. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused\&. The default minimum ratio is 8:1 (2^3:1); an option value of \-1 will disable dirty page purging\&. .RE .PP "opt\&.stats_print" (\fBbool\fR) r\- @@ -612,25 +802,47 @@ Junk filling enabled/disabled\&. If enabled, each byte of uninitialized allocate 0xa5\&. All deallocated memory will be initialized to 0x5a\&. This is intended for debugging and will impact performance negatively\&. This option is disabled by default unless \fB\-\-enable\-debug\fR -is specified during configuration, in which case it is enabled by default\&. +is specified during configuration, in which case it is enabled by default unless running inside +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2\&. +.RE +.PP +"opt\&.quarantine" (\fBsize_t\fR) r\- [\fB\-\-enable\-fill\fR] +.RS 4 +Per thread quarantine size in bytes\&. If non\-zero, each thread maintains a FIFO object quarantine that stores up to the specified number of bytes of memory\&. The quarantined memory is not freed until it is released from quarantine, though it is immediately junk\-filled if the +"opt\&.junk" +option is enabled\&. This feature is of particular use in combination with +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2, which can detect attempts to access quarantined objects\&. This is intended for debugging and will impact performance negatively\&. The default quarantine size is 0 unless running inside Valgrind, in which case the default is 16 MiB\&. +.RE +.PP +"opt\&.redzone" (\fBbool\fR) r\- [\fB\-\-enable\-fill\fR] +.RS 4 +Redzones enabled/disabled\&. If enabled, small allocations have redzones before and after them\&. Furthermore, if the +"opt\&.junk" +option is enabled, the redzones are checked for corruption during deallocation\&. However, the primary intended purpose of this feature is to be used in combination with +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2, which needs redzones in order to do effective buffer overflow/underflow detection\&. This option is intended for debugging and will impact performance negatively\&. This option is disabled by default unless running inside Valgrind\&. .RE .PP "opt\&.zero" (\fBbool\fR) r\- [\fB\-\-enable\-fill\fR] .RS 4 Zero filling enabled/disabled\&. If enabled, each byte of uninitialized allocated memory will be initialized to 0\&. Note that this initialization only happens once for each byte, so -\fBrealloc\fR\fB\fR +\fBrealloc\fR\fB\fR, +\fBrallocx\fR\fB\fR and \fBrallocm\fR\fB\fR calls do not zero memory that was previously allocated\&. This is intended for debugging and will impact performance negatively\&. This option is disabled by default\&. .RE .PP -"opt\&.sysv" (\fBbool\fR) r\- [\fB\-\-enable\-sysv\fR] +"opt\&.utrace" (\fBbool\fR) r\- [\fB\-\-enable\-utrace\fR] .RS 4 -If enabled, attempting to allocate zero bytes will return a -\fBNULL\fR -pointer instead of a valid pointer\&. (The default behavior is to make a minimal allocation and return a pointer to it\&.) This option is provided for System V compatibility\&. This option is incompatible with the -"opt\&.xmalloc" -option\&. This option is disabled by default\&. +Allocation tracing based on +\fButrace\fR(2) +enabled/disabled\&. This option is disabled by default\&. +.RE +.PP +"opt\&.valgrind" (\fBbool\fR) r\- [\fB\-\-enable\-valgrind\fR] +.RS 4 +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2 +support enabled/disabled\&. This option is vestigal because jemalloc auto\-detects whether it is running inside Valgrind\&. This option is disabled by default, unless running inside Valgrind\&. .RE .PP "opt\&.xmalloc" (\fBbool\fR) r\- [\fB\-\-enable\-xmalloc\fR] @@ -656,15 +868,9 @@ This option is disabled by default\&. "opt\&.tcache" (\fBbool\fR) r\- [\fB\-\-enable\-tcache\fR] .RS 4 Thread\-specific caching enabled/disabled\&. When there are multiple threads, each thread uses a thread\-specific cache for objects up to a certain size\&. Thread\-specific caching allows many allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use\&. See the -"opt\&.lg_tcache_gc_sweep" -and "opt\&.lg_tcache_max" -options for related tuning information\&. This option is enabled by default\&. -.RE -.PP -"opt\&.lg_tcache_gc_sweep" (\fBssize_t\fR) r\- [\fB\-\-enable\-tcache\fR] -.RS 4 -Approximate interval (log base 2) between full thread\-specific cache garbage collection sweeps, counted in terms of thread\-specific cache allocation/deallocation events\&. Garbage collection is actually performed incrementally, one size class at a time, in order to avoid large collection pauses\&. The default sweep interval is 8192 (2^13); setting this option to \-1 will disable garbage collection\&. +option for related tuning information\&. This option is enabled by default unless running inside +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2\&. .RE .PP "opt\&.lg_tcache_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-tcache\fR] @@ -674,31 +880,22 @@ Maximum size class (log base 2) to cache in the thread\-specific cache\&. At a m .PP "opt\&.prof" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Memory profiling enabled/disabled\&. If enabled, profile memory allocation activity, and use an -\fBatexit\fR(3) -function to dump final memory usage to a file named according to the pattern -\&.\&.\&.f\&.heap, where - -is controlled by the -"opt\&.prof_prefix" -option\&. See the -"opt\&.lg_prof_bt_max" -option for backtrace depth control\&. See the +Memory profiling enabled/disabled\&. If enabled, profile memory allocation activity\&. See the "opt\&.prof_active" option for on\-the\-fly activation/deactivation\&. See the "opt\&.lg_prof_sample" option for probabilistic sampling control\&. See the "opt\&.prof_accum" option for control of cumulative sample reporting\&. See the -"opt\&.lg_prof_tcmax" -option for control of per thread backtrace caching\&. See the "opt\&.lg_prof_interval" -option for information on interval\-triggered profile dumping, and the +option for information on interval\-triggered profile dumping, the "opt\&.prof_gdump" -option for information on high\-water\-triggered profile dumping\&. Profile output is compatible with the included +option for information on high\-water\-triggered profile dumping, and the +"opt\&.prof_final" +option for final profile dumping\&. Profile output is compatible with the included \fBpprof\fR Perl script, which originates from the -\m[blue]\fBgoogle\-perftools package\fR\m[]\&\s-2\u[2]\d\s+2\&. +\m[blue]\fBgperftools package\fR\m[]\&\s-2\u[3]\d\s+2\&. .RE .PP "opt\&.prof_prefix" (\fBconst char *\fR) r\- [\fB\-\-enable\-prof\fR] @@ -707,12 +904,7 @@ Filename prefix for profile dumps\&. If the prefix is set to the empty string, n jeprof\&. .RE .PP -"opt\&.lg_prof_bt_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-prof\fR] -.RS 4 -Maximum backtrace depth (log base 2) when profiling memory allocation activity\&. The default is 128 (2^7)\&. -.RE -.PP -"opt\&.prof_active" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] +"opt\&.prof_active" (\fBbool\fR) rw [\fB\-\-enable\-prof\fR] .RS 4 Profiling activated/deactivated\&. This is a secondary control mechanism that makes it possible to start the application with profiling enabled (see the "opt\&.prof" @@ -723,21 +915,12 @@ mallctl\&. This option is enabled by default\&. .PP "opt\&.lg_prof_sample" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity\&. Increasing the sampling interval decreases profile fidelity, but also decreases the computational overhead\&. The default sample interval is 1 (2^0) (i\&.e\&. all allocations are sampled)\&. +Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity\&. Increasing the sampling interval decreases profile fidelity, but also decreases the computational overhead\&. The default sample interval is 512 KiB (2^19 B)\&. .RE .PP "opt\&.prof_accum" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Reporting of cumulative object/byte counts in profile dumps enabled/disabled\&. If this option is enabled, every unique backtrace must be stored for the duration of execution\&. Depending on the application, this can impose a large memory overhead, and the cumulative counts are not always of interest\&. See the -"opt\&.lg_prof_tcmax" -option for control of per thread backtrace caching, which has important interactions\&. This option is enabled by default\&. -.RE -.PP -"opt\&.lg_prof_tcmax" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] -.RS 4 -Maximum per thread backtrace cache (log base 2) used for heap profiling\&. A backtrace can only be discarded if the -"opt\&.prof_accum" -option is disabled, and no thread caches currently refer to the backtrace\&. Therefore, a backtrace cache limit should be imposed if the intention is to limit how much memory is used by backtraces\&. By default, no limit is imposed (encoded as \-1)\&. +Reporting of cumulative object/byte counts in profile dumps enabled/disabled\&. If this option is enabled, every unique backtrace must be stored for the duration of execution\&. Depending on the application, this can impose a large memory overhead, and the cumulative counts are not always of interest\&. This option is disabled by default\&. .RE .PP "opt\&.lg_prof_interval" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] @@ -760,38 +943,30 @@ is controlled by the option\&. This option is disabled by default\&. .RE .PP +"opt\&.prof_final" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] +.RS 4 +Use an +\fBatexit\fR(3) +function to dump final memory usage to a file named according to the pattern +\&.\&.\&.f\&.heap, where + +is controlled by the +"opt\&.prof_prefix" +option\&. This option is enabled by default\&. +.RE +.PP "opt\&.prof_leak" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 Leak reporting enabled/disabled\&. If enabled, use an \fBatexit\fR(3) function to report memory leaks detected by allocation sampling\&. See the -"opt\&.lg_prof_bt_max" -option for backtrace depth control\&. See the "opt\&.prof" option for information on analyzing heap profile output\&. This option is disabled by default\&. .RE .PP -"opt\&.overcommit" (\fBbool\fR) r\- [\fB\-\-enable\-swap\fR] -.RS 4 -Over\-commit enabled/disabled\&. If enabled, over\-commit memory as a side effect of using anonymous -\fBmmap\fR(2) -or -\fBsbrk\fR(2) -for virtual memory allocation\&. In order for overcommit to be disabled, the -"swap\&.fds" -mallctl must have been successfully written to\&. This option is enabled by default\&. -.RE -.PP -"tcache\&.flush" (\fBvoid\fR) \-\- [\fB\-\-enable\-tcache\fR] -.RS 4 -Flush calling thread\*(Aqs tcache\&. This interface releases all cached objects and internal data structures associated with the calling thread\*(Aqs thread\-specific cache\&. Ordinarily, this interface need not be called, since automatic periodic incremental garbage collection occurs, and the thread cache is automatically discarded when a thread exits\&. However, garbage collection is triggered by allocation activity, so it is possible for a thread that stops allocating/deallocating to retain its cache indefinitely, in which case the developer may find manual flushing useful\&. -.RE -.PP "thread\&.arena" (\fBunsigned\fR) rw .RS 4 -Get or set the arena associated with the calling thread\&. The arena index must be less than the maximum number of arenas (see the -"arenas\&.narenas" -mallctl)\&. If the specified arena was not initialized beforehand (see the +Get or set the arena associated with the calling thread\&. If the specified arena was not initialized beforehand (see the "arenas\&.initialized" mallctl), it will be automatically initialized as a side effect of calling this interface\&. .RE @@ -824,81 +999,51 @@ mallctl\&. This is useful for avoiding the overhead of repeated calls\&. .RE .PP -"arenas\&.narenas" (\fBunsigned\fR) r\- -.RS 4 -Maximum number of arenas\&. -.RE -.PP -"arenas\&.initialized" (\fBbool *\fR) r\- -.RS 4 -An array of -"arenas\&.narenas" -booleans\&. Each boolean indicates whether the corresponding arena is initialized\&. -.RE -.PP -"arenas\&.quantum" (\fBsize_t\fR) r\- -.RS 4 -Quantum size\&. -.RE -.PP -"arenas\&.cacheline" (\fBsize_t\fR) r\- -.RS 4 -Assumed cacheline size\&. -.RE -.PP -"arenas\&.subpage" (\fBsize_t\fR) r\- -.RS 4 -Subpage size class interval\&. -.RE -.PP -"arenas\&.pagesize" (\fBsize_t\fR) r\- -.RS 4 -Page size\&. -.RE -.PP -"arenas\&.chunksize" (\fBsize_t\fR) r\- -.RS 4 -Chunk size\&. -.RE -.PP -"arenas\&.tspace_min" (\fBsize_t\fR) r\- +"thread\&.tcache\&.enabled" (\fBbool\fR) rw [\fB\-\-enable\-tcache\fR] .RS 4 -Minimum tiny size class\&. Tiny size classes are powers of two\&. +Enable/disable calling thread\*(Aqs tcache\&. The tcache is implicitly flushed as a side effect of becoming disabled (see +"thread\&.tcache\&.flush")\&. .RE .PP -"arenas\&.tspace_max" (\fBsize_t\fR) r\- +"thread\&.tcache\&.flush" (\fBvoid\fR) \-\- [\fB\-\-enable\-tcache\fR] .RS 4 -Maximum tiny size class\&. Tiny size classes are powers of two\&. +Flush calling thread\*(Aqs tcache\&. This interface releases all cached objects and internal data structures associated with the calling thread\*(Aqs thread\-specific cache\&. Ordinarily, this interface need not be called, since automatic periodic incremental garbage collection occurs, and the thread cache is automatically discarded when a thread exits\&. However, garbage collection is triggered by allocation activity, so it is possible for a thread that stops allocating/deallocating to retain its cache indefinitely, in which case the developer may find manual flushing useful\&. .RE .PP -"arenas\&.qspace_min" (\fBsize_t\fR) r\- +"arena\&.\&.purge" (\fBunsigned\fR) \-\- .RS 4 -Minimum quantum\-spaced size class\&. +Purge unused dirty pages for arena , or for all arenas if equals +"arenas\&.narenas"\&. .RE .PP -"arenas\&.qspace_max" (\fBsize_t\fR) r\- +"arena\&.\&.dss" (\fBconst char *\fR) rw .RS 4 -Maximum quantum\-spaced size class\&. +Set the precedence of dss allocation as related to mmap allocation for arena , or for all arenas if equals +"arenas\&.narenas"\&. Note that even during huge allocation this setting is read from the arena that would be chosen for small or large allocation so that applications can depend on consistent dss versus mmap allocation regardless of allocation size\&. See +"opt\&.dss" +for supported settings\&. .RE .PP -"arenas\&.cspace_min" (\fBsize_t\fR) r\- +"arenas\&.narenas" (\fBunsigned\fR) r\- .RS 4 -Minimum cacheline\-spaced size class\&. +Current limit on number of arenas\&. .RE .PP -"arenas\&.cspace_max" (\fBsize_t\fR) r\- +"arenas\&.initialized" (\fBbool *\fR) r\- .RS 4 -Maximum cacheline\-spaced size class\&. +An array of +"arenas\&.narenas" +booleans\&. Each boolean indicates whether the corresponding arena is initialized\&. .RE .PP -"arenas\&.sspace_min" (\fBsize_t\fR) r\- +"arenas\&.quantum" (\fBsize_t\fR) r\- .RS 4 -Minimum subpage\-spaced size class\&. +Quantum size\&. .RE .PP -"arenas\&.sspace_max" (\fBsize_t\fR) r\- +"arenas\&.page" (\fBsize_t\fR) r\- .RS 4 -Maximum subpage\-spaced size class\&. +Page size\&. .RE .PP "arenas\&.tcache_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-tcache\fR] @@ -906,29 +1051,9 @@ Maximum subpage\-spaced size class\&. Maximum thread\-cached size class\&. .RE .PP -"arenas\&.ntbins" (\fBunsigned\fR) r\- -.RS 4 -Number of tiny bin size classes\&. -.RE -.PP -"arenas\&.nqbins" (\fBunsigned\fR) r\- -.RS 4 -Number of quantum\-spaced bin size classes\&. -.RE -.PP -"arenas\&.ncbins" (\fBunsigned\fR) r\- -.RS 4 -Number of cacheline\-spaced bin size classes\&. -.RE -.PP -"arenas\&.nsbins" (\fBunsigned\fR) r\- -.RS 4 -Number of subpage\-spaced bin size classes\&. -.RE -.PP "arenas\&.nbins" (\fBunsigned\fR) r\- .RS 4 -Total number of bin size classes\&. +Number of bin size classes\&. .RE .PP "arenas\&.nhbins" (\fBunsigned\fR) r\- [\fB\-\-enable\-tcache\fR] @@ -966,6 +1091,11 @@ Maximum size supported by this large size class\&. Purge unused dirty pages for the specified arena, or for all arenas if none is specified\&. .RE .PP +"arenas\&.extend" (\fBunsigned\fR) r\- +.RS 4 +Extend the array of arenas by appending a new arena, and returning the new arena index\&. +.RE +.PP "prof\&.active" (\fBbool\fR) rw [\fB\-\-enable\-prof\fR] .RS 4 Control whether sampling is currently active\&. See the @@ -1005,18 +1135,20 @@ Total number of bytes allocated by the application\&. "stats\&.active" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Total number of bytes in active pages allocated by the application\&. This is a multiple of the page size, and greater than or equal to -"stats\&.allocated"\&. +"stats\&.allocated"\&. This does not include +"stats\&.arenas\&.\&.pdirty" +and pages entirely devoted to allocator metadata\&. .RE .PP "stats\&.mapped" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Total number of bytes in chunks mapped on behalf of the application\&. This is a multiple of the chunk size, and is at least as large as -"stats\&.active"\&. This does not include inactive chunks backed by swap files\&. his does not include inactive chunks embedded in the DSS\&. +"stats\&.active"\&. This does not include inactive chunks\&. .RE .PP "stats\&.chunks\&.current" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 -Total number of chunks actively mapped on behalf of the application\&. This does not include inactive chunks backed by swap files\&. This does not include inactive chunks embedded in the DSS\&. +Total number of chunks actively mapped on behalf of the application\&. This does not include inactive chunks\&. .RE .PP "stats\&.chunks\&.total" (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] @@ -1044,6 +1176,15 @@ Cumulative number of huge allocation requests\&. Cumulative number of huge deallocation requests\&. .RE .PP +"stats\&.arenas\&.\&.dss" (\fBconst char *\fR) r\- +.RS 4 +dss (\fBsbrk\fR(2)) allocation precedence as related to +\fBmmap\fR(2) +allocation\&. See +"opt\&.dss" +for details\&. +.RE +.PP "stats\&.arenas\&.\&.nthreads" (\fBunsigned\fR) r\- .RS 4 Number of threads currently assigned to arena\&. @@ -1078,7 +1219,7 @@ Number of or similar calls made to purge dirty pages\&. .RE .PP -"stats\&.arenas\&.\&.npurged" (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] +"stats\&.arenas\&.\&.purged" (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Number of pages purged\&. .RE @@ -1163,11 +1304,6 @@ Cumulative number of runs created\&. Cumulative number of times the current run from which to allocate changed\&. .RE .PP -"stats\&.arenas\&.\&.bins\&.\&.highruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] -.RS 4 -Maximum number of runs at any time thus far\&. -.RE -.PP "stats\&.arenas\&.\&.bins\&.\&.curruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Current number of runs\&. @@ -1188,44 +1324,10 @@ Cumulative number of deallocation requests for this size class served directly b Cumulative number of allocation requests for this size class\&. .RE .PP -"stats\&.arenas\&.\&.lruns\&.\&.highruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] -.RS 4 -Maximum number of runs at any time thus far for this size class\&. -.RE -.PP "stats\&.arenas\&.\&.lruns\&.\&.curruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Current number of runs for this size class\&. .RE -.PP -"swap\&.avail" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats \-\-enable\-swap\fR] -.RS 4 -Number of swap file bytes that are currently not associated with any chunk (i\&.e\&. mapped, but otherwise completely unmanaged)\&. -.RE -.PP -"swap\&.prezeroed" (\fBbool\fR) rw [\fB\-\-enable\-swap\fR] -.RS 4 -If true, the allocator assumes that the swap file(s) contain nothing but nil bytes\&. If this assumption is violated, allocator behavior is undefined\&. This value becomes read\-only after -"swap\&.fds" -is successfully written to\&. -.RE -.PP -"swap\&.nfds" (\fBsize_t\fR) r\- [\fB\-\-enable\-swap\fR] -.RS 4 -Number of file descriptors in use for swap\&. -.RE -.PP -"swap\&.fds" (\fBint *\fR) rw [\fB\-\-enable\-swap\fR] -.RS 4 -When written to, the files associated with the specified file descriptors are contiguously mapped via -\fBmmap\fR(2)\&. The resulting virtual memory region is preferred over anonymous -\fBmmap\fR(2) -and -\fBsbrk\fR(2) -memory\&. Note that if a file\*(Aqs size is not a multiple of the page size, it is automatically truncated to the nearest page size multiple\&. See the -"swap\&.prezeroed" -mallctl for specifying that the files are pre\-zeroed\&. -.RE .SH "DEBUGGING MALLOC PROBLEMS" .PP When debugging, it is a good idea to configure/build jemalloc with the @@ -1240,7 +1342,11 @@ option) tends to expose such bugs in the form of obviously incorrect results and "opt\&.zero" option) eliminates the symptoms of such bugs\&. Between these two options, it is usually possible to quickly detect, diagnose, and eliminate such bugs\&. .PP -This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information would be prohibitive\&. There are a number of allocator implementations available on the Internet which focus on detecting and pinpointing problems by trading performance for extra sanity checks and detailed diagnostics\&. +This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information would be prohibitive\&. However, jemalloc does integrate with the most excellent +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2 +tool if the +\fB\-\-enable\-valgrind\fR +configuration option is enabled\&. .SH "DIAGNOSTIC MESSAGES" .PP If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor @@ -1296,6 +1402,28 @@ Memory allocation error\&. .RE .PP The +\fBaligned_alloc\fR\fB\fR +function returns a pointer to the allocated memory if successful; otherwise a +\fBNULL\fR +pointer is returned and +\fIerrno\fR +is set\&. The +\fBaligned_alloc\fR\fB\fR +function will fail if: +.PP +EINVAL +.RS 4 +The +\fIalignment\fR +parameter is not a power of 2\&. +.RE +.PP +ENOMEM +.RS 4 +Memory allocation error\&. +.RE +.PP +The \fBrealloc\fR\fB\fR function returns a pointer, possibly identical to \fIptr\fR, to the allocated memory if successful; otherwise a @@ -1314,11 +1442,32 @@ function returns no value\&. .SS "Non\-standard API" .PP The -\fBmalloc_usable_size\fR\fB\fR -function returns the usable size of the allocation pointed to by +\fBmallocx\fR\fB\fR +and +\fBrallocx\fR\fB\fR +functions return a pointer to the allocated memory if successful; otherwise a +\fBNULL\fR +pointer is returned to indicate insufficient contiguous memory was available to service the allocation request\&. +.PP +The +\fBxallocx\fR\fB\fR +function returns the real size of the resulting resized allocation pointed to by +\fIptr\fR, which is a value less than +\fIsize\fR +if the allocation could not be adequately grown in place\&. +.PP +The +\fBsallocx\fR\fB\fR +function returns the real size of the allocation pointed to by \fIptr\fR\&. .PP The +\fBnallocx\fR\fB\fR +returns the real size that would result from a successful equivalent +\fBmallocx\fR\fB\fR +function call, or zero if insufficient memory is available to perform the size computation\&. +.PP +The \fBmallctl\fR\fB\fR, \fBmallctlnametomib\fR\fB\fR, and \fBmallctlbymib\fR\fB\fR @@ -1335,12 +1484,6 @@ is too large or too small\&. Alternatively, is too large or too small; in this case as much data as possible are read despite the error\&. .RE .PP -ENOMEM -.RS 4 -\fI*oldlenp\fR -is too short to hold the requested value\&. -.RE -.PP ENOENT .RS 4 \fIname\fR @@ -1365,19 +1508,25 @@ An interface with side effects failed in some way not directly related to \fBmallctl*\fR\fB\fR read/write processing\&. .RE +.PP +The +\fBmalloc_usable_size\fR\fB\fR +function returns the usable size of the allocation pointed to by +\fIptr\fR\&. .SS "Experimental API" .PP The \fBallocm\fR\fB\fR, \fBrallocm\fR\fB\fR, -\fBsallocm\fR\fB\fR, and -\fBdallocm\fR\fB\fR +\fBsallocm\fR\fB\fR, +\fBdallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions return \fBALLOCM_SUCCESS\fR on success; otherwise they return an error value\&. The -\fBallocm\fR\fB\fR -and -\fBrallocm\fR\fB\fR +\fBallocm\fR\fB\fR, +\fBrallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions will fail if: .PP ALLOCM_ERR_OOM @@ -1442,6 +1591,7 @@ malloc_conf = "lg_chunk:24"; \fBmadvise\fR(2), \fBmmap\fR(2), \fBsbrk\fR(2), +\fButrace\fR(2), \fBalloca\fR(3), \fBatexit\fR(3), \fBgetpagesize\fR(3) @@ -1469,7 +1619,12 @@ jemalloc website \%http://www.canonware.com/jemalloc/ .RE .IP " 2." 4 -google-perftools package +Valgrind +.RS 4 +\%http://valgrind.org/ +.RE +.IP " 3." 4 +gperftools package .RS 4 -\%http://code.google.com/p/google-perftools/ +\%http://code.google.com/p/gperftools/ .RE diff --git a/deps/jemalloc/doc/jemalloc.html b/deps/jemalloc/doc/jemalloc.html index fc2ba878e54..5a9fc778997 100644 --- a/deps/jemalloc/doc/jemalloc.html +++ b/deps/jemalloc/doc/jemalloc.html @@ -1,8 +1,8 @@ -JEMALLOC

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760. More information - can be found at the jemalloc website.

SYNOPSIS

#include <stdlib.h>
-#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

size_t malloc_usable_size(const void *ptr);
 
void malloc_stats_print(void (*write_cb) +JEMALLOC

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 3.6.0-0-g46c0af68bd248b04df75e4f92d5fb804c3d75340. More information + can be found at the jemalloc website.

SYNOPSIS

#include <stdlib.h>
+#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *aligned_alloc(size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

void *mallocx(size_t size,
 int flags);
 
void *rallocx(void *ptr,
 size_t size,
 int flags);
 
size_t xallocx(void *ptr,
 size_t size,
 size_t extra,
 int flags);
 
size_t sallocx(void *ptr,
 int flags);
 
void dallocx(void *ptr,
 int flags);
 
size_t nallocx(size_t size,
 int flags);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void malloc_stats_print(void (*write_cb) (void *, const char *) - ,
 void *cbopaque,
 const char *opts);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates + ,

 void *cbopaque,
 const char *opts);
 
size_t malloc_usable_size(const void *ptr);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 
int nallocm(size_t *rsize,
 size_t size,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates size bytes of uninitialized memory. The allocated space is suitably aligned (after possible pointer coercion) for storage of any type of object.

The calloc() function allocates @@ -17,7 +17,13 @@ alignment, and returns the allocation in the value pointed to by ptr. The requested alignment must be a power of 2 at least as large - as sizeof(void *).

The realloc() function changes the + as sizeof(void *).

The aligned_alloc() function + allocates size bytes of memory such that the + allocation's base address is an even multiple of + alignment. The requested + alignment must be a power of 2. Behavior is + undefined if size is not an integral multiple of + alignment.

The realloc() function changes the size of the previously allocated memory referenced by ptr to size bytes. The contents of the memory are unchanged up to the lesser of the new and old @@ -32,38 +38,67 @@ malloc() for the specified size.

The free() function causes the allocated memory referenced by ptr to be made available for future allocations. If ptr is - NULL, no action occurs.

Non-standard API

The malloc_usable_size() function - returns the usable size of the allocation pointed to by - ptr. The return value may be larger than the size - that was requested during allocation. The - malloc_usable_size() function is not a - mechanism for in-place realloc(); rather - it is provided solely as a tool for introspection purposes. Any - discrepancy between the requested allocation size and the size reported - by malloc_usable_size() should not be - depended on, since such behavior is entirely implementation-dependent. -

The malloc_stats_print() function - writes human-readable summary statistics via the - write_cb callback function pointer and - cbopaque data passed to - write_cb, or - malloc_message() if - write_cb is NULL. This - function can be called repeatedly. General information that never - changes during execution can be omitted by specifying "g" as a character - within the opts string. Note that - malloc_message() uses the - mallctl*() functions internally, so - inconsistent statistics can be reported if multiple threads use these - functions simultaneously. If --enable-stats is - specified during configuration, “m” and “a” can - be specified to omit merged arena and per arena statistics, respectively; - “b” and “l” can be specified to omit per size - class statistics for bins and large objects, respectively. Unrecognized - characters are silently ignored. Note that thread caching may prevent - some statistics from being completely up to date, since extra locking - would be required to merge counters that track thread cache operations. -

The mallctl() function provides a + NULL, no action occurs.

Non-standard API

The mallocx(), + rallocx(), + xallocx(), + sallocx(), + dallocx(), and + nallocx() functions all have a + flags argument that can be used to specify + options. The functions only check the options that are contextually + relevant. Use bitwise or (|) operations to + specify one or more of the following: +

MALLOCX_LG_ALIGN(la) +

Align the memory allocation to start at an address + that is a multiple of (1 << + la). This macro does not validate + that la is within the valid + range.

MALLOCX_ALIGN(a) +

Align the memory allocation to start at an address + that is a multiple of a, where + a is a power of two. This macro does not + validate that a is a power of 2. +

MALLOCX_ZERO

Initialize newly allocated memory to contain zero + bytes. In the growing reallocation case, the real size prior to + reallocation defines the boundary between untouched bytes and those + that are initialized to contain zero bytes. If this macro is + absent, newly allocated memory is uninitialized.

MALLOCX_ARENA(a) +

Use the arena specified by the index + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range.

+

The mallocx() function allocates at + least size bytes of memory, and returns a pointer + to the base address of the allocation. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints.

The rallocx() function resizes the + allocation at ptr to be at least + size bytes, and returns a pointer to the base + address of the resulting allocation, which may or may not have moved from + its original location. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints.

The xallocx() function resizes the + allocation at ptr in place to be at least + size bytes, and returns the real size of the + allocation. If extra is non-zero, an attempt is + made to resize the allocation to be at least (size + + extra) bytes, though inability to allocate + the extra byte(s) will not by itself result in failure to resize. + Behavior is undefined if size is + 0, or if (size + extra + > SIZE_T_MAX).

The sallocx() function returns the + real size of the allocation at ptr.

The dallocx() function causes the + memory referenced by ptr to be made available for + future allocations.

The nallocx() function allocates no + memory, but it performs the same size computation as the + mallocx() function, and returns the real + size of the allocation that would result from the equivalent + mallocx() function call. Behavior is + undefined if size is 0, or if + request size overflows due to size class and/or alignment + constraints.

The mallctl() function provides a general interface for introspecting the memory allocator, as well as setting modifiable parameters and triggering actions. The period-separated name argument specifies a @@ -96,15 +131,14 @@ the corresponding MIB component will always be that integer. Therefore, it is legitimate to construct code like the following: 

 unsigned nbins, i;
-
-int mib[4];
+size_t mib[4];
 size_t len, miblen;
 
 len = sizeof(nbins);
 mallctl("arenas.nbins", &nbins, &len, NULL, 0);
 
 miblen = 4;
-mallnametomib("arenas.bin.0.size", mib, &miblen);
+mallctlnametomib("arenas.bin.0.size", mib, &miblen);
 for (i = 0; i < nbins; i++) {
 	size_t bin_size;
 
@@ -112,16 +146,50 @@
 	len = sizeof(bin_size);
 	mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0);
 	/* Do something with bin_size... */
-}

Experimental API

The experimental API is subject to change or removal without regard - for backward compatibility.

The allocm(), +}

The malloc_stats_print() function + writes human-readable summary statistics via the + write_cb callback function pointer and + cbopaque data passed to + write_cb, or + malloc_message() if + write_cb is NULL. This + function can be called repeatedly. General information that never + changes during execution can be omitted by specifying "g" as a character + within the opts string. Note that + malloc_message() uses the + mallctl*() functions internally, so + inconsistent statistics can be reported if multiple threads use these + functions simultaneously. If --enable-stats is + specified during configuration, “m” and “a” can + be specified to omit merged arena and per arena statistics, respectively; + “b” and “l” can be specified to omit per size + class statistics for bins and large objects, respectively. Unrecognized + characters are silently ignored. Note that thread caching may prevent + some statistics from being completely up to date, since extra locking + would be required to merge counters that track thread cache operations. +

The malloc_usable_size() function + returns the usable size of the allocation pointed to by + ptr. The return value may be larger than the size + that was requested during allocation. The + malloc_usable_size() function is not a + mechanism for in-place realloc(); rather + it is provided solely as a tool for introspection purposes. Any + discrepancy between the requested allocation size and the size reported + by malloc_usable_size() should not be + depended on, since such behavior is entirely implementation-dependent. +

Experimental API

The experimental API is subject to change or removal without regard + for backward compatibility. If --disable-experimental + is specified during configuration, the experimental API is + omitted.

The allocm(), rallocm(), - sallocm(), and - dallocm() functions all have a + sallocm(), + dallocm(), and + nallocm() functions all have a flags argument that can be used to specify options. The functions only check the options that are contextually relevant. Use bitwise or (|) operations to specify one or more of the following: -

ALLOCM_LG_ALIGN(la) +

ALLOCM_LG_ALIGN(la)

Align the memory allocation to start at an address that is a multiple of (1 << la). This macro does not validate @@ -134,36 +202,61 @@

ALLOCM_ZERO

Initialize newly allocated memory to contain zero bytes. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those - that are initialized to contain zero bytes. If this option is + that are initialized to contain zero bytes. If this macro is absent, newly allocated memory is uninitialized.

ALLOCM_NO_MOVE

For reallocation, fail rather than moving the object. This constraint can apply to both growth and - shrinkage.

+ shrinkage.

ALLOCM_ARENA(a) +

Use the arena specified by the index + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range.

The allocm() function allocates at least size bytes of memory, sets *ptr to the base address of the allocation, and sets *rsize to the real size of the allocation if - rsize is not NULL.

The rallocm() function resizes the + rsize is not NULL. Behavior + is undefined if size is 0, or + if request size overflows due to size class and/or alignment + constraints.

The rallocm() function resizes the allocation at *ptr to be at least size bytes, sets *ptr to the base address of the allocation if it moved, and sets *rsize to the real size of the allocation if rsize is not NULL. If extra is non-zero, an attempt is made to resize - the allocation to be at least size + + the allocation to be at least (size + extra) bytes, though inability to allocate the extra byte(s) will not by itself result in failure. Behavior is - undefined if (size + + undefined if size is 0, if + request size overflows due to size class and/or alignment constraints, or + if (size + extra > SIZE_T_MAX).

The sallocm() function sets *rsize to the real size of the allocation.

The dallocm() function causes the memory referenced by ptr to be made available for - future allocations.

TUNING

Once, when the first call is made to one of the memory allocation + future allocations.

The nallocm() function allocates no + memory, but it performs the same size computation as the + allocm() function, and if + rsize is not NULL it sets + *rsize to the real size of the allocation that + would result from the equivalent allocm() + function call. Behavior is undefined if size is + 0, or if request size overflows due to size class + and/or alignment constraints.

TUNING

Once, when the first call is made to one of the memory allocation routines, the allocator initializes its internals based in part on various options that can be specified at compile- or run-time.

The string pointed to by the global variable malloc_conf, the “name” of the file referenced by the symbolic link named /etc/malloc.conf, and the value of the environment variable MALLOC_CONF, will be interpreted, in - that order, from left to right as options.

An options string is a comma-separated list of option:value pairs. + that order, from left to right as options. Note that + malloc_conf may be read before + main() is entered, so the declaration of + malloc_conf should specify an initializer that contains + the final value to be read by jemalloc. malloc_conf is + a compile-time setting, whereas /etc/malloc.conf and MALLOC_CONF + can be safely set any time prior to program invocation.

An options string is a comma-separated list of option:value pairs. There is one key corresponding to each "opt.*" mallctl (see the MALLCTL NAMESPACE section for options @@ -175,13 +268,13 @@ options. Some options have boolean values (true/false), others have integer values (base 8, 10, or 16, depending on prefix), and yet others have raw string - values.

IMPLEMENTATION NOTES

Traditionally, allocators have used + values.

IMPLEMENTATION NOTES

Traditionally, allocators have used sbrk(2) to obtain memory, which is suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory. If --enable-dss is specified during configuration, this - allocator uses both sbrk(2) and - mmap(2), in that order of preference; + allocator uses both mmap(2) and + sbrk(2), in that order of preference; otherwise only mmap(2) is used.

This allocator uses multiple arenas in order to reduce lock contention for threaded programs on multi-processor systems. This works well with regard to threading scalability, but incurs some costs. There is @@ -212,26 +305,14 @@ large object). The combination of chunk alignment and chunk page maps makes it possible to determine all metadata regarding small and large allocations in constant time.

Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Unless - --disable-tiny is specified during configuration, - allocation requests that are no more than half the quantum (8 or 16, - depending on architecture) are rounded up to the nearest power of two that - is at least sizeof(void *). - Allocation requests that are more than half the quantum, but no more than - the minimum cacheline-multiple size class (see the - "opt.lg_qspace_max" - - option) are rounded up to the nearest multiple of the quantum. Allocation - requests that are more than the minimum cacheline-multiple size class, but - no more than the minimum subpage-multiple size class (see the - "opt.lg_cspace_max" - - option) are rounded up to the nearest multiple of the cacheline size (64). - Allocation requests that are more than the minimum subpage-multiple size - class, but no more than the maximum subpage-multiple size class are rounded - up to the nearest multiple of the subpage size (256). Allocation requests - that are more than the maximum subpage-multiple size class, but small - enough to fit in an arena-managed chunk (see the + a frontier and free list to track which regions are in use. Allocation + requests that are no more than half the quantum (8 or 16, depending on + architecture) are rounded up to the nearest power of two that is at least + sizeof(double). All other small + object size classes are multiples of the quantum, spaced such that internal + fragmentation is limited to approximately 25% for all but the smallest size + classes. Allocation requests that are larger than the maximum small size + class, but small enough to fit in an arena-managed chunk (see the "opt.lg_chunk" option), are rounded up to the nearest run size. Allocation requests that are too large @@ -241,7 +322,7 @@ suffer from cacheline sharing, round your allocation requests up to the nearest multiple of the cacheline size, or specify cacheline alignment when allocating.

Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit - system, the size classes in each category are as shown in Table 1.

Table 1. Size classes

CategorySubcategorySize
SmallTiny[8]
Quantum-spaced[16, 32, 48, ..., 128]
Cacheline-spaced[192, 256, 320, ..., 512]
Subpage-spaced[768, 1024, 1280, ..., 3840]
Large[4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB]
Huge[4 MiB, 8 MiB, 12 MiB, ...]

MALLCTL NAMESPACE

The following names are defined in the namespace accessible via the + system, the size classes in each category are as shown in Table 1.

Table 1. Size classes

CategorySpacingSize
Smalllg[8]
16[16, 32, 48, ..., 128]
32[160, 192, 224, 256]
64[320, 384, 448, 512]
128[640, 768, 896, 1024]
256[1280, 1536, 1792, 2048]
512[2560, 3072, 3584]
Large4 KiB[4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB]
Huge4 MiB[4 MiB, 8 MiB, 12 MiB, ...]

MALLCTL NAMESPACE

The following names are defined in the namespace accessible via the mallctl*() functions. Value types are specified in parentheses, their readable/writable statuses are encoded as rw, r-, -w, or @@ -259,7 +340,7 @@ note of the "epoch" mallctl, - which controls refreshing of cached dynamic statistics.

+ which controls refreshing of cached dynamic statistics.

"version" @@ -274,105 +355,105 @@

If a value is passed in, refresh the data from which the mallctl*() functions report values, and increment the epoch. Return the current epoch. This is useful for - detecting whether another thread caused a refresh.

+ detecting whether another thread caused a refresh.

"config.debug" (bool) r-

--enable-debug was specified during - build configuration.

+ build configuration.

"config.dss" (bool) r-

--enable-dss was specified during - build configuration.

- - "config.dynamic_page_shift" - - (bool) - r- -

--enable-dynamic-page-shift was - specified during build configuration.

+ build configuration.

"config.fill" (bool) r-

--enable-fill was specified during - build configuration.

+ build configuration.

"config.lazy_lock" (bool) r-

--enable-lazy-lock was specified - during build configuration.

+ during build configuration.

+ + "config.mremap" + + (bool) + r- +

--enable-mremap was specified during + build configuration.

+ + "config.munmap" + + (bool) + r- +

--enable-munmap was specified during + build configuration.

"config.prof" (bool) r-

--enable-prof was specified during - build configuration.

+ build configuration.

"config.prof_libgcc" (bool) r-

--disable-prof-libgcc was not - specified during build configuration.

+ specified during build configuration.

"config.prof_libunwind" (bool) r-

--enable-prof-libunwind was specified - during build configuration.

+ during build configuration.

"config.stats" (bool) r-

--enable-stats was specified during - build configuration.

- - "config.swap" - - (bool) - r- -

--enable-swap was specified during - build configuration.

+ build configuration.

- "config.sysv" + "config.tcache" (bool) r- -

--enable-sysv was specified during - build configuration.

+

--disable-tcache was not specified + during build configuration.

- "config.tcache" + "config.tls" (bool) r- -

--disable-tcache was not specified - during build configuration.

+

--disable-tls was not specified during + build configuration.

- "config.tiny" + "config.utrace" (bool) r- -

--disable-tiny was not specified - during build configuration.

+

--enable-utrace was specified during + build configuration.

- "config.tls" + "config.valgrind" (bool) r- -

--disable-tls was not specified during - build configuration.

+

--enable-valgrind was specified during + build configuration.

"config.xmalloc" @@ -390,40 +471,39 @@ abort(3) in these cases. This option is disabled by default unless --enable-debug is specified during configuration, in which case it is enabled by default. -

- - "opt.lg_qspace_max" - - (size_t) - r- -

Size (log base 2) of the maximum size class that is a - multiple of the quantum (8 or 16 bytes, depending on architecture). - Above this size, cacheline spacing is used for size classes. The - default value is 128 bytes (2^7).

+

- "opt.lg_cspace_max" + "opt.dss" - (size_t) + (const char *) r- -

Size (log base 2) of the maximum size class that is a - multiple of the cacheline size (64). Above this size, subpage spacing - (256 bytes) is used for size classes. The default value is 512 bytes - (2^9).

+

dss (sbrk(2)) allocation precedence as + related to mmap(2) allocation. The following + settings are supported: “disabled”, “primary”, + and “secondary”. The default is “secondary” if + + "config.dss" + is + true, “disabled” otherwise. +

"opt.lg_chunk" (size_t) r- -

Virtual memory chunk size (log base 2). The default - chunk size is 4 MiB (2^22).

+

Virtual memory chunk size (log base 2). If a chunk + size outside the supported size range is specified, the size is + silently clipped to the minimum/maximum supported size. The default + chunk size is 4 MiB (2^22). +

"opt.narenas" (size_t) r- -

Maximum number of arenas to use. The default maximum - number of arenas is four times the number of CPUs, or one if there is a - single CPU.

+

Maximum number of arenas to use for automatic + multiplexing of threads and arenas. The default is four times the + number of CPUs, or one if there is a single CPU.

"opt.lg_dirty_mult" @@ -436,7 +516,7 @@ pages via madvise(2) or a similar system call. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused. The - default minimum ratio is 32:1 (2^5:1); an option value of -1 will + default minimum ratio is 8:1 (2^3:1); an option value of -1 will disable dirty page purging.

"opt.stats_print" @@ -465,7 +545,44 @@ 0x5a. This is intended for debugging and will impact performance negatively. This option is disabled by default unless --enable-debug is specified during - configuration, in which case it is enabled by default.

+ configuration, in which case it is enabled by default unless running + inside Valgrind.

+ + "opt.quarantine" + + (size_t) + r- + [--enable-fill] +

Per thread quarantine size in bytes. If non-zero, each + thread maintains a FIFO object quarantine that stores up to the + specified number of bytes of memory. The quarantined memory is not + freed until it is released from quarantine, though it is immediately + junk-filled if the + "opt.junk" + option is + enabled. This feature is of particular use in combination with Valgrind, which can detect attempts + to access quarantined objects. This is intended for debugging and will + impact performance negatively. The default quarantine size is 0 unless + running inside Valgrind, in which case the default is 16 + MiB.

+ + "opt.redzone" + + (bool) + r- + [--enable-fill] +

Redzones enabled/disabled. If enabled, small + allocations have redzones before and after them. Furthermore, if the + + "opt.junk" + option is + enabled, the redzones are checked for corruption during deallocation. + However, the primary intended purpose of this feature is to be used in + combination with Valgrind, + which needs redzones in order to do effective buffer overflow/underflow + detection. This option is intended for debugging and will impact + performance negatively. This option is disabled by + default unless running inside Valgrind.

"opt.zero" @@ -475,25 +592,31 @@

Zero filling enabled/disabled. If enabled, each byte of uninitialized allocated memory will be initialized to 0. Note that this initialization only happens once for each byte, so - realloc() and + realloc(), + rallocx() and rallocm() calls do not zero memory that was previously allocated. This is intended for debugging and will impact performance negatively. This option is disabled by default. -

+

- "opt.sysv" + "opt.utrace" (bool) r- - [--enable-sysv] -

If enabled, attempting to allocate zero bytes will - return a NULL pointer instead of a valid pointer. - (The default behavior is to make a minimal allocation and return a - pointer to it.) This option is provided for System V compatibility. - This option is incompatible with the - "opt.xmalloc" - option. - This option is disabled by default.

+ [--enable-utrace] +

Allocation tracing based on + utrace(2) enabled/disabled. This option + is disabled by default.

+ + "opt.valgrind" + + (bool) + r- + [--enable-valgrind] +

Valgrind + support enabled/disabled. This option is vestigal because jemalloc + auto-detects whether it is running inside Valgrind. This option is + disabled by default, unless running inside Valgrind.

"opt.xmalloc" @@ -521,27 +644,11 @@ objects up to a certain size. Thread-specific caching allows many allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use. See the - - "opt.lg_tcache_gc_sweep" - - and + "opt.lg_tcache_max" - options for related tuning information. This option is enabled by - default.

- - "opt.lg_tcache_gc_sweep" - - (ssize_t) - r- - [--enable-tcache] -

Approximate interval (log base 2) between full - thread-specific cache garbage collection sweeps, counted in terms of - thread-specific cache allocation/deallocation events. Garbage - collection is actually performed incrementally, one size class at a - time, in order to avoid large collection pauses. The default sweep - interval is 8192 (2^13); setting this option to -1 will disable garbage - collection.

+ option for related tuning information. This option is enabled by + default unless running inside Valgrind.

"opt.lg_tcache_max" @@ -559,17 +666,7 @@ r- [--enable-prof]

Memory profiling enabled/disabled. If enabled, profile - memory allocation activity, and use an - atexit(3) function to dump final memory - usage to a file named according to the pattern - <prefix>.<pid>.<seq>.f.heap, - where <prefix> is controlled by the - "opt.prof_prefix" - - option. See the - "opt.lg_prof_bt_max" - - option for backtrace depth control. See the + memory allocation activity. See the "opt.prof_active" option for on-the-fly activation/deactivation. See the @@ -578,19 +675,19 @@ option for probabilistic sampling control. See the "opt.prof_accum" - option for control of cumulative sample reporting. See the - "opt.lg_prof_tcmax" - - option for control of per thread backtrace caching. See the + option for control of cumulative sample reporting. See the "opt.lg_prof_interval" - option for information on interval-triggered profile dumping, and the - + option for information on interval-triggered profile dumping, the "opt.prof_gdump" - option for information on high-water-triggered profile dumping. - Profile output is compatible with the included pprof - Perl script, which originates from the google-perftools + option for information on high-water-triggered profile dumping, and the + + "opt.prof_final" + + option for final profile dumping. Profile output is compatible with + the included pprof Perl script, which originates + from the gperftools package.

"opt.prof_prefix" @@ -602,20 +699,12 @@ set to the empty string, no automatic dumps will occur; this is primarily useful for disabling the automatic final heap dump (which also disables leak reporting, if enabled). The default prefix is - jeprof.

- - "opt.lg_prof_bt_max" - - (size_t) - r- - [--enable-prof] -

Maximum backtrace depth (log base 2) when profiling - memory allocation activity. The default is 128 (2^7).

+ jeprof.

"opt.prof_active" (bool) - r- + rw [--enable-prof]

Profiling activated/deactivated. This is a secondary control mechanism that makes it possible to start the application with @@ -636,8 +725,8 @@

Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity. Increasing the sampling interval decreases profile fidelity, but also decreases the - computational overhead. The default sample interval is 1 (2^0) (i.e. - all allocations are sampled).

+ computational overhead. The default sample interval is 512 KiB (2^19 + B).

"opt.prof_accum" @@ -648,28 +737,8 @@ dumps enabled/disabled. If this option is enabled, every unique backtrace must be stored for the duration of execution. Depending on the application, this can impose a large memory overhead, and the - cumulative counts are not always of interest. See the - - "opt.lg_prof_tcmax" - - option for control of per thread backtrace caching, which has important - interactions. This option is enabled by default.

- - "opt.lg_prof_tcmax" - - (ssize_t) - r- - [--enable-prof] -

Maximum per thread backtrace cache (log base 2) used - for heap profiling. A backtrace can only be discarded if the - - "opt.prof_accum" - - option is disabled, and no thread caches currently refer to the - backtrace. Therefore, a backtrace cache limit should be imposed if the - intention is to limit how much memory is used by backtraces. By - default, no limit is imposed (encoded as -1). -

+ cumulative counts are not always of interest. This option is disabled + by default.

"opt.lg_prof_interval" @@ -702,7 +771,21 @@ where <prefix> is controlled by the "opt.prof_prefix" - option. This option is disabled by default.

+ option. This option is disabled by default.

+ + "opt.prof_final" + + (bool) + r- + [--enable-prof] +

Use an + atexit(3) function to dump final memory + usage to a file named according to the pattern + <prefix>.<pid>.<seq>.f.heap, + where <prefix> is controlled by the + "opt.prof_prefix" + + option. This option is enabled by default.

"opt.prof_leak" @@ -712,57 +795,19 @@

Leak reporting enabled/disabled. If enabled, use an atexit(3) function to report memory leaks detected by allocation sampling. See the - - "opt.lg_prof_bt_max" - - option for backtrace depth control. See the "opt.prof" option for information on analyzing heap profile output. This option is disabled - by default.

- - "opt.overcommit" - - (bool) - r- - [--enable-swap] -

Over-commit enabled/disabled. If enabled, over-commit - memory as a side effect of using anonymous - mmap(2) or - sbrk(2) for virtual memory allocation. - In order for overcommit to be disabled, the - "swap.fds" - mallctl must have - been successfully written to. This option is enabled by - default.

- - "tcache.flush" - - (void) - -- - [--enable-tcache] -

Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface - need not be called, since automatic periodic incremental garbage - collection occurs, and the thread cache is automatically discarded when - a thread exits. However, garbage collection is triggered by allocation - activity, so it is possible for a thread that stops - allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful.

+ by default.

"thread.arena" (unsigned) rw

Get or set the arena associated with the calling - thread. The arena index must be less than the maximum number of arenas - (see the - "arenas.narenas" - - mallctl). If the specified arena was not initialized beforehand (see - the + thread. If the specified arena was not initialized beforehand (see the + "arenas.initialized" mallctl), it will be automatically initialized as a side effect of @@ -776,7 +821,7 @@

Get the total number of bytes ever allocated by the calling thread. This counter has the potential to wrap around; it is up to the application to appropriately interpret the counter in such - cases.

+ cases.

"thread.allocatedp" @@ -798,7 +843,7 @@

Get the total number of bytes ever deallocated by the calling thread. This counter has the potential to wrap around; it is up to the application to appropriately interpret the counter in such - cases.

+ cases.

"thread.deallocatedp" @@ -810,13 +855,68 @@ "thread.deallocated" mallctl. This is useful for avoiding the overhead of repeated - mallctl*() calls.

+ mallctl*() calls.

+ + "thread.tcache.enabled" + + (bool) + rw + [--enable-tcache] +

Enable/disable calling thread's tcache. The tcache is + implicitly flushed as a side effect of becoming + disabled (see + "thread.tcache.flush" + ). +

+ + "thread.tcache.flush" + + (void) + -- + [--enable-tcache] +

Flush calling thread's tcache. This interface releases + all cached objects and internal data structures associated with the + calling thread's thread-specific cache. Ordinarily, this interface + need not be called, since automatic periodic incremental garbage + collection occurs, and the thread cache is automatically discarded when + a thread exits. However, garbage collection is triggered by allocation + activity, so it is possible for a thread that stops + allocating/deallocating to retain its cache indefinitely, in which case + the developer may find manual flushing useful.

+ + "arena.<i>.purge" + + (unsigned) + -- +

Purge unused dirty pages for arena <i>, or for + all arenas if <i> equals + "arenas.narenas" + . +

+ + "arena.<i>.dss" + + (const char *) + rw +

Set the precedence of dss allocation as related to mmap + allocation for arena <i>, or for all arenas if <i> equals + + "arenas.narenas" + . Note + that even during huge allocation this setting is read from the arena + that would be chosen for small or large allocation so that applications + can depend on consistent dss versus mmap allocation regardless of + allocation size. See + "opt.dss" + for supported + settings. +

"arenas.narenas" (unsigned) r- -

Maximum number of arenas.

+

Current limit on number of arenas.

"arenas.initialized" @@ -826,127 +926,32 @@ "arenas.narenas" booleans. Each boolean indicates whether the corresponding arena is - initialized.

+ initialized.

"arenas.quantum" (size_t) r- -

Quantum size.

- - "arenas.cacheline" - - (size_t) - r- -

Assumed cacheline size.

- - "arenas.subpage" - - (size_t) - r- -

Subpage size class interval.

- - "arenas.pagesize" - - (size_t) - r- -

Page size.

- - "arenas.chunksize" - - (size_t) - r- -

Chunk size.

- - "arenas.tspace_min" - - (size_t) - r- -

Minimum tiny size class. Tiny size classes are powers - of two.

+

Quantum size.

- "arenas.tspace_max" + "arenas.page" (size_t) r- -

Maximum tiny size class. Tiny size classes are powers - of two.

- - "arenas.qspace_min" - - (size_t) - r- -

Minimum quantum-spaced size class.

- - "arenas.qspace_max" - - (size_t) - r- -

Maximum quantum-spaced size class.

- - "arenas.cspace_min" - - (size_t) - r- -

Minimum cacheline-spaced size class.

- - "arenas.cspace_max" - - (size_t) - r- -

Maximum cacheline-spaced size class.

- - "arenas.sspace_min" - - (size_t) - r- -

Minimum subpage-spaced size class.

- - "arenas.sspace_max" - - (size_t) - r- -

Maximum subpage-spaced size class.

+

Page size.

"arenas.tcache_max" (size_t) r- [--enable-tcache] -

Maximum thread-cached size class.

- - "arenas.ntbins" - - (unsigned) - r- -

Number of tiny bin size classes.

- - "arenas.nqbins" - - (unsigned) - r- -

Number of quantum-spaced bin size - classes.

- - "arenas.ncbins" - - (unsigned) - r- -

Number of cacheline-spaced bin size - classes.

- - "arenas.nsbins" - - (unsigned) - r- -

Number of subpage-spaced bin size - classes.

+

Maximum thread-cached size class.

"arenas.nbins" (unsigned) r- -

Total number of bin size classes.

+

Number of bin size classes.

"arenas.nhbins" @@ -960,39 +965,46 @@ (size_t) r- -

Maximum size supported by size class.

+

Maximum size supported by size class.

"arenas.bin.<i>.nregs" (uint32_t) r- -

Number of regions per page run.

+

Number of regions per page run.

"arenas.bin.<i>.run_size" (size_t) r- -

Number of bytes per page run.

+

Number of bytes per page run.

"arenas.nlruns" (size_t) r- -

Total number of large size classes.

+

Total number of large size classes.

"arenas.lrun.<i>.size" (size_t) r-

Maximum size supported by this large size - class.

+ class.

"arenas.purge" (unsigned) -w

Purge unused dirty pages for the specified arena, or - for all arenas if none is specified.

+ for all arenas if none is specified.

+ + "arenas.extend" + + (unsigned) + r- +

Extend the array of arenas by appending a new arena, + and returning the new arena index.

"prof.active" @@ -1004,7 +1016,7 @@ "opt.prof_active" option for additional information. -

+

"prof.dump" @@ -1018,7 +1030,7 @@ "opt.prof_prefix" - option.

+ option.

"prof.interval" @@ -1067,7 +1079,11 @@ equal to "stats.allocated" . -

+ This does not include + + "stats.arenas.<i>.pdirty" + and pages + entirely devoted to allocator metadata.

"stats.mapped" @@ -1079,8 +1095,7 @@ large as "stats.active" . This - does not include inactive chunks backed by swap files. his does not - include inactive chunks embedded in the DSS.

+ does not include inactive chunks.

"stats.chunks.current" @@ -1088,16 +1103,15 @@ r- [--enable-stats]

Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks backed by swap - files. This does not include inactive chunks embedded in the DSS. -

+ application. This does not include inactive chunks. +

"stats.chunks.total" (uint64_t) r- [--enable-stats] -

Cumulative number of chunks allocated.

+

Cumulative number of chunks allocated.

"stats.chunks.high" @@ -1105,7 +1119,7 @@ r- [--enable-stats]

Maximum number of active chunks at any time thus far. -

+

"stats.huge.allocated" @@ -1113,7 +1127,7 @@ r- [--enable-stats]

Number of bytes currently allocated by huge objects. -

+

"stats.huge.nmalloc" @@ -1121,7 +1135,7 @@ r- [--enable-stats]

Cumulative number of huge allocation requests. -

+

"stats.huge.ndalloc" @@ -1129,20 +1143,30 @@ r- [--enable-stats]

Cumulative number of huge deallocation requests. -

+

+ + "stats.arenas.<i>.dss" + + (const char *) + r- +

dss (sbrk(2)) allocation precedence as + related to mmap(2) allocation. See + "opt.dss" + for details. +

"stats.arenas.<i>.nthreads" (unsigned) r-

Number of threads currently assigned to - arena.

+ arena.

"stats.arenas.<i>.pactive" (size_t) r- -

Number of pages in active runs.

+

Number of pages in active runs.

"stats.arenas.<i>.pdirty" @@ -1151,14 +1175,14 @@

Number of pages within unused runs that are potentially dirty, and for which madvise(..., MADV_DONTNEED) or - similar has not been called.

+ similar has not been called.

"stats.arenas.<i>.mapped" (size_t) r- [--enable-stats] -

Number of mapped bytes.

+

Number of mapped bytes.

"stats.arenas.<i>.npurge" @@ -1166,7 +1190,7 @@ r- [--enable-stats]

Number of dirty page purge sweeps performed. -

+

"stats.arenas.<i>.nmadvise" @@ -1175,14 +1199,14 @@ [--enable-stats]

Number of madvise(..., MADV_DONTNEED) or - similar calls made to purge dirty pages.

+ similar calls made to purge dirty pages.

- "stats.arenas.<i>.npurged" + "stats.arenas.<i>.purged" (uint64_t) r- [--enable-stats] -

Number of pages purged.

+

Number of pages purged.

"stats.arenas.<i>.small.allocated" @@ -1190,7 +1214,7 @@ r- [--enable-stats]

Number of bytes currently allocated by small objects. -

+

"stats.arenas.<i>.small.nmalloc" @@ -1198,7 +1222,7 @@ r- [--enable-stats]

Cumulative number of allocation requests served by - small bins.

+ small bins.

"stats.arenas.<i>.small.ndalloc" @@ -1206,7 +1230,7 @@ r- [--enable-stats]

Cumulative number of small objects returned to bins. -

+

"stats.arenas.<i>.small.nrequests" @@ -1214,7 +1238,7 @@ r- [--enable-stats]

Cumulative number of small allocation requests. -

+

"stats.arenas.<i>.large.allocated" @@ -1222,7 +1246,7 @@ r- [--enable-stats]

Number of bytes currently allocated by large objects. -

+

"stats.arenas.<i>.large.nmalloc" @@ -1230,7 +1254,7 @@ r- [--enable-stats]

Cumulative number of large allocation requests served - directly by the arena.

+ directly by the arena.

"stats.arenas.<i>.large.ndalloc" @@ -1238,7 +1262,7 @@ r- [--enable-stats]

Cumulative number of large deallocation requests served - directly by the arena.

+ directly by the arena.

"stats.arenas.<i>.large.nrequests" @@ -1246,7 +1270,7 @@ r- [--enable-stats]

Cumulative number of large allocation requests. -

+

"stats.arenas.<i>.bins.<j>.allocated" @@ -1254,7 +1278,7 @@ r- [--enable-stats]

Current number of bytes allocated by - bin.

+ bin.

"stats.arenas.<i>.bins.<j>.nmalloc" @@ -1262,7 +1286,7 @@ r- [--enable-stats]

Cumulative number of allocations served by bin. -

+

"stats.arenas.<i>.bins.<j>.ndalloc" @@ -1270,7 +1294,7 @@ r- [--enable-stats]

Cumulative number of allocations returned to bin. -

+

"stats.arenas.<i>.bins.<j>.nrequests" @@ -1278,28 +1302,28 @@ r- [--enable-stats]

Cumulative number of allocation - requests.

+ requests.

"stats.arenas.<i>.bins.<j>.nfills" (uint64_t) r- [--enable-stats --enable-tcache] -

Cumulative number of tcache fills.

+

Cumulative number of tcache fills.

"stats.arenas.<i>.bins.<j>.nflushes" (uint64_t) r- [--enable-stats --enable-tcache] -

Cumulative number of tcache flushes.

+

Cumulative number of tcache flushes.

"stats.arenas.<i>.bins.<j>.nruns" (uint64_t) r- [--enable-stats] -

Cumulative number of runs created.

+

Cumulative number of runs created.

"stats.arenas.<i>.bins.<j>.nreruns" @@ -1307,22 +1331,14 @@ r- [--enable-stats]

Cumulative number of times the current run from which - to allocate changed.

- - "stats.arenas.<i>.bins.<j>.highruns" - - (size_t) - r- - [--enable-stats] -

Maximum number of runs at any time thus far. -

+ to allocate changed.

"stats.arenas.<i>.bins.<j>.curruns" (size_t) r- [--enable-stats] -

Current number of runs.

+

Current number of runs.

"stats.arenas.<i>.lruns.<j>.nmalloc" @@ -1330,7 +1346,7 @@ r- [--enable-stats]

Cumulative number of allocation requests for this size - class served directly by the arena.

+ class served directly by the arena.

"stats.arenas.<i>.lruns.<j>.ndalloc" @@ -1338,7 +1354,7 @@ r- [--enable-stats]

Cumulative number of deallocation requests for this - size class served directly by the arena.

+ size class served directly by the arena.

"stats.arenas.<i>.lruns.<j>.nrequests" @@ -1346,15 +1362,7 @@ r- [--enable-stats]

Cumulative number of allocation requests for this size - class.

- - "stats.arenas.<i>.lruns.<j>.highruns" - - (size_t) - r- - [--enable-stats] -

Maximum number of runs at any time thus far for this - size class.

+ class.

"stats.arenas.<i>.lruns.<j>.curruns" @@ -1362,55 +1370,7 @@ r- [--enable-stats]

Current number of runs for this size class. -

- - "swap.avail" - - (size_t) - r- - [--enable-stats --enable-swap] -

Number of swap file bytes that are currently not - associated with any chunk (i.e. mapped, but otherwise completely - unmanaged).

- - "swap.prezeroed" - - (bool) - rw - [--enable-swap] -

If true, the allocator assumes that the swap file(s) - contain nothing but nil bytes. If this assumption is violated, - allocator behavior is undefined. This value becomes read-only after - - "swap.fds" - is - successfully written to.

- - "swap.nfds" - - (size_t) - r- - [--enable-swap] -

Number of file descriptors in use for swap. -

- - "swap.fds" - - (int *) - rw - [--enable-swap] -

When written to, the files associated with the - specified file descriptors are contiguously mapped via - mmap(2). The resulting virtual memory - region is preferred over anonymous - mmap(2) and - sbrk(2) memory. Note that if a file's - size is not a multiple of the page size, it is automatically truncated - to the nearest page size multiple. See the - - "swap.prezeroed" - - mallctl for specifying that the files are pre-zeroed.

DEBUGGING MALLOC PROBLEMS

When debugging, it is a good idea to configure/build jemalloc with +

DEBUGGING MALLOC PROBLEMS

When debugging, it is a good idea to configure/build jemalloc with the --enable-debug and --enable-fill options, and recompile the program with suitable options and symbols for debugger support. When so configured, jemalloc incorporates a wide variety @@ -1428,10 +1388,9 @@ the symptoms of such bugs. Between these two options, it is usually possible to quickly detect, diagnose, and eliminate such bugs.

This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an + would be prohibitive. However, jemalloc does integrate with the most + excellent Valgrind tool if the + --enable-valgrind configuration option is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor STDERR_FILENO. Errors will result in the process dumping core. If the @@ -1447,7 +1406,7 @@ malloc_stats_print(), followed by a string pointer. Please note that doing anything which tries to allocate memory in this function is likely to result in a crash or deadlock.

All messages are prefixed by - “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and + “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and calloc() functions return a pointer to the allocated memory if successful; otherwise a NULL pointer is returned and errno is set to @@ -1455,10 +1414,18 @@ returns the value 0 if successful; otherwise it returns an error value. The posix_memalign() function will fail if: -

EINVAL

The alignment parameter is +

EINVAL

The alignment parameter is not a power of 2 at least as large as sizeof(void *).

ENOMEM

Memory allocation error.

+

The aligned_alloc() function returns + a pointer to the allocated memory if successful; otherwise a + NULL pointer is returned and + errno is set. The + aligned_alloc() function will fail if: +

EINVAL

The alignment parameter is + not a power of 2. +

ENOMEM

Memory allocation error.

The realloc() function returns a pointer, possibly identical to ptr, to the allocated memory if successful; otherwise a NULL @@ -1467,33 +1434,47 @@ allocation failure. The realloc() function always leaves the original buffer intact when an error occurs.

The free() function returns no - value.

Non-standard API

The malloc_usable_size() function - returns the usable size of the allocation pointed to by - ptr.

The mallctl(), + value.

Non-standard API

The mallocx() and + rallocx() functions return a pointer to + the allocated memory if successful; otherwise a NULL + pointer is returned to indicate insufficient contiguous memory was + available to service the allocation request.

The xallocx() function returns the + real size of the resulting resized allocation pointed to by + ptr, which is a value less than + size if the allocation could not be adequately + grown in place.

The sallocx() function returns the + real size of the allocation pointed to by ptr. +

The nallocx() returns the real size + that would result from a successful equivalent + mallocx() function call, or zero if + insufficient memory is available to perform the size computation.

The mallctl(), mallctlnametomib(), and mallctlbymib() functions return 0 on success; otherwise they return an error value. The functions will fail if: -

EINVAL

newp is not +

EINVAL

newp is not NULL, and newlen is too large or too small. Alternatively, *oldlenp is too large or too small; in this case as much data as possible - are read despite the error.

ENOMEM

*oldlenp is too short to - hold the requested value.

ENOENT

name or + are read despite the error.

ENOENT

name or mib specifies an unknown/invalid value.

EPERM

Attempt to read or write void value, or attempt to write read-only value.

EAGAIN

A memory allocation failure occurred.

EFAULT

An interface with side effects failed in some way not directly related to mallctl*() read/write processing.

-

Experimental API

The allocm(), +

The malloc_usable_size() function + returns the usable size of the allocation pointed to by + ptr.

Experimental API

The allocm(), rallocm(), - sallocm(), and - dallocm() functions return + sallocm(), + dallocm(), and + nallocm() functions return ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm() and - rallocm() functions will fail if: -

ALLOCM_ERR_OOM

Out of memory. Insufficient contiguous memory was + error value. The allocm(), + rallocm(), and + nallocm() functions will fail if: +

ALLOCM_ERR_OOM

Out of memory. Insufficient contiguous memory was available to service the allocation request. The allocm() function additionally sets *ptr to NULL, whereas @@ -1501,24 +1482,25 @@ *ptr unmodified.

The rallocm() function will also fail if: -

ALLOCM_ERR_NOT_MOVED

ALLOCM_NO_MOVE was specified, +

ALLOCM_ERR_NOT_MOVED

ALLOCM_NO_MOVE was specified, but the reallocation request could not be serviced without moving the object.

-

ENVIRONMENT

The following environment variable affects the execution of the +

ENVIRONMENT

The following environment variable affects the execution of the allocation functions: -

MALLOC_CONF

If the environment variable +

MALLOC_CONF

If the environment variable MALLOC_CONF is set, the characters it contains will be interpreted as options.

-

EXAMPLES

To dump core whenever a problem occurs: +

EXAMPLES

To dump core whenever a problem occurs: 

ln -s 'abort:true' /etc/malloc.conf

To specify in the source a chunk size that is 16 MiB: 

-malloc_conf = "lg_chunk:24";

SEE ALSO

madvise(2), +malloc_conf = "lg_chunk:24";

SEE ALSO

madvise(2), mmap(2), sbrk(2), + utrace(2), alloca(3), atexit(3), - getpagesize(3)

STANDARDS

The malloc(), + getpagesize(3)

STANDARDS

The malloc(), calloc(), realloc(), and free() functions conform to ISO/IEC diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 7a32879abee..d8e2e711f2c 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -30,17 +30,25 @@ malloc calloc posix_memalign + aligned_alloc realloc free - malloc_usable_size - malloc_stats_print + mallocx + rallocx + xallocx + sallocx + dallocx + nallocx mallctl mallctlnametomib mallctlbymib + malloc_stats_print + malloc_usable_size allocm rallocm sallocm dallocm + nallocm --> general purpose memory allocation functions @@ -72,6 +80,11 @@ size_t alignment size_t size + + void *aligned_alloc + size_t alignment + size_t size + void *realloc void *ptr @@ -85,16 +98,37 @@ Non-standard API - size_t malloc_usable_size - const void *ptr + void *mallocx + size_t size + int flags - void malloc_stats_print - void (*write_cb) - void *, const char * - - void *cbopaque - const char *opts + void *rallocx + void *ptr + size_t size + int flags + + + size_t xallocx + void *ptr + size_t size + size_t extra + int flags + + + size_t sallocx + void *ptr + int flags + + + void dallocx + void *ptr + int flags + + + size_t nallocx + size_t size + int flags int mallctl @@ -119,6 +153,18 @@ void *newp size_t newlen + + void malloc_stats_print + void (*write_cb) + void *, const char * + + void *cbopaque + const char *opts + + + size_t malloc_usable_size + const void *ptr + void (*malloc_message) void *cbopaque @@ -154,6 +200,12 @@ void *ptr int flags + + int nallocm + size_t *rsize + size_t size + int flags + @@ -183,6 +235,14 @@ alignment must be a power of 2 at least as large as sizeof(void *). + The aligned_alloc function + allocates size bytes of memory such that the + allocation's base address is an even multiple of + alignment. The requested + alignment must be a power of 2. Behavior is + undefined if size is not an integral multiple of + alignment. + The realloc function changes the size of the previously allocated memory referenced by ptr to size bytes. The @@ -204,42 +264,103 @@ Non-standard API + The mallocx, + rallocx, + xallocx, + sallocx, + dallocx, and + nallocx functions all have a + flags argument that can be used to specify + options. The functions only check the options that are contextually + relevant. Use bitwise or (|) operations to + specify one or more of the following: + + + MALLOCX_LG_ALIGN(la) + - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. The return value may be larger than the size - that was requested during allocation. The - malloc_usable_size function is not a - mechanism for in-place realloc; rather - it is provided solely as a tool for introspection purposes. Any - discrepancy between the requested allocation size and the size reported - by malloc_usable_size should not be - depended on, since such behavior is entirely implementation-dependent. - + Align the memory allocation to start at an address + that is a multiple of (1 << + la). This macro does not validate + that la is within the valid + range. + + + MALLOCX_ALIGN(a) + - The malloc_stats_print function - writes human-readable summary statistics via the - write_cb callback function pointer and - cbopaque data passed to - write_cb, or - malloc_message if - write_cb is NULL. This - function can be called repeatedly. General information that never - changes during execution can be omitted by specifying "g" as a character - within the opts string. Note that - malloc_message uses the - mallctl* functions internally, so - inconsistent statistics can be reported if multiple threads use these - functions simultaneously. If is - specified during configuration, “m” and “a” can - be specified to omit merged arena and per arena statistics, respectively; - “b” and “l” can be specified to omit per size - class statistics for bins and large objects, respectively. Unrecognized - characters are silently ignored. Note that thread caching may prevent - some statistics from being completely up to date, since extra locking - would be required to merge counters that track thread cache operations. + Align the memory allocation to start at an address + that is a multiple of a, where + a is a power of two. This macro does not + validate that a is a power of 2. + + + + MALLOCX_ZERO + + Initialize newly allocated memory to contain zero + bytes. In the growing reallocation case, the real size prior to + reallocation defines the boundary between untouched bytes and those + that are initialized to contain zero bytes. If this macro is + absent, newly allocated memory is uninitialized. + + + MALLOCX_ARENA(a) + + + Use the arena specified by the index + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range. + + + The mallocx function allocates at + least size bytes of memory, and returns a pointer + to the base address of the allocation. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints. + + The rallocx function resizes the + allocation at ptr to be at least + size bytes, and returns a pointer to the base + address of the resulting allocation, which may or may not have moved from + its original location. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints. + + The xallocx function resizes the + allocation at ptr in place to be at least + size bytes, and returns the real size of the + allocation. If extra is non-zero, an attempt is + made to resize the allocation to be at least (size + + extra) bytes, though inability to allocate + the extra byte(s) will not by itself result in failure to resize. + Behavior is undefined if size is + 0, or if (size + extra + > SIZE_T_MAX). + + The sallocx function returns the + real size of the allocation at ptr. + + The dallocx function causes the + memory referenced by ptr to be made available for + future allocations. + + The nallocx function allocates no + memory, but it performs the same size computation as the + mallocx function, and returns the real + size of the allocation that would result from the equivalent + mallocx function call. Behavior is + undefined if size is 0, or if + request size overflows due to size class and/or alignment + constraints. + The mallctl function provides a general interface for introspecting the memory allocator, as well as setting modifiable parameters and triggering actions. The @@ -276,15 +397,14 @@ it is legitimate to construct code like the following: + + The malloc_stats_print function + writes human-readable summary statistics via the + write_cb callback function pointer and + cbopaque data passed to + write_cb, or + malloc_message if + write_cb is NULL. This + function can be called repeatedly. General information that never + changes during execution can be omitted by specifying "g" as a character + within the opts string. Note that + malloc_message uses the + mallctl* functions internally, so + inconsistent statistics can be reported if multiple threads use these + functions simultaneously. If is + specified during configuration, “m” and “a” can + be specified to omit merged arena and per arena statistics, respectively; + “b” and “l” can be specified to omit per size + class statistics for bins and large objects, respectively. Unrecognized + characters are silently ignored. Note that thread caching may prevent + some statistics from being completely up to date, since extra locking + would be required to merge counters that track thread cache operations. + + + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. The return value may be larger than the size + that was requested during allocation. The + malloc_usable_size function is not a + mechanism for in-place realloc; rather + it is provided solely as a tool for introspection purposes. Any + discrepancy between the requested allocation size and the size reported + by malloc_usable_size should not be + depended on, since such behavior is entirely implementation-dependent. + Experimental API The experimental API is subject to change or removal without regard - for backward compatibility. + for backward compatibility. If + is specified during configuration, the experimental API is + omitted. The allocm, rallocm, - sallocm, and - dallocm functions all have a + sallocm, + dallocm, and + nallocm functions all have a flags argument that can be used to specify options. The functions only check the options that are contextually relevant. Use bitwise or (|) operations to @@ -334,7 +492,7 @@ for (i = 0; i < nbins; i++) { Initialize newly allocated memory to contain zero bytes. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those - that are initialized to contain zero bytes. If this option is + that are initialized to contain zero bytes. If this macro is absent, newly allocated memory is uninitialized. @@ -344,6 +502,17 @@ for (i = 0; i < nbins; i++) { object. This constraint can apply to both growth and shrinkage. + + ALLOCM_ARENA(a) + + + Use the arena specified by the index + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range. + @@ -351,7 +520,10 @@ for (i = 0; i < nbins; i++) { least size bytes of memory, sets *ptr to the base address of the allocation, and sets *rsize to the real size of the allocation if - rsize is not NULL. + rsize is not NULL. Behavior + is undefined if size is 0, or + if request size overflows due to size class and/or alignment + constraints. The rallocm function resizes the allocation at *ptr to be at least @@ -361,10 +533,12 @@ for (i = 0; i < nbins; i++) { rsize is not NULL. If extra is non-zero, an attempt is made to resize the allocation to be at least size + + language="C">(size + extra) bytes, though inability to allocate the extra byte(s) will not by itself result in failure. Behavior is - undefined if (size + + undefined if size is 0, if + request size overflows due to size class and/or alignment constraints, or + if (size + extra > SIZE_T_MAX). @@ -374,6 +548,16 @@ for (i = 0; i < nbins; i++) { The dallocm function causes the memory referenced by ptr to be made available for future allocations. + + The nallocm function allocates no + memory, but it performs the same size computation as the + allocm function, and if + rsize is not NULL it sets + *rsize to the real size of the allocation that + would result from the equivalent allocm + function call. Behavior is undefined if size is + 0, or if request size overflows due to size class + and/or alignment constraints. @@ -387,7 +571,14 @@ for (i = 0; i < nbins; i++) { referenced by the symbolic link named /etc/malloc.conf, and the value of the environment variable MALLOC_CONF, will be interpreted, in - that order, from left to right as options. + that order, from left to right as options. Note that + malloc_conf may be read before + main is entered, so the declaration of + malloc_conf should specify an initializer that contains + the final value to be read by jemalloc. malloc_conf is + a compile-time setting, whereas /etc/malloc.conf and MALLOC_CONF + can be safely set any time prior to program invocation. An options string is a comma-separated list of option:value pairs. There is one key corresponding to each --enable-dss is specified during configuration, this - allocator uses both sbrk + allocator uses both mmap 2 and - mmap + sbrk 2, in that order of preference; otherwise only mmap 2 is used. @@ -455,24 +646,14 @@ for (i = 0; i < nbins; i++) { allocations in constant time. Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Unless - is specified during configuration, - allocation requests that are no more than half the quantum (8 or 16, - depending on architecture) are rounded up to the nearest power of two that - is at least sizeof(void *). - Allocation requests that are more than half the quantum, but no more than - the minimum cacheline-multiple size class (see the opt.lg_qspace_max - option) are rounded up to the nearest multiple of the quantum. Allocation - requests that are more than the minimum cacheline-multiple size class, but - no more than the minimum subpage-multiple size class (see the opt.lg_cspace_max - option) are rounded up to the nearest multiple of the cacheline size (64). - Allocation requests that are more than the minimum subpage-multiple size - class, but no more than the maximum subpage-multiple size class are rounded - up to the nearest multiple of the subpage size (256). Allocation requests - that are more than the maximum subpage-multiple size class, but small - enough to fit in an arena-managed chunk (see the sizeof(double). All other small + object size classes are multiples of the quantum, spaced such that internal + fragmentation is limited to approximately 25% for all but the smallest size + classes. Allocation requests that are larger than the maximum small size + class, but small enough to fit in an arena-managed chunk (see the opt.lg_chunk option), are rounded up to the nearest run size. Allocation requests that are too large to fit in an arena-managed chunk are rounded up to the nearest multiple of @@ -490,41 +671,55 @@ for (i = 0; i < nbins; i++) { Size classes - - - - + + + + Category - Subcategory + Spacing Size - Small - Tiny + Small + lg [8] - Quantum-spaced + 16 [16, 32, 48, ..., 128] - Cacheline-spaced - [192, 256, 320, ..., 512] + 32 + [160, 192, 224, 256] + + + 64 + [320, 384, 448, 512] + + + 128 + [640, 768, 896, 1024] - Subpage-spaced - [768, 1024, 1280, ..., 3840] + 256 + [1280, 1536, 1792, 2048] - Large + 512 + [2560, 3072, 3584] + + + Large + 4 KiB [4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB] - Huge + Huge + 4 MiB [4 MiB, 8 MiB, 12 MiB, ...] @@ -549,7 +744,7 @@ for (i = 0; i < nbins; i++) { which controls refreshing of cached dynamic statistics. - + version (const char *) @@ -570,7 +765,7 @@ for (i = 0; i < nbins; i++) { detecting whether another thread caused a refresh. - + config.debug (bool) @@ -580,7 +775,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.dss (bool) @@ -590,37 +785,47 @@ for (i = 0; i < nbins; i++) { build configuration. - + - config.dynamic_page_shift + config.fill (bool) r- - was - specified during build configuration. + was specified during + build configuration. - + - config.fill + config.lazy_lock (bool) r- - was specified during + was specified + during build configuration. + + + + + config.mremap + (bool) + r- + + was specified during build configuration. - + - config.lazy_lock + config.munmap (bool) r- - was specified - during build configuration. + was specified during + build configuration. - + config.prof (bool) @@ -630,7 +835,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.prof_libgcc (bool) @@ -640,7 +845,7 @@ for (i = 0; i < nbins; i++) { specified during build configuration. - + config.prof_libunwind (bool) @@ -650,7 +855,7 @@ for (i = 0; i < nbins; i++) { during build configuration. - + config.stats (bool) @@ -660,57 +865,47 @@ for (i = 0; i < nbins; i++) { build configuration. - + - config.swap + config.tcache (bool) r- - was specified during - build configuration. + was not specified + during build configuration. - + - config.sysv + config.tls (bool) r- - was specified during + was not specified during build configuration. - + - config.tcache + config.utrace (bool) r- - was not specified - during build configuration. - - - - - config.tiny - (bool) - r- - - was not specified - during build configuration. + was specified during + build configuration. - + - config.tls + config.valgrind (bool) r- - was not specified during + was specified during build configuration. - + config.xmalloc (bool) @@ -735,28 +930,21 @@ for (i = 0; i < nbins; i++) { - - - opt.lg_qspace_max - (size_t) - r- - - Size (log base 2) of the maximum size class that is a - multiple of the quantum (8 or 16 bytes, depending on architecture). - Above this size, cacheline spacing is used for size classes. The - default value is 128 bytes (2^7). - - - + - opt.lg_cspace_max - (size_t) + opt.dss + (const char *) r- - Size (log base 2) of the maximum size class that is a - multiple of the cacheline size (64). Above this size, subpage spacing - (256 bytes) is used for size classes. The default value is 512 bytes - (2^9). + dss (sbrk + 2) allocation precedence as + related to mmap + 2 allocation. The following + settings are supported: “disabled”, “primary”, + and “secondary”. The default is “secondary” if + config.dss is + true, “disabled” otherwise. + @@ -765,8 +953,11 @@ for (i = 0; i < nbins; i++) { (size_t) r- - Virtual memory chunk size (log base 2). The default - chunk size is 4 MiB (2^22). + Virtual memory chunk size (log base 2). If a chunk + size outside the supported size range is specified, the size is + silently clipped to the minimum/maximum supported size. The default + chunk size is 4 MiB (2^22). + @@ -775,9 +966,9 @@ for (i = 0; i < nbins; i++) { (size_t) r- - Maximum number of arenas to use. The default maximum - number of arenas is four times the number of CPUs, or one if there is a - single CPU. + Maximum number of arenas to use for automatic + multiplexing of threads and arenas. The default is four times the + number of CPUs, or one if there is a single CPU. @@ -794,7 +985,7 @@ for (i = 0; i < nbins; i++) { 2 or a similar system call. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused. The - default minimum ratio is 32:1 (2^5:1); an option value of -1 will + default minimum ratio is 8:1 (2^3:1); an option value of -1 will disable dirty page purging. @@ -830,7 +1021,49 @@ for (i = 0; i < nbins; i++) { 0x5a. This is intended for debugging and will impact performance negatively. This option is disabled by default unless is specified during - configuration, in which case it is enabled by default. + configuration, in which case it is enabled by default unless running + inside Valgrind. + + + + + opt.quarantine + (size_t) + r- + [] + + Per thread quarantine size in bytes. If non-zero, each + thread maintains a FIFO object quarantine that stores up to the + specified number of bytes of memory. The quarantined memory is not + freed until it is released from quarantine, though it is immediately + junk-filled if the opt.junk option is + enabled. This feature is of particular use in combination with Valgrind, which can detect attempts + to access quarantined objects. This is intended for debugging and will + impact performance negatively. The default quarantine size is 0 unless + running inside Valgrind, in which case the default is 16 + MiB. + + + + + opt.redzone + (bool) + r- + [] + + Redzones enabled/disabled. If enabled, small + allocations have redzones before and after them. Furthermore, if the + opt.junk option is + enabled, the redzones are checked for corruption during deallocation. + However, the primary intended purpose of this feature is to be used in + combination with Valgrind, + which needs redzones in order to do effective buffer overflow/underflow + detection. This option is intended for debugging and will impact + performance negatively. This option is disabled by + default unless running inside Valgrind. @@ -843,27 +1076,38 @@ for (i = 0; i < nbins; i++) { Zero filling enabled/disabled. If enabled, each byte of uninitialized allocated memory will be initialized to 0. Note that this initialization only happens once for each byte, so - realloc and + realloc, + rallocx and rallocm calls do not zero memory that was previously allocated. This is intended for debugging and will impact performance negatively. This option is disabled by default. - + - opt.sysv + opt.utrace (bool) r- - [] + [] - If enabled, attempting to allocate zero bytes will - return a NULL pointer instead of a valid pointer. - (The default behavior is to make a minimal allocation and return a - pointer to it.) This option is provided for System V compatibility. - This option is incompatible with the opt.xmalloc option. - This option is disabled by default. + Allocation tracing based on + utrace + 2 enabled/disabled. This option + is disabled by default. + + + + + opt.valgrind + (bool) + r- + [] + + Valgrind + support enabled/disabled. This option is vestigal because jemalloc + auto-detects whether it is running inside Valgrind. This option is + disabled by default, unless running inside Valgrind. @@ -899,27 +1143,10 @@ malloc_conf = "xmalloc:true";]]> allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use. See the opt.lg_tcache_gc_sweep - and opt.lg_tcache_max - options for related tuning information. This option is enabled by - default. - - - - - opt.lg_tcache_gc_sweep - (ssize_t) - r- - [] - - Approximate interval (log base 2) between full - thread-specific cache garbage collection sweeps, counted in terms of - thread-specific cache allocation/deallocation events. Garbage - collection is actually performed incrementally, one size class at a - time, in order to avoid large collection pauses. The default sweep - interval is 8192 (2^13); setting this option to -1 will disable garbage - collection. + option for related tuning information. This option is enabled by + default unless running inside Valgrind. @@ -943,31 +1170,21 @@ malloc_conf = "xmalloc:true";]]> [] Memory profiling enabled/disabled. If enabled, profile - memory allocation activity, and use an - atexit - 3 function to dump final memory - usage to a file named according to the pattern - <prefix>.<pid>.<seq>.f.heap, - where <prefix> is controlled by the opt.prof_prefix - option. See the opt.lg_prof_bt_max - option for backtrace depth control. See the opt.prof_active option for on-the-fly activation/deactivation. See the opt.lg_prof_sample option for probabilistic sampling control. See the opt.prof_accum option for control of cumulative sample reporting. See the opt.lg_prof_tcmax - option for control of per thread backtrace caching. See the opt.lg_prof_interval - option for information on interval-triggered profile dumping, and the - opt.prof_gdump - option for information on high-water-triggered profile dumping. - Profile output is compatible with the included pprof - Perl script, which originates from the google-perftools + option for information on interval-triggered profile dumping, the opt.prof_gdump + option for information on high-water-triggered profile dumping, and the + opt.prof_final + option for final profile dumping. Profile output is compatible with + the included pprof Perl script, which originates + from the gperftools package. @@ -985,22 +1202,11 @@ malloc_conf = "xmalloc:true";]]> jeprof. - - - opt.lg_prof_bt_max - (size_t) - r- - [] - - Maximum backtrace depth (log base 2) when profiling - memory allocation activity. The default is 128 (2^7). - - opt.prof_active (bool) - r- + rw [] Profiling activated/deactivated. This is a secondary @@ -1023,8 +1229,8 @@ malloc_conf = "xmalloc:true";]]> Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity. Increasing the sampling interval decreases profile fidelity, but also decreases the - computational overhead. The default sample interval is 1 (2^0) (i.e. - all allocations are sampled). + computational overhead. The default sample interval is 512 KiB (2^19 + B). @@ -1038,28 +1244,8 @@ malloc_conf = "xmalloc:true";]]> dumps enabled/disabled. If this option is enabled, every unique backtrace must be stored for the duration of execution. Depending on the application, this can impose a large memory overhead, and the - cumulative counts are not always of interest. See the - opt.lg_prof_tcmax - option for control of per thread backtrace caching, which has important - interactions. This option is enabled by default. - - - - - opt.lg_prof_tcmax - (ssize_t) - r- - [] - - Maximum per thread backtrace cache (log base 2) used - for heap profiling. A backtrace can only be discarded if the - opt.prof_accum - option is disabled, and no thread caches currently refer to the - backtrace. Therefore, a backtrace cache limit should be imposed if the - intention is to limit how much memory is used by backtraces. By - default, no limit is imposed (encoded as -1). - + cumulative counts are not always of interest. This option is disabled + by default. @@ -1099,6 +1285,23 @@ malloc_conf = "xmalloc:true";]]> option. This option is disabled by default. + + + opt.prof_final + (bool) + r- + [] + + Use an + atexit + 3 function to dump final memory + usage to a file named according to the pattern + <prefix>.<pid>.<seq>.f.heap, + where <prefix> is controlled by the opt.prof_prefix + option. This option is enabled by default. + + opt.prof_leak @@ -1110,63 +1313,20 @@ malloc_conf = "xmalloc:true";]]> atexit 3 function to report memory leaks detected by allocation sampling. See the - opt.lg_prof_bt_max - option for backtrace depth control. See the opt.prof option for information on analyzing heap profile output. This option is disabled by default. - - - opt.overcommit - (bool) - r- - [] - - Over-commit enabled/disabled. If enabled, over-commit - memory as a side effect of using anonymous - mmap - 2 or - sbrk - 2 for virtual memory allocation. - In order for overcommit to be disabled, the swap.fds mallctl must have - been successfully written to. This option is enabled by - default. - - - - - tcache.flush - (void) - -- - [] - - Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface - need not be called, since automatic periodic incremental garbage - collection occurs, and the thread cache is automatically discarded when - a thread exits. However, garbage collection is triggered by allocation - activity, so it is possible for a thread that stops - allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful. - - - + thread.arena (unsigned) rw Get or set the arena associated with the calling - thread. The arena index must be less than the maximum number of arenas - (see the arenas.narenas - mallctl). If the specified arena was not initialized beforehand (see - the arenas.initialized mallctl), it will be automatically initialized as a side effect of calling this interface. @@ -1185,7 +1345,7 @@ malloc_conf = "xmalloc:true";]]> cases. - + thread.allocatedp (uint64_t *) @@ -1212,7 +1372,7 @@ malloc_conf = "xmalloc:true";]]> cases. - + thread.deallocatedp (uint64_t *) @@ -1226,147 +1386,109 @@ malloc_conf = "xmalloc:true";]]> mallctl* calls. - - - arenas.narenas - (unsigned) - r- - - Maximum number of arenas. - - - - - arenas.initialized - (bool *) - r- - - An array of arenas.narenas - booleans. Each boolean indicates whether the corresponding arena is - initialized. - - - - - arenas.quantum - (size_t) - r- - - Quantum size. - - - - - arenas.cacheline - (size_t) - r- - - Assumed cacheline size. - - - - - arenas.subpage - (size_t) - r- - - Subpage size class interval. - - - - - arenas.pagesize - (size_t) - r- - - Page size. - - - + - arenas.chunksize - (size_t) - r- - - Chunk size. - - - - - arenas.tspace_min - (size_t) - r- + thread.tcache.enabled + (bool) + rw + [] - Minimum tiny size class. Tiny size classes are powers - of two. + Enable/disable calling thread's tcache. The tcache is + implicitly flushed as a side effect of becoming + disabled (see thread.tcache.flush). + - + - arenas.tspace_max - (size_t) - r- + thread.tcache.flush + (void) + -- + [] - Maximum tiny size class. Tiny size classes are powers - of two. + Flush calling thread's tcache. This interface releases + all cached objects and internal data structures associated with the + calling thread's thread-specific cache. Ordinarily, this interface + need not be called, since automatic periodic incremental garbage + collection occurs, and the thread cache is automatically discarded when + a thread exits. However, garbage collection is triggered by allocation + activity, so it is possible for a thread that stops + allocating/deallocating to retain its cache indefinitely, in which case + the developer may find manual flushing useful. - + - arenas.qspace_min - (size_t) - r- + arena.<i>.purge + (unsigned) + -- - Minimum quantum-spaced size class. + Purge unused dirty pages for arena <i>, or for + all arenas if <i> equals arenas.narenas. + - + - arenas.qspace_max - (size_t) - r- + arena.<i>.dss + (const char *) + rw - Maximum quantum-spaced size class. + Set the precedence of dss allocation as related to mmap + allocation for arena <i>, or for all arenas if <i> equals + arenas.narenas. Note + that even during huge allocation this setting is read from the arena + that would be chosen for small or large allocation so that applications + can depend on consistent dss versus mmap allocation regardless of + allocation size. See opt.dss for supported + settings. + - + - arenas.cspace_min - (size_t) + arenas.narenas + (unsigned) r- - Minimum cacheline-spaced size class. + Current limit on number of arenas. - + - arenas.cspace_max - (size_t) + arenas.initialized + (bool *) r- - Maximum cacheline-spaced size class. + An array of arenas.narenas + booleans. Each boolean indicates whether the corresponding arena is + initialized. - + - arenas.sspace_min + arenas.quantum (size_t) r- - Minimum subpage-spaced size class. + Quantum size. - + - arenas.sspace_max + arenas.page (size_t) r- - Maximum subpage-spaced size class. + Page size. - + arenas.tcache_max (size_t) @@ -1376,55 +1498,16 @@ malloc_conf = "xmalloc:true";]]> Maximum thread-cached size class. - - - arenas.ntbins - (unsigned) - r- - - Number of tiny bin size classes. - - - - - arenas.nqbins - (unsigned) - r- - - Number of quantum-spaced bin size - classes. - - - - - arenas.ncbins - (unsigned) - r- - - Number of cacheline-spaced bin size - classes. - - - - - arenas.nsbins - (unsigned) - r- - - Number of subpage-spaced bin size - classes. - - - + arenas.nbins (unsigned) r- - Total number of bin size classes. + Number of bin size classes. - + arenas.nhbins (unsigned) @@ -1444,7 +1527,7 @@ malloc_conf = "xmalloc:true";]]> Maximum size supported by size class. - + arenas.bin.<i>.nregs (uint32_t) @@ -1453,7 +1536,7 @@ malloc_conf = "xmalloc:true";]]> Number of regions per page run. - + arenas.bin.<i>.run_size (size_t) @@ -1462,7 +1545,7 @@ malloc_conf = "xmalloc:true";]]> Number of bytes per page run. - + arenas.nlruns (size_t) @@ -1471,7 +1554,7 @@ malloc_conf = "xmalloc:true";]]> Total number of large size classes. - + arenas.lrun.<i>.size (size_t) @@ -1481,7 +1564,7 @@ malloc_conf = "xmalloc:true";]]> class. - + arenas.purge (unsigned) @@ -1491,6 +1574,16 @@ malloc_conf = "xmalloc:true";]]> for all arenas if none is specified. + + + arenas.extend + (unsigned) + r- + + Extend the array of arenas by appending a new arena, + and returning the new arena index. + + prof.active @@ -1505,7 +1598,7 @@ malloc_conf = "xmalloc:true";]]> - + prof.dump (const char *) @@ -1521,7 +1614,7 @@ malloc_conf = "xmalloc:true";]]> option. - + prof.interval (uint64_t) @@ -1576,10 +1669,12 @@ malloc_conf = "xmalloc:true";]]> application. This is a multiple of the page size, and greater than or equal to stats.allocated. - + This does not include + stats.arenas.<i>.pdirty and pages + entirely devoted to allocator metadata. - + stats.mapped (size_t) @@ -1590,11 +1685,10 @@ malloc_conf = "xmalloc:true";]]> application. This is a multiple of the chunk size, and is at least as large as stats.active. This - does not include inactive chunks backed by swap files. his does not - include inactive chunks embedded in the DSS. + does not include inactive chunks. - + stats.chunks.current (size_t) @@ -1602,12 +1696,11 @@ malloc_conf = "xmalloc:true";]]> [] Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks backed by swap - files. This does not include inactive chunks embedded in the DSS. + application. This does not include inactive chunks. - + stats.chunks.total (uint64_t) @@ -1617,7 +1710,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of chunks allocated. - + stats.chunks.high (size_t) @@ -1628,7 +1721,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.allocated (size_t) @@ -1639,7 +1732,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.nmalloc (uint64_t) @@ -1650,7 +1743,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.ndalloc (uint64_t) @@ -1661,7 +1754,21 @@ malloc_conf = "xmalloc:true";]]> - + + + stats.arenas.<i>.dss + (const char *) + r- + + dss (sbrk + 2) allocation precedence as + related to mmap + 2 allocation. See opt.dss for details. + + + + stats.arenas.<i>.nthreads (unsigned) @@ -1671,7 +1778,7 @@ malloc_conf = "xmalloc:true";]]> arena. - + stats.arenas.<i>.pactive (size_t) @@ -1680,7 +1787,7 @@ malloc_conf = "xmalloc:true";]]> Number of pages in active runs. - + stats.arenas.<i>.pdirty (size_t) @@ -1692,7 +1799,7 @@ malloc_conf = "xmalloc:true";]]> similar has not been called. - + stats.arenas.<i>.mapped (size_t) @@ -1702,7 +1809,7 @@ malloc_conf = "xmalloc:true";]]> Number of mapped bytes. - + stats.arenas.<i>.npurge (uint64_t) @@ -1713,7 +1820,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.nmadvise (uint64_t) @@ -1725,9 +1832,9 @@ malloc_conf = "xmalloc:true";]]> similar calls made to purge dirty pages. - + - stats.arenas.<i>.npurged + stats.arenas.<i>.purged (uint64_t) r- [] @@ -1735,7 +1842,7 @@ malloc_conf = "xmalloc:true";]]> Number of pages purged. - + stats.arenas.<i>.small.allocated (size_t) @@ -1746,7 +1853,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.small.nmalloc (uint64_t) @@ -1757,7 +1864,7 @@ malloc_conf = "xmalloc:true";]]> small bins. - + stats.arenas.<i>.small.ndalloc (uint64_t) @@ -1768,7 +1875,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.small.nrequests (uint64_t) @@ -1779,7 +1886,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.large.allocated (size_t) @@ -1790,7 +1897,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.large.nmalloc (uint64_t) @@ -1801,7 +1908,7 @@ malloc_conf = "xmalloc:true";]]> directly by the arena. - + stats.arenas.<i>.large.ndalloc (uint64_t) @@ -1812,7 +1919,7 @@ malloc_conf = "xmalloc:true";]]> directly by the arena. - + stats.arenas.<i>.large.nrequests (uint64_t) @@ -1823,7 +1930,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.allocated (size_t) @@ -1834,7 +1941,7 @@ malloc_conf = "xmalloc:true";]]> bin. - + stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) @@ -1845,7 +1952,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) @@ -1856,7 +1963,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.nrequests (uint64_t) @@ -1867,7 +1974,7 @@ malloc_conf = "xmalloc:true";]]> requests. - + stats.arenas.<i>.bins.<j>.nfills (uint64_t) @@ -1877,7 +1984,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of tcache fills. - + stats.arenas.<i>.bins.<j>.nflushes (uint64_t) @@ -1887,7 +1994,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of tcache flushes. - + stats.arenas.<i>.bins.<j>.nruns (uint64_t) @@ -1897,7 +2004,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of runs created. - + stats.arenas.<i>.bins.<j>.nreruns (uint64_t) @@ -1908,18 +2015,7 @@ malloc_conf = "xmalloc:true";]]> to allocate changed. - - - stats.arenas.<i>.bins.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far. - - - - + stats.arenas.<i>.bins.<j>.curruns (size_t) @@ -1929,7 +2025,7 @@ malloc_conf = "xmalloc:true";]]> Current number of runs. - + stats.arenas.<i>.lruns.<j>.nmalloc (uint64_t) @@ -1940,7 +2036,7 @@ malloc_conf = "xmalloc:true";]]> class served directly by the arena. - + stats.arenas.<i>.lruns.<j>.ndalloc (uint64_t) @@ -1951,7 +2047,7 @@ malloc_conf = "xmalloc:true";]]> size class served directly by the arena. - + stats.arenas.<i>.lruns.<j>.nrequests (uint64_t) @@ -1962,18 +2058,7 @@ malloc_conf = "xmalloc:true";]]> class. - - - stats.arenas.<i>.lruns.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far for this - size class. - - - + stats.arenas.<i>.lruns.<j>.curruns (size_t) @@ -1983,65 +2068,6 @@ malloc_conf = "xmalloc:true";]]> Current number of runs for this size class. - - - - swap.avail - (size_t) - r- - [] - - Number of swap file bytes that are currently not - associated with any chunk (i.e. mapped, but otherwise completely - unmanaged). - - - - - swap.prezeroed - (bool) - rw - [] - - If true, the allocator assumes that the swap file(s) - contain nothing but nil bytes. If this assumption is violated, - allocator behavior is undefined. This value becomes read-only after - swap.fds is - successfully written to. - - - - - swap.nfds - (size_t) - r- - [] - - Number of file descriptors in use for swap. - - - - - - swap.fds - (int *) - rw - [] - - When written to, the files associated with the - specified file descriptors are contiguously mapped via - mmap - 2. The resulting virtual memory - region is preferred over anonymous - mmap - 2 and - sbrk - 2 memory. Note that if a file's - size is not a multiple of the page size, it is automatically truncated - to the nearest page size multiple. See the - swap.prezeroed - mallctl for specifying that the files are pre-zeroed. - @@ -2065,10 +2091,9 @@ malloc_conf = "xmalloc:true";]]> This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics. + would be prohibitive. However, jemalloc does integrate with the most + excellent Valgrind tool if the + configuration option is enabled. DIAGNOSTIC MESSAGES @@ -2124,6 +2149,27 @@ malloc_conf = "xmalloc:true";]]> + The aligned_alloc function returns + a pointer to the allocated memory if successful; otherwise a + NULL pointer is returned and + errno is set. The + aligned_alloc function will fail if: + + + EINVAL + + The alignment parameter is + not a power of 2. + + + + ENOMEM + + Memory allocation error. + + + + The realloc function returns a pointer, possibly identical to ptr, to the allocated memory if successful; otherwise a NULL @@ -2138,9 +2184,26 @@ malloc_conf = "xmalloc:true";]]> Non-standard API - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. + The mallocx and + rallocx functions return a pointer to + the allocated memory if successful; otherwise a NULL + pointer is returned to indicate insufficient contiguous memory was + available to service the allocation request. + + The xallocx function returns the + real size of the resulting resized allocation pointed to by + ptr, which is a value less than + size if the allocation could not be adequately + grown in place. + + The sallocx function returns the + real size of the allocation pointed to by ptr. + + + The nallocx returns the real size + that would result from a successful equivalent + mallocx function call, or zero if + insufficient memory is available to perform the size computation. The mallctl, mallctlnametomib, and @@ -2157,12 +2220,6 @@ malloc_conf = "xmalloc:true";]]> is too large or too small; in this case as much data as possible are read despite the error. - - ENOMEM - - *oldlenp is too short to - hold the requested value. - ENOENT @@ -2191,16 +2248,22 @@ malloc_conf = "xmalloc:true";]]> + + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. Experimental API The allocm, rallocm, - sallocm, and - dallocm functions return + sallocm, + dallocm, and + nallocm functions return ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm and - rallocm functions will fail if: + error value. The allocm, + rallocm, and + nallocm functions will fail if: ALLOCM_ERR_OOM @@ -2259,6 +2322,8 @@ malloc_conf = "lg_chunk:24";]]> 2, sbrk 2, + utrace + 2, alloca 3, atexit diff --git a/deps/jemalloc/include/jemalloc/internal/arena.h b/deps/jemalloc/include/jemalloc/internal/arena.h index b80c118d811..9d000c03dec 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena.h +++ b/deps/jemalloc/include/jemalloc/internal/arena.h @@ -1,41 +1,6 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -/* - * Subpages are an artificially designated partitioning of pages. Their only - * purpose is to support subpage-spaced size classes. - * - * There must be at least 4 subpages per page, due to the way size classes are - * handled. - */ -#define LG_SUBPAGE 8 -#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) -#define SUBPAGE_MASK (SUBPAGE - 1) - -/* Return the smallest subpage multiple that is >= s. */ -#define SUBPAGE_CEILING(s) \ - (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) - -#ifdef JEMALLOC_TINY - /* Smallest size class to support. */ -# define LG_TINY_MIN LG_SIZEOF_PTR -# define TINY_MIN (1U << LG_TINY_MIN) -#endif - -/* - * Maximum size class that is a multiple of the quantum, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_QSPACE_MAX_DEFAULT 7 - -/* - * Maximum size class that is a multiple of the cacheline, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_CSPACE_MAX_DEFAULT 9 - /* * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized * as small as possible such that this setting is still honored, without @@ -51,7 +16,7 @@ * constraint is relaxed (ignored) for runs that are so small that the * per-region overhead is greater than: * - * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) + * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) */ #define RUN_BFP 12 /* \/ Implicit binary fixed point. */ @@ -62,15 +27,21 @@ #define LG_RUN_MAXREGS 11 #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) +/* + * Minimum redzone size. Redzones may be larger than this if necessary to + * preserve region alignment. + */ +#define REDZONE_MINSIZE 16 + /* * The minimum ratio of active:dirty pages per arena is computed as: * * (nactive >> opt_lg_dirty_mult) >= ndirty * - * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 - * times as many active pages as dirty pages. + * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times + * as many active pages as dirty pages. */ -#define LG_DIRTY_MULT_DEFAULT 5 +#define LG_DIRTY_MULT_DEFAULT 3 typedef struct arena_chunk_map_s arena_chunk_map_t; typedef struct arena_chunk_s arena_chunk_t; @@ -85,11 +56,20 @@ typedef struct arena_s arena_t; /* Each element of the chunk map corresponds to one page within the chunk. */ struct arena_chunk_map_s { +#ifndef JEMALLOC_PROF + /* + * Overlay prof_ctx in order to allow it to be referenced by dead code. + * Such antics aren't warranted for per arena data structures, but + * chunk map overhead accounts for a percentage of memory, rather than + * being just a fixed cost. + */ + union { +#endif union { /* * Linkage for run trees. There are two disjoint uses: * - * 1) arena_t's runs_avail_{clean,dirty} trees. + * 1) arena_t's runs_avail tree. * 2) arena_run_t conceptually uses this linkage for in-use * non-full runs, rather than directly embedding linkage. */ @@ -103,22 +83,23 @@ struct arena_chunk_map_s { ql_elm(arena_chunk_map_t) ql_link; } u; -#ifdef JEMALLOC_PROF /* Profile counters, used for large object runs. */ prof_ctx_t *prof_ctx; +#ifndef JEMALLOC_PROF + }; /* union { ... }; */ #endif /* * Run address (or size) and various flags are stored together. The bit * layout looks like (assuming 32-bit system): * - * ???????? ???????? ????---- ----dula + * ???????? ???????? ????nnnn nnnndula * * ? : Unallocated: Run address for first/last pages, unset for internal * pages. * Small: Run page offset. * Large: Run size for first page, unset for trailing pages. - * - : Unused. + * n : binind for small size class, BININD_INVALID for large size class. * d : dirty? * u : unzeroed? * l : large? @@ -128,7 +109,8 @@ struct arena_chunk_map_s { * * p : run page offset * s : run size - * c : (binind+1) for size class (used only if prof_promote is true) + * n : binind for size class; large objects set these to BININD_INVALID + * except for promoted allocations (see prof_promote) * x : don't care * - : 0 * + : 1 @@ -136,37 +118,38 @@ struct arena_chunk_map_s { * [dula] : bit unset * * Unallocated (clean): - * ssssssss ssssssss ssss---- ----du-a - * xxxxxxxx xxxxxxxx xxxx---- -----Uxx - * ssssssss ssssssss ssss---- ----dU-a + * ssssssss ssssssss ssss++++ ++++du-a + * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx + * ssssssss ssssssss ssss++++ ++++dU-a * * Unallocated (dirty): - * ssssssss ssssssss ssss---- ----D--a - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * ssssssss ssssssss ssss---- ----D--a + * ssssssss ssssssss ssss++++ ++++D--a + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * ssssssss ssssssss ssss++++ ++++D--a * * Small: - * pppppppp pppppppp pppp---- ----d--A - * pppppppp pppppppp pppp---- -------A - * pppppppp pppppppp pppp---- ----d--A + * pppppppp pppppppp ppppnnnn nnnnd--A + * pppppppp pppppppp ppppnnnn nnnn---A + * pppppppp pppppppp ppppnnnn nnnnd--A * * Large: - * ssssssss ssssssss ssss---- ----D-LA - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * -------- -------- -------- ----D-LA + * ssssssss ssssssss ssss++++ ++++D-LA + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * -------- -------- ----++++ ++++D-LA * - * Large (sampled, size <= PAGE_SIZE): - * ssssssss ssssssss sssscccc ccccD-LA + * Large (sampled, size <= PAGE): + * ssssssss ssssssss ssssnnnn nnnnD-LA * - * Large (not sampled, size == PAGE_SIZE): - * ssssssss ssssssss ssss---- ----D-LA + * Large (not sampled, size == PAGE): + * ssssssss ssssssss ssss++++ ++++D-LA */ size_t bits; -#ifdef JEMALLOC_PROF -#define CHUNK_MAP_CLASS_SHIFT 4 -#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) -#endif -#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) +#define CHUNK_MAP_BININD_SHIFT 4 +#define BININD_INVALID ((size_t)0xffU) +/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ +#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) +#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK +#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) #define CHUNK_MAP_DIRTY ((size_t)0x8U) #define CHUNK_MAP_UNZEROED ((size_t)0x4U) #define CHUNK_MAP_LARGE ((size_t)0x2U) @@ -175,24 +158,30 @@ struct arena_chunk_map_s { }; typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; +typedef ql_head(arena_chunk_map_t) arena_chunk_mapelms_t; /* Arena chunk header. */ struct arena_chunk_s { /* Arena that owns the chunk. */ - arena_t *arena; + arena_t *arena; + + /* Linkage for tree of arena chunks that contain dirty runs. */ + rb_node(arena_chunk_t) dirty_link; + + /* Number of dirty pages. */ + size_t ndirty; - /* Linkage for the arena's chunks_dirty list. */ - ql_elm(arena_chunk_t) link_dirty; + /* Number of available runs. */ + size_t nruns_avail; /* - * True if the chunk is currently in the chunks_dirty list, due to - * having at some point contained one or more dirty pages. Removal - * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. + * Number of available run adjacencies that purging could coalesce. + * Clean and dirty available runs are not coalesced, which causes + * virtual memory fragmentation. The ratio of + * (nruns_avail-nruns_adjac):nruns_adjac is used for tracking this + * fragmentation. */ - bool dirtied; - - /* Number of dirty pages. */ - size_t ndirty; + size_t nruns_adjac; /* * Map of pages within chunk that keeps track of free/large/small. The @@ -200,16 +189,11 @@ struct arena_chunk_s { * need to be tracked in the map. This omission saves a header page * for common chunk sizes (e.g. 4 MiB). */ - arena_chunk_map_t map[1]; /* Dynamically sized. */ + arena_chunk_map_t map[1]; /* Dynamically sized. */ }; typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; struct arena_run_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_RUN_MAGIC 0x384adf93 -#endif - /* Bin this run is associated with. */ arena_bin_t *bin; @@ -224,11 +208,50 @@ struct arena_run_s { * Read-only information associated with each element of arena_t's bins array * is stored separately, partly to reduce memory usage (only one copy, rather * than one per arena), but mainly to avoid false cacheline sharing. + * + * Each run has the following layout: + * + * /--------------------\ + * | arena_run_t header | + * | ... | + * bitmap_offset | bitmap | + * | ... | + * ctx0_offset | ctx map | + * | ... | + * |--------------------| + * | redzone | + * reg0_offset | region 0 | + * | redzone | + * |--------------------| \ + * | redzone | | + * | region 1 | > reg_interval + * | redzone | / + * |--------------------| + * | ... | + * | ... | + * | ... | + * |--------------------| + * | redzone | + * | region nregs-1 | + * | redzone | + * |--------------------| + * | alignment pad? | + * \--------------------/ + * + * reg_interval has at least the same minimum alignment as reg_size; this + * preserves the alignment constraint that sa2u() depends on. Alignment pad is + * either 0 or redzone_size; it is present only if needed to align reg0_offset. */ struct arena_bin_info_s { /* Size of regions in a run for this bin's size class. */ size_t reg_size; + /* Redzone size. */ + size_t redzone_size; + + /* Interval between regions (reg_size + (redzone_size << 1)). */ + size_t reg_interval; + /* Total size of a run for this bin's size class. */ size_t run_size; @@ -247,13 +270,11 @@ struct arena_bin_info_s { */ bitmap_info_t bitmap_info; -#ifdef JEMALLOC_PROF /* * Offset of first (prof_ctx_t *) in a run header for this bin's size - * class, or 0 if (opt_prof == false). + * class, or 0 if (config_prof == false || opt_prof == false). */ uint32_t ctx0_offset; -#endif /* Offset of first region in a run for this bin's size class. */ uint32_t reg0_offset; @@ -283,18 +304,11 @@ struct arena_bin_s { */ arena_run_tree_t runs; -#ifdef JEMALLOC_STATS /* Bin statistics. */ malloc_bin_stats_t stats; -#endif }; struct arena_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_MAGIC 0x947d3d24 -#endif - /* This arena's index within the arenas array. */ unsigned ind; @@ -314,23 +328,19 @@ struct arena_s { */ malloc_mutex_t lock; -#ifdef JEMALLOC_STATS arena_stats_t stats; -# ifdef JEMALLOC_TCACHE /* * List of tcaches for extant threads associated with this arena. * Stats from these are merged incrementally, and at exit. */ ql_head(tcache_t) tcache_ql; -# endif -#endif -#ifdef JEMALLOC_PROF uint64_t prof_accumbytes; -#endif - /* List of dirty-page-containing chunks this arena manages. */ - ql_head(arena_chunk_t) chunks_dirty; + dss_prec_t dss_prec; + + /* Tree of dirty-page-containing chunks this arena manages. */ + arena_chunk_tree_t chunks_dirty; /* * In order to avoid rapid chunk allocation/deallocation when an arena @@ -365,153 +375,421 @@ struct arena_s { /* * Size/address-ordered trees of this arena's available runs. The trees - * are used for first-best-fit run allocation. The dirty tree contains - * runs with dirty pages (i.e. very likely to have been touched and - * therefore have associated physical pages), whereas the clean tree - * contains runs with pages that either have no associated physical - * pages, or have pages that the kernel may recycle at any time due to - * previous madvise(2) calls. The dirty tree is used in preference to - * the clean tree for allocations, because using dirty pages reduces - * the amount of dirty purging necessary to keep the active:dirty page - * ratio below the purge threshold. + * are used for first-best-fit run allocation. */ - arena_avail_tree_t runs_avail_clean; - arena_avail_tree_t runs_avail_dirty; + arena_avail_tree_t runs_avail; - /* - * bins is used to store trees of free regions of the following sizes, - * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and - * default MALLOC_CONF. - * - * bins[i] | size | - * --------+--------+ - * 0 | 8 | - * --------+--------+ - * 1 | 16 | - * 2 | 32 | - * 3 | 48 | - * : : - * 6 | 96 | - * 7 | 112 | - * 8 | 128 | - * --------+--------+ - * 9 | 192 | - * 10 | 256 | - * 11 | 320 | - * 12 | 384 | - * 13 | 448 | - * 14 | 512 | - * --------+--------+ - * 15 | 768 | - * 16 | 1024 | - * 17 | 1280 | - * : : - * 25 | 3328 | - * 26 | 3584 | - * 27 | 3840 | - * --------+--------+ - */ - arena_bin_t bins[1]; /* Dynamically sized. */ + /* bins is used to store trees of free regions. */ + arena_bin_t bins[NBINS]; }; #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -extern size_t opt_lg_qspace_max; -extern size_t opt_lg_cspace_max; extern ssize_t opt_lg_dirty_mult; /* * small_size2bin is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via the SMALL_SIZE2BIN macro. */ -extern uint8_t const *small_size2bin; +extern uint8_t const small_size2bin[]; #define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) -extern arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ -# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) -#else -# define ntbins 0 -#endif -extern unsigned nqbins; /* Number of quantum-spaced bins. */ -extern unsigned ncbins; /* Number of cacheline-spaced bins. */ -extern unsigned nsbins; /* Number of subpage-spaced bins. */ -extern unsigned nbins; -#ifdef JEMALLOC_TINY -# define tspace_max ((size_t)(QUANTUM >> 1)) -#endif -#define qspace_min QUANTUM -extern size_t qspace_max; -extern size_t cspace_min; -extern size_t cspace_max; -extern size_t sspace_min; -extern size_t sspace_max; -#define small_maxclass sspace_max +extern arena_bin_info_t arena_bin_info[NBINS]; +/* Number of large size classes. */ #define nlclasses (chunk_npages - map_bias) void arena_purge_all(arena_t *arena); -#ifdef JEMALLOC_PROF -void arena_prof_accum(arena_t *arena, uint64_t accumbytes); -#endif -#ifdef JEMALLOC_TCACHE void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, - size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ); + size_t binind, uint64_t prof_accumbytes); +void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, + bool zero); +#ifdef JEMALLOC_JET +typedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, + uint8_t); +extern arena_redzone_corruption_t *arena_redzone_corruption; +typedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); +extern arena_dalloc_junk_small_t *arena_dalloc_junk_small; +#else +void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); #endif +void arena_quarantine_junk_small(void *ptr, size_t usize); void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); -void *arena_malloc(size_t size, bool zero); -void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, - size_t alignment, bool zero); -size_t arena_salloc(const void *ptr); -#ifdef JEMALLOC_PROF +void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); void arena_prof_promoted(const void *ptr, size_t size); -size_t arena_salloc_demote(const void *ptr); -#endif -void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, +void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm); +void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind, arena_chunk_map_t *mapelm); +void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind); +#ifdef JEMALLOC_JET +typedef void (arena_dalloc_junk_large_t)(void *, size_t); +extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; +#endif +void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, + void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); -#ifdef JEMALLOC_STATS -void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats); +#ifdef JEMALLOC_JET +typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); +extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; #endif -void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, +bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); +void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc); +dss_prec_t arena_dss_prec_get(arena_t *arena); +void arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, + size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats); bool arena_new(arena_t *arena, unsigned ind); -bool arena_boot(void); +void arena_boot(void); +void arena_prefork(arena_t *arena); +void arena_postfork_parent(arena_t *arena); +void arena_postfork_child(arena_t *arena); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); +size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbitsp_read(size_t *mapbitsp); +size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, + size_t pageind); +size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); +void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); +void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, + size_t size, size_t flags); +void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, + size_t size); +void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, + size_t size, size_t flags); +void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, + size_t binind); +void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, + size_t runind, size_t binind, size_t flags); +void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, + size_t unzeroed); +bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); +bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); +bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); +size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); -# ifdef JEMALLOC_PROF prof_ctx_t *arena_prof_ctx_get(const void *ptr); -void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -# endif -void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); +void arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); +void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); +size_t arena_salloc(const void *ptr, bool demote); +void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, + bool try_tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) +# ifdef JEMALLOC_ARENA_INLINE_A +JEMALLOC_ALWAYS_INLINE arena_chunk_map_t * +arena_mapp_get(arena_chunk_t *chunk, size_t pageind) +{ + + assert(pageind >= map_bias); + assert(pageind < chunk_npages); + + return (&chunk->map[pageind-map_bias]); +} + +JEMALLOC_ALWAYS_INLINE size_t * +arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) +{ + + return (&arena_mapp_get(chunk, pageind)->bits); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbitsp_read(size_t *mapbitsp) +{ + + return (*mapbitsp); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_get(arena_chunk_t *chunk, size_t pageind) +{ + + return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); + return (mapbits & ~PAGE_MASK); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == + (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); + return (mapbits & ~PAGE_MASK); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == + CHUNK_MAP_ALLOCATED); + return (mapbits >> LG_PAGE); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + size_t binind; + + mapbits = arena_mapbits_get(chunk, pageind); + binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; + assert(binind < NBINS || binind == BININD_INVALID); + return (binind); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_DIRTY); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_UNZEROED); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_LARGE); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_ALLOCATED); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) +{ + + *mapbitsp = mapbits; +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, + size_t flags) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + + assert((size & PAGE_MASK) == 0); + assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); + assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); + arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, + size_t size) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); + + assert((size & PAGE_MASK) == 0); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); + arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK)); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, + size_t flags) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); + size_t unzeroed; + + assert((size & PAGE_MASK) == 0); + assert((flags & CHUNK_MAP_DIRTY) == flags); + unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags + | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, + size_t binind) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); + + assert(binind <= BININD_INVALID); + assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); + arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | + (binind << CHUNK_MAP_BININD_SHIFT)); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, + size_t binind, size_t flags) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); + size_t unzeroed; + + assert(binind < BININD_INVALID); + assert(pageind - runind >= map_bias); + assert((flags & CHUNK_MAP_DIRTY) == flags); + unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind << + CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED); +} + +JEMALLOC_ALWAYS_INLINE void +arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, + size_t unzeroed) +{ + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); + + arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) | + unzeroed); +} + +JEMALLOC_INLINE bool +arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + assert(prof_interval != 0); + + arena->prof_accumbytes += accumbytes; + if (arena->prof_accumbytes >= prof_interval) { + arena->prof_accumbytes -= prof_interval; + return (true); + } + return (false); +} + +JEMALLOC_INLINE bool +arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + + if (prof_interval == 0) + return (false); + return (arena_prof_accum_impl(arena, accumbytes)); +} + +JEMALLOC_INLINE bool +arena_prof_accum(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + + if (prof_interval == 0) + return (false); + + { + bool ret; + + malloc_mutex_lock(&arena->lock); + ret = arena_prof_accum_impl(arena, accumbytes); + malloc_mutex_unlock(&arena->lock); + return (ret); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_ptr_small_binind_get(const void *ptr, size_t mapbits) +{ + size_t binind; + + binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; + + if (config_debug) { + arena_chunk_t *chunk; + arena_t *arena; + size_t pageind; + size_t actual_mapbits; + arena_run_t *run; + arena_bin_t *bin; + size_t actual_binind; + arena_bin_info_t *bin_info; + + assert(binind != BININD_INVALID); + assert(binind < NBINS); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = chunk->arena; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + actual_mapbits = arena_mapbits_get(chunk, pageind); + assert(mapbits == actual_mapbits); + assert(arena_mapbits_large_get(chunk, pageind) == 0); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (actual_mapbits >> LG_PAGE)) << LG_PAGE)); + bin = run->bin; + actual_binind = bin - arena->bins; + assert(binind == actual_binind); + bin_info = &arena_bin_info[actual_binind]; + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval + == 0); + } + + return (binind); +} +# endif /* JEMALLOC_ARENA_INLINE_A */ + +# ifdef JEMALLOC_ARENA_INLINE_B JEMALLOC_INLINE size_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { size_t binind = bin - arena->bins; - assert(binind < nbins); + assert(binind < NBINS); return (binind); } @@ -519,9 +797,8 @@ JEMALLOC_INLINE unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) { unsigned shift, diff, regind; - size_t size; + size_t interval; - dassert(run->magic == ARENA_RUN_MAGIC); /* * Freeing a pointer lower than region zero can cause assertion * failure. @@ -537,12 +814,12 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) bin_info->reg0_offset); /* Rescale (factor powers of 2 out of the numerator and denominator). */ - size = bin_info->reg_size; - shift = ffs(size) - 1; + interval = bin_info->reg_interval; + shift = ffs(interval) - 1; diff >>= shift; - size >>= shift; + interval >>= shift; - if (size == 1) { + if (interval == 1) { /* The divisor was a power of 2. */ regind = diff; } else { @@ -554,7 +831,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) * * becomes * - * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT + * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT * * We can omit the first three elements, because we never * divide by 0, and 1 and 2 are both powers of two, which are @@ -562,7 +839,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) */ #define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) - static const unsigned size_invs[] = { + static const unsigned interval_invs[] = { SIZE_INV(3), SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), @@ -573,20 +850,21 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) }; - if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) - regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; - else - regind = diff / size; + if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + + 2)) { + regind = (diff * interval_invs[interval - 3]) >> + SIZE_INV_SHIFT; + } else + regind = diff / interval; #undef SIZE_INV #undef SIZE_INV_SHIFT } - assert(diff == regind * size); + assert(diff == regind * interval); assert(regind < bin_info->nregs); return (regind); } -#ifdef JEMALLOC_PROF JEMALLOC_INLINE prof_ctx_t * arena_prof_ctx_get(const void *ptr) { @@ -594,149 +872,191 @@ arena_prof_ctx_get(const void *ptr) arena_chunk_t *chunk; size_t pageind, mapbits; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); if ((mapbits & CHUNK_MAP_LARGE) == 0) { if (prof_promote) ret = (prof_ctx_t *)(uintptr_t)1U; else { arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - size_t binind = arena_bin_index(chunk->arena, run->bin); + (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << + LG_PAGE)); + size_t binind = arena_ptr_small_binind_get(ptr, + mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind; - dassert(run->magic == ARENA_RUN_MAGIC); regind = arena_run_regind(run, bin_info, ptr); ret = *(prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t *))); } } else - ret = chunk->map[pageind-map_bias].prof_ctx; + ret = arena_mapp_get(chunk, pageind)->prof_ctx; return (ret); } JEMALLOC_INLINE void -arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) { arena_chunk_t *chunk; - size_t pageind, mapbits; + size_t pageind; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + + if (usize > SMALL_MAXCLASS || (prof_promote && + ((uintptr_t)ctx != (uintptr_t)1U || arena_mapbits_large_get(chunk, + pageind) != 0))) { + assert(arena_mapbits_large_get(chunk, pageind) != 0); + arena_mapp_get(chunk, pageind)->prof_ctx = ctx; + } else { + assert(arena_mapbits_large_get(chunk, pageind) == 0); if (prof_promote == false) { + size_t mapbits = arena_mapbits_get(chunk, pageind); arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; + (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << + LG_PAGE)); size_t binind; arena_bin_info_t *bin_info; unsigned regind; - dassert(run->magic == ARENA_RUN_MAGIC); - binind = arena_bin_index(chunk->arena, bin); + binind = arena_ptr_small_binind_get(ptr, mapbits); bin_info = &arena_bin_info[binind]; regind = arena_run_regind(run, bin_info, ptr); - *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset - + (regind * sizeof(prof_ctx_t *)))) = ctx; - } else - assert((uintptr_t)ctx == (uintptr_t)1U); - } else - chunk->map[pageind-map_bias].prof_ctx = ctx; + *((prof_ctx_t **)((uintptr_t)run + + bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t + *)))) = ctx; + } + } } -#endif -JEMALLOC_INLINE void -arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) +JEMALLOC_ALWAYS_INLINE void * +arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) { - size_t pageind; - arena_chunk_map_t *mapelm; + tcache_t *tcache; + + assert(size != 0); + assert(size <= arena_maxclass); + + if (size <= SMALL_MAXCLASS) { + if (try_tcache && (tcache = tcache_get(true)) != NULL) + return (tcache_alloc_small(tcache, size, zero)); + else { + return (arena_malloc_small(choose_arena(arena), size, + zero)); + } + } else { + /* + * Initialize tcache after checking size in order to avoid + * infinite recursion during tcache initialization. + */ + if (try_tcache && size <= tcache_maxclass && (tcache = + tcache_get(true)) != NULL) + return (tcache_alloc_large(tcache, size, zero)); + else { + return (arena_malloc_large(choose_arena(arena), size, + zero)); + } + } +} + +/* Return the size of the allocation pointed to by ptr. */ +JEMALLOC_ALWAYS_INLINE size_t +arena_salloc(const void *ptr, bool demote) +{ + size_t ret; + arena_chunk_t *chunk; + size_t pageind, binind; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + binind = arena_mapbits_binind_get(chunk, pageind); + if (binind == BININD_INVALID || (config_prof && demote == false && + prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { + /* + * Large allocation. In the common case (demote == true), and + * as this is an inline function, most callers will only end up + * looking at binind to determine that ptr is a small + * allocation. + */ + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + ret = arena_mapbits_large_size_get(chunk, pageind); + assert(ret != 0); + assert(pageind + (ret>>LG_PAGE) <= chunk_npages); + assert(ret == PAGE || arena_mapbits_large_size_get(chunk, + pageind+(ret>>LG_PAGE)-1) == 0); + assert(binind == arena_mapbits_binind_get(chunk, + pageind+(ret>>LG_PAGE)-1)); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); + } else { + /* + * Small allocation (possibly promoted to a large object due to + * prof_promote). + */ + assert(arena_mapbits_large_get(chunk, pageind) != 0 || + arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)) == binind); + ret = arena_bin_info[binind].reg_size; + } + + return (ret); +} + +JEMALLOC_ALWAYS_INLINE void +arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) +{ + size_t pageind, mapbits; + tcache_t *tcache; assert(arena != NULL); - dassert(arena->magic == ARENA_MAGIC); assert(chunk->arena == arena); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { /* Small allocation. */ -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; + if (try_tcache && (tcache = tcache_get(false)) != NULL) { + size_t binind; - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_small(tcache, ptr); - else { -#endif - arena_run_t *run; - arena_bin_t *bin; - - run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> - PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; -#ifdef JEMALLOC_DEBUG - { - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % - bin_info->reg_size == 0); - } -#endif - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, ptr, mapelm); - malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_TCACHE - } -#endif + binind = arena_ptr_small_binind_get(ptr, mapbits); + tcache_dalloc_small(tcache, ptr, binind); + } else + arena_dalloc_small(arena, chunk, ptr, pageind); } else { -#ifdef JEMALLOC_TCACHE - size_t size = mapelm->bits & ~PAGE_MASK; + size_t size = arena_mapbits_large_size_get(chunk, pageind); assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_large(tcache, ptr, size); - else { - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } - } else { - malloc_mutex_lock(&arena->lock); + + if (try_tcache && size <= tcache_maxclass && (tcache = + tcache_get(false)) != NULL) { + tcache_dalloc_large(tcache, ptr, size); + } else arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } -#else - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); -#endif } } +# endif /* JEMALLOC_ARENA_INLINE_B */ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic.h index 9a298623f8a..11a7b47fe0f 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic.h @@ -11,22 +11,8 @@ #define atomic_read_uint64(p) atomic_add_uint64(p, 0) #define atomic_read_uint32(p) atomic_add_uint32(p, 0) - -#if (LG_SIZEOF_PTR == 3) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x) -#elif (LG_SIZEOF_PTR == 2) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x) -#endif +#define atomic_read_z(p) atomic_add_z(p, 0) +#define atomic_read_u(p) atomic_add_u(p, 0) #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -37,12 +23,17 @@ uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); +size_t atomic_add_z(size_t *p, size_t x); +size_t atomic_sub_z(size_t *p, size_t x); +unsigned atomic_add_u(unsigned *p, unsigned x); +unsigned atomic_sub_u(unsigned *p, unsigned x); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) /******************************************************************************/ /* 64-bit operations. */ -#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 +#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) +# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -56,6 +47,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (__sync_sub_and_fetch(p, x)); } +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, x)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, -((int64_t)x))); +} #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) @@ -70,7 +75,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); } -#elif (defined(__amd64_) || defined(__x86_64__)) +# elif (defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -97,8 +102,43 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (x); } -#else -# if (LG_SIZEOF_PTR == 3) +# elif (defined(JEMALLOC_ATOMIC9)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + /* + * atomic_fetchadd_64() doesn't exist, but we only ever use this + * function on LP64 systems, so atomic_fetchadd_long() will do. + */ + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + return (atomic_fetchadd_long(p, (unsigned long)x) + x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); +} +# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} +# else # error "Missing implementation for 64-bit atomic operations" # endif #endif @@ -119,6 +159,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (__sync_sub_and_fetch(p, x)); } +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, x)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, -((int32_t)x))); +} #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) @@ -133,7 +187,7 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); } -#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) +#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { @@ -160,9 +214,90 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (x); } +#elif (defined(JEMALLOC_ATOMIC9)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (atomic_fetchadd_32(p, x) + x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); +} +#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} #else # error "Missing implementation for 32-bit atomic operations" #endif + +/******************************************************************************/ +/* size_t operations. */ +JEMALLOC_INLINE size_t +atomic_add_z(size_t *p, size_t x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); +#elif (LG_SIZEOF_PTR == 2) + return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); +#endif +} + +JEMALLOC_INLINE size_t +atomic_sub_z(size_t *p, size_t x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((size_t)atomic_add_uint64((uint64_t *)p, + (uint64_t)-((int64_t)x))); +#elif (LG_SIZEOF_PTR == 2) + return ((size_t)atomic_add_uint32((uint32_t *)p, + (uint32_t)-((int32_t)x))); +#endif +} + +/******************************************************************************/ +/* unsigned operations. */ +JEMALLOC_INLINE unsigned +atomic_add_u(unsigned *p, unsigned x) +{ + +#if (LG_SIZEOF_INT == 3) + return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); +#elif (LG_SIZEOF_INT == 2) + return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); +#endif +} + +JEMALLOC_INLINE unsigned +atomic_sub_u(unsigned *p, unsigned x) +{ + +#if (LG_SIZEOF_INT == 3) + return ((unsigned)atomic_add_uint64((uint64_t *)p, + (uint64_t)-((int64_t)x))); +#elif (LG_SIZEOF_INT == 2) + return ((unsigned)atomic_add_uint32((uint32_t *)p, + (uint32_t)-((int32_t)x))); +#endif +} +/******************************************************************************/ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/deps/jemalloc/include/jemalloc/internal/base.h b/deps/jemalloc/include/jemalloc/internal/base.h index e353f309bd2..9cf75ffb0b3 100644 --- a/deps/jemalloc/include/jemalloc/internal/base.h +++ b/deps/jemalloc/include/jemalloc/internal/base.h @@ -9,12 +9,14 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -extern malloc_mutex_t base_mtx; - void *base_alloc(size_t size); +void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dealloc(extent_node_t *node); bool base_boot(void); +void base_prefork(void); +void base_postfork_parent(void); +void base_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk.h b/deps/jemalloc/include/jemalloc/internal/chunk.h index 54b6a3ec886..87d8700dac8 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk.h @@ -28,20 +28,14 @@ #ifdef JEMALLOC_H_EXTERNS extern size_t opt_lg_chunk; -#ifdef JEMALLOC_SWAP -extern bool opt_overcommit; -#endif +extern const char *opt_dss; -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) /* Protects stats_chunks; currently not used for any other purpose. */ extern malloc_mutex_t chunks_mtx; /* Chunk statistics. */ extern chunk_stats_t stats_chunks; -#endif -#ifdef JEMALLOC_IVSALLOC extern rtree_t *chunks_rtree; -#endif extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ @@ -49,9 +43,14 @@ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -void *chunk_alloc(size_t size, bool base, bool *zero); +void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, + dss_prec_t dss_prec); +void chunk_unmap(void *chunk, size_t size); void chunk_dealloc(void *chunk, size_t size, bool unmap); bool chunk_boot(void); +void chunk_prefork(void); +void chunk_postfork_parent(void); +void chunk_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -60,6 +59,5 @@ bool chunk_boot(void); #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#include "jemalloc/internal/chunk_swap.h" #include "jemalloc/internal/chunk_dss.h" #include "jemalloc/internal/chunk_mmap.h" diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h index 6f005222181..4535ce09c09 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h @@ -1,25 +1,34 @@ -#ifdef JEMALLOC_DSS /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +typedef enum { + dss_prec_disabled = 0, + dss_prec_primary = 1, + dss_prec_secondary = 2, + + dss_prec_limit = 3 +} dss_prec_t; +#define DSS_PREC_DEFAULT dss_prec_secondary +#define DSS_DEFAULT "secondary" + #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +extern const char *dss_prec_names[]; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -/* - * Protects sbrk() calls. This avoids malloc races among threads, though it - * does not protect against races with threads that call sbrk() directly. - */ -extern malloc_mutex_t dss_mtx; - -void *chunk_alloc_dss(size_t size, bool *zero); +dss_prec_t chunk_dss_prec_get(void); +bool chunk_dss_prec_set(dss_prec_t dss_prec); +void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero); bool chunk_in_dss(void *chunk); -bool chunk_dealloc_dss(void *chunk, size_t size); bool chunk_dss_boot(void); +void chunk_dss_prefork(void); +void chunk_dss_postfork_parent(void); +void chunk_dss_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -27,4 +36,3 @@ bool chunk_dss_boot(void); #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h index 07b50a4dc37..f24abac7538 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h @@ -9,11 +9,10 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -void *chunk_alloc_mmap(size_t size); -void *chunk_alloc_mmap_noreserve(size_t size); -void chunk_dealloc_mmap(void *chunk, size_t size); +bool pages_purge(void *addr, size_t length); -bool chunk_mmap_boot(void); +void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero); +bool chunk_dealloc_mmap(void *chunk, size_t size); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_swap.h b/deps/jemalloc/include/jemalloc/internal/chunk_swap.h deleted file mode 100644 index 9faa739f713..00000000000 --- a/deps/jemalloc/include/jemalloc/internal/chunk_swap.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifdef JEMALLOC_SWAP -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern malloc_mutex_t swap_mtx; -extern bool swap_enabled; -extern bool swap_prezeroed; -extern size_t swap_nfds; -extern int *swap_fds; -#ifdef JEMALLOC_STATS -extern size_t swap_avail; -#endif - -void *chunk_alloc_swap(size_t size, bool *zero); -bool chunk_in_swap(void *chunk); -bool chunk_dealloc_swap(void *chunk, size_t size); -bool chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed); -bool chunk_swap_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ -#endif /* JEMALLOC_SWAP */ diff --git a/deps/jemalloc/include/jemalloc/internal/ckh.h b/deps/jemalloc/include/jemalloc/internal/ckh.h index 3e4ad4c85f9..58712a6a763 100644 --- a/deps/jemalloc/include/jemalloc/internal/ckh.h +++ b/deps/jemalloc/include/jemalloc/internal/ckh.h @@ -5,7 +5,7 @@ typedef struct ckh_s ckh_t; typedef struct ckhc_s ckhc_t; /* Typedefs to allow easy function pointer passing. */ -typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *); +typedef void ckh_hash_t (const void *, size_t[2]); typedef bool ckh_keycomp_t (const void *, const void *); /* Maintain counters used to get an idea of performance. */ @@ -17,7 +17,7 @@ typedef bool ckh_keycomp_t (const void *, const void *); * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket. Try to fit * one bucket per L1 cache line. */ -#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) +#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -30,11 +30,6 @@ struct ckhc_s { }; struct ckh_s { -#ifdef JEMALLOC_DEBUG -#define CKH_MAGIC 0x3af2489d - uint32_t magic; -#endif - #ifdef CKH_COUNT /* Counters used to get an idea of performance. */ uint64_t ngrows; @@ -47,7 +42,7 @@ struct ckh_s { /* Used for pseudo-random number generation. */ #define CKH_A 1103515241 #define CKH_C 12347 - uint32_t prn_state; + uint32_t prng_state; /* Total number of items. */ size_t count; @@ -80,11 +75,9 @@ bool ckh_insert(ckh_t *ckh, const void *key, const void *data); bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data); -void ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); +void ckh_string_hash(const void *key, size_t r_hash[2]); bool ckh_string_keycomp(const void *k1, const void *k2); -void ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); +void ckh_pointer_hash(const void *key, size_t r_hash[2]); bool ckh_pointer_keycomp(const void *k1, const void *k2); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/deps/jemalloc/include/jemalloc/internal/ctl.h b/deps/jemalloc/include/jemalloc/internal/ctl.h index f1f5eb70a2a..0ffecc5f2a2 100644 --- a/deps/jemalloc/include/jemalloc/internal/ctl.h +++ b/deps/jemalloc/include/jemalloc/internal/ctl.h @@ -2,6 +2,8 @@ #ifdef JEMALLOC_H_TYPES typedef struct ctl_node_s ctl_node_t; +typedef struct ctl_named_node_s ctl_named_node_t; +typedef struct ctl_indexed_node_s ctl_indexed_node_t; typedef struct ctl_arena_stats_s ctl_arena_stats_t; typedef struct ctl_stats_s ctl_stats_t; @@ -11,28 +13,29 @@ typedef struct ctl_stats_s ctl_stats_t; struct ctl_node_s { bool named; - union { - struct { - const char *name; - /* If (nchildren == 0), this is a terminal node. */ - unsigned nchildren; - const ctl_node_t *children; - } named; - struct { - const ctl_node_t *(*index)(const size_t *, size_t, - size_t); - } indexed; - } u; - int (*ctl)(const size_t *, size_t, void *, size_t *, void *, - size_t); +}; + +struct ctl_named_node_s { + struct ctl_node_s node; + const char *name; + /* If (nchildren == 0), this is a terminal node. */ + unsigned nchildren; + const ctl_node_t *children; + int (*ctl)(const size_t *, size_t, void *, size_t *, + void *, size_t); +}; + +struct ctl_indexed_node_s { + struct ctl_node_s node; + const ctl_named_node_t *(*index)(const size_t *, size_t, size_t); }; struct ctl_arena_stats_s { bool initialized; unsigned nthreads; + const char *dss; size_t pactive; size_t pdirty; -#ifdef JEMALLOC_STATS arena_stats_t astats; /* Aggregate stats for small size classes, based on bin stats. */ @@ -41,13 +44,11 @@ struct ctl_arena_stats_s { uint64_t ndalloc_small; uint64_t nrequests_small; - malloc_bin_stats_t *bstats; /* nbins elements. */ + malloc_bin_stats_t bstats[NBINS]; malloc_large_stats_t *lstats; /* nlclasses elements. */ -#endif }; struct ctl_stats_s { -#ifdef JEMALLOC_STATS size_t allocated; size_t active; size_t mapped; @@ -61,11 +62,8 @@ struct ctl_stats_s { uint64_t nmalloc; /* huge_nmalloc */ uint64_t ndalloc; /* huge_ndalloc */ } huge; -#endif + unsigned narenas; ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ -#ifdef JEMALLOC_SWAP - size_t swap_avail; -#endif }; #endif /* JEMALLOC_H_STRUCTS */ @@ -79,29 +77,30 @@ int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp); int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); bool ctl_boot(void); +void ctl_prefork(void); +void ctl_postfork_parent(void); +void ctl_postfork_child(void); #define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen) \ + if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ != 0) { \ - malloc_write(": Failure in xmallctl(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ + malloc_printf( \ + ": Failure in xmallctl(\"%s\", ...)\n", \ + name); \ abort(); \ } \ } while (0) #define xmallctlnametomib(name, mibp, miblenp) do { \ - if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) { \ - malloc_write( \ - ": Failure in xmallctlnametomib(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ + if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \ + malloc_printf(": Failure in " \ + "xmallctlnametomib(\"%s\", ...)\n", name); \ abort(); \ } \ } while (0) #define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp, \ + if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, \ newlen) != 0) { \ malloc_write( \ ": Failure in xmallctlbymib()\n"); \ diff --git a/deps/jemalloc/include/jemalloc/internal/extent.h b/deps/jemalloc/include/jemalloc/internal/extent.h index 6fe9702b5f6..ba95ca816bd 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent.h +++ b/deps/jemalloc/include/jemalloc/internal/extent.h @@ -9,24 +9,23 @@ typedef struct extent_node_s extent_node_t; /* Tree of extents. */ struct extent_node_s { -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) /* Linkage for the size/address-ordered tree. */ rb_node(extent_node_t) link_szad; -#endif /* Linkage for the address-ordered tree. */ rb_node(extent_node_t) link_ad; -#ifdef JEMALLOC_PROF /* Profile counters, used for huge objects. */ prof_ctx_t *prof_ctx; -#endif /* Pointer to the extent that this tree node is responsible for. */ void *addr; /* Total region size. */ size_t size; + + /* True if zero-filled; used by chunk recycling code. */ + bool zeroed; }; typedef rb_tree(extent_node_t) extent_tree_t; @@ -34,9 +33,7 @@ typedef rb_tree(extent_node_t) extent_tree_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) -#endif rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) diff --git a/deps/jemalloc/include/jemalloc/internal/hash.h b/deps/jemalloc/include/jemalloc/internal/hash.h index 8a46ce30803..c7183ede82d 100644 --- a/deps/jemalloc/include/jemalloc/internal/hash.h +++ b/deps/jemalloc/include/jemalloc/internal/hash.h @@ -1,3 +1,8 @@ +/* + * The following hash function is based on MurmurHash3, placed into the public + * domain by Austin Appleby. See http://code.google.com/p/smhasher/ for + * details. + */ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -14,55 +19,315 @@ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -uint64_t hash(const void *key, size_t len, uint64_t seed); +uint32_t hash_x86_32(const void *key, int len, uint32_t seed); +void hash_x86_128(const void *key, const int len, uint32_t seed, + uint64_t r_out[2]); +void hash_x64_128(const void *key, const int len, const uint32_t seed, + uint64_t r_out[2]); +void hash(const void *key, size_t len, const uint32_t seed, + size_t r_hash[2]); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_)) -/* - * The following hash function is based on MurmurHash64A(), placed into the - * public domain by Austin Appleby. See http://murmurhash.googlepages.com/ for - * details. - */ +/******************************************************************************/ +/* Internal implementation. */ +JEMALLOC_INLINE uint32_t +hash_rotl_32(uint32_t x, int8_t r) +{ + + return (x << r) | (x >> (32 - r)); +} + +JEMALLOC_INLINE uint64_t +hash_rotl_64(uint64_t x, int8_t r) +{ + return (x << r) | (x >> (64 - r)); +} + +JEMALLOC_INLINE uint32_t +hash_get_block_32(const uint32_t *p, int i) +{ + + return (p[i]); +} + JEMALLOC_INLINE uint64_t -hash(const void *key, size_t len, uint64_t seed) +hash_get_block_64(const uint64_t *p, int i) { - const uint64_t m = 0xc6a4a7935bd1e995LLU; - const int r = 47; - uint64_t h = seed ^ (len * m); - const uint64_t *data = (const uint64_t *)key; - const uint64_t *end = data + (len/8); - const unsigned char *data2; - assert(((uintptr_t)key & 0x7) == 0); + return (p[i]); +} + +JEMALLOC_INLINE uint32_t +hash_fmix_32(uint32_t h) +{ - while(data != end) { - uint64_t k = *data++; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; - k *= m; - k ^= k >> r; - k *= m; + return (h); +} - h ^= k; - h *= m; +JEMALLOC_INLINE uint64_t +hash_fmix_64(uint64_t k) +{ + + k ^= k >> 33; + k *= QU(0xff51afd7ed558ccdLLU); + k ^= k >> 33; + k *= QU(0xc4ceb9fe1a85ec53LLU); + k ^= k >> 33; + + return (k); +} + +JEMALLOC_INLINE uint32_t +hash_x86_32(const void *key, int len, uint32_t seed) +{ + const uint8_t *data = (const uint8_t *) key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + /* body */ + { + const uint32_t *blocks = (const uint32_t *) (data + nblocks*4); + int i; + + for (i = -nblocks; i; i++) { + uint32_t k1 = hash_get_block_32(blocks, i); + + k1 *= c1; + k1 = hash_rotl_32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = hash_rotl_32(h1, 13); + h1 = h1*5 + 0xe6546b64; + } } - data2 = (const unsigned char *)data; - switch(len & 7) { - case 7: h ^= ((uint64_t)(data2[6])) << 48; - case 6: h ^= ((uint64_t)(data2[5])) << 40; - case 5: h ^= ((uint64_t)(data2[4])) << 32; - case 4: h ^= ((uint64_t)(data2[3])) << 24; - case 3: h ^= ((uint64_t)(data2[2])) << 16; - case 2: h ^= ((uint64_t)(data2[1])) << 8; - case 1: h ^= ((uint64_t)(data2[0])); - h *= m; + /* tail */ + { + const uint8_t *tail = (const uint8_t *) (data + nblocks*4); + + uint32_t k1 = 0; + + switch (len & 3) { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); + k1 *= c2; h1 ^= k1; + } } - h ^= h >> r; - h *= m; - h ^= h >> r; + /* finalization */ + h1 ^= len; - return (h); + h1 = hash_fmix_32(h1); + + return (h1); +} + +UNUSED JEMALLOC_INLINE void +hash_x86_128(const void *key, const int len, uint32_t seed, + uint64_t r_out[2]) +{ + const uint8_t * data = (const uint8_t *) key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t c1 = 0x239b961b; + const uint32_t c2 = 0xab0e9789; + const uint32_t c3 = 0x38b34ae5; + const uint32_t c4 = 0xa1e38b93; + + /* body */ + { + const uint32_t *blocks = (const uint32_t *) (data + nblocks*16); + int i; + + for (i = -nblocks; i; i++) { + uint32_t k1 = hash_get_block_32(blocks, i*4 + 0); + uint32_t k2 = hash_get_block_32(blocks, i*4 + 1); + uint32_t k3 = hash_get_block_32(blocks, i*4 + 2); + uint32_t k4 = hash_get_block_32(blocks, i*4 + 3); + + k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + + h1 = hash_rotl_32(h1, 19); h1 += h2; + h1 = h1*5 + 0x561ccd1b; + + k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + + h2 = hash_rotl_32(h2, 17); h2 += h3; + h2 = h2*5 + 0x0bcaa747; + + k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + + h3 = hash_rotl_32(h3, 15); h3 += h4; + h3 = h3*5 + 0x96cd1c35; + + k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + + h4 = hash_rotl_32(h4, 13); h4 += h1; + h4 = h4*5 + 0x32ac3b17; + } + } + + /* tail */ + { + const uint8_t *tail = (const uint8_t *) (data + nblocks*16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch (len & 15) { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + } + } + + /* finalization */ + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = hash_fmix_32(h1); + h2 = hash_fmix_32(h2); + h3 = hash_fmix_32(h3); + h4 = hash_fmix_32(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + r_out[0] = (((uint64_t) h2) << 32) | h1; + r_out[1] = (((uint64_t) h4) << 32) | h3; +} + +UNUSED JEMALLOC_INLINE void +hash_x64_128(const void *key, const int len, const uint32_t seed, + uint64_t r_out[2]) +{ + const uint8_t *data = (const uint8_t *) key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + const uint64_t c1 = QU(0x87c37b91114253d5LLU); + const uint64_t c2 = QU(0x4cf5ad432745937fLLU); + + /* body */ + { + const uint64_t *blocks = (const uint64_t *) (data); + int i; + + for (i = 0; i < nblocks; i++) { + uint64_t k1 = hash_get_block_64(blocks, i*2 + 0); + uint64_t k2 = hash_get_block_64(blocks, i*2 + 1); + + k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + + h1 = hash_rotl_64(h1, 27); h1 += h2; + h1 = h1*5 + 0x52dce729; + + k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + + h2 = hash_rotl_64(h2, 31); h2 += h1; + h2 = h2*5 + 0x38495ab5; + } + } + + /* tail */ + { + const uint8_t *tail = (const uint8_t*)(data + nblocks*16); + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) { + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; + case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; + k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; + case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; + k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + } + } + + /* finalization */ + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = hash_fmix_64(h1); + h2 = hash_fmix_64(h2); + + h1 += h2; + h2 += h1; + + r_out[0] = h1; + r_out[1] = h2; +} + +/******************************************************************************/ +/* API. */ +JEMALLOC_INLINE void +hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) +{ +#if (LG_SIZEOF_PTR == 3 && !defined(JEMALLOC_BIG_ENDIAN)) + hash_x64_128(key, len, seed, (uint64_t *)r_hash); +#else + uint64_t hashes[2]; + hash_x86_128(key, len, seed, hashes); + r_hash[0] = (size_t)hashes[0]; + r_hash[1] = (size_t)hashes[1]; +#endif } #endif diff --git a/deps/jemalloc/include/jemalloc/internal/huge.h b/deps/jemalloc/include/jemalloc/internal/huge.h index 66544cf8d97..a2b9c779191 100644 --- a/deps/jemalloc/include/jemalloc/internal/huge.h +++ b/deps/jemalloc/include/jemalloc/internal/huge.h @@ -9,29 +9,34 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -#ifdef JEMALLOC_STATS /* Huge allocation statistics. */ extern uint64_t huge_nmalloc; extern uint64_t huge_ndalloc; extern size_t huge_allocated; -#endif /* Protects chunk-related data structures. */ extern malloc_mutex_t huge_mtx; -void *huge_malloc(size_t size, bool zero); -void *huge_palloc(size_t size, size_t alignment, bool zero); -void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, +void *huge_malloc(size_t size, bool zero, dss_prec_t dss_prec); +void *huge_palloc(size_t size, size_t alignment, bool zero, + dss_prec_t dss_prec); +bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); + size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec); +#ifdef JEMALLOC_JET +typedef void (huge_dalloc_junk_t)(void *, size_t); +extern huge_dalloc_junk_t *huge_dalloc_junk; +#endif void huge_dalloc(void *ptr, bool unmap); size_t huge_salloc(const void *ptr); -#ifdef JEMALLOC_PROF +dss_prec_t huge_dss_prec_get(arena_t *arena); prof_ctx_t *huge_prof_ctx_get(const void *ptr); void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -#endif bool huge_boot(void); +void huge_prefork(void); +void huge_postfork_parent(void); +void huge_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in index a44f0978ae4..574bbb14186 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in @@ -1,17 +1,33 @@ -#include -#include -#include +#ifndef JEMALLOC_INTERNAL_H +#define JEMALLOC_INTERNAL_H +#include +#ifdef _WIN32 +# include +# define ENOENT ERROR_PATH_NOT_FOUND +# define EINVAL ERROR_BAD_ARGUMENTS +# define EAGAIN ERROR_OUTOFMEMORY +# define EPERM ERROR_WRITE_FAULT +# define EFAULT ERROR_INVALID_ADDRESS +# define ENOMEM ERROR_NOT_ENOUGH_MEMORY +# undef ERANGE +# define ERANGE ERROR_INVALID_DATA +#else +# include +# include +# include +# if !defined(SYS_write) && defined(__NR_write) +# define SYS_write __NR_write +# endif +# include +# include +# include +#endif #include -#include -#include -#include #include #ifndef SIZE_T_MAX # define SIZE_T_MAX SIZE_MAX #endif -#include -#include #include #include #include @@ -25,16 +41,160 @@ #include #include #include -#include +#ifdef _MSC_VER +# include +typedef intptr_t ssize_t; +# define PATH_MAX 1024 +# define STDERR_FILENO 2 +# define __func__ __FUNCTION__ +/* Disable warnings about deprecated system functions */ +# pragma warning(disable: 4996) +#else +# include +#endif #include -#include -#include -#define JEMALLOC_MANGLE -#include "../jemalloc@install_suffix@.h" +#include "jemalloc_internal_defs.h" +#ifdef JEMALLOC_UTRACE +#include +#endif + +#ifdef JEMALLOC_VALGRIND +#include +#include +#endif + +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "../jemalloc@install_suffix@.h" +# undef JEMALLOC_NO_RENAME +#else +# define JEMALLOC_N(n) @private_namespace@##n +# include "../jemalloc@install_suffix@.h" +#endif #include "jemalloc/internal/private_namespace.h" +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool config_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool config_mremap = +#ifdef JEMALLOC_MREMAP + true +#else + false +#endif + ; +static const bool config_munmap = +#ifdef JEMALLOC_MUNMAP + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_valgrind = +#ifdef JEMALLOC_VALGRIND + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_ivsalloc = +#ifdef JEMALLOC_IVSALLOC + true +#else + false +#endif + ; + +#ifdef JEMALLOC_ATOMIC9 +#include +#endif + #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) #include #endif @@ -46,48 +206,11 @@ #include #endif -#ifdef JEMALLOC_LAZY_LOCK -#include -#endif - #define RB_COMPACT #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" #include "jemalloc/internal/ql.h" -extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); - -/* - * Define a custom assert() in order to reduce the chances of deadlock during - * assertion failure. - */ -#ifndef assert -# ifdef JEMALLOC_DEBUG -# define assert(e) do { \ - if (!(e)) { \ - char line_buf[UMAX2S_BUFSIZE]; \ - malloc_write(": "); \ - malloc_write(__FILE__); \ - malloc_write(":"); \ - malloc_write(u2s(__LINE__, 10, line_buf)); \ - malloc_write(": Failed assertion: "); \ - malloc_write("\""); \ - malloc_write(#e); \ - malloc_write("\"\n"); \ - abort(); \ - } \ -} while (0) -# else -# define assert(e) -# endif -#endif - -#ifdef JEMALLOC_DEBUG -# define dassert(e) assert(e) -#else -# define dassert(e) -#endif - /* * jemalloc can conceptually be broken into components (arena, tcache, etc.), * but there are circular dependencies that cannot be broken without @@ -102,55 +225,64 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); * JEMALLOC_H_INLINES : Inline functions. */ /******************************************************************************/ -#define JEMALLOC_H_TYPES +#define JEMALLOC_H_TYPES -#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) +#include "jemalloc/internal/jemalloc_internal_macros.h" -#define ZU(z) ((size_t)z) - -#ifndef __DECONST -# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) -#endif - -#ifdef JEMALLOC_DEBUG - /* Disable inlining to make debugging easier. */ -# define JEMALLOC_INLINE -# define inline -#else -# define JEMALLOC_ENABLE_INLINE -# define JEMALLOC_INLINE static inline -#endif +#define MALLOCX_LG_ALIGN_MASK ((int)0x3f) +#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) -/* Size of stack-allocated buffer passed to buferror(). */ -#define BUFERROR_BUF 64 +/* Smallest size class to support. */ +#define LG_TINY_MIN 3 +#define TINY_MIN (1U << LG_TINY_MIN) -/* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */ -#ifdef __i386__ -# define LG_QUANTUM 4 -#endif -#ifdef __ia64__ -# define LG_QUANTUM 4 -#endif -#ifdef __alpha__ -# define LG_QUANTUM 4 -#endif -#ifdef __sparc64__ -# define LG_QUANTUM 4 -#endif -#if (defined(__amd64__) || defined(__x86_64__)) -# define LG_QUANTUM 4 -#endif -#ifdef __arm__ -# define LG_QUANTUM 3 -#endif -#ifdef __mips__ -# define LG_QUANTUM 3 -#endif -#ifdef __powerpc__ -# define LG_QUANTUM 4 -#endif -#ifdef __s390x__ -# define LG_QUANTUM 4 +/* + * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# ifdef __sparc64__ +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# ifdef __SH4__ +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" +# endif #endif #define QUANTUM ((size_t)(1U << LG_QUANTUM)) @@ -164,67 +296,157 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #define LONG_MASK (LONG - 1) /* Return the smallest long multiple that is >= a. */ -#define LONG_CEILING(a) \ +#define LONG_CEILING(a) \ (((a) + LONG_MASK) & ~LONG_MASK) #define SIZEOF_PTR (1U << LG_SIZEOF_PTR) #define PTR_MASK (SIZEOF_PTR - 1) /* Return the smallest (void *) multiple that is >= a. */ -#define PTR_CEILING(a) \ +#define PTR_CEILING(a) \ (((a) + PTR_MASK) & ~PTR_MASK) /* * Maximum size of L1 cache line. This is used to avoid cache line aliasing. * In addition, this controls the spacing of cacheline-spaced size classes. + * + * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can + * only handle raw constants. */ #define LG_CACHELINE 6 -#define CACHELINE ((size_t)(1U << LG_CACHELINE)) +#define CACHELINE 64 #define CACHELINE_MASK (CACHELINE - 1) /* Return the smallest cacheline multiple that is >= s. */ #define CACHELINE_CEILING(s) \ (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) -/* - * Page size. STATIC_PAGE_SHIFT is determined by the configure script. If - * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where - * compile-time values are required for the purposes of defining data - * structures. - */ -#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) -#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) - -#ifdef PAGE_SHIFT -# undef PAGE_SHIFT -#endif -#ifdef PAGE_SIZE -# undef PAGE_SIZE -#endif +/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */ #ifdef PAGE_MASK # undef PAGE_MASK #endif - -#ifdef DYNAMIC_PAGE_SHIFT -# define PAGE_SHIFT lg_pagesize -# define PAGE_SIZE pagesize -# define PAGE_MASK pagesize_mask -#else -# define PAGE_SHIFT STATIC_PAGE_SHIFT -# define PAGE_SIZE STATIC_PAGE_SIZE -# define PAGE_MASK STATIC_PAGE_MASK -#endif +#define LG_PAGE STATIC_PAGE_SHIFT +#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT)) +#define PAGE_MASK ((size_t)(PAGE - 1)) /* Return the smallest pagesize multiple that is >= s. */ #define PAGE_CEILING(s) \ (((s) + PAGE_MASK) & ~PAGE_MASK) +/* Return the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2BASE(a, alignment) \ + ((void *)((uintptr_t)(a) & (-(alignment)))) + +/* Return the offset between a and the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ + ((size_t)((uintptr_t)(a) & (alignment - 1))) + +/* Return the smallest alignment multiple that is >= s. */ +#define ALIGNMENT_CEILING(s, alignment) \ + (((s) + (alignment - 1)) & (-(alignment))) + +/* Declare a variable length array */ +#if __STDC_VERSION__ < 199901L +# ifdef _MSC_VER +# include +# define alloca _alloca +# else +# ifdef JEMALLOC_HAS_ALLOCA_H +# include +# else +# include +# endif +# endif +# define VARIABLE_ARRAY(type, name, count) \ + type *name = alloca(sizeof(type) * count) +#else +# define VARIABLE_ARRAY(type, name, count) type name[count] +#endif + +#ifdef JEMALLOC_VALGRIND +/* + * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions + * so that when Valgrind reports errors, there are no extra stack frames + * in the backtraces. + * + * The size that is reported to valgrind must be consistent through a chain of + * malloc..realloc..realloc calls. Request size isn't recorded anywhere in + * jemalloc, so it is critical that all callers of these macros provide usize + * rather than request size. As a result, buffer overflow detection is + * technically weakened for the standard API, though it is generally accepted + * practice to consider any extra bytes reported by malloc_usable_size() as + * usable space. + */ +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ + if (config_valgrind && opt_valgrind && cond) \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ +} while (0) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) do { \ + if (config_valgrind && opt_valgrind) { \ + size_t rzsize = p2rz(ptr); \ + \ + if (ptr == old_ptr) { \ + VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ + usize, rzsize); \ + if (zero && old_usize < usize) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + old_usize), usize - old_usize); \ + } \ + } else { \ + if (old_ptr != NULL) { \ + VALGRIND_FREELIKE_BLOCK(old_ptr, \ + old_rzsize); \ + } \ + if (ptr != NULL) { \ + size_t copy_size = (old_usize < usize) \ + ? old_usize : usize; \ + size_t tail_size = usize - copy_size; \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ + rzsize, false); \ + if (copy_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED(ptr, \ + copy_size); \ + } \ + if (zero && tail_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + copy_size), tail_size); \ + } \ + } \ + } \ + } \ +} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ + if (config_valgrind && opt_valgrind) \ + VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ +} while (0) +#else +#define RUNNING_ON_VALGRIND ((unsigned)0) +#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ + do {} while (0) +#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ + do {} while (0) +#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0) +#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0) +#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0) +#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0) +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) do {} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) +#endif + +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" @@ -235,21 +457,22 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_TYPES /******************************************************************************/ -#define JEMALLOC_H_STRUCTS +#define JEMALLOC_H_STRUCTS +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -260,112 +483,66 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" -#ifdef JEMALLOC_STATS typedef struct { uint64_t allocated; uint64_t deallocated; } thread_allocated_t; -#endif +/* + * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro + * argument. + */ +#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0}) #undef JEMALLOC_H_STRUCTS /******************************************************************************/ -#define JEMALLOC_H_EXTERNS +#define JEMALLOC_H_EXTERNS extern bool opt_abort; -#ifdef JEMALLOC_FILL extern bool opt_junk; -#endif -#ifdef JEMALLOC_SYSV -extern bool opt_sysv; -#endif -#ifdef JEMALLOC_XMALLOC +extern size_t opt_quarantine; +extern bool opt_redzone; +extern bool opt_utrace; +extern bool opt_valgrind; extern bool opt_xmalloc; -#endif -#ifdef JEMALLOC_FILL extern bool opt_zero; -#endif extern size_t opt_narenas; -#ifdef DYNAMIC_PAGE_SHIFT -extern size_t pagesize; -extern size_t pagesize_mask; -extern size_t lg_pagesize; -#endif - /* Number of CPUs. */ extern unsigned ncpus; -extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ -extern pthread_key_t arenas_tsd; -#ifndef NO_TLS -/* - * Map of pthread_self() --> arenas[???], used for selecting an arena to use - * for allocations. - */ -extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); -# define ARENA_GET() arenas_tls -# define ARENA_SET(v) do { \ - arenas_tls = (v); \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#else -# define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) -# define ARENA_SET(v) do { \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#endif - +/* Protects arenas initialization (arenas, arenas_total). */ +extern malloc_mutex_t arenas_lock; /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. + * + * arenas[0..narenas_auto) are used for automatic multiplexing of threads and + * arenas. arenas[narenas_auto..narenas_total) are only used if the application + * takes some action to create them and allocate from them. */ extern arena_t **arenas; -extern unsigned narenas; - -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -extern __thread thread_allocated_t thread_allocated_tls; -# define ALLOCATED_GET() (thread_allocated_tls.allocated) -# define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) -# define DEALLOCATED_GET() (thread_allocated_tls.deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_tls.allocated += a; \ - thread_allocated_tls.deallocated += d; \ -} while (0) -# else -extern pthread_key_t thread_allocated_tsd; -thread_allocated_t *thread_allocated_get_hard(void); - -# define ALLOCATED_GET() (thread_allocated_get()->allocated) -# define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) -# define DEALLOCATED_GET() (thread_allocated_get()->deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_t *thread_allocated = thread_allocated_get(); \ - thread_allocated->allocated += (a); \ - thread_allocated->deallocated += (d); \ -} while (0) -# endif -#endif +extern unsigned narenas_total; +extern unsigned narenas_auto; /* Read-only after initialization. */ arena_t *arenas_extend(unsigned ind); +void arenas_cleanup(void *arg); arena_t *choose_arena_hard(void); -int buferror(int errnum, char *buf, size_t buflen); void jemalloc_prefork(void); -void jemalloc_postfork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -376,21 +553,22 @@ void jemalloc_postfork(void); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_EXTERNS /******************************************************************************/ -#define JEMALLOC_H_INLINES +#define JEMALLOC_H_INLINES +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" @@ -398,44 +576,32 @@ void jemalloc_postfork(void); #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE -size_t pow2_ceil(size_t x); +malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) + size_t s2u(size_t size); -size_t sa2u(size_t size, size_t alignment, size_t *run_size_p); -void malloc_write(const char *s); -arena_t *choose_arena(void); -# if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t *thread_allocated_get(void); -# endif +size_t sa2u(size_t size, size_t alignment); +unsigned narenas_total_get(void); +arena_t *choose_arena(arena_t *arena); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -/* Compute the smallest power of 2 that is >= x. */ -JEMALLOC_INLINE size_t -pow2_ceil(size_t x) -{ - - x--; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; -#if (LG_SIZEOF_PTR == 3) - x |= x >> 32; -#endif - x++; - return (x); -} +/* + * Map of pthread_self() --> arenas[???], used for selecting an arena to use + * for allocations. + */ +malloc_tsd_externs(arenas, arena_t *) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL, + arenas_cleanup) /* * Compute usable size that would result from allocating an object with the * specified size. */ -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t s2u(size_t size) { - if (size <= small_maxclass) + if (size <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); if (size <= arena_maxclass) return (PAGE_CEILING(size)); @@ -446,11 +612,13 @@ s2u(size_t size) * Compute usable size that would result from allocating an object with the * specified size and alignment. */ -JEMALLOC_INLINE size_t -sa2u(size_t size, size_t alignment, size_t *run_size_p) +JEMALLOC_ALWAYS_INLINE size_t +sa2u(size_t size, size_t alignment) { size_t usize; + assert(alignment != 0 && ((alignment - 1) & alignment) == 0); + /* * Round size up to the nearest multiple of alignment. * @@ -464,12 +632,8 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) * 96 | 1100000 | 32 * 144 | 10100000 | 32 * 192 | 11000000 | 64 - * - * Depending on runtime settings, it is possible that arena_malloc() - * will further round up to a power of two, but that never causes - * correctness issues. */ - usize = (size + (alignment - 1)) & (-alignment); + usize = ALIGNMENT_CEILING(size, alignment); /* * (usize < size) protects against the combination of maximal * alignment and size greater than maximal alignment. @@ -479,8 +643,8 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) return (0); } - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { - if (usize <= small_maxclass) + if (usize <= arena_maxclass && alignment <= PAGE) { + if (usize <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); return (PAGE_CEILING(usize)); } else { @@ -494,7 +658,7 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) usize = PAGE_CEILING(size); /* * (usize < size) protects against very large sizes within - * PAGE_SIZE of SIZE_T_MAX. + * PAGE of SIZE_T_MAX. * * (usize + alignment < usize) protects against the * combination of maximal alignment and usize large enough @@ -512,196 +676,218 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) /* * Calculate the size of the over-size run that arena_palloc() * would need to allocate in order to guarantee the alignment. + * If the run wouldn't fit within a chunk, round up to a huge + * allocation size. */ - if (usize >= alignment) - run_size = usize + alignment - PAGE_SIZE; - else { - /* - * It is possible that (alignment << 1) will cause - * overflow, but it doesn't matter because we also - * subtract PAGE_SIZE, which in the case of overflow - * leaves us with a very large run_size. That causes - * the first conditional below to fail, which means - * that the bogus run_size value never gets used for - * anything important. - */ - run_size = (alignment << 1) - PAGE_SIZE; - } - if (run_size_p != NULL) - *run_size_p = run_size; - + run_size = usize + alignment - PAGE; if (run_size <= arena_maxclass) return (PAGE_CEILING(usize)); return (CHUNK_CEILING(usize)); } } -/* - * Wrapper around malloc_message() that avoids the need for - * JEMALLOC_P(malloc_message)(...) throughout the code. - */ -JEMALLOC_INLINE void -malloc_write(const char *s) +JEMALLOC_INLINE unsigned +narenas_total_get(void) { + unsigned narenas; + + malloc_mutex_lock(&arenas_lock); + narenas = narenas_total; + malloc_mutex_unlock(&arenas_lock); - JEMALLOC_P(malloc_message)(NULL, s); + return (narenas); } -/* - * Choose an arena based on a per-thread value (fast-path code, calls slow-path - * code if necessary). - */ +/* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * -choose_arena(void) +choose_arena(arena_t *arena) { arena_t *ret; - ret = ARENA_GET(); - if (ret == NULL) { + if (arena != NULL) + return (arena); + + if ((ret = *arenas_tsd_get()) == NULL) { ret = choose_arena_hard(); assert(ret != NULL); } return (ret); } - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -JEMALLOC_INLINE thread_allocated_t * -thread_allocated_get(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - pthread_getspecific(thread_allocated_tsd); - - if (thread_allocated == NULL) - return (thread_allocated_get_hard()); - return (thread_allocated); -} -#endif #endif #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/rtree.h" +/* + * Include arena.h twice in order to resolve circular dependencies with + * tcache.h. + */ +#define JEMALLOC_ARENA_INLINE_A +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_A #include "jemalloc/internal/tcache.h" +#define JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE +void *imalloct(size_t size, bool try_tcache, arena_t *arena); void *imalloc(size_t size); +void *icalloct(size_t size, bool try_tcache, arena_t *arena); void *icalloc(size_t size); +void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena); void *ipalloc(size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr); -# ifdef JEMALLOC_IVSALLOC -size_t ivsalloc(const void *ptr); -# endif +size_t isalloc(const void *ptr, bool demote); +size_t ivsalloc(const void *ptr, bool demote); +size_t u2rz(size_t usize); +size_t p2rz(const void *ptr); +void idalloct(void *ptr, bool try_tcache); void idalloc(void *ptr); +void iqalloct(void *ptr, bool try_tcache); +void iqalloc(void *ptr); +void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena); +void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero, bool no_move); + bool zero); +bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero); +malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_INLINE void * -imalloc(size_t size) +JEMALLOC_ALWAYS_INLINE void * +imalloct(size_t size, bool try_tcache, arena_t *arena) { assert(size != 0); if (size <= arena_maxclass) - return (arena_malloc(size, false)); + return (arena_malloc(arena, size, false, try_tcache)); else - return (huge_malloc(size, false)); + return (huge_malloc(size, false, huge_dss_prec_get(arena))); } -JEMALLOC_INLINE void * -icalloc(size_t size) +JEMALLOC_ALWAYS_INLINE void * +imalloc(size_t size) +{ + + return (imalloct(size, true, NULL)); +} + +JEMALLOC_ALWAYS_INLINE void * +icalloct(size_t size, bool try_tcache, arena_t *arena) { if (size <= arena_maxclass) - return (arena_malloc(size, true)); + return (arena_malloc(arena, size, true, try_tcache)); else - return (huge_malloc(size, true)); + return (huge_malloc(size, true, huge_dss_prec_get(arena))); } -JEMALLOC_INLINE void * -ipalloc(size_t usize, size_t alignment, bool zero) +JEMALLOC_ALWAYS_INLINE void * +icalloc(size_t size) +{ + + return (icalloct(size, true, NULL)); +} + +JEMALLOC_ALWAYS_INLINE void * +ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { void *ret; assert(usize != 0); - assert(usize == sa2u(usize, alignment, NULL)); + assert(usize == sa2u(usize, alignment)); - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) - ret = arena_malloc(usize, zero); + if (usize <= arena_maxclass && alignment <= PAGE) + ret = arena_malloc(arena, usize, zero, try_tcache); else { - size_t run_size -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; - - /* - * Ideally we would only ever call sa2u() once per aligned - * allocation request, and the caller of this function has - * already done so once. However, it's rather burdensome to - * require every caller to pass in run_size, especially given - * that it's only relevant to large allocations. Therefore, - * just call it again here in order to get run_size. - */ - sa2u(usize, alignment, &run_size); - if (run_size <= arena_maxclass) { - ret = arena_palloc(choose_arena(), usize, run_size, + if (usize <= arena_maxclass) { + ret = arena_palloc(choose_arena(arena), usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(usize, zero); + ret = huge_malloc(usize, zero, huge_dss_prec_get(arena)); else - ret = huge_palloc(usize, alignment, zero); + ret = huge_palloc(usize, alignment, zero, huge_dss_prec_get(arena)); } - assert(((uintptr_t)ret & (alignment - 1)) == 0); + assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); return (ret); } -JEMALLOC_INLINE size_t -isalloc(const void *ptr) +JEMALLOC_ALWAYS_INLINE void * +ipalloc(size_t usize, size_t alignment, bool zero) +{ + + return (ipalloct(usize, alignment, zero, true, NULL)); +} + +/* + * Typical usage: + * void *ptr = [...] + * size_t sz = isalloc(ptr, config_prof); + */ +JEMALLOC_ALWAYS_INLINE size_t +isalloc(const void *ptr, bool demote) { size_t ret; arena_chunk_t *chunk; assert(ptr != NULL); + /* Demotion only makes sense if config_prof is true. */ + assert(config_prof || demote == false); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { - /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - -#ifdef JEMALLOC_PROF - ret = arena_salloc_demote(ptr); -#else - ret = arena_salloc(ptr); -#endif - } else + if (chunk != ptr) + ret = arena_salloc(ptr, demote); + else ret = huge_salloc(ptr); return (ret); } -#ifdef JEMALLOC_IVSALLOC -JEMALLOC_INLINE size_t -ivsalloc(const void *ptr) +JEMALLOC_ALWAYS_INLINE size_t +ivsalloc(const void *ptr, bool demote) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) + if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0) return (0); - return (isalloc(ptr)); + return (isalloc(ptr, demote)); } -#endif -JEMALLOC_INLINE void -idalloc(void *ptr) +JEMALLOC_INLINE size_t +u2rz(size_t usize) +{ + size_t ret; + + if (usize <= SMALL_MAXCLASS) { + size_t binind = SMALL_SIZE2BIN(usize); + ret = arena_bin_info[binind].redzone_size; + } else + ret = 0; + + return (ret); +} + +JEMALLOC_INLINE size_t +p2rz(const void *ptr) +{ + size_t usize = isalloc(ptr, false); + + return (u2rz(usize)); +} + +JEMALLOC_ALWAYS_INLINE void +idalloct(void *ptr, bool try_tcache) { arena_chunk_t *chunk; @@ -709,80 +895,134 @@ idalloc(void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr); + arena_dalloc(chunk->arena, chunk, ptr, try_tcache); else huge_dalloc(ptr, true); } -JEMALLOC_INLINE void * -iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, - bool no_move) +JEMALLOC_ALWAYS_INLINE void +idalloc(void *ptr) +{ + + idalloct(ptr, true); +} + +JEMALLOC_ALWAYS_INLINE void +iqalloct(void *ptr, bool try_tcache) +{ + + if (config_fill && opt_quarantine) + quarantine(ptr); + else + idalloct(ptr, try_tcache); +} + +JEMALLOC_ALWAYS_INLINE void +iqalloc(void *ptr) +{ + + iqalloct(ptr, true); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena) +{ + void *p; + size_t usize, copysize; + + usize = sa2u(size + extra, alignment); + if (usize == 0) + return (NULL); + p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + if (p == NULL) { + if (extra == 0) + return (NULL); + /* Try again, without extra this time. */ + usize = sa2u(size, alignment); + if (usize == 0) + return (NULL); + p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + if (p == NULL) + return (NULL); + } + /* + * Copy at most size bytes (not size+extra), since the caller has no + * expectation that the extra bytes will be reliably preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + memcpy(p, ptr, copysize); + iqalloct(ptr, try_tcache_dalloc); + return (p); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, + bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { - void *ret; size_t oldsize; assert(ptr != NULL); assert(size != 0); - oldsize = isalloc(ptr); + oldsize = isalloc(ptr, config_prof); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { - size_t usize, copysize; - /* - * Existing object alignment is inadquate; allocate new space + * Existing object alignment is inadequate; allocate new space * and copy. */ - if (no_move) - return (NULL); - usize = sa2u(size + extra, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, without extra this time. */ - usize = sa2u(size, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - if (ret == NULL) - return (NULL); - } - /* - * Copy at most size bytes (not size+extra), since the caller - * has no expectation that the extra bytes will be reliably - * preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - memcpy(ret, ptr, copysize); - idalloc(ptr); - return (ret); + return (iralloct_realign(ptr, oldsize, size, extra, alignment, + zero, try_tcache_alloc, try_tcache_dalloc, arena)); } - if (no_move) { - if (size <= arena_maxclass) { - return (arena_ralloc_no_move(ptr, oldsize, size, - extra, zero)); - } else { - return (huge_ralloc_no_move(ptr, oldsize, size, - extra)); - } + if (size + extra <= arena_maxclass) { + return (arena_ralloc(arena, ptr, oldsize, size, extra, + alignment, zero, try_tcache_alloc, + try_tcache_dalloc)); } else { - if (size + extra <= arena_maxclass) { - return (arena_ralloc(ptr, oldsize, size, extra, - alignment, zero)); - } else { - return (huge_ralloc(ptr, oldsize, size, extra, - alignment, zero)); - } + return (huge_ralloc(ptr, oldsize, size, extra, + alignment, zero, try_tcache_dalloc, huge_dss_prec_get(arena))); } } + +JEMALLOC_ALWAYS_INLINE void * +iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) +{ + + return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL)); +} + +JEMALLOC_ALWAYS_INLINE bool +ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) +{ + size_t oldsize; + + assert(ptr != NULL); + assert(size != 0); + + oldsize = isalloc(ptr, config_prof); + if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) + != 0) { + /* Existing object alignment is inadequate. */ + return (true); + } + + if (size <= arena_maxclass) + return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); + else + return (huge_ralloc_no_move(ptr, oldsize, size, extra)); +} + +malloc_tsd_externs(thread_allocated, thread_allocated_t) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t, + THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) #endif #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_INLINES /******************************************************************************/ +#endif /* JEMALLOC_INTERNAL_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in new file mode 100644 index 00000000000..c166fbd9e41 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -0,0 +1,205 @@ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +#undef JEMALLOC_PREFIX +#undef JEMALLOC_CPREFIX + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#undef JEMALLOC_PRIVATE_NAMESPACE + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#undef CPU_SPINWAIT + +/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */ +#undef JEMALLOC_ATOMIC9 + +/* + * Defined if OSAtomic*() functions are available, as provided by Darwin, and + * documented in the atomic(3) manual page. + */ +#undef JEMALLOC_OSATOMIC + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +#undef JEMALLOC_OSSPIN + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +#undef JEMALLOC_MALLOC_THREAD_CLEANUP + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#undef JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +#undef JEMALLOC_MUTEX_INIT_CB + +/* Defined if sbrk() is supported. */ +#undef JEMALLOC_HAVE_SBRK + +/* Non-empty if the tls_model attribute is supported. */ +#undef JEMALLOC_TLS_MODEL + +/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */ +#undef JEMALLOC_CC_SILENCE + +/* JEMALLOC_CODE_COVERAGE enables test code coverage analysis. */ +#undef JEMALLOC_CODE_COVERAGE + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +#undef JEMALLOC_DEBUG + +/* JEMALLOC_STATS enables statistics calculation. */ +#undef JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +#undef JEMALLOC_PROF + +/* Use libunwind for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBUNWIND + +/* Use libgcc for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBGCC + +/* Use gcc intrinsics for profile backtracing if defined. */ +#undef JEMALLOC_PROF_GCC + +/* + * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. + * This makes it possible to allocate/deallocate objects without any locking + * when the cache is in the steady state. + */ +#undef JEMALLOC_TCACHE + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage + * segment (DSS). + */ +#undef JEMALLOC_DSS + +/* Support memory filling (junk/zero/quarantine/redzone). */ +#undef JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +#undef JEMALLOC_UTRACE + +/* Support Valgrind. */ +#undef JEMALLOC_VALGRIND + +/* Support optional abort() on OOM. */ +#undef JEMALLOC_XMALLOC + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +#undef JEMALLOC_LAZY_LOCK + +/* One page is 2^STATIC_PAGE_SHIFT bytes. */ +#undef STATIC_PAGE_SHIFT + +/* + * If defined, use munmap() to unmap freed chunks, rather than storing them for + * later reuse. This is disabled by default on Linux because common sequences + * of mmap()/munmap() calls will cause virtual memory map holes. + */ +#undef JEMALLOC_MUNMAP + +/* + * If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is + * disabled by default because it is Linux-specific and it will cause virtual + * memory map holes, much like munmap(2) does. + */ +#undef JEMALLOC_MREMAP + +/* TLS is used to map arenas and magazine caches to threads. */ +#undef JEMALLOC_TLS + +/* + * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside + * within jemalloc-owned chunks before dereferencing them. + */ +#undef JEMALLOC_IVSALLOC + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +#undef JEMALLOC_ZONE +#undef JEMALLOC_ZONE_VERSION + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched. + * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being + * unused, such that they will be discarded rather + * than swapped out. + */ +#undef JEMALLOC_PURGE_MADVISE_DONTNEED +#undef JEMALLOC_PURGE_MADVISE_FREE + +/* + * Define if operating system has alloca.h header. + */ +#undef JEMALLOC_HAS_ALLOCA_H + +/* C99 restrict keyword supported. */ +#undef JEMALLOC_HAS_RESTRICT + +/* For use by hash code. */ +#undef JEMALLOC_BIG_ENDIAN + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#undef LG_SIZEOF_INT + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#undef LG_SIZEOF_LONG + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#undef LG_SIZEOF_INTMAX_T + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h new file mode 100644 index 00000000000..4e2392302c7 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -0,0 +1,51 @@ +/* + * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for + * functions that are static inline functions if inlining is enabled, and + * single-definition library-private functions if inlining is disabled. + * + * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in + * which case the denoted functions are always static, regardless of whether + * inlining is enabled. + */ +#if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE) + /* Disable inlining to make debugging/profiling easier. */ +# define JEMALLOC_ALWAYS_INLINE +# define JEMALLOC_ALWAYS_INLINE_C static +# define JEMALLOC_INLINE +# define JEMALLOC_INLINE_C static +# define inline +#else +# define JEMALLOC_ENABLE_INLINE +# ifdef JEMALLOC_HAVE_ATTR +# define JEMALLOC_ALWAYS_INLINE \ + static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline) +# define JEMALLOC_ALWAYS_INLINE_C \ + static inline JEMALLOC_ATTR(always_inline) +# else +# define JEMALLOC_ALWAYS_INLINE static inline +# define JEMALLOC_ALWAYS_INLINE_C static inline +# endif +# define JEMALLOC_INLINE static inline +# define JEMALLOC_INLINE_C static inline +# ifdef _MSC_VER +# define inline _inline +# endif +#endif + +#ifdef JEMALLOC_CC_SILENCE +# define UNUSED JEMALLOC_ATTR(unused) +#else +# define UNUSED +#endif + +#define ZU(z) ((size_t)z) +#define QU(q) ((uint64_t)q) +#define QI(q) ((int64_t)q) + +#ifndef __DECONST +# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) +#endif + +#ifndef JEMALLOC_HAS_RESTRICT +# define restrict +#endif diff --git a/deps/jemalloc/include/jemalloc/internal/mb.h b/deps/jemalloc/include/jemalloc/internal/mb.h index dc9f2a54262..3cfa7872942 100644 --- a/deps/jemalloc/include/jemalloc/internal/mb.h +++ b/deps/jemalloc/include/jemalloc/internal/mb.h @@ -54,7 +54,7 @@ mb_write(void) ); #endif } -#elif (defined(__amd64_) || defined(__x86_64__)) +#elif (defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE void mb_write(void) { @@ -87,6 +87,13 @@ mb_write(void) : "memory" /* Clobbers. */ ); } +#elif defined(__tile__) +JEMALLOC_INLINE void +mb_write(void) +{ + + __sync_synchronize(); +} #else /* * This is much slower than a simple memory barrier, but the semantics of mutex diff --git a/deps/jemalloc/include/jemalloc/internal/mutex.h b/deps/jemalloc/include/jemalloc/internal/mutex.h index 62947ced55e..de44e1435ad 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex.h @@ -1,22 +1,42 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -#ifdef JEMALLOC_OSSPIN -typedef OSSpinLock malloc_mutex_t; -#else -typedef pthread_mutex_t malloc_mutex_t; -#endif +typedef struct malloc_mutex_s malloc_mutex_t; -#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP -# define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +#ifdef _WIN32 +# define MALLOC_MUTEX_INITIALIZER +#elif (defined(JEMALLOC_OSSPIN)) +# define MALLOC_MUTEX_INITIALIZER {0} +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL} #else -# define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ + defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP} +# else +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER} +# endif #endif #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct malloc_mutex_s { +#ifdef _WIN32 + CRITICAL_SECTION lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + pthread_mutex_t lock; + malloc_mutex_t *postponed_next; +#else + pthread_mutex_t lock; +#endif +}; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -24,11 +44,15 @@ typedef pthread_mutex_t malloc_mutex_t; #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; #else +# undef isthreaded /* Undo private_namespace.h definition. */ # define isthreaded true #endif bool malloc_mutex_init(malloc_mutex_t *mutex); -void malloc_mutex_destroy(malloc_mutex_t *mutex); +void malloc_mutex_prefork(malloc_mutex_t *mutex); +void malloc_mutex_postfork_parent(malloc_mutex_t *mutex); +void malloc_mutex_postfork_child(malloc_mutex_t *mutex); +bool mutex_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -36,7 +60,6 @@ void malloc_mutex_destroy(malloc_mutex_t *mutex); #ifndef JEMALLOC_ENABLE_INLINE void malloc_mutex_lock(malloc_mutex_t *mutex); -bool malloc_mutex_trylock(malloc_mutex_t *mutex); void malloc_mutex_unlock(malloc_mutex_t *mutex); #endif @@ -46,37 +69,27 @@ malloc_mutex_lock(malloc_mutex_t *mutex) { if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockLock(mutex); +#ifdef _WIN32 + EnterCriticalSection(&mutex->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockLock(&mutex->lock); #else - pthread_mutex_lock(mutex); + pthread_mutex_lock(&mutex->lock); #endif } } -JEMALLOC_INLINE bool -malloc_mutex_trylock(malloc_mutex_t *mutex) -{ - - if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - return (OSSpinLockTry(mutex) == false); -#else - return (pthread_mutex_trylock(mutex) != 0); -#endif - } else - return (false); -} - JEMALLOC_INLINE void malloc_mutex_unlock(malloc_mutex_t *mutex) { if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockUnlock(mutex); +#ifdef _WIN32 + LeaveCriticalSection(&mutex->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockUnlock(&mutex->lock); #else - pthread_mutex_unlock(mutex); + pthread_mutex_unlock(&mutex->lock); #endif } } diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.h b/deps/jemalloc/include/jemalloc/internal/private_namespace.h deleted file mode 100644 index d4f5f96d7b2..00000000000 --- a/deps/jemalloc/include/jemalloc/internal/private_namespace.h +++ /dev/null @@ -1,195 +0,0 @@ -#define arena_bin_index JEMALLOC_N(arena_bin_index) -#define arena_boot JEMALLOC_N(arena_boot) -#define arena_dalloc JEMALLOC_N(arena_dalloc) -#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) -#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) -#define arena_malloc JEMALLOC_N(arena_malloc) -#define arena_malloc_large JEMALLOC_N(arena_malloc_large) -#define arena_malloc_small JEMALLOC_N(arena_malloc_small) -#define arena_new JEMALLOC_N(arena_new) -#define arena_palloc JEMALLOC_N(arena_palloc) -#define arena_prof_accum JEMALLOC_N(arena_prof_accum) -#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) -#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) -#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) -#define arena_purge_all JEMALLOC_N(arena_purge_all) -#define arena_ralloc JEMALLOC_N(arena_ralloc) -#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) -#define arena_run_regind JEMALLOC_N(arena_run_regind) -#define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote) -#define arena_stats_merge JEMALLOC_N(arena_stats_merge) -#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) -#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) -#define arenas_extend JEMALLOC_N(arenas_extend) -#define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) -#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) -#define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) -#define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) -#define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) -#define base_alloc JEMALLOC_N(base_alloc) -#define base_boot JEMALLOC_N(base_boot) -#define base_node_alloc JEMALLOC_N(base_node_alloc) -#define base_node_dealloc JEMALLOC_N(base_node_dealloc) -#define bitmap_full JEMALLOC_N(bitmap_full) -#define bitmap_get JEMALLOC_N(bitmap_get) -#define bitmap_info_init JEMALLOC_N(bitmap_info_init) -#define bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups) -#define bitmap_init JEMALLOC_N(bitmap_init) -#define bitmap_set JEMALLOC_N(bitmap_set) -#define bitmap_sfu JEMALLOC_N(bitmap_sfu) -#define bitmap_size JEMALLOC_N(bitmap_size) -#define bitmap_unset JEMALLOC_N(bitmap_unset) -#define bt_init JEMALLOC_N(bt_init) -#define buferror JEMALLOC_N(buferror) -#define choose_arena JEMALLOC_N(choose_arena) -#define choose_arena_hard JEMALLOC_N(choose_arena_hard) -#define chunk_alloc JEMALLOC_N(chunk_alloc) -#define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) -#define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) -#define chunk_alloc_mmap_noreserve JEMALLOC_N(chunk_alloc_mmap_noreserve) -#define chunk_alloc_swap JEMALLOC_N(chunk_alloc_swap) -#define chunk_boot JEMALLOC_N(chunk_boot) -#define chunk_dealloc JEMALLOC_N(chunk_dealloc) -#define chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss) -#define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) -#define chunk_dealloc_swap JEMALLOC_N(chunk_dealloc_swap) -#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) -#define chunk_in_dss JEMALLOC_N(chunk_in_dss) -#define chunk_in_swap JEMALLOC_N(chunk_in_swap) -#define chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot) -#define chunk_swap_boot JEMALLOC_N(chunk_swap_boot) -#define chunk_swap_enable JEMALLOC_N(chunk_swap_enable) -#define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) -#define ckh_count JEMALLOC_N(ckh_count) -#define ckh_delete JEMALLOC_N(ckh_delete) -#define ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert) -#define ckh_insert JEMALLOC_N(ckh_insert) -#define ckh_isearch JEMALLOC_N(ckh_isearch) -#define ckh_iter JEMALLOC_N(ckh_iter) -#define ckh_new JEMALLOC_N(ckh_new) -#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) -#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) -#define ckh_rebuild JEMALLOC_N(ckh_rebuild) -#define ckh_remove JEMALLOC_N(ckh_remove) -#define ckh_search JEMALLOC_N(ckh_search) -#define ckh_string_hash JEMALLOC_N(ckh_string_hash) -#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) -#define ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert) -#define ckh_try_insert JEMALLOC_N(ckh_try_insert) -#define create_zone JEMALLOC_N(create_zone) -#define ctl_boot JEMALLOC_N(ctl_boot) -#define ctl_bymib JEMALLOC_N(ctl_bymib) -#define ctl_byname JEMALLOC_N(ctl_byname) -#define ctl_nametomib JEMALLOC_N(ctl_nametomib) -#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) -#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) -#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) -#define extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse) -#define extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start) -#define extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last) -#define extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new) -#define extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next) -#define extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch) -#define extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev) -#define extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch) -#define extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove) -#define extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter) -#define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) -#define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) -#define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) -#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) -#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) -#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) -#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) -#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) -#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) -#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) -#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) -#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) -#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) -#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) -#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) -#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) -#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) -#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) -#define hash JEMALLOC_N(hash) -#define huge_boot JEMALLOC_N(huge_boot) -#define huge_dalloc JEMALLOC_N(huge_dalloc) -#define huge_malloc JEMALLOC_N(huge_malloc) -#define huge_palloc JEMALLOC_N(huge_palloc) -#define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) -#define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) -#define huge_ralloc JEMALLOC_N(huge_ralloc) -#define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) -#define huge_salloc JEMALLOC_N(huge_salloc) -#define iallocm JEMALLOC_N(iallocm) -#define icalloc JEMALLOC_N(icalloc) -#define idalloc JEMALLOC_N(idalloc) -#define imalloc JEMALLOC_N(imalloc) -#define ipalloc JEMALLOC_N(ipalloc) -#define iralloc JEMALLOC_N(iralloc) -#define isalloc JEMALLOC_N(isalloc) -#define ivsalloc JEMALLOC_N(ivsalloc) -#define jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init) -#define jemalloc_postfork JEMALLOC_N(jemalloc_postfork) -#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) -#define malloc_cprintf JEMALLOC_N(malloc_cprintf) -#define malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy) -#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) -#define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) -#define malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock) -#define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) -#define malloc_printf JEMALLOC_N(malloc_printf) -#define malloc_write JEMALLOC_N(malloc_write) -#define mb_write JEMALLOC_N(mb_write) -#define pow2_ceil JEMALLOC_N(pow2_ceil) -#define prof_backtrace JEMALLOC_N(prof_backtrace) -#define prof_boot0 JEMALLOC_N(prof_boot0) -#define prof_boot1 JEMALLOC_N(prof_boot1) -#define prof_boot2 JEMALLOC_N(prof_boot2) -#define prof_ctx_get JEMALLOC_N(prof_ctx_get) -#define prof_ctx_set JEMALLOC_N(prof_ctx_set) -#define prof_free JEMALLOC_N(prof_free) -#define prof_gdump JEMALLOC_N(prof_gdump) -#define prof_idump JEMALLOC_N(prof_idump) -#define prof_lookup JEMALLOC_N(prof_lookup) -#define prof_malloc JEMALLOC_N(prof_malloc) -#define prof_mdump JEMALLOC_N(prof_mdump) -#define prof_realloc JEMALLOC_N(prof_realloc) -#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) -#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) -#define prof_tdata_init JEMALLOC_N(prof_tdata_init) -#define pthread_create JEMALLOC_N(pthread_create) -#define rtree_get JEMALLOC_N(rtree_get) -#define rtree_get_locked JEMALLOC_N(rtree_get_locked) -#define rtree_new JEMALLOC_N(rtree_new) -#define rtree_set JEMALLOC_N(rtree_set) -#define s2u JEMALLOC_N(s2u) -#define sa2u JEMALLOC_N(sa2u) -#define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) -#define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) -#define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) -#define stats_cactive_add JEMALLOC_N(stats_cactive_add) -#define stats_cactive_get JEMALLOC_N(stats_cactive_get) -#define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) -#define stats_print JEMALLOC_N(stats_print) -#define szone2ozone JEMALLOC_N(szone2ozone) -#define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) -#define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) -#define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) -#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) -#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) -#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) -#define tcache_boot JEMALLOC_N(tcache_boot) -#define tcache_create JEMALLOC_N(tcache_create) -#define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) -#define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) -#define tcache_destroy JEMALLOC_N(tcache_destroy) -#define tcache_event JEMALLOC_N(tcache_event) -#define tcache_get JEMALLOC_N(tcache_get) -#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define thread_allocated_get JEMALLOC_N(thread_allocated_get) -#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard) -#define u2s JEMALLOC_N(u2s) diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.sh b/deps/jemalloc/include/jemalloc/internal/private_namespace.sh new file mode 100755 index 00000000000..cd25eb3061e --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_namespace.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +for symbol in `cat $1` ; do + echo "#define ${symbol} JEMALLOC_N(${symbol})" +done diff --git a/deps/jemalloc/include/jemalloc/internal/private_symbols.txt b/deps/jemalloc/include/jemalloc/internal/private_symbols.txt new file mode 100644 index 00000000000..93516d242b6 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_symbols.txt @@ -0,0 +1,413 @@ +a0calloc +a0free +a0malloc +arena_alloc_junk_small +arena_bin_index +arena_bin_info +arena_boot +arena_dalloc +arena_dalloc_bin +arena_dalloc_bin_locked +arena_dalloc_junk_large +arena_dalloc_junk_small +arena_dalloc_large +arena_dalloc_large_locked +arena_dalloc_small +arena_dss_prec_get +arena_dss_prec_set +arena_malloc +arena_malloc_large +arena_malloc_small +arena_mapbits_allocated_get +arena_mapbits_binind_get +arena_mapbits_dirty_get +arena_mapbits_get +arena_mapbits_large_binind_set +arena_mapbits_large_get +arena_mapbits_large_set +arena_mapbits_large_size_get +arena_mapbits_small_runind_get +arena_mapbits_small_set +arena_mapbits_unallocated_set +arena_mapbits_unallocated_size_get +arena_mapbits_unallocated_size_set +arena_mapbits_unzeroed_get +arena_mapbits_unzeroed_set +arena_mapbitsp_get +arena_mapbitsp_read +arena_mapbitsp_write +arena_mapp_get +arena_maxclass +arena_new +arena_palloc +arena_postfork_child +arena_postfork_parent +arena_prefork +arena_prof_accum +arena_prof_accum_impl +arena_prof_accum_locked +arena_prof_ctx_get +arena_prof_ctx_set +arena_prof_promoted +arena_ptr_small_binind_get +arena_purge_all +arena_quarantine_junk_small +arena_ralloc +arena_ralloc_junk_large +arena_ralloc_no_move +arena_redzone_corruption +arena_run_regind +arena_salloc +arena_stats_merge +arena_tcache_fill_small +arenas +arenas_booted +arenas_cleanup +arenas_extend +arenas_initialized +arenas_lock +arenas_tls +arenas_tsd +arenas_tsd_boot +arenas_tsd_cleanup_wrapper +arenas_tsd_get +arenas_tsd_get_wrapper +arenas_tsd_init_head +arenas_tsd_set +atomic_add_u +atomic_add_uint32 +atomic_add_uint64 +atomic_add_z +atomic_sub_u +atomic_sub_uint32 +atomic_sub_uint64 +atomic_sub_z +base_alloc +base_boot +base_calloc +base_node_alloc +base_node_dealloc +base_postfork_child +base_postfork_parent +base_prefork +bitmap_full +bitmap_get +bitmap_info_init +bitmap_info_ngroups +bitmap_init +bitmap_set +bitmap_sfu +bitmap_size +bitmap_unset +bt_init +buferror +choose_arena +choose_arena_hard +chunk_alloc +chunk_alloc_dss +chunk_alloc_mmap +chunk_boot +chunk_dealloc +chunk_dealloc_mmap +chunk_dss_boot +chunk_dss_postfork_child +chunk_dss_postfork_parent +chunk_dss_prec_get +chunk_dss_prec_set +chunk_dss_prefork +chunk_in_dss +chunk_npages +chunk_postfork_child +chunk_postfork_parent +chunk_prefork +chunk_unmap +chunks_mtx +chunks_rtree +chunksize +chunksize_mask +ckh_bucket_search +ckh_count +ckh_delete +ckh_evict_reloc_insert +ckh_insert +ckh_isearch +ckh_iter +ckh_new +ckh_pointer_hash +ckh_pointer_keycomp +ckh_rebuild +ckh_remove +ckh_search +ckh_string_hash +ckh_string_keycomp +ckh_try_bucket_insert +ckh_try_insert +ctl_boot +ctl_bymib +ctl_byname +ctl_nametomib +ctl_postfork_child +ctl_postfork_parent +ctl_prefork +dss_prec_names +extent_tree_ad_first +extent_tree_ad_insert +extent_tree_ad_iter +extent_tree_ad_iter_recurse +extent_tree_ad_iter_start +extent_tree_ad_last +extent_tree_ad_new +extent_tree_ad_next +extent_tree_ad_nsearch +extent_tree_ad_prev +extent_tree_ad_psearch +extent_tree_ad_remove +extent_tree_ad_reverse_iter +extent_tree_ad_reverse_iter_recurse +extent_tree_ad_reverse_iter_start +extent_tree_ad_search +extent_tree_szad_first +extent_tree_szad_insert +extent_tree_szad_iter +extent_tree_szad_iter_recurse +extent_tree_szad_iter_start +extent_tree_szad_last +extent_tree_szad_new +extent_tree_szad_next +extent_tree_szad_nsearch +extent_tree_szad_prev +extent_tree_szad_psearch +extent_tree_szad_remove +extent_tree_szad_reverse_iter +extent_tree_szad_reverse_iter_recurse +extent_tree_szad_reverse_iter_start +extent_tree_szad_search +get_errno +hash +hash_fmix_32 +hash_fmix_64 +hash_get_block_32 +hash_get_block_64 +hash_rotl_32 +hash_rotl_64 +hash_x64_128 +hash_x86_128 +hash_x86_32 +huge_allocated +huge_boot +huge_dalloc +huge_dalloc_junk +huge_dss_prec_get +huge_malloc +huge_mtx +huge_ndalloc +huge_nmalloc +huge_palloc +huge_postfork_child +huge_postfork_parent +huge_prefork +huge_prof_ctx_get +huge_prof_ctx_set +huge_ralloc +huge_ralloc_no_move +huge_salloc +iallocm +icalloc +icalloct +idalloc +idalloct +imalloc +imalloct +ipalloc +ipalloct +iqalloc +iqalloct +iralloc +iralloct +iralloct_realign +isalloc +isthreaded +ivsalloc +ixalloc +jemalloc_postfork_child +jemalloc_postfork_parent +jemalloc_prefork +malloc_cprintf +malloc_mutex_init +malloc_mutex_lock +malloc_mutex_postfork_child +malloc_mutex_postfork_parent +malloc_mutex_prefork +malloc_mutex_unlock +malloc_printf +malloc_snprintf +malloc_strtoumax +malloc_tsd_boot +malloc_tsd_cleanup_register +malloc_tsd_dalloc +malloc_tsd_malloc +malloc_tsd_no_cleanup +malloc_vcprintf +malloc_vsnprintf +malloc_write +map_bias +mb_write +mutex_boot +narenas_auto +narenas_total +narenas_total_get +ncpus +nhbins +opt_abort +opt_dss +opt_junk +opt_lg_chunk +opt_lg_dirty_mult +opt_lg_prof_interval +opt_lg_prof_sample +opt_lg_tcache_max +opt_narenas +opt_prof +opt_prof_accum +opt_prof_active +opt_prof_final +opt_prof_gdump +opt_prof_leak +opt_prof_prefix +opt_quarantine +opt_redzone +opt_stats_print +opt_tcache +opt_utrace +opt_valgrind +opt_xmalloc +opt_zero +p2rz +pages_purge +pow2_ceil +prof_backtrace +prof_boot0 +prof_boot1 +prof_boot2 +prof_bt_count +prof_ctx_get +prof_ctx_set +prof_dump_open +prof_free +prof_gdump +prof_idump +prof_interval +prof_lookup +prof_malloc +prof_mdump +prof_postfork_child +prof_postfork_parent +prof_prefork +prof_promote +prof_realloc +prof_sample_accum_update +prof_sample_threshold_update +prof_tdata_booted +prof_tdata_cleanup +prof_tdata_get +prof_tdata_init +prof_tdata_initialized +prof_tdata_tls +prof_tdata_tsd +prof_tdata_tsd_boot +prof_tdata_tsd_cleanup_wrapper +prof_tdata_tsd_get +prof_tdata_tsd_get_wrapper +prof_tdata_tsd_init_head +prof_tdata_tsd_set +quarantine +quarantine_alloc_hook +quarantine_boot +quarantine_booted +quarantine_cleanup +quarantine_init +quarantine_tls +quarantine_tsd +quarantine_tsd_boot +quarantine_tsd_cleanup_wrapper +quarantine_tsd_get +quarantine_tsd_get_wrapper +quarantine_tsd_init_head +quarantine_tsd_set +register_zone +rtree_delete +rtree_get +rtree_get_locked +rtree_new +rtree_postfork_child +rtree_postfork_parent +rtree_prefork +rtree_set +s2u +sa2u +set_errno +small_size2bin +stats_cactive +stats_cactive_add +stats_cactive_get +stats_cactive_sub +stats_chunks +stats_print +tcache_alloc_easy +tcache_alloc_large +tcache_alloc_small +tcache_alloc_small_hard +tcache_arena_associate +tcache_arena_dissociate +tcache_bin_flush_large +tcache_bin_flush_small +tcache_bin_info +tcache_boot0 +tcache_boot1 +tcache_booted +tcache_create +tcache_dalloc_large +tcache_dalloc_small +tcache_destroy +tcache_enabled_booted +tcache_enabled_get +tcache_enabled_initialized +tcache_enabled_set +tcache_enabled_tls +tcache_enabled_tsd +tcache_enabled_tsd_boot +tcache_enabled_tsd_cleanup_wrapper +tcache_enabled_tsd_get +tcache_enabled_tsd_get_wrapper +tcache_enabled_tsd_init_head +tcache_enabled_tsd_set +tcache_event +tcache_event_hard +tcache_flush +tcache_get +tcache_initialized +tcache_maxclass +tcache_salloc +tcache_stats_merge +tcache_thread_cleanup +tcache_tls +tcache_tsd +tcache_tsd_boot +tcache_tsd_cleanup_wrapper +tcache_tsd_get +tcache_tsd_get_wrapper +tcache_tsd_init_head +tcache_tsd_set +thread_allocated_booted +thread_allocated_initialized +thread_allocated_tls +thread_allocated_tsd +thread_allocated_tsd_boot +thread_allocated_tsd_cleanup_wrapper +thread_allocated_tsd_get +thread_allocated_tsd_get_wrapper +thread_allocated_tsd_init_head +thread_allocated_tsd_set +tsd_init_check_recursion +tsd_init_finish +u2rz diff --git a/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh b/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh new file mode 100755 index 00000000000..23fed8e8034 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +for symbol in `cat $1` ; do + echo "#undef ${symbol}" +done diff --git a/deps/jemalloc/include/jemalloc/internal/prn.h b/deps/jemalloc/include/jemalloc/internal/prng.h similarity index 89% rename from deps/jemalloc/include/jemalloc/internal/prn.h rename to deps/jemalloc/include/jemalloc/internal/prng.h index 0709d708012..7b2b06512ff 100644 --- a/deps/jemalloc/include/jemalloc/internal/prn.h +++ b/deps/jemalloc/include/jemalloc/internal/prng.h @@ -4,7 +4,7 @@ /* * Simple linear congruential pseudo-random number generator: * - * prn(y) = (a*x + c) % m + * prng(y) = (a*x + c) % m * * where the following constants ensure maximal period: * @@ -25,7 +25,7 @@ * uint32_t state : Seed value. * const uint32_t a, c : See above discussion. */ -#define prn32(r, lg_range, state, a, c) do { \ +#define prng32(r, lg_range, state, a, c) do { \ assert(lg_range > 0); \ assert(lg_range <= 32); \ \ @@ -34,8 +34,8 @@ r >>= (32 - lg_range); \ } while (false) -/* Same as prn32(), but 64 bits of pseudo-randomness, using uint64_t. */ -#define prn64(r, lg_range, state, a, c) do { \ +/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */ +#define prng64(r, lg_range, state, a, c) do { \ assert(lg_range > 0); \ assert(lg_range <= 64); \ \ diff --git a/deps/jemalloc/include/jemalloc/internal/prof.h b/deps/jemalloc/include/jemalloc/internal/prof.h index e9064ba6e73..6f162d21e84 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof.h +++ b/deps/jemalloc/include/jemalloc/internal/prof.h @@ -1,4 +1,3 @@ -#ifdef JEMALLOC_PROF /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -9,29 +8,46 @@ typedef struct prof_ctx_s prof_ctx_t; typedef struct prof_tdata_s prof_tdata_t; /* Option defaults. */ -#define PROF_PREFIX_DEFAULT "jeprof" -#define LG_PROF_BT_MAX_DEFAULT 7 -#define LG_PROF_SAMPLE_DEFAULT 0 +#ifdef JEMALLOC_PROF +# define PROF_PREFIX_DEFAULT "jeprof" +#else +# define PROF_PREFIX_DEFAULT "" +#endif +#define LG_PROF_SAMPLE_DEFAULT 19 #define LG_PROF_INTERVAL_DEFAULT -1 -#define LG_PROF_TCMAX_DEFAULT -1 /* - * Hard limit on stack backtrace depth. Note that the version of - * prof_backtrace() that is based on __builtin_return_address() necessarily has - * a hard-coded number of backtrace frame handlers. + * Hard limit on stack backtrace depth. The version of prof_backtrace() that + * is based on __builtin_return_address() necessarily has a hard-coded number + * of backtrace frame handlers, and should be kept in sync with this setting. */ -#if (defined(JEMALLOC_PROF_LIBGCC) || defined(JEMALLOC_PROF_LIBUNWIND)) -# define LG_PROF_BT_MAX ((ZU(1) << (LG_SIZEOF_PTR+3)) - 1) -#else -# define LG_PROF_BT_MAX 7 /* >= LG_PROF_BT_MAX_DEFAULT */ -#endif -#define PROF_BT_MAX (1U << LG_PROF_BT_MAX) +#define PROF_BT_MAX 128 + +/* Maximum number of backtraces to store in each per thread LRU cache. */ +#define PROF_TCMAX 1024 /* Initial hash table size. */ -#define PROF_CKH_MINITEMS 64 +#define PROF_CKH_MINITEMS 64 /* Size of memory buffer to use when writing dump files. */ -#define PROF_DUMP_BUF_SIZE 65536 +#define PROF_DUMP_BUFSIZE 65536 + +/* Size of stack-allocated buffer used by prof_printf(). */ +#define PROF_PRINTF_BUFSIZE 128 + +/* + * Number of mutexes shared among all ctx's. No space is allocated for these + * unless profiling is enabled, so it's okay to over-provision. + */ +#define PROF_NCTX_LOCKS 1024 + +/* + * prof_tdata pointers close to NULL are used to encode state information that + * is used for cleaning up during thread shutdown. + */ +#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) +#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) +#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -109,8 +125,19 @@ struct prof_ctx_s { /* Associated backtrace. */ prof_bt_t *bt; - /* Protects cnt_merged and cnts_ql. */ - malloc_mutex_t lock; + /* Protects nlimbo, cnt_merged, and cnts_ql. */ + malloc_mutex_t *lock; + + /* + * Number of threads that currently cause this ctx to be in a state of + * limbo due to one of: + * - Initializing per thread counters associated with this ctx. + * - Preparing to destroy this ctx. + * - Dumping a heap profile that includes this ctx. + * nlimbo must be 1 (single destroyer) in order to safely destroy the + * ctx. + */ + unsigned nlimbo; /* Temporary storage for summation during dump. */ prof_cnt_t cnt_summed; @@ -123,7 +150,11 @@ struct prof_ctx_s { * this context. */ ql_head(prof_thr_cnt_t) cnts_ql; + + /* Linkage for list of contexts to be dumped. */ + ql_elm(prof_ctx_t) dump_link; }; +typedef ql_head(prof_ctx_t) prof_ctx_list_t; struct prof_tdata_s { /* @@ -145,9 +176,14 @@ struct prof_tdata_s { void **vec; /* Sampling state. */ - uint64_t prn_state; + uint64_t prng_state; uint64_t threshold; uint64_t accum; + + /* State used to avoid dumping while operating on prof internals. */ + bool enq; + bool enq_idump; + bool enq_gdump; }; #endif /* JEMALLOC_H_STRUCTS */ @@ -162,14 +198,18 @@ extern bool opt_prof; * to notice state changes. */ extern bool opt_prof_active; -extern size_t opt_lg_prof_bt_max; /* Maximum backtrace depth. */ extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ extern bool opt_prof_gdump; /* High-water memory dumping. */ +extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern ssize_t opt_lg_prof_tcmax; /* lg(max per thread bactrace cache) */ -extern char opt_prof_prefix[PATH_MAX + 1]; +extern char opt_prof_prefix[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; /* * Profile dump interval, measured in bytes allocated. Each arena triggers a @@ -186,42 +226,25 @@ extern uint64_t prof_interval; */ extern bool prof_promote; -/* (1U << opt_lg_prof_bt_max). */ -extern unsigned prof_bt_max; - -/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ -#ifndef NO_TLS -extern __thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define PROF_TCACHE_GET() prof_tdata_tls -# define PROF_TCACHE_SET(v) do { \ - prof_tdata_tls = (v); \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#else -# define PROF_TCACHE_GET() \ - ((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd)) -# define PROF_TCACHE_SET(v) do { \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#endif -/* - * Same contents as b2cnt_tls, but initialized such that the TSD destructor is - * called when a thread exits, so that prof_tdata_tls contents can be merged, - * unlinked, and deallocated. - */ -extern pthread_key_t prof_tdata_tsd; - void bt_init(prof_bt_t *bt, void **vec); -void prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max); +void prof_backtrace(prof_bt_t *bt, unsigned nignore); prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); +#ifdef JEMALLOC_JET +size_t prof_bt_count(void); +typedef int (prof_dump_open_t)(bool, const char *); +extern prof_dump_open_t *prof_dump_open; +#endif void prof_idump(void); bool prof_mdump(const char *filename); void prof_gdump(void); prof_tdata_t *prof_tdata_init(void); +void prof_tdata_cleanup(void *arg); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); +void prof_prefork(void); +void prof_postfork_parent(void); +void prof_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -233,13 +256,13 @@ bool prof_boot2(void); \ assert(size == s2u(size)); \ \ - prof_tdata = PROF_TCACHE_GET(); \ - if (prof_tdata == NULL) { \ - prof_tdata = prof_tdata_init(); \ - if (prof_tdata == NULL) { \ + prof_tdata = prof_tdata_get(true); \ + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \ + if (prof_tdata != NULL) \ + ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ + else \ ret = NULL; \ - break; \ - } \ + break; \ } \ \ if (opt_prof_active == false) { \ @@ -249,13 +272,13 @@ bool prof_boot2(void); /* Don't bother with sampling logic, since sampling */\ /* interval is 1. */\ bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ + prof_backtrace(&bt, nignore); \ ret = prof_lookup(&bt); \ } else { \ if (prof_tdata->threshold == 0) { \ /* Initialize. Seed the prng differently for */\ /* each thread. */\ - prof_tdata->prn_state = \ + prof_tdata->prng_state = \ (uint64_t)(uintptr_t)&size; \ prof_sample_threshold_update(prof_tdata); \ } \ @@ -272,7 +295,7 @@ bool prof_boot2(void); if (size >= prof_tdata->threshold - \ prof_tdata->accum) { \ bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ + prof_backtrace(&bt, nignore); \ ret = prof_lookup(&bt); \ } else \ ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ @@ -280,23 +303,61 @@ bool prof_boot2(void); } while (0) #ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) + +prof_tdata_t *prof_tdata_get(bool create); void prof_sample_threshold_update(prof_tdata_t *prof_tdata); prof_ctx_t *prof_ctx_get(const void *ptr); -void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +void prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); bool prof_sample_accum_update(size_t size); -void prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt); -void prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx); +void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); +void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, + size_t old_usize, prof_ctx_t *old_ctx); void prof_free(const void *ptr, size_t size); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) +/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ +malloc_tsd_externs(prof_tdata, prof_tdata_t *) +malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, + prof_tdata_cleanup) + +JEMALLOC_INLINE prof_tdata_t * +prof_tdata_get(bool create) +{ + prof_tdata_t *prof_tdata; + + cassert(config_prof); + + prof_tdata = *prof_tdata_tsd_get(); + if (create && prof_tdata == NULL) + prof_tdata = prof_tdata_init(); + + return (prof_tdata); +} + JEMALLOC_INLINE void prof_sample_threshold_update(prof_tdata_t *prof_tdata) { + /* + * The body of this function is compiled out unless heap profiling is + * enabled, so that it is possible to compile jemalloc with floating + * point support completely disabled. Avoiding floating point code is + * important on memory-constrained systems, but it also enables a + * workaround for versions of glibc that don't properly save/restore + * floating point registers during dynamic lazy symbol loading (which + * internally calls into whatever malloc implementation happens to be + * integrated into the application). Note that some compilers (e.g. + * gcc 4.8) may use floating point registers for fast memory moves, so + * jemalloc must be compiled with such optimizations disabled (e.g. + * -mno-sse) in order for the workaround to be complete. + */ +#ifdef JEMALLOC_PROF uint64_t r; double u; + cassert(config_prof); + /* * Compute sample threshold as a geometrically distributed random * variable with mean (2^opt_lg_prof_sample). @@ -313,14 +374,15 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) * Luc Devroye * Springer-Verlag, New York, 1986 * pp 500 - * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) + * (http://luc.devroye.org/rnbookindex.html) */ - prn64(r, 53, prof_tdata->prn_state, - (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU); + prng64(r, 53, prof_tdata->prng_state, + UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); u = (double)r * (1.0/9007199254740992.0L); prof_tdata->threshold = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) + (uint64_t)1U; +#endif } JEMALLOC_INLINE prof_ctx_t * @@ -329,13 +391,12 @@ prof_ctx_get(const void *ptr) prof_ctx_t *ret; arena_chunk_t *chunk; + cassert(config_prof); assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - ret = arena_prof_ctx_get(ptr); } else ret = huge_prof_ctx_get(ptr); @@ -344,18 +405,17 @@ prof_ctx_get(const void *ptr) } JEMALLOC_INLINE void -prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) { arena_chunk_t *chunk; + cassert(config_prof); assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - - arena_prof_ctx_set(ptr, ctx); + arena_prof_ctx_set(ptr, usize, ctx); } else huge_prof_ctx_set(ptr, ctx); } @@ -365,11 +425,13 @@ prof_sample_accum_update(size_t size) { prof_tdata_t *prof_tdata; + cassert(config_prof); /* Sampling logic is unnecessary if the interval is 1. */ assert(opt_lg_prof_sample != 0); - prof_tdata = PROF_TCACHE_GET(); - assert(prof_tdata != NULL); + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (true); /* Take care to avoid integer overflow. */ if (size >= prof_tdata->threshold - prof_tdata->accum) { @@ -388,19 +450,20 @@ prof_sample_accum_update(size_t size) } JEMALLOC_INLINE void -prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) +prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { + cassert(config_prof); assert(ptr != NULL); - assert(size == isalloc(ptr)); + assert(usize == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { + if (prof_sample_accum_update(usize)) { /* * Don't sample. For malloc()-like allocation, it is * always possible to tell in advance how large an * object's usable size will be, so there should never - * be a difference between the size passed to + * be a difference between the usize passed to * PROF_ALLOC_PREP() and prof_malloc(). */ assert((uintptr_t)cnt == (uintptr_t)1U); @@ -408,17 +471,17 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) } if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); + prof_ctx_set(ptr, usize, cnt->ctx); cnt->epoch++; /*********/ mb_write(); /*********/ cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; + cnt->cnts.curbytes += usize; if (opt_prof_accum) { cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; + cnt->cnts.accumbytes += usize; } /*********/ mb_write(); @@ -428,27 +491,28 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) mb_write(); /*********/ } else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx) +prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, + size_t old_usize, prof_ctx_t *old_ctx) { prof_thr_cnt_t *told_cnt; + cassert(config_prof); assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); if (ptr != NULL) { - assert(size == isalloc(ptr)); + assert(usize == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { + if (prof_sample_accum_update(usize)) { /* - * Don't sample. The size passed to + * Don't sample. The usize passed to * PROF_ALLOC_PREP() was larger than what * actually got allocated, so a backtrace was * captured for this allocation, even though - * its actual size was insufficient to cross + * its actual usize was insufficient to cross * the sample threshold. */ cnt = (prof_thr_cnt_t *)(uintptr_t)1U; @@ -463,10 +527,10 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, * It's too late to propagate OOM for this realloc(), * so operate directly on old_cnt->ctx->cnt_merged. */ - malloc_mutex_lock(&old_ctx->lock); + malloc_mutex_lock(old_ctx->lock); old_ctx->cnt_merged.curobjs--; - old_ctx->cnt_merged.curbytes -= old_size; - malloc_mutex_unlock(&old_ctx->lock); + old_ctx->cnt_merged.curbytes -= old_usize; + malloc_mutex_unlock(old_ctx->lock); told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; } } else @@ -475,23 +539,23 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, if ((uintptr_t)told_cnt > (uintptr_t)1U) told_cnt->epoch++; if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); + prof_ctx_set(ptr, usize, cnt->ctx); cnt->epoch++; - } else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + } else if (ptr != NULL) + prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); /*********/ mb_write(); /*********/ if ((uintptr_t)told_cnt > (uintptr_t)1U) { told_cnt->cnts.curobjs--; - told_cnt->cnts.curbytes -= old_size; + told_cnt->cnts.curbytes -= old_usize; } if ((uintptr_t)cnt > (uintptr_t)1U) { cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; + cnt->cnts.curbytes += usize; if (opt_prof_accum) { cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; + cnt->cnts.accumbytes += usize; } } /*********/ @@ -510,9 +574,12 @@ prof_free(const void *ptr, size_t size) { prof_ctx_t *ctx = prof_ctx_get(ptr); + cassert(config_prof); + if ((uintptr_t)ctx > (uintptr_t)1) { - assert(size == isalloc(ptr)); - prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); + prof_thr_cnt_t *tcnt; + assert(size == isalloc(ptr, true)); + tcnt = prof_lookup(ctx->bt); if (tcnt != NULL) { tcnt->epoch++; @@ -533,10 +600,10 @@ prof_free(const void *ptr, size_t size) * OOM during free() cannot be propagated, so operate * directly on cnt->ctx->cnt_merged. */ - malloc_mutex_lock(&ctx->lock); + malloc_mutex_lock(ctx->lock); ctx->cnt_merged.curobjs--; ctx->cnt_merged.curbytes -= size; - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); } } } @@ -544,4 +611,3 @@ prof_free(const void *ptr, size_t size) #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc/include/jemalloc/internal/public_namespace.sh b/deps/jemalloc/include/jemalloc/internal/public_namespace.sh new file mode 100755 index 00000000000..362109f7127 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/public_namespace.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +for nm in `cat $1` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'` + echo "#define je_${n} JEMALLOC_N(${n})" +done diff --git a/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh b/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh new file mode 100755 index 00000000000..4239d17754c --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +for nm in `cat $1` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'` + echo "#undef je_${n}" +done diff --git a/deps/jemalloc/include/jemalloc/internal/ql.h b/deps/jemalloc/include/jemalloc/internal/ql.h index a9ed2393f0c..f70c5f6f391 100644 --- a/deps/jemalloc/include/jemalloc/internal/ql.h +++ b/deps/jemalloc/include/jemalloc/internal/ql.h @@ -1,61 +1,61 @@ /* * List definitions. */ -#define ql_head(a_type) \ +#define ql_head(a_type) \ struct { \ a_type *qlh_first; \ } -#define ql_head_initializer(a_head) {NULL} +#define ql_head_initializer(a_head) {NULL} -#define ql_elm(a_type) qr(a_type) +#define ql_elm(a_type) qr(a_type) /* List functions. */ -#define ql_new(a_head) do { \ +#define ql_new(a_head) do { \ (a_head)->qlh_first = NULL; \ } while (0) -#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) +#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) -#define ql_first(a_head) ((a_head)->qlh_first) +#define ql_first(a_head) ((a_head)->qlh_first) -#define ql_last(a_head, a_field) \ +#define ql_last(a_head, a_field) \ ((ql_first(a_head) != NULL) \ ? qr_prev(ql_first(a_head), a_field) : NULL) -#define ql_next(a_head, a_elm, a_field) \ +#define ql_next(a_head, a_elm, a_field) \ ((ql_last(a_head, a_field) != (a_elm)) \ ? qr_next((a_elm), a_field) : NULL) -#define ql_prev(a_head, a_elm, a_field) \ +#define ql_prev(a_head, a_elm, a_field) \ ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \ : NULL) -#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ +#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ qr_before_insert((a_qlelm), (a_elm), a_field); \ if (ql_first(a_head) == (a_qlelm)) { \ ql_first(a_head) = (a_elm); \ } \ } while (0) -#define ql_after_insert(a_qlelm, a_elm, a_field) \ +#define ql_after_insert(a_qlelm, a_elm, a_field) \ qr_after_insert((a_qlelm), (a_elm), a_field) -#define ql_head_insert(a_head, a_elm, a_field) do { \ +#define ql_head_insert(a_head, a_elm, a_field) do { \ if (ql_first(a_head) != NULL) { \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \ } \ ql_first(a_head) = (a_elm); \ } while (0) -#define ql_tail_insert(a_head, a_elm, a_field) do { \ +#define ql_tail_insert(a_head, a_elm, a_field) do { \ if (ql_first(a_head) != NULL) { \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \ } \ ql_first(a_head) = qr_next((a_elm), a_field); \ } while (0) -#define ql_remove(a_head, a_elm, a_field) do { \ +#define ql_remove(a_head, a_elm, a_field) do { \ if (ql_first(a_head) == (a_elm)) { \ ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ } \ @@ -66,18 +66,18 @@ struct { \ } \ } while (0) -#define ql_head_remove(a_head, a_type, a_field) do { \ +#define ql_head_remove(a_head, a_type, a_field) do { \ a_type *t = ql_first(a_head); \ ql_remove((a_head), t, a_field); \ } while (0) -#define ql_tail_remove(a_head, a_type, a_field) do { \ +#define ql_tail_remove(a_head, a_type, a_field) do { \ a_type *t = ql_last(a_head, a_field); \ ql_remove((a_head), t, a_field); \ } while (0) -#define ql_foreach(a_var, a_head, a_field) \ +#define ql_foreach(a_var, a_head, a_field) \ qr_foreach((a_var), ql_first(a_head), a_field) -#define ql_reverse_foreach(a_var, a_head, a_field) \ +#define ql_reverse_foreach(a_var, a_head, a_field) \ qr_reverse_foreach((a_var), ql_first(a_head), a_field) diff --git a/deps/jemalloc/include/jemalloc/internal/qr.h b/deps/jemalloc/include/jemalloc/internal/qr.h index fe22352fedd..602944b9b4f 100644 --- a/deps/jemalloc/include/jemalloc/internal/qr.h +++ b/deps/jemalloc/include/jemalloc/internal/qr.h @@ -1,28 +1,28 @@ /* Ring definitions. */ -#define qr(a_type) \ +#define qr(a_type) \ struct { \ a_type *qre_next; \ a_type *qre_prev; \ } /* Ring functions. */ -#define qr_new(a_qr, a_field) do { \ +#define qr_new(a_qr, a_field) do { \ (a_qr)->a_field.qre_next = (a_qr); \ (a_qr)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) +#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) -#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) +#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) -#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ +#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ (a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev; \ (a_qr)->a_field.qre_next = (a_qrelm); \ (a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr); \ (a_qrelm)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_after_insert(a_qrelm, a_qr, a_field) \ +#define qr_after_insert(a_qrelm, a_qr, a_field) \ do \ { \ (a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next; \ @@ -31,7 +31,7 @@ struct { \ (a_qrelm)->a_field.qre_next = (a_qr); \ } while (0) -#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ +#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ void *t; \ (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ @@ -42,10 +42,10 @@ struct { \ /* qr_meld() and qr_split() are functionally equivalent, so there's no need to * have two copies of the code. */ -#define qr_split(a_qr_a, a_qr_b, a_field) \ +#define qr_split(a_qr_a, a_qr_b, a_field) \ qr_meld((a_qr_a), (a_qr_b), a_field) -#define qr_remove(a_qr, a_field) do { \ +#define qr_remove(a_qr, a_field) do { \ (a_qr)->a_field.qre_prev->a_field.qre_next \ = (a_qr)->a_field.qre_next; \ (a_qr)->a_field.qre_next->a_field.qre_prev \ @@ -54,13 +54,13 @@ struct { \ (a_qr)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_foreach(var, a_qr, a_field) \ +#define qr_foreach(var, a_qr, a_field) \ for ((var) = (a_qr); \ (var) != NULL; \ (var) = (((var)->a_field.qre_next != (a_qr)) \ ? (var)->a_field.qre_next : NULL)) -#define qr_reverse_foreach(var, a_qr, a_field) \ +#define qr_reverse_foreach(var, a_qr, a_field) \ for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ (var) != NULL; \ (var) = (((var) != (a_qr)) \ diff --git a/deps/jemalloc/include/jemalloc/internal/quarantine.h b/deps/jemalloc/include/jemalloc/internal/quarantine.h new file mode 100644 index 00000000000..16f677f73da --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/quarantine.h @@ -0,0 +1,67 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct quarantine_obj_s quarantine_obj_t; +typedef struct quarantine_s quarantine_t; + +/* Default per thread quarantine size if valgrind is enabled. */ +#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct quarantine_obj_s { + void *ptr; + size_t usize; +}; + +struct quarantine_s { + size_t curbytes; + size_t curobjs; + size_t first; +#define LG_MAXOBJS_INIT 10 + size_t lg_maxobjs; + quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +quarantine_t *quarantine_init(size_t lg_maxobjs); +void quarantine(void *ptr); +void quarantine_cleanup(void *arg); +bool quarantine_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *) + +void quarantine_alloc_hook(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_)) +malloc_tsd_externs(quarantine, quarantine_t *) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL, + quarantine_cleanup) + +JEMALLOC_ALWAYS_INLINE void +quarantine_alloc_hook(void) +{ + quarantine_t *quarantine; + + assert(config_fill && opt_quarantine); + + quarantine = *quarantine_tsd_get(); + if (quarantine == NULL) + quarantine_init(LG_MAXOBJS_INIT); +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/deps/jemalloc/include/jemalloc/internal/rb.h b/deps/jemalloc/include/jemalloc/internal/rb.h index ee9b009d235..423802eb2dc 100644 --- a/deps/jemalloc/include/jemalloc/internal/rb.h +++ b/deps/jemalloc/include/jemalloc/internal/rb.h @@ -22,10 +22,6 @@ #ifndef RB_H_ #define RB_H_ -#if 0 -__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $"); -#endif - #ifdef RB_COMPACT /* Node structure. */ #define rb_node(a_type) \ @@ -223,88 +219,88 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ * The following API is generated: * * static void - * ex_new(ex_t *extree); + * ex_new(ex_t *tree); * Description: Initialize a red-black tree structure. * Args: - * extree: Pointer to an uninitialized red-black tree object. + * tree: Pointer to an uninitialized red-black tree object. * * static ex_node_t * - * ex_first(ex_t *extree); + * ex_first(ex_t *tree); * static ex_node_t * - * ex_last(ex_t *extree); - * Description: Get the first/last node in extree. + * ex_last(ex_t *tree); + * Description: Get the first/last node in tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * Ret: First/last node in extree, or NULL if extree is empty. + * tree: Pointer to an initialized red-black tree object. + * Ret: First/last node in tree, or NULL if tree is empty. * * static ex_node_t * - * ex_next(ex_t *extree, ex_node_t *node); + * ex_next(ex_t *tree, ex_node_t *node); * static ex_node_t * - * ex_prev(ex_t *extree, ex_node_t *node); + * ex_prev(ex_t *tree, ex_node_t *node); * Description: Get node's successor/predecessor. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : A node in extree. - * Ret: node's successor/predecessor in extree, or NULL if node is + * tree: Pointer to an initialized red-black tree object. + * node: A node in tree. + * Ret: node's successor/predecessor in tree, or NULL if node is * last/first. * * static ex_node_t * - * ex_search(ex_t *extree, ex_node_t *key); + * ex_search(ex_t *tree, ex_node_t *key); * Description: Search for node that matches key. * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or NULL if no match. + * tree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in tree that matches key, or NULL if no match. * * static ex_node_t * - * ex_nsearch(ex_t *extree, ex_node_t *key); + * ex_nsearch(ex_t *tree, ex_node_t *key); * static ex_node_t * - * ex_psearch(ex_t *extree, ex_node_t *key); + * ex_psearch(ex_t *tree, ex_node_t *key); * Description: Search for node that matches key. If no match is found, * return what would be key's successor/predecessor, were - * key in extree. + * key in tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or if no match, hypothetical - * node's successor/predecessor (NULL if no successor/predecessor). + * tree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in tree that matches key, or if no match, hypothetical node's + * successor/predecessor (NULL if no successor/predecessor). * * static void - * ex_insert(ex_t *extree, ex_node_t *node); - * Description: Insert node into extree. + * ex_insert(ex_t *tree, ex_node_t *node); + * Description: Insert node into tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node to be inserted into extree. + * tree: Pointer to an initialized red-black tree object. + * node: Node to be inserted into tree. * * static void - * ex_remove(ex_t *extree, ex_node_t *node); - * Description: Remove node from extree. + * ex_remove(ex_t *tree, ex_node_t *node); + * Description: Remove node from tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node in extree to be removed. + * tree: Pointer to an initialized red-black tree object. + * node: Node in tree to be removed. * * static ex_node_t * - * ex_iter(ex_t *extree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, + * ex_iter(ex_t *tree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, * ex_node_t *, void *), void *arg); * static ex_node_t * - * ex_reverse_iter(ex_t *extree, ex_node_t *start, ex_node *(*cb)(ex_t *, + * ex_reverse_iter(ex_t *tree, ex_node_t *start, ex_node *(*cb)(ex_t *, * ex_node_t *, void *), void *arg); - * Description: Iterate forward/backward over extree, starting at node. - * If extree is modified, iteration must be immediately + * Description: Iterate forward/backward over tree, starting at node. If + * tree is modified, iteration must be immediately * terminated by the callback function that causes the * modification. * Args: - * extree: Pointer to an initialized red-black tree object. - * start : Node at which to start iteration, or NULL to start at - * first/last node. - * cb : Callback function, which is called for each node during - * iteration. Under normal circumstances the callback function - * should return NULL, which causes iteration to continue. If a - * callback function returns non-NULL, iteration is immediately - * terminated and the non-NULL return value is returned by the - * iterator. This is useful for re-starting iteration after - * modifying extree. - * arg : Opaque pointer passed to cb(). + * tree : Pointer to an initialized red-black tree object. + * start: Node at which to start iteration, or NULL to start at + * first/last node. + * cb : Callback function, which is called for each node during + * iteration. Under normal circumstances the callback function + * should return NULL, which causes iteration to continue. If a + * callback function returns non-NULL, iteration is immediately + * terminated and the non-NULL return value is returned by the + * iterator. This is useful for re-starting iteration after + * modifying tree. + * arg : Opaque pointer passed to cb(). * Ret: NULL if iteration completed, or the non-NULL callback return value * that caused termination of the iteration. */ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree.h b/deps/jemalloc/include/jemalloc/internal/rtree.h index 95d6355a5f4..bc74769f50e 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree.h @@ -14,17 +14,18 @@ typedef struct rtree_s rtree_t; * Size of each radix tree node (must be a power of 2). This impacts tree * depth. */ -#if (LG_SIZEOF_PTR == 2) -# define RTREE_NODESIZE (1U << 14) -#else -# define RTREE_NODESIZE CACHELINE -#endif +#define RTREE_NODESIZE (1U << 16) + +typedef void *(rtree_alloc_t)(size_t); +typedef void (rtree_dalloc_t)(void *); #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS struct rtree_s { + rtree_alloc_t *alloc; + rtree_dalloc_t *dalloc; malloc_mutex_t mutex; void **root; unsigned height; @@ -35,27 +36,31 @@ struct rtree_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -rtree_t *rtree_new(unsigned bits); +rtree_t *rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc); +void rtree_delete(rtree_t *rtree); +void rtree_prefork(rtree_t *rtree); +void rtree_postfork_parent(rtree_t *rtree); +void rtree_postfork_child(rtree_t *rtree); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -#ifndef JEMALLOC_DEBUG -void *rtree_get_locked(rtree_t *rtree, uintptr_t key); +#ifdef JEMALLOC_DEBUG +uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key); #endif -void *rtree_get(rtree_t *rtree, uintptr_t key); -bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); +uint8_t rtree_get(rtree_t *rtree, uintptr_t key); +bool rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) #define RTREE_GET_GENERATE(f) \ /* The least significant bits of the key are ignored. */ \ -JEMALLOC_INLINE void * \ +JEMALLOC_INLINE uint8_t \ f(rtree_t *rtree, uintptr_t key) \ { \ - void *ret; \ + uint8_t ret; \ uintptr_t subkey; \ unsigned i, lshift, height, bits; \ void **node, **child; \ @@ -65,12 +70,12 @@ f(rtree_t *rtree, uintptr_t key) \ i < height - 1; \ i++, lshift += bits, node = child) { \ bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ 3)) - bits); \ child = (void**)node[subkey]; \ if (child == NULL) { \ RTREE_UNLOCK(&rtree->mutex); \ - return (NULL); \ + return (0); \ } \ } \ \ @@ -81,7 +86,10 @@ f(rtree_t *rtree, uintptr_t key) \ bits = rtree->level2bits[i]; \ subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \ bits); \ - ret = node[subkey]; \ + { \ + uint8_t *leaf = (uint8_t *)node; \ + ret = leaf[subkey]; \ + } \ RTREE_UNLOCK(&rtree->mutex); \ \ RTREE_GET_VALIDATE \ @@ -120,7 +128,7 @@ RTREE_GET_GENERATE(rtree_get) #undef RTREE_GET_VALIDATE JEMALLOC_INLINE bool -rtree_set(rtree_t *rtree, uintptr_t key, void *val) +rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val) { uintptr_t subkey; unsigned i, lshift, height, bits; @@ -135,14 +143,14 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val) bits); child = (void**)node[subkey]; if (child == NULL) { - child = (void**)base_alloc(sizeof(void *) << - rtree->level2bits[i+1]); + size_t size = ((i + 1 < height - 1) ? sizeof(void *) + : (sizeof(uint8_t))) << rtree->level2bits[i+1]; + child = (void**)rtree->alloc(size); if (child == NULL) { malloc_mutex_unlock(&rtree->mutex); return (true); } - memset(child, 0, sizeof(void *) << - rtree->level2bits[i+1]); + memset(child, 0, size); node[subkey] = child; } } @@ -150,7 +158,10 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val) /* node is a leaf, so it contains values rather than node pointers. */ bits = rtree->level2bits[i]; subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits); - node[subkey] = val; + { + uint8_t *leaf = (uint8_t *)node; + leaf[subkey] = val; + } malloc_mutex_unlock(&rtree->mutex); return (false); diff --git a/deps/jemalloc/include/jemalloc/internal/size_classes.sh b/deps/jemalloc/include/jemalloc/internal/size_classes.sh new file mode 100755 index 00000000000..29c80c1fb8d --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/size_classes.sh @@ -0,0 +1,122 @@ +#!/bin/sh + +# The following limits are chosen such that they cover all supported platforms. + +# Range of quanta. +lg_qmin=3 +lg_qmax=4 + +# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. +lg_tmin=3 + +# Range of page sizes. +lg_pmin=12 +lg_pmax=16 + +pow2() { + e=$1 + pow2_result=1 + while [ ${e} -gt 0 ] ; do + pow2_result=$((${pow2_result} + ${pow2_result})) + e=$((${e} - 1)) + done +} + +cat < 255) +# error "Too many small size classes" +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ +EOF diff --git a/deps/jemalloc/include/jemalloc/internal/stats.h b/deps/jemalloc/include/jemalloc/internal/stats.h index 2a9b31d9ffc..27f68e3681c 100644 --- a/deps/jemalloc/include/jemalloc/internal/stats.h +++ b/deps/jemalloc/include/jemalloc/internal/stats.h @@ -1,25 +1,16 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -#define UMAX2S_BUFSIZE 65 - -#ifdef JEMALLOC_STATS typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t; typedef struct arena_stats_s arena_stats_t; -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) typedef struct chunk_stats_s chunk_stats_t; -#endif #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS -#ifdef JEMALLOC_STATS - -#ifdef JEMALLOC_TCACHE struct tcache_bin_stats_s { /* * Number of allocation requests that corresponded to the size of this @@ -27,7 +18,6 @@ struct tcache_bin_stats_s { */ uint64_t nrequests; }; -#endif struct malloc_bin_stats_s { /* @@ -52,13 +42,11 @@ struct malloc_bin_stats_s { */ uint64_t nrequests; -#ifdef JEMALLOC_TCACHE /* Number of tcache fills from this bin. */ uint64_t nfills; /* Number of tcache flushes to this bin. */ uint64_t nflushes; -#endif /* Total number of runs created for this bin's size class. */ uint64_t nruns; @@ -69,9 +57,6 @@ struct malloc_bin_stats_s { */ uint64_t reruns; - /* High-water mark for this bin. */ - size_t highruns; - /* Current number of runs in this bin. */ size_t curruns; }; @@ -93,9 +78,6 @@ struct malloc_large_stats_s { */ uint64_t nrequests; - /* High-water mark for this size class. */ - size_t highruns; - /* Current number of runs of this size class. */ size_t curruns; }; @@ -127,14 +109,10 @@ struct arena_stats_s { */ malloc_large_stats_t *lstats; }; -#endif /* JEMALLOC_STATS */ -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) struct chunk_stats_s { -# ifdef JEMALLOC_STATS /* Number of chunks that were allocated. */ uint64_t nchunks; -# endif /* High-water mark for number of chunks allocated. */ size_t highchunks; @@ -146,7 +124,6 @@ struct chunk_stats_s { */ size_t curchunks; }; -#endif /* JEMALLOC_STATS */ #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ @@ -154,24 +131,14 @@ struct chunk_stats_s { extern bool opt_stats_print; -#ifdef JEMALLOC_STATS extern size_t stats_cactive; -#endif -char *u2s(uint64_t x, unsigned base, char *s); -#ifdef JEMALLOC_STATS -void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, - const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); -void malloc_printf(const char *format, ...) - JEMALLOC_ATTR(format(printf, 1, 2)); -#endif void stats_print(void (*write)(void *, const char *), void *cbopaque, const char *opts); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES -#ifdef JEMALLOC_STATS #ifndef JEMALLOC_ENABLE_INLINE size_t stats_cactive_get(void); @@ -202,6 +169,5 @@ stats_cactive_sub(size_t size) } #endif -#endif /* JEMALLOC_STATS */ #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/tcache.h b/deps/jemalloc/include/jemalloc/internal/tcache.h index da3c68c5770..c3d4b58d4dc 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache.h @@ -1,4 +1,3 @@ -#ifdef JEMALLOC_TCACHE /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -6,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t; typedef struct tcache_bin_s tcache_bin_t; typedef struct tcache_s tcache_t; +/* + * tcache pointers close to NULL are used to encode state information that is + * used for two purposes: preventing thread caching on a per thread basis and + * cleaning up during thread shutdown. + */ +#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1) +#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2) +#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3) +#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY + /* * Absolute maximum number of cache slots for each small bin in the thread * cache. This is an additional constraint beyond that imposed as: twice the @@ -22,17 +31,26 @@ typedef struct tcache_s tcache_t; #define LG_TCACHE_MAXCLASS_DEFAULT 15 /* - * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation - * events between full GC sweeps (-1: disabled). Integer rounding may cause - * the actual number to be slightly higher, since GC is performed - * incrementally. + * TCACHE_GC_SWEEP is the approximate number of allocation events between + * full GC sweeps. Integer rounding may cause the actual number to be + * slightly higher, since GC is performed incrementally. */ -#define LG_TCACHE_GC_SWEEP_DEFAULT 13 +#define TCACHE_GC_SWEEP 8192 + +/* Number of tcache allocation/deallocation events between incremental GCs. */ +#define TCACHE_GC_INCR \ + ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +typedef enum { + tcache_enabled_false = 0, /* Enable cast to/from bool. */ + tcache_enabled_true = 1, + tcache_enabled_default = 2 +} tcache_enabled_t; + /* * Read-only information associated with each element of tcache_t's tbins array * is stored separately, mainly to reduce memory usage. @@ -42,9 +60,7 @@ struct tcache_bin_info_s { }; struct tcache_bin_s { -# ifdef JEMALLOC_STATS tcache_bin_stats_t tstats; -# endif int low_water; /* Min # cached since last GC. */ unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */ unsigned ncached; /* # of cached objects. */ @@ -52,12 +68,8 @@ struct tcache_bin_s { }; struct tcache_s { -# ifdef JEMALLOC_STATS ql_elm(tcache_t) link; /* Used for aggregating stats. */ -# endif -# ifdef JEMALLOC_PROF uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ -# endif arena_t *arena; /* This thread's arena. */ unsigned ev_cnt; /* Event count since incremental GC. */ unsigned next_gc_bin; /* Next bin to GC. */ @@ -76,29 +88,11 @@ struct tcache_s { extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; -extern ssize_t opt_lg_tcache_gc_sweep; extern tcache_bin_info_t *tcache_bin_info; -/* Map of thread-specific caches. */ -#ifndef NO_TLS -extern __thread tcache_t *tcache_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define TCACHE_GET() tcache_tls -# define TCACHE_SET(v) do { \ - tcache_tls = (tcache_t *)(v); \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#else -# define TCACHE_GET() ((tcache_t *)pthread_getspecific(tcache_tsd)) -# define TCACHE_SET(v) do { \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#endif -extern pthread_key_t tcache_tsd; - /* - * Number of tcache bins. There are nbins small-object bins, plus 0 or more + * Number of tcache bins. There are NBINS small-object bins, plus 0 or more * large-object bins. */ extern size_t nhbins; @@ -106,134 +100,178 @@ extern size_t nhbins; /* Maximum cached size class. */ extern size_t tcache_maxclass; -/* Number of tcache allocation/deallocation events between incremental GCs. */ -extern unsigned tcache_gc_incr; - -void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -tcache_t *tcache_create(arena_t *arena); +size_t tcache_salloc(const void *ptr); +void tcache_event_hard(tcache_t *tcache); void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind); +void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache); +void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache); +void tcache_arena_associate(tcache_t *tcache, arena_t *arena); +void tcache_arena_dissociate(tcache_t *tcache); +tcache_t *tcache_create(arena_t *arena); void tcache_destroy(tcache_t *tcache); -#ifdef JEMALLOC_STATS +void tcache_thread_cleanup(void *arg); void tcache_stats_merge(tcache_t *tcache, arena_t *arena); -#endif -bool tcache_boot(void); +bool tcache_boot0(void); +bool tcache_boot1(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *) +malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t) + void tcache_event(tcache_t *tcache); -tcache_t *tcache_get(void); +void tcache_flush(void); +bool tcache_enabled_get(void); +tcache_t *tcache_get(bool create); +void tcache_enabled_set(bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin); void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); -void tcache_dalloc_small(tcache_t *tcache, void *ptr); +void tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind); void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) -JEMALLOC_INLINE tcache_t * -tcache_get(void) +/* Map of thread-specific caches. */ +malloc_tsd_externs(tcache, tcache_t *) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL, + tcache_thread_cleanup) +/* Per thread flag that allows thread caches to be disabled. */ +malloc_tsd_externs(tcache_enabled, tcache_enabled_t) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t, + tcache_enabled_default, malloc_tsd_no_cleanup) + +JEMALLOC_INLINE void +tcache_flush(void) { tcache_t *tcache; - if ((isthreaded & opt_tcache) == false) + cassert(config_tcache); + + tcache = *tcache_tsd_get(); + if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) + return; + tcache_destroy(tcache); + tcache = NULL; + tcache_tsd_set(&tcache); +} + +JEMALLOC_INLINE bool +tcache_enabled_get(void) +{ + tcache_enabled_t tcache_enabled; + + cassert(config_tcache); + + tcache_enabled = *tcache_enabled_tsd_get(); + if (tcache_enabled == tcache_enabled_default) { + tcache_enabled = (tcache_enabled_t)opt_tcache; + tcache_enabled_tsd_set(&tcache_enabled); + } + + return ((bool)tcache_enabled); +} + +JEMALLOC_INLINE void +tcache_enabled_set(bool enabled) +{ + tcache_enabled_t tcache_enabled; + tcache_t *tcache; + + cassert(config_tcache); + + tcache_enabled = (tcache_enabled_t)enabled; + tcache_enabled_tsd_set(&tcache_enabled); + tcache = *tcache_tsd_get(); + if (enabled) { + if (tcache == TCACHE_STATE_DISABLED) { + tcache = NULL; + tcache_tsd_set(&tcache); + } + } else /* disabled */ { + if (tcache > TCACHE_STATE_MAX) { + tcache_destroy(tcache); + tcache = NULL; + } + if (tcache == NULL) { + tcache = TCACHE_STATE_DISABLED; + tcache_tsd_set(&tcache); + } + } +} + +JEMALLOC_ALWAYS_INLINE tcache_t * +tcache_get(bool create) +{ + tcache_t *tcache; + + if (config_tcache == false) + return (NULL); + if (config_lazy_lock && isthreaded == false) return (NULL); - tcache = TCACHE_GET(); - if ((uintptr_t)tcache <= (uintptr_t)2) { + tcache = *tcache_tsd_get(); + if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) { + if (tcache == TCACHE_STATE_DISABLED) + return (NULL); if (tcache == NULL) { - tcache = tcache_create(choose_arena()); - if (tcache == NULL) - return (NULL); - } else { - if (tcache == (void *)(uintptr_t)1) { + if (create == false) { /* - * Make a note that an allocator function was - * called after the tcache_thread_cleanup() was - * called. + * Creating a tcache here would cause + * allocation as a side effect of free(). + * Ordinarily that would be okay since + * tcache_create() failure is a soft failure + * that doesn't propagate. However, if TLS + * data are freed via free() as in glibc, + * subtle corruption could result from setting + * a TLS variable after its backing memory is + * freed. */ - TCACHE_SET((uintptr_t)2); + return (NULL); } + if (tcache_enabled_get() == false) { + tcache_enabled_set(false); /* Memoize. */ + return (NULL); + } + return (tcache_create(choose_arena(NULL))); + } + if (tcache == TCACHE_STATE_PURGATORY) { + /* + * Make a note that an allocator function was called + * after tcache_thread_cleanup() was called. + */ + tcache = TCACHE_STATE_REINCARNATED; + tcache_tsd_set(&tcache); return (NULL); } + if (tcache == TCACHE_STATE_REINCARNATED) + return (NULL); + not_reached(); } return (tcache); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void tcache_event(tcache_t *tcache) { - if (tcache_gc_incr == 0) + if (TCACHE_GC_INCR == 0) return; tcache->ev_cnt++; - assert(tcache->ev_cnt <= tcache_gc_incr); - if (tcache->ev_cnt == tcache_gc_incr) { - size_t binind = tcache->next_gc_bin; - tcache_bin_t *tbin = &tcache->tbins[binind]; - tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; - - if (tbin->low_water > 0) { - /* - * Flush (ceiling) 3/4 of the objects below the low - * water mark. - */ - if (binind < nbins) { - tcache_bin_flush_small(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } else { - tcache_bin_flush_large(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } - /* - * Reduce fill count by 2X. Limit lg_fill_div such that - * the fill count is always at least 1. - */ - if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) - >= 1) - tbin->lg_fill_div++; - } else if (tbin->low_water < 0) { - /* - * Increase fill count by 2X. Make sure lg_fill_div - * stays greater than 0. - */ - if (tbin->lg_fill_div > 1) - tbin->lg_fill_div--; - } - tbin->low_water = tbin->ncached; - - tcache->next_gc_bin++; - if (tcache->next_gc_bin == nhbins) - tcache->next_gc_bin = 0; - tcache->ev_cnt = 0; - } + assert(tcache->ev_cnt <= TCACHE_GC_INCR); + if (tcache->ev_cnt == TCACHE_GC_INCR) + tcache_event_hard(tcache); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_easy(tcache_bin_t *tbin) { void *ret; @@ -249,7 +287,7 @@ tcache_alloc_easy(tcache_bin_t *tbin) return (ret); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) { void *ret; @@ -257,37 +295,44 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) tcache_bin_t *tbin; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); + assert(binind < NBINS); tbin = &tcache->tbins[binind]; + size = arena_bin_info[binind].reg_size; ret = tcache_alloc_easy(tbin); if (ret == NULL) { ret = tcache_alloc_small_hard(tcache, tbin, binind); if (ret == NULL) return (NULL); } - assert(arena_salloc(ret) == arena_bin_info[binind].reg_size); + assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size); if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) + memset(ret, 0, size); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += arena_bin_info[binind].reg_size; -#endif + if (config_stats) + tbin->tstats.nrequests++; + if (config_prof) + tcache->prof_accumbytes += arena_bin_info[binind].reg_size; tcache_event(tcache); return (ret); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) { void *ret; @@ -296,7 +341,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) size = PAGE_CEILING(size); assert(size <= tcache_maxclass); - binind = nbins + (size >> PAGE_SHIFT) - 1; + binind = NBINS + (size >> LG_PAGE) - 1; assert(binind < nhbins); tbin = &tcache->tbins[binind]; ret = tcache_alloc_easy(tbin); @@ -309,74 +354,53 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) if (ret == NULL) return (NULL); } else { -#ifdef JEMALLOC_PROF - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> - PAGE_SHIFT); - chunk->map[pageind-map_bias].bits &= ~CHUNK_MAP_CLASS_MASK; -#endif + if (config_prof && prof_promote && size == PAGE) { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(ret); + size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> + LG_PAGE); + arena_mapbits_large_binind_set(chunk, pageind, + BININD_INVALID); + } if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + } else { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += size; -#endif + if (config_stats) + tbin->tstats.nrequests++; + if (config_prof) + tcache->prof_accumbytes += size; } tcache_event(tcache); return (ret); } -JEMALLOC_INLINE void -tcache_dalloc_small(tcache_t *tcache, void *ptr) +JEMALLOC_ALWAYS_INLINE void +tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) { - arena_t *arena; - arena_chunk_t *chunk; - arena_run_t *run; - arena_bin_t *bin; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; - size_t pageind, binind; - arena_chunk_map_t *mapelm; - - assert(arena_salloc(ptr) <= small_maxclass); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; - binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) / - sizeof(arena_bin_t); - assert(binind < nbins); - -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, arena_bin_info[binind].reg_size); -#endif + + assert(tcache_salloc(ptr) <= SMALL_MAXCLASS); + + if (config_fill && opt_junk) + arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (tbin->ncached == tbin_info->ncached_max) { tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); + 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; @@ -385,38 +409,27 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr) tcache_event(tcache); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) { - arena_t *arena; - arena_chunk_t *chunk; - size_t pageind, binind; + size_t binind; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; assert((size & PAGE_MASK) == 0); - assert(arena_salloc(ptr) > small_maxclass); - assert(arena_salloc(ptr) <= tcache_maxclass); + assert(tcache_salloc(ptr) > SMALL_MAXCLASS); + assert(tcache_salloc(ptr) <= tcache_maxclass); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - binind = nbins + (size >> PAGE_SHIFT) - 1; + binind = NBINS + (size >> LG_PAGE) - 1; -#ifdef JEMALLOC_FILL - if (opt_junk) + if (config_fill && opt_junk) memset(ptr, 0x5a, size); -#endif tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (tbin->ncached == tbin_info->ncached_max) { tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); + 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; @@ -428,4 +441,3 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc/include/jemalloc/internal/tsd.h b/deps/jemalloc/include/jemalloc/internal/tsd.h new file mode 100644 index 00000000000..9fb4a23ec6b --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/tsd.h @@ -0,0 +1,434 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Maximum number of malloc_tsd users with cleanup functions. */ +#define MALLOC_TSD_CLEANUPS_MAX 8 + +typedef bool (*malloc_tsd_cleanup_t)(void); + +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +typedef struct tsd_init_block_s tsd_init_block_t; +typedef struct tsd_init_head_s tsd_init_head_t; +#endif + +/* + * TLS/TSD-agnostic macro-based implementation of thread-specific data. There + * are four macros that support (at least) three use cases: file-private, + * library-private, and library-private inlined. Following is an example + * library-private tsd variable: + * + * In example.h: + * typedef struct { + * int x; + * int y; + * } example_t; + * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) + * malloc_tsd_protos(, example, example_t *) + * malloc_tsd_externs(example, example_t *) + * In example.c: + * malloc_tsd_data(, example, example_t *, EX_INITIALIZER) + * malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER, + * example_tsd_cleanup) + * + * The result is a set of generated functions, e.g.: + * + * bool example_tsd_boot(void) {...} + * example_t **example_tsd_get() {...} + * void example_tsd_set(example_t **val) {...} + * + * Note that all of the functions deal in terms of (a_type *) rather than + * (a_type) so that it is possible to support non-pointer types (unlike + * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is + * cast to (void *). This means that the cleanup function needs to cast *and* + * dereference the function argument, e.g.: + * + * void + * example_tsd_cleanup(void *arg) + * { + * example_t *example = *(example_t **)arg; + * + * [...] + * if ([want the cleanup function to be called again]) { + * example_tsd_set(&example); + * } + * } + * + * If example_tsd_set() is called within example_tsd_cleanup(), it will be + * called again. This is similar to how pthreads TSD destruction works, except + * that pthreads only calls the cleanup function again if the value was set to + * non-NULL. + */ + +/* malloc_tsd_protos(). */ +#define malloc_tsd_protos(a_attr, a_name, a_type) \ +a_attr bool \ +a_name##_tsd_boot(void); \ +a_attr a_type * \ +a_name##_tsd_get(void); \ +a_attr void \ +a_name##_tsd_set(a_type *val); + +/* malloc_tsd_externs(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_externs(a_name, a_type) \ +extern __thread a_type a_name##_tls; \ +extern __thread bool a_name##_initialized; \ +extern bool a_name##_booted; +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_externs(a_name, a_type) \ +extern __thread a_type a_name##_tls; \ +extern pthread_key_t a_name##_tsd; \ +extern bool a_name##_booted; +#elif (defined(_WIN32)) +#define malloc_tsd_externs(a_name, a_type) \ +extern DWORD a_name##_tsd; \ +extern bool a_name##_booted; +#else +#define malloc_tsd_externs(a_name, a_type) \ +extern pthread_key_t a_name##_tsd; \ +extern tsd_init_head_t a_name##_tsd_init_head; \ +extern bool a_name##_booted; +#endif + +/* malloc_tsd_data(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr __thread a_type JEMALLOC_TLS_MODEL \ + a_name##_tls = a_initializer; \ +a_attr __thread bool JEMALLOC_TLS_MODEL \ + a_name##_initialized = false; \ +a_attr bool a_name##_booted = false; +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr __thread a_type JEMALLOC_TLS_MODEL \ + a_name##_tls = a_initializer; \ +a_attr pthread_key_t a_name##_tsd; \ +a_attr bool a_name##_booted = false; +#elif (defined(_WIN32)) +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr DWORD a_name##_tsd; \ +a_attr bool a_name##_booted = false; +#else +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr pthread_key_t a_name##_tsd; \ +a_attr tsd_init_head_t a_name##_tsd_init_head = { \ + ql_head_initializer(blocks), \ + MALLOC_MUTEX_INITIALIZER \ +}; \ +a_attr bool a_name##_booted = false; +#endif + +/* malloc_tsd_funcs(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_cleanup_wrapper(void) \ +{ \ + \ + if (a_name##_initialized) { \ + a_name##_initialized = false; \ + a_cleanup(&a_name##_tls); \ + } \ + return (a_name##_initialized); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + malloc_tsd_cleanup_register( \ + &a_name##_tsd_cleanup_wrapper); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + \ + assert(a_name##_booted); \ + return (&a_name##_tls); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + \ + assert(a_name##_booted); \ + a_name##_tls = (*val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + a_name##_initialized = true; \ +} +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \ + return (true); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + \ + assert(a_name##_booted); \ + return (&a_name##_tls); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + \ + assert(a_name##_booted); \ + a_name##_tls = (*val); \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)(&a_name##_tls))) { \ + malloc_write(": Error" \ + " setting TSD for "#a_name"\n"); \ + if (opt_abort) \ + abort(); \ + } \ + } \ +} +#elif (defined(_WIN32)) +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Data structure. */ \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##_tsd_wrapper_t; \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_cleanup_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \ + if (wrapper == NULL) \ + return (false); \ + if (a_cleanup != malloc_tsd_no_cleanup && \ + wrapper->initialized) { \ + a_type val = wrapper->val; \ + a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + a_cleanup(&val); \ + if (wrapper->initialized) { \ + /* Trigger another cleanup round. */ \ + return (true); \ + } \ + } \ + malloc_tsd_dalloc(wrapper); \ + return (false); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + a_name##_tsd = TlsAlloc(); \ + if (a_name##_tsd == TLS_OUT_OF_INDEXES) \ + return (true); \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + malloc_tsd_cleanup_register( \ + &a_name##_tsd_cleanup_wrapper); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_name##_tsd_wrapper_t * \ +a_name##_tsd_get_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ + TlsGetValue(a_name##_tsd); \ + \ + if (wrapper == NULL) { \ + wrapper = (a_name##_tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } else { \ + static a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + } \ + if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + } \ + return (wrapper); \ +} \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + return (&wrapper->val); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + wrapper->val = *(val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + wrapper->initialized = true; \ +} +#else +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Data structure. */ \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##_tsd_wrapper_t; \ +/* Initialization/cleanup. */ \ +a_attr void \ +a_name##_tsd_cleanup_wrapper(void *arg) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\ + \ + if (a_cleanup != malloc_tsd_no_cleanup && \ + wrapper->initialized) { \ + wrapper->initialized = false; \ + a_cleanup(&wrapper->val); \ + if (wrapper->initialized) { \ + /* Trigger another cleanup round. */ \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)wrapper)) { \ + malloc_write(": Error" \ + " setting TSD for "#a_name"\n"); \ + if (opt_abort) \ + abort(); \ + } \ + return; \ + } \ + } \ + malloc_tsd_dalloc(wrapper); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (pthread_key_create(&a_name##_tsd, \ + a_name##_tsd_cleanup_wrapper) != 0) \ + return (true); \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_name##_tsd_wrapper_t * \ +a_name##_tsd_get_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ + pthread_getspecific(a_name##_tsd); \ + \ + if (wrapper == NULL) { \ + tsd_init_block_t block; \ + wrapper = tsd_init_check_recursion( \ + &a_name##_tsd_init_head, &block); \ + if (wrapper) \ + return (wrapper); \ + wrapper = (a_name##_tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + block.data = wrapper; \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } else { \ + static a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + } \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + tsd_init_finish(&a_name##_tsd_init_head, &block); \ + } \ + return (wrapper); \ +} \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + return (&wrapper->val); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + wrapper->val = *(val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + wrapper->initialized = true; \ +} +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +struct tsd_init_block_s { + ql_elm(tsd_init_block_t) link; + pthread_t thread; + void *data; +}; +struct tsd_init_head_s { + ql_head(tsd_init_block_t) blocks; + malloc_mutex_t lock; +}; +#endif + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_no_cleanup(void *); +void malloc_tsd_cleanup_register(bool (*f)(void)); +void malloc_tsd_boot(void); +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +void *tsd_init_check_recursion(tsd_init_head_t *head, + tsd_init_block_t *block); +void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); +#endif + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/util.h b/deps/jemalloc/include/jemalloc/internal/util.h new file mode 100644 index 00000000000..6b938f74688 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/util.h @@ -0,0 +1,162 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Size of stack-allocated buffer passed to buferror(). */ +#define BUFERROR_BUF 64 + +/* + * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be + * large enough for all possible uses within jemalloc. + */ +#define MALLOC_PRINTF_BUFSIZE 4096 + +/* + * Wrap a cpp argument that contains commas such that it isn't broken up into + * multiple arguments. + */ +#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ + +/* + * Silence compiler warnings due to uninitialized values. This is used + * wherever the compiler fails to recognize that the variable is never used + * uninitialized. + */ +#ifdef JEMALLOC_CC_SILENCE +# define JEMALLOC_CC_SILENCE_INIT(v) = v +#else +# define JEMALLOC_CC_SILENCE_INIT(v) +#endif + +/* + * Define a custom assert() in order to reduce the chances of deadlock during + * assertion failure. + */ +#ifndef assert +#define assert(e) do { \ + if (config_debug && !(e)) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ +} while (0) +#endif + +#ifndef not_reached +#define not_reached() do { \ + if (config_debug) { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ +} while (0) +#endif + +#ifndef not_implemented +#define not_implemented() do { \ + if (config_debug) { \ + malloc_printf(": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ +} while (0) +#endif + +#ifndef assert_not_implemented +#define assert_not_implemented(e) do { \ + if (config_debug && !(e)) \ + not_implemented(); \ +} while (0) +#endif + +/* Use to assert a particular configuration, e.g., cassert(config_debug). */ +#define cassert(c) do { \ + if ((c) == false) \ + not_reached(); \ +} while (0) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +int buferror(int err, char *buf, size_t buflen); +uintmax_t malloc_strtoumax(const char *restrict nptr, + char **restrict endptr, int base); +void malloc_write(const char *s); + +/* + * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating + * point math. + */ +int malloc_vsnprintf(char *str, size_t size, const char *format, + va_list ap); +int malloc_snprintf(char *str, size_t size, const char *format, ...) + JEMALLOC_ATTR(format(printf, 3, 4)); +void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, va_list ap); +void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, + const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); +void malloc_printf(const char *format, ...) + JEMALLOC_ATTR(format(printf, 1, 2)); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +size_t pow2_ceil(size_t x); +void set_errno(int errnum); +int get_errno(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) +/* Compute the smallest power of 2 that is >= x. */ +JEMALLOC_INLINE size_t +pow2_ceil(size_t x) +{ + + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; +#if (LG_SIZEOF_PTR == 3) + x |= x >> 32; +#endif + x++; + return (x); +} + +/* Sets error code */ +JEMALLOC_INLINE void +set_errno(int errnum) +{ + +#ifdef _WIN32 + SetLastError(errnum); +#else + errno = errnum; +#endif +} + +/* Get last error code */ +JEMALLOC_INLINE int +get_errno(void) +{ + +#ifdef _WIN32 + return (GetLastError()); +#else + return (errno); +#endif +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/zone.h b/deps/jemalloc/include/jemalloc/internal/zone.h deleted file mode 100644 index 859b529d443..00000000000 --- a/deps/jemalloc/include/jemalloc/internal/zone.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef JEMALLOC_ZONE -# error "This source file is for zones on Darwin (OS X)." -#endif -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -malloc_zone_t *create_zone(void); -void szone2ozone(malloc_zone_t *zone); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/jemalloc.h.in b/deps/jemalloc/include/jemalloc/jemalloc.h.in deleted file mode 100644 index 580a5ec5d5f..00000000000 --- a/deps/jemalloc/include/jemalloc/jemalloc.h.in +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef JEMALLOC_H_ -#define JEMALLOC_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#define JEMALLOC_VERSION "@jemalloc_version@" -#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ -#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ -#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ -#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ -#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" - -#include "jemalloc_defs@install_suffix@.h" -#ifndef JEMALLOC_P -# define JEMALLOC_P(s) s -#endif - -#define ALLOCM_LG_ALIGN(la) (la) -#if LG_SIZEOF_PTR == 2 -#define ALLOCM_ALIGN(a) (ffs(a)-1) -#else -#define ALLOCM_ALIGN(a) ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) -#endif -#define ALLOCM_ZERO ((int)0x40) -#define ALLOCM_NO_MOVE ((int)0x80) - -#define ALLOCM_SUCCESS 0 -#define ALLOCM_ERR_OOM 1 -#define ALLOCM_ERR_NOT_MOVED 2 - -extern const char *JEMALLOC_P(malloc_conf); -extern void (*JEMALLOC_P(malloc_message))(void *, const char *); - -void *JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc); -void *JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc); -int JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) - JEMALLOC_ATTR(nonnull(1)); -void *JEMALLOC_P(realloc)(void *ptr, size_t size); -void JEMALLOC_P(free)(void *ptr); - -size_t JEMALLOC_P(malloc_usable_size)(const void *ptr); -void JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts); -int JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -int JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, - size_t *miblenp); -int JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen); - -int JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) - JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, - size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) - JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(dallocm)(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1)); - -#ifdef __cplusplus -}; -#endif -#endif /* JEMALLOC_H_ */ diff --git a/deps/jemalloc/include/jemalloc/jemalloc.sh b/deps/jemalloc/include/jemalloc/jemalloc.sh new file mode 100755 index 00000000000..e4738ebae99 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/jemalloc.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +objroot=$1 + +cat < +#include + +#define JEMALLOC_VERSION "@jemalloc_version@" +#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ +#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ +#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ +#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ +#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" + +# define MALLOCX_LG_ALIGN(la) (la) +# if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) (ffs(a)-1) +# else +# define MALLOCX_ALIGN(a) \ + ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) +# endif +# define MALLOCX_ZERO ((int)0x40) +/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */ +# define MALLOCX_ARENA(a) ((int)(((a)+1) << 8)) + +#ifdef JEMALLOC_EXPERIMENTAL +# define ALLOCM_LG_ALIGN(la) (la) +# if LG_SIZEOF_PTR == 2 +# define ALLOCM_ALIGN(a) (ffs(a)-1) +# else +# define ALLOCM_ALIGN(a) \ + ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) +# endif +# define ALLOCM_ZERO ((int)0x40) +# define ALLOCM_NO_MOVE ((int)0x80) +/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */ +# define ALLOCM_ARENA(a) ((int)(((a)+1) << 8)) +# define ALLOCM_SUCCESS 0 +# define ALLOCM_ERR_OOM 1 +# define ALLOCM_ERR_NOT_MOVED 2 +#endif + +#ifdef JEMALLOC_HAVE_ATTR +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +#elif _MSC_VER +# define JEMALLOC_ATTR(s) +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_NOINLINE __declspec(noinline) +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_EXPORT +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_SECTION(s) +# define JEMALLOC_NOINLINE +#endif diff --git a/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh b/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh new file mode 100755 index 00000000000..df328b78dac --- /dev/null +++ b/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +public_symbols_txt=$1 +symbol_prefix=$2 + +cat < 1000 +#pragma once +#endif + +#include "stdint.h" + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +#ifdef _WIN64 +# define __PRI64_PREFIX "l" +# define __PRIPTR_PREFIX "l" +#else +# define __PRI64_PREFIX "ll" +# define __PRIPTR_PREFIX +#endif + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "d" +#define PRIi32 "i" +#define PRIdLEAST32 "d" +#define PRIiLEAST32 "i" +#define PRIdFAST32 "d" +#define PRIiFAST32 "i" + +#define PRId64 __PRI64_PREFIX "d" +#define PRIi64 __PRI64_PREFIX "i" +#define PRIdLEAST64 __PRI64_PREFIX "d" +#define PRIiLEAST64 __PRI64_PREFIX "i" +#define PRIdFAST64 __PRI64_PREFIX "d" +#define PRIiFAST64 __PRI64_PREFIX "i" + +#define PRIdMAX __PRI64_PREFIX "d" +#define PRIiMAX __PRI64_PREFIX "i" + +#define PRIdPTR __PRIPTR_PREFIX "d" +#define PRIiPTR __PRIPTR_PREFIX "i" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "o" +#define PRIu32 "u" +#define PRIx32 "x" +#define PRIX32 "X" +#define PRIoLEAST32 "o" +#define PRIuLEAST32 "u" +#define PRIxLEAST32 "x" +#define PRIXLEAST32 "X" +#define PRIoFAST32 "o" +#define PRIuFAST32 "u" +#define PRIxFAST32 "x" +#define PRIXFAST32 "X" + +#define PRIo64 __PRI64_PREFIX "o" +#define PRIu64 __PRI64_PREFIX "u" +#define PRIx64 __PRI64_PREFIX "x" +#define PRIX64 __PRI64_PREFIX "X" +#define PRIoLEAST64 __PRI64_PREFIX "o" +#define PRIuLEAST64 __PRI64_PREFIX "u" +#define PRIxLEAST64 __PRI64_PREFIX "x" +#define PRIXLEAST64 __PRI64_PREFIX "X" +#define PRIoFAST64 __PRI64_PREFIX "o" +#define PRIuFAST64 __PRI64_PREFIX "u" +#define PRIxFAST64 __PRI64_PREFIX "x" +#define PRIXFAST64 __PRI64_PREFIX "X" + +#define PRIoMAX __PRI64_PREFIX "o" +#define PRIuMAX __PRI64_PREFIX "u" +#define PRIxMAX __PRI64_PREFIX "x" +#define PRIXMAX __PRI64_PREFIX "X" + +#define PRIoPTR __PRIPTR_PREFIX "o" +#define PRIuPTR __PRIPTR_PREFIX "u" +#define PRIxPTR __PRIPTR_PREFIX "x" +#define PRIXPTR __PRIPTR_PREFIX "X" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + + +#endif // _MSC_INTTYPES_H_ ] diff --git a/deps/jemalloc/include/msvc_compat/stdbool.h b/deps/jemalloc/include/msvc_compat/stdbool.h new file mode 100644 index 00000000000..da9ee8b809b --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/stdbool.h @@ -0,0 +1,16 @@ +#ifndef stdbool_h +#define stdbool_h + +#include + +/* MSVC doesn't define _Bool or bool in C, but does have BOOL */ +/* Note this doesn't pass autoconf's test because (bool) 0.5 != true */ +typedef BOOL _Bool; + +#define bool _Bool +#define true 1 +#define false 0 + +#define __bool_true_false_are_defined 1 + +#endif /* stdbool_h */ diff --git a/deps/jemalloc/include/msvc_compat/stdint.h b/deps/jemalloc/include/msvc_compat/stdint.h new file mode 100644 index 00000000000..d02608a5972 --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/stdint.h @@ -0,0 +1,247 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +#define INTMAX_C INT64_C +#define UINTMAX_C UINT64_C + +#endif // __STDC_CONSTANT_MACROS ] + + +#endif // _MSC_STDINT_H_ ] diff --git a/deps/jemalloc/include/msvc_compat/strings.h b/deps/jemalloc/include/msvc_compat/strings.h new file mode 100644 index 00000000000..c84975b6b8e --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/strings.h @@ -0,0 +1,23 @@ +#ifndef strings_h +#define strings_h + +/* MSVC doesn't define ffs/ffsl. This dummy strings.h header is provided + * for both */ +#include +#pragma intrinsic(_BitScanForward) +static __forceinline int ffsl(long x) +{ + unsigned long i; + + if (_BitScanForward(&i, x)) + return (i + 1); + return (0); +} + +static __forceinline int ffs(int x) +{ + + return (ffsl(x)); +} + +#endif diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c index d166ca1ec4d..dad707b63d0 100644 --- a/deps/jemalloc/src/arena.c +++ b/deps/jemalloc/src/arena.c @@ -4,197 +4,52 @@ /******************************************************************************/ /* Data. */ -size_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; -size_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; -uint8_t const *small_size2bin; -arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -unsigned nqbins; -unsigned ncbins; -unsigned nsbins; -unsigned nbins; -size_t qspace_max; -size_t cspace_min; -size_t cspace_max; -size_t sspace_min; -size_t sspace_max; - -size_t lg_mspace; -size_t mspace_mask; +arena_bin_info_t arena_bin_info[NBINS]; -/* - * const_small_size2bin is a static constant lookup table that in the common - * case can be used as-is for small_size2bin. - */ -#if (LG_TINY_MIN == 2) -#define S2B_4(i) i, -#define S2B_8(i) S2B_4(i) S2B_4(i) -#elif (LG_TINY_MIN == 3) +JEMALLOC_ALIGNED(CACHELINE) +const uint8_t small_size2bin[] = { #define S2B_8(i) i, -#else -# error "Unsupported LG_TINY_MIN" -#endif #define S2B_16(i) S2B_8(i) S2B_8(i) #define S2B_32(i) S2B_16(i) S2B_16(i) #define S2B_64(i) S2B_32(i) S2B_32(i) #define S2B_128(i) S2B_64(i) S2B_64(i) #define S2B_256(i) S2B_128(i) S2B_128(i) -/* - * The number of elements in const_small_size2bin is dependent on the - * definition for SUBPAGE. - */ -static JEMALLOC_ATTR(aligned(CACHELINE)) - const uint8_t const_small_size2bin[] = { -#if (LG_QUANTUM == 4) -/* 16-byte quantum **********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ - S2B_8(2) /* 16 */ -# define S2B_QMIN 2 -# elif (LG_TINY_MIN == 3) - S2B_8(0) /* 8 */ - S2B_8(1) /* 16 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_16(0) /* 16 */ -# define S2B_QMIN 0 -# endif - S2B_16(S2B_QMIN + 1) /* 32 */ - S2B_16(S2B_QMIN + 2) /* 48 */ - S2B_16(S2B_QMIN + 3) /* 64 */ - S2B_16(S2B_QMIN + 4) /* 80 */ - S2B_16(S2B_QMIN + 5) /* 96 */ - S2B_16(S2B_QMIN + 6) /* 112 */ - S2B_16(S2B_QMIN + 7) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 8) -#else -/* 8-byte quantum ***********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_8(0) /* 8 */ -# define S2B_QMIN 0 -# endif - S2B_8(S2B_QMIN + 1) /* 16 */ - S2B_8(S2B_QMIN + 2) /* 24 */ - S2B_8(S2B_QMIN + 3) /* 32 */ - S2B_8(S2B_QMIN + 4) /* 40 */ - S2B_8(S2B_QMIN + 5) /* 48 */ - S2B_8(S2B_QMIN + 6) /* 56 */ - S2B_8(S2B_QMIN + 7) /* 64 */ - S2B_8(S2B_QMIN + 8) /* 72 */ - S2B_8(S2B_QMIN + 9) /* 80 */ - S2B_8(S2B_QMIN + 10) /* 88 */ - S2B_8(S2B_QMIN + 11) /* 96 */ - S2B_8(S2B_QMIN + 12) /* 104 */ - S2B_8(S2B_QMIN + 13) /* 112 */ - S2B_8(S2B_QMIN + 14) /* 120 */ - S2B_8(S2B_QMIN + 15) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 16) -#endif -/****************************************/ - S2B_64(S2B_CMIN + 0) /* 192 */ - S2B_64(S2B_CMIN + 1) /* 256 */ - S2B_64(S2B_CMIN + 2) /* 320 */ - S2B_64(S2B_CMIN + 3) /* 384 */ - S2B_64(S2B_CMIN + 4) /* 448 */ - S2B_64(S2B_CMIN + 5) /* 512 */ -# define S2B_SMIN (S2B_CMIN + 6) - S2B_256(S2B_SMIN + 0) /* 768 */ - S2B_256(S2B_SMIN + 1) /* 1024 */ - S2B_256(S2B_SMIN + 2) /* 1280 */ - S2B_256(S2B_SMIN + 3) /* 1536 */ - S2B_256(S2B_SMIN + 4) /* 1792 */ - S2B_256(S2B_SMIN + 5) /* 2048 */ - S2B_256(S2B_SMIN + 6) /* 2304 */ - S2B_256(S2B_SMIN + 7) /* 2560 */ - S2B_256(S2B_SMIN + 8) /* 2816 */ - S2B_256(S2B_SMIN + 9) /* 3072 */ - S2B_256(S2B_SMIN + 10) /* 3328 */ - S2B_256(S2B_SMIN + 11) /* 3584 */ - S2B_256(S2B_SMIN + 12) /* 3840 */ -#if (STATIC_PAGE_SHIFT == 13) - S2B_256(S2B_SMIN + 13) /* 4096 */ - S2B_256(S2B_SMIN + 14) /* 4352 */ - S2B_256(S2B_SMIN + 15) /* 4608 */ - S2B_256(S2B_SMIN + 16) /* 4864 */ - S2B_256(S2B_SMIN + 17) /* 5120 */ - S2B_256(S2B_SMIN + 18) /* 5376 */ - S2B_256(S2B_SMIN + 19) /* 5632 */ - S2B_256(S2B_SMIN + 20) /* 5888 */ - S2B_256(S2B_SMIN + 21) /* 6144 */ - S2B_256(S2B_SMIN + 22) /* 6400 */ - S2B_256(S2B_SMIN + 23) /* 6656 */ - S2B_256(S2B_SMIN + 24) /* 6912 */ - S2B_256(S2B_SMIN + 25) /* 7168 */ - S2B_256(S2B_SMIN + 26) /* 7424 */ - S2B_256(S2B_SMIN + 27) /* 7680 */ - S2B_256(S2B_SMIN + 28) /* 7936 */ -#endif -}; -#undef S2B_1 -#undef S2B_2 -#undef S2B_4 +#define S2B_512(i) S2B_256(i) S2B_256(i) +#define S2B_1024(i) S2B_512(i) S2B_512(i) +#define S2B_2048(i) S2B_1024(i) S2B_1024(i) +#define S2B_4096(i) S2B_2048(i) S2B_2048(i) +#define S2B_8192(i) S2B_4096(i) S2B_4096(i) +#define SIZE_CLASS(bin, delta, size) \ + S2B_##delta(bin) + SIZE_CLASSES #undef S2B_8 #undef S2B_16 #undef S2B_32 #undef S2B_64 #undef S2B_128 #undef S2B_256 -#undef S2B_QMIN -#undef S2B_CMIN -#undef S2B_SMIN +#undef S2B_512 +#undef S2B_1024 +#undef S2B_2048 +#undef S2B_4096 +#undef S2B_8192 +#undef SIZE_CLASS +}; /******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, - bool large, bool zero); -static arena_chunk_t *arena_chunk_alloc(arena_t *arena); -static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); -static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, - bool zero); +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + static void arena_purge(arena_t *arena, bool all); -static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); -static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, size_t oldsize, size_t newsize); -static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); -static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); -static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); -static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin); +static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, + bool cleaned); static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); -static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, - void *ptr, size_t oldsize, size_t size); -static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, - void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); -static bool small_size2bin_init(void); -#ifdef JEMALLOC_DEBUG -static void small_size2bin_validate(void); -#endif -static bool small_size2bin_init_hard(void); -static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, - size_t min_run_size); -static bool bin_info_init(void); /******************************************************************************/ @@ -211,8 +66,8 @@ arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) } /* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, - arena_chunk_map_t, u.rb_link, arena_run_comp) +rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, + u.rb_link, arena_run_comp) static inline int arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) @@ -221,9 +76,6 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) size_t a_size = a->bits & ~PAGE_MASK; size_t b_size = b->bits & ~PAGE_MASK; - assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & - CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); - ret = (a_size > b_size) - (a_size < b_size); if (ret == 0) { uintptr_t a_mapelm, b_mapelm; @@ -246,8 +98,184 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) } /* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, - arena_chunk_map_t, u.rb_link, arena_avail_comp) +rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, + u.rb_link, arena_avail_comp) + +static inline int +arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) +{ + + assert(a != NULL); + assert(b != NULL); + + /* + * Short-circuit for self comparison. The following comparison code + * would come to the same result, but at the cost of executing the slow + * path. + */ + if (a == b) + return (0); + + /* + * Order such that chunks with higher fragmentation are "less than" + * those with lower fragmentation -- purging order is from "least" to + * "greatest". Fragmentation is measured as: + * + * mean current avail run size + * -------------------------------- + * mean defragmented avail run size + * + * navail + * ----------- + * nruns_avail nruns_avail-nruns_adjac + * = ========================= = ----------------------- + * navail nruns_avail + * ----------------------- + * nruns_avail-nruns_adjac + * + * The following code multiplies away the denominator prior to + * comparison, in order to avoid division. + * + */ + { + size_t a_val = (a->nruns_avail - a->nruns_adjac) * + b->nruns_avail; + size_t b_val = (b->nruns_avail - b->nruns_adjac) * + a->nruns_avail; + + if (a_val < b_val) + return (1); + if (a_val > b_val) + return (-1); + } + /* + * Break ties by chunk address. For fragmented chunks, report lower + * addresses as "lower", so that fragmentation reduction happens first + * at lower addresses. However, use the opposite ordering for + * unfragmented chunks, in order to increase the chances of + * re-allocating dirty runs. + */ + { + uintptr_t a_chunk = (uintptr_t)a; + uintptr_t b_chunk = (uintptr_t)b; + int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); + if (a->nruns_adjac == 0) { + assert(b->nruns_adjac == 0); + ret = -ret; + } + return (ret); + } +} + +/* Generate red-black tree functions. */ +rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, + dirty_link, arena_chunk_dirty_comp) + +static inline bool +arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) +{ + bool ret; + + if (pageind-1 < map_bias) + ret = false; + else { + ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); + assert(ret == false || arena_mapbits_dirty_get(chunk, + pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); + } + return (ret); +} + +static inline bool +arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) +{ + bool ret; + + if (pageind+npages == chunk_npages) + ret = false; + else { + assert(pageind+npages < chunk_npages); + ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); + assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) + != arena_mapbits_dirty_get(chunk, pageind+npages)); + } + return (ret); +} + +static inline bool +arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) +{ + + return (arena_avail_adjac_pred(chunk, pageind) || + arena_avail_adjac_succ(chunk, pageind, npages)); +} + +static void +arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) +{ + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + + /* + * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be + * removed and reinserted even if the run to be inserted is clean. + */ + if (chunk->ndirty != 0) + arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); + + if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) + chunk->nruns_adjac++; + if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) + chunk->nruns_adjac++; + chunk->nruns_avail++; + assert(chunk->nruns_avail > chunk->nruns_adjac); + + if (arena_mapbits_dirty_get(chunk, pageind) != 0) { + arena->ndirty += npages; + chunk->ndirty += npages; + } + if (chunk->ndirty != 0) + arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); + + arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, + pageind)); +} + +static void +arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) +{ + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + + /* + * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be + * removed and reinserted even if the run to be removed is clean. + */ + if (chunk->ndirty != 0) + arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); + + if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) + chunk->nruns_adjac--; + if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) + chunk->nruns_adjac--; + chunk->nruns_avail--; + assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail + == 0 && chunk->nruns_adjac == 0)); + + if (arena_mapbits_dirty_get(chunk, pageind) != 0) { + arena->ndirty -= npages; + chunk->ndirty -= npages; + } + if (chunk->ndirty != 0) + arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); + + arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, + pageind)); +} static inline void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) @@ -257,13 +285,12 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); - dassert(run->magic == ARENA_RUN_MAGIC); assert(run->nfree > 0); assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + - (uintptr_t)(bin_info->reg_size * regind)); + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; if (regind == run->nextind) run->nextind++; @@ -275,7 +302,9 @@ static inline void arena_run_reg_dalloc(arena_run_t *run, void *ptr) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - size_t binind = arena_bin_index(chunk->arena, run->bin); + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t mapbits = arena_mapbits_get(chunk, pageind); + size_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + @@ -284,8 +313,8 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) assert(run->nfree < bin_info->nregs); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size - == 0); + (uintptr_t)bin_info->reg0_offset)) % + (uintptr_t)bin_info->reg_interval == 0); assert((uintptr_t)ptr >= (uintptr_t)run + (uintptr_t)bin_info->reg0_offset); /* Freeing an unallocated pointer can cause assertion failure. */ @@ -295,302 +324,349 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) run->nfree++; } -#ifdef JEMALLOC_DEBUG static inline void -arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) +arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) +{ + + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << + LG_PAGE)), (npages << LG_PAGE)); + memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, + (npages << LG_PAGE)); +} + +static inline void +arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) +{ + + VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << + LG_PAGE)), PAGE); +} + +static inline void +arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) { size_t i; - size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << PAGE_SHIFT)); + UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); - for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) + arena_run_page_mark_zeroed(chunk, run_ind); + for (i = 0; i < PAGE / sizeof(size_t); i++) assert(p[i] == 0); } -#endif static void -arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, - bool zero) +arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) { - arena_chunk_t *chunk; - size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; - size_t flag_dirty; - arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - old_ndirty = chunk->ndirty; - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); - flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; - runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; - total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> - PAGE_SHIFT; - assert((chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty); - need_pages = (size >> PAGE_SHIFT); - assert(need_pages > 0); + if (config_stats) { + ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + + add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - + sub_pages) << LG_PAGE); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); + } +} + +static void +arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, + size_t flag_dirty, size_t need_pages) +{ + size_t total_pages, rem_pages; + + total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> + LG_PAGE; + assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == + flag_dirty); assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; - arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING((arena->nactive + need_pages) << - PAGE_SHIFT) - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif + arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); + arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; /* Keep track of trailing unused pages for later use. */ if (rem_pages > 0) { if (flag_dirty != 0) { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; + arena_mapbits_unallocated_set(chunk, + run_ind+need_pages, (rem_pages << LG_PAGE), + flag_dirty); + arena_mapbits_unallocated_set(chunk, + run_ind+total_pages-1, (rem_pages << LG_PAGE), + flag_dirty); } else { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-map_bias].bits & - CHUNK_MAP_UNZEROED); - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); + arena_mapbits_unallocated_set(chunk, run_ind+need_pages, + (rem_pages << LG_PAGE), + arena_mapbits_unzeroed_get(chunk, + run_ind+need_pages)); + arena_mapbits_unallocated_set(chunk, + run_ind+total_pages-1, (rem_pages << LG_PAGE), + arena_mapbits_unzeroed_get(chunk, + run_ind+total_pages-1)); } - arena_avail_tree_insert(runs_avail, - &chunk->map[run_ind+need_pages-map_bias]); + arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, + false, true); } +} + +static void +arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, + bool remove, bool zero) +{ + arena_chunk_t *chunk; + size_t flag_dirty, run_ind, need_pages, i; - /* Update dirty page accounting. */ - if (flag_dirty != 0) { - chunk->ndirty -= need_pages; - arena->ndirty -= need_pages; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + if (remove) { + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, + need_pages); } - /* - * Update the page map separately for large vs. small runs, since it is - * possible to avoid iteration for large mallocs. - */ - if (large) { - if (zero) { - if (flag_dirty == 0) { - /* - * The run is clean, so some pages may be - * zeroed (i.e. never before touched). - */ - for (i = 0; i < need_pages; i++) { - if ((chunk->map[run_ind+i-map_bias].bits - & CHUNK_MAP_UNZEROED) != 0) { - memset((void *)((uintptr_t) - chunk + ((run_ind+i) << - PAGE_SHIFT)), 0, - PAGE_SIZE); - } -#ifdef JEMALLOC_DEBUG - else { - arena_chunk_validate_zeroed( - chunk, run_ind+i); - } -#endif + if (zero) { + if (flag_dirty == 0) { + /* + * The run is clean, so some pages may be zeroed (i.e. + * never before touched). + */ + for (i = 0; i < need_pages; i++) { + if (arena_mapbits_unzeroed_get(chunk, run_ind+i) + != 0) + arena_run_zero(chunk, run_ind+i, 1); + else if (config_debug) { + arena_run_page_validate_zeroed(chunk, + run_ind+i); + } else { + arena_run_page_mark_zeroed(chunk, + run_ind+i); } - } else { - /* - * The run is dirty, so all pages must be - * zeroed. - */ - memset((void *)((uintptr_t)chunk + (run_ind << - PAGE_SHIFT)), 0, (need_pages << - PAGE_SHIFT)); } + } else { + /* The run is dirty, so all pages must be zeroed. */ + arena_run_zero(chunk, run_ind, need_pages); } - - /* - * Set the last element first, in case the run only contains one - * page (i.e. both statements set the same element). - */ - chunk->map[run_ind+need_pages-1-map_bias].bits = - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; - chunk->map[run_ind-map_bias].bits = size | flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; } else { - assert(zero == false); - /* - * Propagate the dirty and unzeroed flags to the allocated - * small run, so that arena_dalloc_bin_run() has the ability to - * conditionally trim clean pages. - */ - chunk->map[run_ind-map_bias].bits = - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG - /* - * The first page will always be dirtied during small run - * initialization, so a validation failure here would not - * actually cause an observable failure. - */ - if (flag_dirty == 0 && - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) - == 0) - arena_chunk_validate_zeroed(chunk, run_ind); -#endif - for (i = 1; i < need_pages - 1; i++) { - chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) - | (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) - arena_chunk_validate_zeroed(chunk, run_ind+i); -#endif - } - chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages - - 1) << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) { - arena_chunk_validate_zeroed(chunk, - run_ind+need_pages-1); - } -#endif + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } + + /* + * Set the last element first, in case the run only contains one page + * (i.e. both statements set the same element). + */ + arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); +} + +static void +arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + arena_run_split_large_helper(arena, run, size, true, zero); +} + +static void +arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + arena_run_split_large_helper(arena, run, size, false, zero); +} + +static void +arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, + size_t binind) +{ + arena_chunk_t *chunk; + size_t flag_dirty, run_ind, need_pages, i; + + assert(binind != BININD_INVALID); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); + + /* + * Propagate the dirty and unzeroed flags to the allocated small run, + * so that arena_dalloc_bin_run() has the ability to conditionally trim + * clean pages. + */ + arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); + /* + * The first page will always be dirtied during small run + * initialization, so a validation failure here would not actually + * cause an observable failure. + */ + if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, + run_ind) == 0) + arena_run_page_validate_zeroed(chunk, run_ind); + for (i = 1; i < need_pages - 1; i++) { + arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); + if (config_debug && flag_dirty == 0 && + arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) + arena_run_page_validate_zeroed(chunk, run_ind+i); + } + arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, + binind, flag_dirty); + if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, + run_ind+need_pages-1) == 0) + arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } static arena_chunk_t * -arena_chunk_alloc(arena_t *arena) +arena_chunk_init_spare(arena_t *arena) { arena_chunk_t *chunk; - size_t i; - if (arena->spare != NULL) { - arena_avail_tree_t *runs_avail; + assert(arena->spare != NULL); - chunk = arena->spare; - arena->spare = NULL; + chunk = arena->spare; + arena->spare = NULL; - /* Insert the run into the appropriate runs_avail_* tree. */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); - assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) - == arena_maxclass); - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == - (chunk->map[chunk_npages-1-map_bias].bits & - CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[0]); - } else { - bool zero; - size_t unzeroed; + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxclass); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxclass); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); - zero = false; - malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); - malloc_mutex_lock(&arena->lock); - if (chunk == NULL) - return (NULL); -#ifdef JEMALLOC_STATS + return (chunk); +} + +static arena_chunk_t * +arena_chunk_init_hard(arena_t *arena) +{ + arena_chunk_t *chunk; + bool zero; + size_t unzeroed, i; + + assert(arena->spare == NULL); + + zero = false; + malloc_mutex_unlock(&arena->lock); + chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, + &zero, arena->dss_prec); + malloc_mutex_lock(&arena->lock); + if (chunk == NULL) + return (NULL); + if (config_stats) arena->stats.mapped += chunksize; -#endif - chunk->arena = arena; - ql_elm_new(chunk, link_dirty); - chunk->dirtied = false; + chunk->arena = arena; - /* - * Claim that no pages are in use, since the header is merely - * overhead. - */ - chunk->ndirty = 0; + /* + * Claim that no pages are in use, since the header is merely overhead. + */ + chunk->ndirty = 0; - /* - * Initialize the map to contain one maximal free untouched run. - * Mark the pages as zeroed iff chunk_alloc() returned a zeroed - * chunk. - */ - unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; - chunk->map[0].bits = arena_maxclass | unzeroed; - /* - * There is no need to initialize the internal page map entries - * unless the chunk is not zeroed. - */ - if (zero == false) { - for (i = map_bias+1; i < chunk_npages-1; i++) - chunk->map[i-map_bias].bits = unzeroed; - } -#ifdef JEMALLOC_DEBUG - else { - for (i = map_bias+1; i < chunk_npages-1; i++) - assert(chunk->map[i-map_bias].bits == unzeroed); + chunk->nruns_avail = 0; + chunk->nruns_adjac = 0; + + /* + * Initialize the map to contain one maximal free untouched run. Mark + * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. + */ + unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; + arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, + unzeroed); + /* + * There is no need to initialize the internal page map entries unless + * the chunk is not zeroed. + */ + if (zero == false) { + VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, + map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); + for (i = map_bias+1; i < chunk_npages-1; i++) + arena_mapbits_unzeroed_set(chunk, i, unzeroed); + } else { + VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, + map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); + if (config_debug) { + for (i = map_bias+1; i < chunk_npages-1; i++) { + assert(arena_mapbits_unzeroed_get(chunk, i) == + unzeroed); + } } -#endif - chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | - unzeroed; + } + arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, + unzeroed); - /* Insert the run into the runs_avail_clean tree. */ - arena_avail_tree_insert(&arena->runs_avail_clean, - &chunk->map[0]); + return (chunk); +} + +static arena_chunk_t * +arena_chunk_alloc(arena_t *arena) +{ + arena_chunk_t *chunk; + + if (arena->spare != NULL) + chunk = arena_chunk_init_spare(arena); + else { + chunk = arena_chunk_init_hard(arena); + if (chunk == NULL) + return (NULL); } + /* Insert the run into the runs_avail tree. */ + arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, + false, false); + return (chunk); } static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) { - arena_avail_tree_t *runs_avail; + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxclass); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxclass); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); /* - * Remove run from the appropriate runs_avail_* tree, so that the arena - * does not use it. + * Remove run from the runs_avail tree, so that the arena does not use + * it. */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - arena_avail_tree_remove(runs_avail, &chunk->map[0]); + arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, + false, false); if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; arena->spare = chunk; - if (spare->dirtied) { - ql_remove(&chunk->arena->chunks_dirty, spare, - link_dirty); - arena->ndirty -= spare->ndirty; - } malloc_mutex_unlock(&arena->lock); chunk_dealloc((void *)spare, chunksize, true); malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.mapped -= chunksize; -#endif + if (config_stats) + arena->stats.mapped -= chunksize; } else arena->spare = chunk; } static arena_run_t * -arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) +arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { - arena_chunk_t *chunk; arena_run_t *run; arena_chunk_map_t *mapelm, key; - assert(size <= arena_maxclass); - assert((size & PAGE_MASK) == 0); - - /* Search the arena's chunks for the lowest best fit. */ key.bits = size | CHUNK_MAP_KEY; - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); size_t pageind = (((uintptr_t)mapelm - @@ -598,31 +674,35 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) + map_bias; run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + LG_PAGE)); + arena_run_split_large(arena, run, size, zero); return (run); } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + return (NULL); +} + +static arena_run_t * +arena_run_alloc_large(arena_t *arena, size_t size, bool zero) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxclass); + assert((size & PAGE_MASK) == 0); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_large_helper(arena, size, zero); + if (run != NULL) return (run); - } /* * No usable runs. Create a new chunk from which to allocate the run. */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { - run = (arena_run_t *)((uintptr_t)chunk + (map_bias << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + arena_run_split_large(arena, run, size, zero); return (run); } @@ -631,7 +711,17 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) * sufficient memory available while this one dropped arena->lock in * arena_chunk_alloc(), so search one more time. */ - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); + return (arena_run_alloc_large_helper(arena, size, zero)); +} + +static arena_run_t * +arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) +{ + arena_run_t *run; + arena_chunk_map_t *mapelm, key; + + key.bits = size | CHUNK_MAP_KEY; + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); size_t pageind = (((uintptr_t)mapelm - @@ -639,219 +729,273 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) + map_bias; run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + LG_PAGE)); + arena_run_split_small(arena, run, size, binind); return (run); } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + return (NULL); +} + +static arena_run_t * +arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxclass); + assert((size & PAGE_MASK) == 0); + assert(binind != BININD_INVALID); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_small_helper(arena, size, binind); + if (run != NULL) + return (run); + + /* + * No usable runs. Create a new chunk from which to allocate the run. + */ + chunk = arena_chunk_alloc(arena); + if (chunk != NULL) { + run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + arena_run_split_small(arena, run, size, binind); return (run); } - return (NULL); + /* + * arena_chunk_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped arena->lock in + * arena_chunk_alloc(), so search one more time. + */ + return (arena_run_alloc_small_helper(arena, size, binind)); } static inline void arena_maybe_purge(arena_t *arena) { + size_t npurgeable, threshold; + + /* Don't purge if the option is disabled. */ + if (opt_lg_dirty_mult < 0) + return; + /* Don't purge if all dirty pages are already being purged. */ + if (arena->ndirty <= arena->npurgatory) + return; + npurgeable = arena->ndirty - arena->npurgatory; + threshold = (arena->nactive >> opt_lg_dirty_mult); + /* + * Don't purge unless the number of purgeable pages exceeds the + * threshold. + */ + if (npurgeable <= threshold) + return; - /* Enforce opt_lg_dirty_mult. */ - if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && - (arena->ndirty - arena->npurgatory) > chunk_npages && - (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - - arena->npurgatory)) - arena_purge(arena, false); + arena_purge(arena, false); } -static inline void -arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) +static arena_chunk_t * +chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) { - ql_head(arena_chunk_map_t) mapelms; - arena_chunk_map_t *mapelm; - size_t pageind, flag_unzeroed; -#ifdef JEMALLOC_DEBUG - size_t ndirty; -#endif -#ifdef JEMALLOC_STATS - size_t nmadvise; -#endif + size_t *ndirty = (size_t *)arg; - ql_new(&mapelms); + assert(chunk->ndirty != 0); + *ndirty += chunk->ndirty; + return (NULL); +} - flag_unzeroed = -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - /* - * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous - * mappings, but not for file-backed mappings. - */ -# ifdef JEMALLOC_SWAP - swap_enabled ? CHUNK_MAP_UNZEROED : -# endif - 0; -#else - CHUNK_MAP_UNZEROED; -#endif +static size_t +arena_compute_npurgatory(arena_t *arena, bool all) +{ + size_t npurgatory, npurgeable; /* - * If chunk is the spare, temporarily re-allocate it, 1) so that its - * run is reinserted into runs_avail_dirty, and 2) so that it cannot be - * completely discarded by another thread while arena->lock is dropped - * by this thread. Note that the arena_run_dalloc() call will - * implicitly deallocate the chunk, so no explicit action is required - * in this function to deallocate the chunk. - * - * Note that once a chunk contains dirty pages, it cannot again contain - * a single run unless 1) it is a dirty run, or 2) this function purges - * dirty pages and causes the transition to a single clean run. Thus - * (chunk == arena->spare) is possible, but it is not possible for - * this function to be called on the spare unless it contains a dirty - * run. + * Compute the minimum number of pages that this thread should try to + * purge. */ - if (chunk == arena->spare) { - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); - arena_chunk_alloc(arena); - } + npurgeable = arena->ndirty - arena->npurgatory; - /* Temporarily allocate all free dirty runs within chunk. */ - for (pageind = map_bias; pageind < chunk_npages;) { - mapelm = &chunk->map[pageind-map_bias]; - if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { - size_t npages; + if (all == false) { + size_t threshold = (arena->nactive >> opt_lg_dirty_mult); - npages = mapelm->bits >> PAGE_SHIFT; - assert(pageind + npages <= chunk_npages); - if (mapelm->bits & CHUNK_MAP_DIRTY) { - size_t i; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif + npurgatory = npurgeable - threshold; + } else + npurgatory = npurgeable; - arena_avail_tree_remove( - &arena->runs_avail_dirty, mapelm); + return (npurgatory); +} - mapelm->bits = (npages << PAGE_SHIFT) | - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; - /* - * Update internal elements in the page map, so - * that CHUNK_MAP_UNZEROED is properly set. - */ - for (i = 1; i < npages - 1; i++) { - chunk->map[pageind+i-map_bias].bits = - flag_unzeroed; - } - if (npages > 1) { - chunk->map[ - pageind+npages-1-map_bias].bits = - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; - } +static void +arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, + arena_chunk_mapelms_t *mapelms) +{ + size_t pageind, npages; -#ifdef JEMALLOC_STATS - /* - * Update stats_cactive if nactive is crossing a - * chunk multiple. - */ - cactive_diff = CHUNK_CEILING((arena->nactive + - npages) << PAGE_SHIFT) - - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif - arena->nactive += npages; + /* + * Temporarily allocate free dirty runs within chunk. If all is false, + * only operate on dirty runs that are fragments; otherwise operate on + * all dirty runs. + */ + for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + if (arena_mapbits_allocated_get(chunk, pageind) == 0) { + size_t run_size = + arena_mapbits_unallocated_size_get(chunk, pageind); + + npages = run_size >> LG_PAGE; + assert(pageind + npages <= chunk_npages); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+npages-1)); + + if (arena_mapbits_dirty_get(chunk, pageind) != 0 && + (all || arena_avail_adjac(chunk, pageind, + npages))) { + arena_run_t *run = (arena_run_t *)((uintptr_t) + chunk + (uintptr_t)(pageind << LG_PAGE)); + + arena_run_split_large(arena, run, run_size, + false); /* Append to list for later processing. */ ql_elm_new(mapelm, u.ql_link); - ql_tail_insert(&mapelms, mapelm, u.ql_link); + ql_tail_insert(mapelms, mapelm, u.ql_link); } - - pageind += npages; } else { - /* Skip allocated run. */ - if (mapelm->bits & CHUNK_MAP_LARGE) - pageind += mapelm->bits >> PAGE_SHIFT; - else { + /* Skip run. */ + if (arena_mapbits_large_get(chunk, pageind) != 0) { + npages = arena_mapbits_large_size_get(chunk, + pageind) >> LG_PAGE; + } else { + size_t binind; + arena_bin_info_t *bin_info; arena_run_t *run = (arena_run_t *)((uintptr_t) - chunk + (uintptr_t)(pageind << PAGE_SHIFT)); - - assert((mapelm->bits >> PAGE_SHIFT) == 0); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(arena, - run->bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - pageind += bin_info->run_size >> PAGE_SHIFT; + chunk + (uintptr_t)(pageind << LG_PAGE)); + + assert(arena_mapbits_small_runind_get(chunk, + pageind) == 0); + binind = arena_bin_index(arena, run->bin); + bin_info = &arena_bin_info[binind]; + npages = bin_info->run_size >> LG_PAGE; } } } assert(pageind == chunk_npages); + assert(chunk->ndirty == 0 || all == false); + assert(chunk->nruns_adjac == 0); +} -#ifdef JEMALLOC_DEBUG - ndirty = chunk->ndirty; -#endif -#ifdef JEMALLOC_STATS - arena->stats.purged += chunk->ndirty; -#endif - arena->ndirty -= chunk->ndirty; - chunk->ndirty = 0; - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; +static size_t +arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, + arena_chunk_mapelms_t *mapelms) +{ + size_t npurged, pageind, npages, nmadvise; + arena_chunk_map_t *mapelm; malloc_mutex_unlock(&arena->lock); -#ifdef JEMALLOC_STATS - nmadvise = 0; -#endif - ql_foreach(mapelm, &mapelms, u.ql_link) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + if (config_stats) + nmadvise = 0; + npurged = 0; + ql_foreach(mapelm, mapelms, u.ql_link) { + bool unzeroed; + size_t flag_unzeroed, i; + + pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; - size_t npages = mapelm->bits >> PAGE_SHIFT; - + npages = arena_mapbits_large_size_get(chunk, pageind) >> + LG_PAGE; assert(pageind + npages <= chunk_npages); -#ifdef JEMALLOC_DEBUG - assert(ndirty >= npages); - ndirty -= npages; -#endif - -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_DONTNEED); -#elif defined(JEMALLOC_PURGE_MADVISE_FREE) - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_FREE); -#else -# error "No method defined for purging unused dirty pages." -#endif - -#ifdef JEMALLOC_STATS - nmadvise++; -#endif + unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << + LG_PAGE)), (npages << LG_PAGE)); + flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; + /* + * Set the unzeroed flag for all pages, now that pages_purge() + * has returned whether the pages were zeroed as a side effect + * of purging. This chunk map modification is safe even though + * the arena mutex isn't currently owned by this thread, + * because the run is marked as allocated, thus protecting it + * from being modified by any other thread. As long as these + * writes don't perturb the first and last elements' + * CHUNK_MAP_ALLOCATED bits, behavior is well defined. + */ + for (i = 0; i < npages; i++) { + arena_mapbits_unzeroed_set(chunk, pageind+i, + flag_unzeroed); + } + npurged += npages; + if (config_stats) + nmadvise++; } -#ifdef JEMALLOC_DEBUG - assert(ndirty == 0); -#endif malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.nmadvise += nmadvise; -#endif + if (config_stats) + arena->stats.nmadvise += nmadvise; + + return (npurged); +} + +static void +arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, + arena_chunk_mapelms_t *mapelms) +{ + arena_chunk_map_t *mapelm; + size_t pageind; /* Deallocate runs. */ - for (mapelm = ql_first(&mapelms); mapelm != NULL; - mapelm = ql_first(&mapelms)) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + for (mapelm = ql_first(mapelms); mapelm != NULL; + mapelm = ql_first(mapelms)) { + arena_run_t *run; + + pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << PAGE_SHIFT)); + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << + LG_PAGE)); + ql_remove(mapelms, mapelm, u.ql_link); + arena_run_dalloc(arena, run, false, true); + } +} + +static inline size_t +arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) +{ + size_t npurged; + arena_chunk_mapelms_t mapelms; + + ql_new(&mapelms); + + /* + * If chunk is the spare, temporarily re-allocate it, 1) so that its + * run is reinserted into runs_avail, and 2) so that it cannot be + * completely discarded by another thread while arena->lock is dropped + * by this thread. Note that the arena_run_dalloc() call will + * implicitly deallocate the chunk, so no explicit action is required + * in this function to deallocate the chunk. + * + * Note that once a chunk contains dirty pages, it cannot again contain + * a single run unless 1) it is a dirty run, or 2) this function purges + * dirty pages and causes the transition to a single clean run. Thus + * (chunk == arena->spare) is possible, but it is not possible for + * this function to be called on the spare unless it contains a dirty + * run. + */ + if (chunk == arena->spare) { + assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); + assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); - ql_remove(&mapelms, mapelm, u.ql_link); - arena_run_dalloc(arena, run, false); + arena_chunk_alloc(arena); } + + if (config_stats) + arena->stats.purged += chunk->ndirty; + + /* + * Operate on all dirty runs if there is no clean/dirty run + * fragmentation. + */ + if (chunk->nruns_adjac == 0) + all = true; + + arena_chunk_stash_dirty(arena, chunk, all, &mapelms); + npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); + arena_chunk_unstash_purged(arena, chunk, &mapelms); + + return (npurged); } static void @@ -859,39 +1003,33 @@ arena_purge(arena_t *arena, bool all) { arena_chunk_t *chunk; size_t npurgatory; -#ifdef JEMALLOC_DEBUG - size_t ndirty = 0; + if (config_debug) { + size_t ndirty = 0; - ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { - assert(chunk->dirtied); - ndirty += chunk->ndirty; + arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, + chunks_dirty_iter_cb, (void *)&ndirty); + assert(ndirty == arena->ndirty); } - assert(ndirty == arena->ndirty); -#endif assert(arena->ndirty > arena->npurgatory || all); - assert(arena->ndirty - arena->npurgatory > chunk_npages || all); assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - arena->npurgatory) || all); -#ifdef JEMALLOC_STATS - arena->stats.npurge++; -#endif + if (config_stats) + arena->stats.npurge++; /* - * Compute the minimum number of pages that this thread should try to - * purge, and add the result to arena->npurgatory. This will keep - * multiple threads from racing to reduce ndirty below the threshold. + * Add the minimum number of pages this thread should try to purge to + * arena->npurgatory. This will keep multiple threads from racing to + * reduce ndirty below the threshold. */ - npurgatory = arena->ndirty - arena->npurgatory; - if (all == false) { - assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); - npurgatory -= arena->nactive >> opt_lg_dirty_mult; - } + npurgatory = arena_compute_npurgatory(arena, all); arena->npurgatory += npurgatory; while (npurgatory > 0) { + size_t npurgeable, npurged, nunpurged; + /* Get next chunk with dirty pages. */ - chunk = ql_first(&arena->chunks_dirty); + chunk = arena_chunk_dirty_first(&arena->chunks_dirty); if (chunk == NULL) { /* * This thread was unable to purge as many pages as @@ -902,23 +1040,15 @@ arena_purge(arena_t *arena, bool all) arena->npurgatory -= npurgatory; return; } - while (chunk->ndirty == 0) { - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; - chunk = ql_first(&arena->chunks_dirty); - if (chunk == NULL) { - /* Same logic as for above. */ - arena->npurgatory -= npurgatory; - return; - } - } + npurgeable = chunk->ndirty; + assert(npurgeable != 0); - if (chunk->ndirty > npurgatory) { + if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { /* - * This thread will, at a minimum, purge all the dirty - * pages in chunk, so set npurgatory to reflect this - * thread's commitment to purge the pages. This tends - * to reduce the chances of the following scenario: + * This thread will purge all the dirty pages in chunk, + * so set npurgatory to reflect this thread's intent to + * purge the pages. This tends to reduce the chances + * of the following scenario: * * 1) This thread sets arena->npurgatory such that * (arena->ndirty - arena->npurgatory) is at the @@ -932,13 +1062,20 @@ arena_purge(arena_t *arena, bool all) * because all of the purging work being done really * needs to happen. */ - arena->npurgatory += chunk->ndirty - npurgatory; - npurgatory = chunk->ndirty; + arena->npurgatory += npurgeable - npurgatory; + npurgatory = npurgeable; } - arena->npurgatory -= chunk->ndirty; - npurgatory -= chunk->ndirty; - arena_chunk_purge(arena, chunk); + /* + * Keep track of how many pages are purgeable, versus how many + * actually get purged, and adjust counters accordingly. + */ + arena->npurgatory -= npurgeable; + npurgatory -= npurgeable; + npurged = arena_chunk_purge(arena, chunk, all); + nunpurged = npurgeable - npurged; + arena->npurgatory += nunpurged; + npurgatory += nunpurged; } } @@ -952,109 +1089,47 @@ arena_purge_all(arena_t *arena) } static void -arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) +arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, + size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) { - arena_chunk_t *chunk; - size_t size, run_ind, run_pages, flag_dirty; - arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); - assert(run_ind >= map_bias); - assert(run_ind < chunk_npages); - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { - size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; - assert(size == PAGE_SIZE || - (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); - } else { - size_t binind = arena_bin_index(arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - size = bin_info->run_size; - } - run_pages = (size >> PAGE_SHIFT); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING(arena->nactive << PAGE_SHIFT) - - CHUNK_CEILING((arena->nactive - run_pages) << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_sub(cactive_diff); -#endif - arena->nactive -= run_pages; - - /* - * The run is dirty if the caller claims to have dirtied it, as well as - * if it was already dirty before being allocated. - */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) - dirty = true; - flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; - runs_avail = dirty ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; - - /* Mark pages as unallocated in the chunk map. */ - if (dirty) { - chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - CHUNK_MAP_DIRTY; - - chunk->ndirty += run_pages; - arena->ndirty += run_pages; - } else { - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); - } + size_t size = *p_size; + size_t run_ind = *p_run_ind; + size_t run_pages = *p_run_pages; /* Try to coalesce forward. */ if (run_ind + run_pages < chunk_npages && - (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & - ~PAGE_MASK; - size_t nrun_pages = nrun_size >> PAGE_SHIFT; + arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && + arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { + size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages); + size_t nrun_pages = nrun_size >> LG_PAGE; /* * Remove successor from runs_avail; the coalesced run is * inserted later. */ - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & ~PAGE_MASK) == nrun_size); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_DIRTY) == flag_dirty); - arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind+run_pages-map_bias]); + assert(arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages+nrun_pages-1) == nrun_size); + assert(arena_mapbits_dirty_get(chunk, + run_ind+run_pages+nrun_pages-1) == flag_dirty); + arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, + false, true); size += nrun_size; run_pages += nrun_pages; - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); } /* Try to coalesce backward. */ - if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t prun_size = chunk->map[run_ind-1-map_bias].bits & - ~PAGE_MASK; - size_t prun_pages = prun_size >> PAGE_SHIFT; + if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, + run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == + flag_dirty) { + size_t prun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind-1); + size_t prun_pages = prun_size >> LG_PAGE; run_ind -= prun_pages; @@ -1062,52 +1137,89 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) * Remove predecessor from runs_avail; the coalesced run is * inserted later. */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) - == prun_size); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) - == flag_dirty); - arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind-map_bias]); + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + prun_size); + assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); + arena_avail_remove(arena, chunk, run_ind, prun_pages, true, + false); size += prun_size; run_pages += prun_pages; - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); } - /* Insert into runs_avail, now that coalescing is complete. */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); + *p_size = size; + *p_run_ind = run_ind; + *p_run_pages = run_pages; +} - if (dirty) { - /* - * Insert into chunks_dirty before potentially calling - * arena_chunk_dealloc(), so that chunks_dirty and - * arena->ndirty are consistent. - */ - if (chunk->dirtied == false) { - ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = true; - } +static void +arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) +{ + arena_chunk_t *chunk; + size_t size, run_ind, run_pages, flag_dirty; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + assert(run_ind >= map_bias); + assert(run_ind < chunk_npages); + if (arena_mapbits_large_get(chunk, run_ind) != 0) { + size = arena_mapbits_large_size_get(chunk, run_ind); + assert(size == PAGE || + arena_mapbits_large_size_get(chunk, + run_ind+(size>>LG_PAGE)-1) == 0); + } else { + size_t binind = arena_bin_index(arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + size = bin_info->run_size; } + run_pages = (size >> LG_PAGE); + arena_cactive_update(arena, 0, run_pages); + arena->nactive -= run_pages; /* - * Deallocate chunk if it is now completely unused. The bit - * manipulation checks whether the first run is unallocated and extends - * to the end of the chunk. + * The run is dirty if the caller claims to have dirtied it, as well as + * if it was already dirty before being allocated and the caller + * doesn't claim to have cleaned it. */ - if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == - arena_maxclass) + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) + dirty = true; + flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; + + /* Mark pages as unallocated in the chunk map. */ + if (dirty) { + arena_mapbits_unallocated_set(chunk, run_ind, size, + CHUNK_MAP_DIRTY); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + CHUNK_MAP_DIRTY); + } else { + arena_mapbits_unallocated_set(chunk, run_ind, size, + arena_mapbits_unzeroed_get(chunk, run_ind)); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); + } + + arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, + flag_dirty); + + /* Insert into runs_avail, now that coalescing is complete. */ + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); + + /* Deallocate chunk if it is now completely unused. */ + if (size == arena_maxclass) { + assert(run_ind == map_bias); + assert(run_pages == (arena_maxclass >> LG_PAGE)); arena_chunk_dealloc(arena, chunk); + } /* * It is okay to do dirty page processing here even if the chunk was @@ -1124,9 +1236,9 @@ static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + size_t head_npages = (oldsize - newsize) >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); @@ -1135,44 +1247,30 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * leading run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = (oldsize - newsize) - | flag_dirty | (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_DEBUG - { - size_t tail_npages = newsize >> PAGE_SHIFT; - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_DIRTY) == flag_dirty); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_ALLOCATED) != 0); + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); + + if (config_debug) { + UNUSED size_t tail_npages = newsize >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); } -#endif - chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + arena_mapbits_large_set(chunk, pageind+head_npages, newsize, + flag_dirty); - arena_run_dalloc(arena, run, false); + arena_run_dalloc(arena, run, false, false); } static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = newsize >> PAGE_SHIFT; - size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY; + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + size_t head_npages = newsize >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); @@ -1181,61 +1279,92 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * trailing run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = - flag_dirty | - (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | - flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); + + if (config_debug) { + UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); + } + arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, + flag_dirty); arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), - dirty); + dirty, false); } static arena_run_t * -arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) +arena_bin_runs_first(arena_bin_t *bin) { - arena_chunk_map_t *mapelm; - arena_run_t *run; - size_t binind; - arena_bin_info_t *bin_info; - - /* Look for a usable run. */ - mapelm = arena_run_tree_first(&bin->runs); + arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); if (mapelm != NULL) { arena_chunk_t *chunk; size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); + arena_run_t *run; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t))) + map_bias; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif + arena_mapbits_small_runind_get(chunk, pageind)) << + LG_PAGE)); return (run); } + + return (NULL); +} + +static void +arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) +{ + arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + + assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); + + arena_run_tree_insert(&bin->runs, mapelm); +} + +static void +arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + + assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); + + arena_run_tree_remove(&bin->runs, mapelm); +} + +static arena_run_t * +arena_bin_nonfull_run_tryget(arena_bin_t *bin) +{ + arena_run_t *run = arena_bin_runs_first(bin); + if (run != NULL) { + arena_bin_runs_remove(bin, run); + if (config_stats) + bin->stats.reruns++; + } + return (run); +} + +static arena_run_t * +arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) +{ + arena_run_t *run; + size_t binind; + arena_bin_info_t *bin_info; + + /* Look for a usable run. */ + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) + return (run); /* No existing runs have any space available. */ binind = arena_bin_index(arena, bin); @@ -1245,7 +1374,7 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) malloc_mutex_unlock(&bin->lock); /******************************/ malloc_mutex_lock(&arena->lock); - run = arena_run_alloc(arena, bin_info->run_size, false, false); + run = arena_run_alloc_small(arena, bin_info->run_size, binind); if (run != NULL) { bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); @@ -1255,47 +1384,26 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) run->nextind = 0; run->nfree = bin_info->nregs; bitmap_init(bitmap, &bin_info->bitmap_info); -#ifdef JEMALLOC_DEBUG - run->magic = ARENA_RUN_MAGIC; -#endif } malloc_mutex_unlock(&arena->lock); /********************************/ malloc_mutex_lock(&bin->lock); if (run != NULL) { -#ifdef JEMALLOC_STATS - bin->stats.nruns++; - bin->stats.curruns++; - if (bin->stats.curruns > bin->stats.highruns) - bin->stats.highruns = bin->stats.curruns; -#endif + if (config_stats) { + bin->stats.nruns++; + bin->stats.curruns++; + } return (run); } /* - * arena_run_alloc() failed, but another thread may have made + * arena_run_alloc_small() failed, but another thread may have made * sufficient memory available while this one dropped bin->lock above, * so search one more time. */ - mapelm = arena_run_tree_first(&bin->runs); - if (mapelm != NULL) { - arena_chunk_t *chunk; - size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t))) + map_bias; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) return (run); - } return (NULL); } @@ -1318,19 +1426,18 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) * Another thread updated runcur while this one ran without the * bin lock in arena_bin_nonfull_run_get(). */ - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); assert(bin->runcur->nfree > 0); ret = arena_run_reg_alloc(bin->runcur, bin_info); if (run != NULL) { arena_chunk_t *chunk; /* - * arena_run_alloc() may have allocated run, or it may - * have pulled run from the bin's run tree. Therefore - * it is unsafe to make any assumptions about how run - * has previously been used, and arena_bin_lower_run() - * must be called, as if a region were just deallocated - * from the run. + * arena_run_alloc_small() may have allocated run, or + * it may have pulled run from the bin's run tree. + * Therefore it is unsafe to make any assumptions about + * how run has previously been used, and + * arena_bin_lower_run() must be called, as if a region + * were just deallocated from the run. */ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); if (run->nfree == bin_info->nregs) @@ -1346,34 +1453,14 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) bin->runcur = run; - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); assert(bin->runcur->nfree > 0); return (arena_run_reg_alloc(bin->runcur, bin_info)); } -#ifdef JEMALLOC_PROF -void -arena_prof_accum(arena_t *arena, uint64_t accumbytes) -{ - - if (prof_interval != 0) { - arena->prof_accumbytes += accumbytes; - if (arena->prof_accumbytes >= prof_interval) { - prof_idump(); - arena->prof_accumbytes -= prof_interval; - } - } -} -#endif - -#ifdef JEMALLOC_TCACHE void -arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ) +arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, + uint64_t prof_accumbytes) { unsigned i, nfill; arena_bin_t *bin; @@ -1382,11 +1469,8 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind assert(tbin->ncached == 0); -#ifdef JEMALLOC_PROF - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, prof_accumbytes); - malloc_mutex_unlock(&arena->lock); -#endif + if (config_prof && arena_prof_accum(arena, prof_accumbytes)) + prof_idump(); bin = &arena->bins[binind]; malloc_mutex_lock(&bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> @@ -1397,21 +1481,125 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind ptr = arena_bin_malloc_hard(arena, bin); if (ptr == NULL) break; + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, &arena_bin_info[binind], + true); + } /* Insert such that low regions get used first. */ tbin->avail[nfill - 1 - i] = ptr; } -#ifdef JEMALLOC_STATS - bin->stats.allocated += i * arena_bin_info[binind].reg_size; - bin->stats.nmalloc += i; - bin->stats.nrequests += tbin->tstats.nrequests; - bin->stats.nfills++; - tbin->tstats.nrequests = 0; -#endif + if (config_stats) { + bin->stats.allocated += i * arena_bin_info[binind].reg_size; + bin->stats.nmalloc += i; + bin->stats.nrequests += tbin->tstats.nrequests; + bin->stats.nfills++; + tbin->tstats.nrequests = 0; + } malloc_mutex_unlock(&bin->lock); tbin->ncached = i; } + +void +arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) +{ + + if (zero) { + size_t redzone_size = bin_info->redzone_size; + memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, + redzone_size); + memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, + redzone_size); + } else { + memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, + bin_info->reg_interval); + } +} + +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) +#endif +static void +arena_redzone_corruption(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + malloc_printf(": Corrupt redzone %zu byte%s %s %p " + "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", + after ? "after" : "before", ptr, usize, byte); +} +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) +arena_redzone_corruption_t *arena_redzone_corruption = + JEMALLOC_N(arena_redzone_corruption_impl); +#endif + +static void +arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) +{ + size_t size = bin_info->reg_size; + size_t redzone_size = bin_info->redzone_size; + size_t i; + bool error = false; + + for (i = 1; i <= redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); + if (*byte != 0xa5) { + error = true; + arena_redzone_corruption(ptr, size, false, i, *byte); + if (reset) + *byte = 0xa5; + } + } + for (i = 0; i < redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); + if (*byte != 0xa5) { + error = true; + arena_redzone_corruption(ptr, size, true, i, *byte); + if (reset) + *byte = 0xa5; + } + } + if (opt_abort && error) + abort(); +} + +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) +#endif +void +arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +{ + size_t redzone_size = bin_info->redzone_size; + + arena_redzones_validate(ptr, bin_info, false); + memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, + bin_info->reg_interval); +} +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) +arena_dalloc_junk_small_t *arena_dalloc_junk_small = + JEMALLOC_N(arena_dalloc_junk_small_impl); #endif +void +arena_quarantine_junk_small(void *ptr, size_t usize) +{ + size_t binind; + arena_bin_info_t *bin_info; + cassert(config_fill); + assert(opt_junk); + assert(opt_quarantine); + assert(usize <= SMALL_MAXCLASS); + + binind = SMALL_SIZE2BIN(usize); + bin_info = &arena_bin_info[binind]; + arena_redzones_validate(ptr, bin_info, true); +} + void * arena_malloc_small(arena_t *arena, size_t size, bool zero) { @@ -1421,7 +1609,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) size_t binind; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); + assert(binind < NBINS); bin = &arena->bins[binind]; size = arena_bin_info[binind].reg_size; @@ -1436,29 +1624,32 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) return (NULL); } -#ifdef JEMALLOC_STATS - bin->stats.allocated += size; - bin->stats.nmalloc++; - bin->stats.nrequests++; -#endif - malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_PROF - if (isthreaded == false) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, size); - malloc_mutex_unlock(&arena->lock); + if (config_stats) { + bin->stats.allocated += size; + bin->stats.nmalloc++; + bin->stats.nrequests++; } -#endif + malloc_mutex_unlock(&bin->lock); + if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) + prof_idump(); if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) + memset(ret, 0, size); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } return (ret); } @@ -1467,247 +1658,120 @@ void * arena_malloc_large(arena_t *arena, size_t size, bool zero) { void *ret; + UNUSED bool idump; /* Large allocation. */ size = PAGE_CEILING(size); malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, size, true, zero); + ret = (void *)arena_run_alloc_large(arena, size, zero); if (ret == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, size); -#endif + if (config_prof) + idump = arena_prof_accum_locked(arena, size); malloc_mutex_unlock(&arena->lock); + if (config_prof && idump) + prof_idump(); if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif + if (config_fill) { + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); + } } return (ret); } -void * -arena_malloc(size_t size, bool zero) -{ - - assert(size != 0); - assert(QUANTUM_CEILING(size) <= arena_maxclass); - - if (size <= small_maxclass) { -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_small(tcache, size, zero)); - else - -#endif - return (arena_malloc_small(choose_arena(), size, zero)); - } else { -#ifdef JEMALLOC_TCACHE - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_large(tcache, size, zero)); - else { - return (arena_malloc_large(choose_arena(), - size, zero)); - } - } else -#endif - return (arena_malloc_large(choose_arena(), size, zero)); - } -} - /* Only handles large allocations that require more than page alignment. */ void * -arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, - bool zero) +arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) { void *ret; - size_t offset; + size_t alloc_size, leadsize, trailsize; + arena_run_t *run; arena_chunk_t *chunk; assert((size & PAGE_MASK) == 0); alignment = PAGE_CEILING(alignment); + alloc_size = size + alignment - PAGE; malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); - if (ret == NULL) { + run = arena_run_alloc_large(arena, alloc_size, false); + if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & PAGE_MASK) == 0); - assert(offset < alloc_size); - if (offset == 0) - arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); - else { - size_t leadsize, trailsize; - - leadsize = alignment - offset; - if (leadsize > 0) { - arena_run_trim_head(arena, chunk, ret, alloc_size, - alloc_size - leadsize); - ret = (void *)((uintptr_t)ret + leadsize); - } - - trailsize = alloc_size - leadsize - size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - arena_run_trim_tail(arena, chunk, ret, size + trailsize, - size, false); - } + leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - + (uintptr_t)run; + assert(alloc_size >= leadsize + size); + trailsize = alloc_size - leadsize - size; + ret = (void *)((uintptr_t)run + leadsize); + if (leadsize != 0) { + arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - + leadsize); + } + if (trailsize != 0) { + arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, + false); } + arena_run_init_large(arena, (arena_run_t *)ret, size, zero); -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); -#ifdef JEMALLOC_FILL - if (zero == false) { + if (config_fill && zero == false) { if (opt_junk) memset(ret, 0xa5, size); else if (opt_zero) memset(ret, 0, size); } -#endif - return (ret); -} - -/* Return the size of the allocation pointed to by ptr. */ -size_t -arena_salloc(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - assert(ret != 0); - } - return (ret); } -#ifdef JEMALLOC_PROF void arena_prof_promoted(const void *ptr, size_t size) { arena_chunk_t *chunk; size_t pageind, binind; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - assert(isalloc(ptr) == PAGE_SIZE); - assert(size <= small_maxclass); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == PAGE); + assert(size <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); - chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & - ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); -} - -size_t -arena_salloc_demote(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - if (prof_promote && ret == PAGE_SIZE && (mapbits & - CHUNK_MAP_CLASS_MASK) != 0) { - size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> - CHUNK_MAP_CLASS_SHIFT) - 1; - assert(binind < nbins); - ret = arena_bin_info[binind].reg_size; - } - assert(ret != 0); - } + assert(binind < NBINS); + arena_mapbits_large_binind_set(chunk, pageind, binind); - return (ret); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == size); } -#endif static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, @@ -1722,16 +1786,12 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; /* * This block's conditional is necessary because if the * run only contains one region, then it never gets * inserted into the non-full runs tree. */ - arena_run_tree_remove(&bin->runs, run_mapelm); + arena_bin_runs_remove(bin, run); } } } @@ -1745,19 +1805,21 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t npages, run_ind, past; assert(run != bin->runcur); - assert(arena_run_tree_search(&bin->runs, &chunk->map[ - (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); + assert(arena_run_tree_search(&bin->runs, + arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) + == NULL); binind = arena_bin_index(chunk->arena, run->bin); bin_info = &arena_bin_info[binind]; malloc_mutex_unlock(&bin->lock); /******************************/ - npages = bin_info->run_size >> PAGE_SHIFT; - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); + npages = bin_info->run_size >> LG_PAGE; + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); past = (size_t)(PAGE_CEILING((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * - bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); + bin_info->reg_interval - bin_info->redzone_size) - + (uintptr_t)chunk) >> LG_PAGE); malloc_mutex_lock(&arena->lock); /* @@ -1765,32 +1827,24 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * trim the clean pages before deallocating the dirty portion of the * run. */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past - - run_ind < npages) { - /* - * Trim clean pages. Convert to large run beforehand. Set the - * last map element first, in case this is a one-page run. - */ - chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | - (chunk->map[run_ind+npages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind-map_bias].bits = bin_info->run_size | - CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), - ((past - run_ind) << PAGE_SHIFT), false); + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+npages-1)); + if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < + npages) { + /* Trim clean pages. Convert to large run beforehand. */ + assert(npages > 0); + arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); + arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); + arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), + ((past - run_ind) << LG_PAGE), false); /* npages = past - run_ind; */ } -#ifdef JEMALLOC_DEBUG - run->magic = 0; -#endif - arena_run_dalloc(arena, run, true); + arena_run_dalloc(arena, run, true, false); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - bin->stats.curruns--; -#endif + if (config_stats) + bin->stats.curruns--; } static void @@ -1799,62 +1853,42 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, { /* - * Make sure that bin->runcur always refers to the lowest non-full run, - * if one exists. + * Make sure that if bin->runcur is non-NULL, it refers to the lowest + * non-full run. It is okay to NULL runcur out rather than proactively + * keeping it pointing at the lowest non-full run. */ - if (bin->runcur == NULL) - bin->runcur = run; - else if ((uintptr_t)run < (uintptr_t)bin->runcur) { + if ((uintptr_t)run < (uintptr_t)bin->runcur) { /* Switch runcur. */ - if (bin->runcur->nfree > 0) { - arena_chunk_t *runcur_chunk = - CHUNK_ADDR2BASE(bin->runcur); - size_t runcur_pageind = (((uintptr_t)bin->runcur - - (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *runcur_mapelm = - &runcur_chunk->map[runcur_pageind-map_bias]; - - /* Insert runcur. */ - arena_run_tree_insert(&bin->runs, runcur_mapelm); - } + if (bin->runcur->nfree > 0) + arena_bin_runs_insert(bin, bin->runcur); bin->runcur = run; - } else { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; - - assert(arena_run_tree_search(&bin->runs, run_mapelm) == NULL); - arena_run_tree_insert(&bin->runs, run_mapelm); - } + if (config_stats) + bin->stats.reruns++; + } else + arena_bin_runs_insert(bin, run); } void -arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, +arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm) { size_t pageind; arena_run_t *run; arena_bin_t *bin; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t size; -#endif + arena_bin_info_t *bin_info; + size_t size, binind; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); + arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size = bin_info->reg_size; -#endif + binind = arena_ptr_small_binind_get(ptr, mapelm->bits); + bin_info = &arena_bin_info[binind]; + if (config_fill || config_stats) + size = bin_info->reg_size; -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, size); -#endif + if (config_fill && opt_junk) + arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); if (run->nfree == bin_info->nregs) { @@ -1863,95 +1897,87 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, } else if (run->nfree == 1 && run != bin->runcur) arena_bin_lower_run(arena, chunk, run, bin); -#ifdef JEMALLOC_STATS - bin->stats.allocated -= size; - bin->stats.ndalloc++; -#endif + if (config_stats) { + bin->stats.allocated -= size; + bin->stats.ndalloc++; + } } -#ifdef JEMALLOC_STATS void -arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats) +arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind, arena_chunk_map_t *mapelm) { - unsigned i; + arena_run_t *run; + arena_bin_t *bin; - malloc_mutex_lock(&arena->lock); - *nactive += arena->nactive; - *ndirty += arena->ndirty; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); + bin = run->bin; + malloc_mutex_lock(&bin->lock); + arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); + malloc_mutex_unlock(&bin->lock); +} - astats->mapped += arena->stats.mapped; - astats->npurge += arena->stats.npurge; - astats->nmadvise += arena->stats.nmadvise; - astats->purged += arena->stats.purged; - astats->allocated_large += arena->stats.allocated_large; - astats->nmalloc_large += arena->stats.nmalloc_large; - astats->ndalloc_large += arena->stats.ndalloc_large; - astats->nrequests_large += arena->stats.nrequests_large; +void +arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind) +{ + arena_chunk_map_t *mapelm; - for (i = 0; i < nlclasses; i++) { - lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; - lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; - lstats[i].nrequests += arena->stats.lstats[i].nrequests; - lstats[i].highruns += arena->stats.lstats[i].highruns; - lstats[i].curruns += arena->stats.lstats[i].curruns; + if (config_debug) { + /* arena_ptr_small_binind_get() does extra sanity checking. */ + assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)) != BININD_INVALID); } - malloc_mutex_unlock(&arena->lock); - - for (i = 0; i < nbins; i++) { - arena_bin_t *bin = &arena->bins[i]; + mapelm = arena_mapp_get(chunk, pageind); + arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); +} - malloc_mutex_lock(&bin->lock); - bstats[i].allocated += bin->stats.allocated; - bstats[i].nmalloc += bin->stats.nmalloc; - bstats[i].ndalloc += bin->stats.ndalloc; - bstats[i].nrequests += bin->stats.nrequests; -#ifdef JEMALLOC_TCACHE - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) #endif - bstats[i].nruns += bin->stats.nruns; - bstats[i].reruns += bin->stats.reruns; - bstats[i].highruns += bin->stats.highruns; - bstats[i].curruns += bin->stats.curruns; - malloc_mutex_unlock(&bin->lock); - } +static void +arena_dalloc_junk_large(void *ptr, size_t usize) +{ + + if (config_fill && opt_junk) + memset(ptr, 0x5a, usize); } +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) +arena_dalloc_junk_large_t *arena_dalloc_junk_large = + JEMALLOC_N(arena_dalloc_junk_large_impl); #endif void -arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) +arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) { - /* Large allocation. */ -#ifdef JEMALLOC_FILL -# ifndef JEMALLOC_STATS - if (opt_junk) -# endif -#endif - { -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> - PAGE_SHIFT; - size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; -#endif + if (config_fill || config_stats) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t usize = arena_mapbits_large_size_get(chunk, pageind); -#ifdef JEMALLOC_FILL -# ifdef JEMALLOC_STATS - if (opt_junk) -# endif - memset(ptr, 0x5a, size); -#endif -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; -#endif + arena_dalloc_junk_large(ptr, usize); + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= usize; + arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; + } } - arena_run_dalloc(arena, (arena_run_t *)ptr, true); + arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); +} + +void +arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) +{ + + malloc_mutex_lock(&arena->lock); + arena_dalloc_large_locked(arena, chunk, ptr); + malloc_mutex_unlock(&arena->lock); } static void @@ -1968,24 +1994,19 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, malloc_mutex_lock(&arena->lock); arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, true); -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); } @@ -1993,20 +2014,19 @@ static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t npages = oldsize >> PAGE_SHIFT; + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t npages = oldsize >> LG_PAGE; size_t followsize; - assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); + assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); /* Try to extend the run. */ assert(size + extra > oldsize); malloc_mutex_lock(&arena->lock); if (pageind + npages < chunk_npages && - (chunk->map[pageind+npages-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0 && (followsize = - chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - - oldsize) { + arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && + (followsize = arena_mapbits_unallocated_size_get(chunk, + pageind+npages)) >= size - oldsize) { /* * The next run is available and sufficiently large. Split the * following run, then merge the first part with the existing @@ -2015,11 +2035,11 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t flag_dirty; size_t splitsize = (oldsize + followsize <= size + extra) ? followsize : size + extra - oldsize; - arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + - ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); + arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + + ((pageind+npages) << LG_PAGE)), splitsize, zero); size = oldsize + splitsize; - npages = size >> PAGE_SHIFT; + npages = size >> LG_PAGE; /* * Mark the extended run as dirty if either portion of the run @@ -2029,34 +2049,24 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, * arena_run_dalloc() with the dirty argument set to false * (which is when dirty flag consistency would really matter). */ - flag_dirty = (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY) | - (chunk->map[pageind+npages-1-map_bias].bits & - CHUNK_MAP_DIRTY); - chunk->map[pageind-map_bias].bits = size | flag_dirty - | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - - 1].curruns; + flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | + arena_mapbits_dirty_get(chunk, pageind+npages-1); + arena_mapbits_large_set(chunk, pageind, size, flag_dirty); + arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); + + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); return (false); } @@ -2065,6 +2075,26 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, return (true); } +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) +#endif +static void +arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) +{ + + if (config_fill && opt_junk) { + memset((void *)((uintptr_t)ptr + usize), 0x5a, + old_usize - usize); + } +} +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) +arena_ralloc_junk_large_t *arena_ralloc_junk_large = + JEMALLOC_N(arena_ralloc_junk_large_impl); +#endif + /* * Try to resize a large allocation, in order to avoid copying. This will * always fail if growing an object, and the following run is already in use. @@ -2078,12 +2108,6 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, psize = PAGE_CEILING(size + extra); if (psize == oldsize) { /* Same size class. */ -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - - size); - } -#endif return (false); } else { arena_chunk_t *chunk; @@ -2091,16 +2115,10 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = chunk->arena; - dassert(arena->magic == ARENA_MAGIC); if (psize < oldsize) { -#ifdef JEMALLOC_FILL /* Fill before shrinking in order avoid a race. */ - if (opt_junk) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } -#endif + arena_ralloc_junk_large(ptr, oldsize, psize); arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, psize); return (false); @@ -2108,18 +2126,23 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); -#ifdef JEMALLOC_FILL - if (ret == false && zero == false && opt_zero) { - memset((void *)((uintptr_t)ptr + oldsize), 0, - size - oldsize); + if (config_fill && ret == false && zero == false) { + if (opt_junk) { + memset((void *)((uintptr_t)ptr + + oldsize), 0xa5, isalloc(ptr, + config_prof) - oldsize); + } else if (opt_zero) { + memset((void *)((uintptr_t)ptr + + oldsize), 0, isalloc(ptr, + config_prof) - oldsize); + } } -#endif return (ret); } } } -void * +bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { @@ -2128,46 +2151,39 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, * Avoid moving the allocation if the size class can be left the same. */ if (oldsize <= arena_maxclass) { - if (oldsize <= small_maxclass) { + if (oldsize <= SMALL_MAXCLASS) { assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size == oldsize); - if ((size + extra <= small_maxclass && + if ((size + extra <= SMALL_MAXCLASS && SMALL_SIZE2BIN(size + extra) == SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && - size + extra >= oldsize)) { -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), - 0x5a, oldsize - size); - } -#endif - return (ptr); - } + size + extra >= oldsize)) + return (false); } else { assert(size <= arena_maxclass); - if (size + extra > small_maxclass) { + if (size + extra > SMALL_MAXCLASS) { if (arena_ralloc_large(ptr, oldsize, size, extra, zero) == false) - return (ptr); + return (false); } } } /* Reallocation would require a move. */ - return (NULL); + return (true); } void * -arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) +arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc) { void *ret; size_t copysize; /* Try to avoid moving the allocation. */ - ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); - if (ret != NULL) - return (ret); + if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) + return (ptr); /* * size and oldsize are different enough that we need to move the @@ -2175,24 +2191,25 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * copying. */ if (alignment != 0) { - size_t usize = sa2u(size + extra, alignment, NULL); + size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); } else - ret = arena_malloc(size + extra, zero); + ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment != 0) { - size_t usize = sa2u(size, alignment, NULL); + size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipalloct(usize, alignment, zero, try_tcache_alloc, + arena); } else - ret = arena_malloc(size, zero); + ret = arena_malloc(arena, size, zero, try_tcache_alloc); if (ret == NULL) return (NULL); @@ -2205,11 +2222,80 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * expectation that the extra bytes will be reliably preserved. */ copysize = (size < oldsize) ? size : oldsize; + VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloct(ptr, try_tcache_dalloc); + return (ret); +} + +dss_prec_t +arena_dss_prec_get(arena_t *arena) +{ + dss_prec_t ret; + + malloc_mutex_lock(&arena->lock); + ret = arena->dss_prec; + malloc_mutex_unlock(&arena->lock); return (ret); } +void +arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) +{ + + malloc_mutex_lock(&arena->lock); + arena->dss_prec = dss_prec; + malloc_mutex_unlock(&arena->lock); +} + +void +arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, + size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats) +{ + unsigned i; + + malloc_mutex_lock(&arena->lock); + *dss = dss_prec_names[arena->dss_prec]; + *nactive += arena->nactive; + *ndirty += arena->ndirty; + + astats->mapped += arena->stats.mapped; + astats->npurge += arena->stats.npurge; + astats->nmadvise += arena->stats.nmadvise; + astats->purged += arena->stats.purged; + astats->allocated_large += arena->stats.allocated_large; + astats->nmalloc_large += arena->stats.nmalloc_large; + astats->ndalloc_large += arena->stats.ndalloc_large; + astats->nrequests_large += arena->stats.nrequests_large; + + for (i = 0; i < nlclasses; i++) { + lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; + lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; + lstats[i].nrequests += arena->stats.lstats[i].nrequests; + lstats[i].curruns += arena->stats.lstats[i].curruns; + } + malloc_mutex_unlock(&arena->lock); + + for (i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + + malloc_mutex_lock(&bin->lock); + bstats[i].allocated += bin->stats.allocated; + bstats[i].nmalloc += bin->stats.nmalloc; + bstats[i].ndalloc += bin->stats.ndalloc; + bstats[i].nrequests += bin->stats.nrequests; + if (config_tcache) { + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; + } + bstats[i].nruns += bin->stats.nruns; + bstats[i].reruns += bin->stats.reruns; + bstats[i].curruns += bin->stats.curruns; + malloc_mutex_unlock(&bin->lock); + } +} + bool arena_new(arena_t *arena, unsigned ind) { @@ -2222,212 +2308,46 @@ arena_new(arena_t *arena, unsigned ind) if (malloc_mutex_init(&arena->lock)) return (true); -#ifdef JEMALLOC_STATS - memset(&arena->stats, 0, sizeof(arena_stats_t)); - arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * - sizeof(malloc_large_stats_t)); - if (arena->stats.lstats == NULL) - return (true); - memset(arena->stats.lstats, 0, nlclasses * - sizeof(malloc_large_stats_t)); -# ifdef JEMALLOC_TCACHE - ql_new(&arena->tcache_ql); -# endif -#endif + if (config_stats) { + memset(&arena->stats, 0, sizeof(arena_stats_t)); + arena->stats.lstats = + (malloc_large_stats_t *)base_alloc(nlclasses * + sizeof(malloc_large_stats_t)); + if (arena->stats.lstats == NULL) + return (true); + memset(arena->stats.lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); + if (config_tcache) + ql_new(&arena->tcache_ql); + } -#ifdef JEMALLOC_PROF - arena->prof_accumbytes = 0; -#endif + if (config_prof) + arena->prof_accumbytes = 0; + + arena->dss_prec = chunk_dss_prec_get(); /* Initialize chunks. */ - ql_new(&arena->chunks_dirty); + arena_chunk_dirty_new(&arena->chunks_dirty); arena->spare = NULL; arena->nactive = 0; arena->ndirty = 0; arena->npurgatory = 0; - arena_avail_tree_new(&arena->runs_avail_clean); - arena_avail_tree_new(&arena->runs_avail_dirty); + arena_avail_tree_new(&arena->runs_avail); /* Initialize bins. */ - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } -#endif - - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { bin = &arena->bins[i]; if (malloc_mutex_init(&bin->lock)) return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - -#ifdef JEMALLOC_DEBUG - arena->magic = ARENA_MAGIC; -#endif - - return (false); -} - -#ifdef JEMALLOC_DEBUG -static void -small_size2bin_validate(void) -{ - size_t i, size, binind; - - i = 1; -# ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i++) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } - for (; i < qspace_min; i++) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } -# endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i++) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i++) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Sub-page. */ - for (; i <= sspace_max; i++) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) - >> LG_SUBPAGE); - assert(SMALL_SIZE2BIN(i) == binind); + if (config_stats) + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } -} -#endif - -static bool -small_size2bin_init(void) -{ - - if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)) - return (small_size2bin_init_hard()); - - small_size2bin = const_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif - return (false); -} - -static bool -small_size2bin_init_hard(void) -{ - size_t i, size, binind; - uint8_t *custom_small_size2bin; -#define CUSTOM_SMALL_SIZE2BIN(s) \ - custom_small_size2bin[(s-1) >> LG_TINY_MIN] - - assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)); - - custom_small_size2bin = (uint8_t *) - base_alloc(small_maxclass >> LG_TINY_MIN); - if (custom_small_size2bin == NULL) - return (true); - i = 1; -#ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - for (; i < qspace_min; i += TINY_MIN) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } -#endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i += TINY_MIN) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i += TINY_MIN) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Sub-page. */ - for (; i <= sspace_max; i += TINY_MIN) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> - LG_SUBPAGE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - - small_size2bin = custom_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif return (false); -#undef CUSTOM_SMALL_SIZE2BIN } /* @@ -2444,18 +2364,40 @@ small_size2bin_init_hard(void) static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) { + size_t pad_size; size_t try_run_size, good_run_size; uint32_t try_nregs, good_nregs; uint32_t try_hdr_size, good_hdr_size; uint32_t try_bitmap_offset, good_bitmap_offset; -#ifdef JEMALLOC_PROF uint32_t try_ctx0_offset, good_ctx0_offset; -#endif - uint32_t try_reg0_offset, good_reg0_offset; + uint32_t try_redzone0_offset, good_redzone0_offset; - assert(min_run_size >= PAGE_SIZE); + assert(min_run_size >= PAGE); assert(min_run_size <= arena_maxclass); + /* + * Determine redzone size based on minimum alignment and minimum + * redzone size. Add padding to the end of the run if it is needed to + * align the regions. The padding allows each redzone to be half the + * minimum alignment; without the padding, each redzone would have to + * be twice as large in order to maintain alignment. + */ + if (config_fill && opt_redzone) { + size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); + if (align_min <= REDZONE_MINSIZE) { + bin_info->redzone_size = REDZONE_MINSIZE; + pad_size = 0; + } else { + bin_info->redzone_size = align_min >> 1; + pad_size = bin_info->redzone_size; + } + } else { + bin_info->redzone_size = 0; + pad_size = 0; + } + bin_info->reg_interval = bin_info->reg_size + + (bin_info->redzone_size << 1); + /* * Calculate known-valid settings before entering the run_size * expansion loop, so that the first part of the loop always copies @@ -2467,7 +2409,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * header's mask length and the number of regions. */ try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + try_nregs = ((try_run_size - sizeof(arena_run_t)) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2481,8 +2424,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { + if (config_prof && opt_prof && prof_promote == false) { /* Pad to a quantum boundary. */ try_hdr_size = QUANTUM_CEILING(try_hdr_size); try_ctx0_offset = try_hdr_size; @@ -2490,10 +2432,9 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } else try_ctx0_offset = 0; -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); /* run_size expansion loop. */ do { @@ -2504,15 +2445,13 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) good_nregs = try_nregs; good_hdr_size = try_hdr_size; good_bitmap_offset = try_bitmap_offset; -#ifdef JEMALLOC_PROF good_ctx0_offset = try_ctx0_offset; -#endif - good_reg0_offset = try_reg0_offset; + good_redzone0_offset = try_redzone0_offset; /* Try more aggressive settings. */ - try_run_size += PAGE_SIZE; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin_info->reg_size) + try_run_size += PAGE; + try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2526,8 +2465,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { + if (config_prof && opt_prof && prof_promote == false) { /* Pad to a quantum boundary. */ try_hdr_size = QUANTUM_CEILING(try_hdr_size); try_ctx0_offset = try_hdr_size; @@ -2537,140 +2475,51 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); } while (try_run_size <= arena_maxclass - && try_run_size <= arena_maxclass - && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX - && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size + && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > + RUN_MAX_OVRHD_RELAX + && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size && try_nregs < RUN_MAXREGS); - assert(good_hdr_size <= good_reg0_offset); + assert(good_hdr_size <= good_redzone0_offset); /* Copy final settings. */ bin_info->run_size = good_run_size; bin_info->nregs = good_nregs; bin_info->bitmap_offset = good_bitmap_offset; -#ifdef JEMALLOC_PROF bin_info->ctx0_offset = good_ctx0_offset; -#endif - bin_info->reg0_offset = good_reg0_offset; + bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; + + assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs + * bin_info->reg_interval) + pad_size == bin_info->run_size); return (good_run_size); } -static bool +static void bin_info_init(void) { arena_bin_info_t *bin_info; - unsigned i; - size_t prev_run_size; - - arena_bin_info = base_alloc(sizeof(arena_bin_info_t) * nbins); - if (arena_bin_info == NULL) - return (true); - - prev_run_size = PAGE_SIZE; - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (1U << (LG_TINY_MIN + i)); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } -#endif - - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (i - ntbins + 1) << LG_QUANTUM; - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = cspace_min + ((i - (ntbins + nqbins)) << - LG_CACHELINE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = sspace_min + ((i - (ntbins + nqbins + - ncbins)) << LG_SUBPAGE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - return (false); + size_t prev_run_size = PAGE; + +#define SIZE_CLASS(bin, delta, size) \ + bin_info = &arena_bin_info[bin]; \ + bin_info->reg_size = size; \ + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + SIZE_CLASSES +#undef SIZE_CLASS } -bool +void arena_boot(void) { size_t header_size; unsigned i; - /* Set variables according to the value of opt_lg_[qc]space_max. */ - qspace_max = (1U << opt_lg_qspace_max); - cspace_min = CACHELINE_CEILING(qspace_max); - if (cspace_min == qspace_max) - cspace_min += CACHELINE; - cspace_max = (1U << opt_lg_cspace_max); - sspace_min = SUBPAGE_CEILING(cspace_max); - if (sspace_min == cspace_max) - sspace_min += SUBPAGE; - assert(sspace_min < PAGE_SIZE); - sspace_max = PAGE_SIZE - SUBPAGE; - -#ifdef JEMALLOC_TINY - assert(LG_QUANTUM >= LG_TINY_MIN); -#endif - assert(ntbins <= LG_QUANTUM); - nqbins = qspace_max >> LG_QUANTUM; - ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; - nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; - nbins = ntbins + nqbins + ncbins + nsbins; - - /* - * The small_size2bin lookup table uses uint8_t to encode each bin - * index, so we cannot support more than 256 small size classes. This - * limit is difficult to exceed (not even possible with 16B quantum and - * 4KiB pages), and such configurations are impractical, but - * nonetheless we need to protect against this case in order to avoid - * undefined behavior. - * - * Further constrain nbins to 255 if prof_promote is true, since all - * small size classes, plus a "not small" size class must be stored in - * 8 bits of arena_chunk_map_t's bits field. - */ -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote) { - if (nbins > 255) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 255)\n"); - abort(); - } - } else -#endif - if (nbins > 256) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 256)\n"); - abort(); - } - /* * Compute the header size such that it is large enough to contain the * page map. The page map is biased to omit entries for the header @@ -2685,20 +2534,44 @@ arena_boot(void) */ map_bias = 0; for (i = 0; i < 3; i++) { - header_size = offsetof(arena_chunk_t, map) - + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); - map_bias = (header_size >> PAGE_SHIFT) + ((header_size & - PAGE_MASK) != 0); + header_size = offsetof(arena_chunk_t, map) + + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); + map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) + != 0); } assert(map_bias > 0); - arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); + arena_maxclass = chunksize - (map_bias << LG_PAGE); - if (small_size2bin_init()) - return (true); + bin_info_init(); +} - if (bin_info_init()) - return (true); +void +arena_prefork(arena_t *arena) +{ + unsigned i; - return (false); + malloc_mutex_prefork(&arena->lock); + for (i = 0; i < NBINS; i++) + malloc_mutex_prefork(&arena->bins[i].lock); +} + +void +arena_postfork_parent(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_parent(&arena->bins[i].lock); + malloc_mutex_postfork_parent(&arena->lock); +} + +void +arena_postfork_child(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_child(&arena->bins[i].lock); + malloc_mutex_postfork_child(&arena->lock); } diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index cc85e8494ec..4e62e8fa918 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -4,7 +4,7 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t base_mtx; +static malloc_mutex_t base_mtx; /* * Current pages that are being used for internal memory allocations. These @@ -32,7 +32,8 @@ base_pages_alloc(size_t minsize) assert(minsize != 0); csize = CHUNK_CEILING(minsize); zero = false; - base_pages = chunk_alloc(csize, true, &zero); + base_pages = chunk_alloc(csize, chunksize, true, &zero, + chunk_dss_prec_get()); if (base_pages == NULL) return (true); base_next_addr = base_pages; @@ -62,6 +63,18 @@ base_alloc(size_t size) ret = base_next_addr; base_next_addr = (void *)((uintptr_t)base_next_addr + csize); malloc_mutex_unlock(&base_mtx); + VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); + + return (ret); +} + +void * +base_calloc(size_t number, size_t size) +{ + void *ret = base_alloc(number * size); + + if (ret != NULL) + memset(ret, 0, number * size); return (ret); } @@ -76,6 +89,7 @@ base_node_alloc(void) ret = base_nodes; base_nodes = *(extent_node_t **)ret; malloc_mutex_unlock(&base_mtx); + VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t)); } else { malloc_mutex_unlock(&base_mtx); ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); @@ -88,6 +102,7 @@ void base_node_dealloc(extent_node_t *node) { + VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); malloc_mutex_lock(&base_mtx); *(extent_node_t **)node = base_nodes; base_nodes = node; @@ -104,3 +119,24 @@ base_boot(void) return (false); } + +void +base_prefork(void) +{ + + malloc_mutex_prefork(&base_mtx); +} + +void +base_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&base_mtx); +} + +void +base_postfork_child(void) +{ + + malloc_mutex_postfork_child(&base_mtx); +} diff --git a/deps/jemalloc/src/bitmap.c b/deps/jemalloc/src/bitmap.c index b47e2629093..e2bd907d558 100644 --- a/deps/jemalloc/src/bitmap.c +++ b/deps/jemalloc/src/bitmap.c @@ -1,4 +1,4 @@ -#define JEMALLOC_BITMAP_C_ +#define JEMALLOC_BITMAP_C_ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ diff --git a/deps/jemalloc/src/chunk.c b/deps/jemalloc/src/chunk.c index d190c6f49b3..90ab116ae5f 100644 --- a/deps/jemalloc/src/chunk.c +++ b/deps/jemalloc/src/chunk.c @@ -4,19 +4,24 @@ /******************************************************************************/ /* Data. */ -size_t opt_lg_chunk = LG_CHUNK_DEFAULT; -#ifdef JEMALLOC_SWAP -bool opt_overcommit = true; -#endif +const char *opt_dss = DSS_DEFAULT; +size_t opt_lg_chunk = LG_CHUNK_DEFAULT; -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) malloc_mutex_t chunks_mtx; chunk_stats_t stats_chunks; -#endif -#ifdef JEMALLOC_IVSALLOC +/* + * Trees of chunks that were previously allocated (trees differ only in node + * ordering). These are used when allocating chunks, in an attempt to re-use + * address space. Depending on function, different tree orderings are needed, + * which is why there are two trees with the same contents. + */ +static extent_tree_t chunks_szad_mmap; +static extent_tree_t chunks_ad_mmap; +static extent_tree_t chunks_szad_dss; +static extent_tree_t chunks_ad_dss; + rtree_t *chunks_rtree; -#endif /* Various chunk-related settings. */ size_t chunksize; @@ -26,6 +31,109 @@ size_t map_bias; size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void *chunk_recycle(extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, size_t size, size_t alignment, bool base, + bool *zero); +static void chunk_record(extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, void *chunk, size_t size); + +/******************************************************************************/ + +static void * +chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, + size_t alignment, bool base, bool *zero) +{ + void *ret; + extent_node_t *node; + extent_node_t key; + size_t alloc_size, leadsize, trailsize; + bool zeroed; + + if (base) { + /* + * This function may need to call base_node_{,de}alloc(), but + * the current chunk allocation request is on behalf of the + * base allocator. Avoid deadlock (and if that weren't an + * issue, potential for infinite recursion) by returning NULL. + */ + return (NULL); + } + + alloc_size = size + alignment - chunksize; + /* Beware size_t wrap-around. */ + if (alloc_size < size) + return (NULL); + key.addr = NULL; + key.size = alloc_size; + malloc_mutex_lock(&chunks_mtx); + node = extent_tree_szad_nsearch(chunks_szad, &key); + if (node == NULL) { + malloc_mutex_unlock(&chunks_mtx); + return (NULL); + } + leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - + (uintptr_t)node->addr; + assert(node->size >= leadsize + size); + trailsize = node->size - leadsize - size; + ret = (void *)((uintptr_t)node->addr + leadsize); + zeroed = node->zeroed; + if (zeroed) + *zero = true; + /* Remove node from the tree. */ + extent_tree_szad_remove(chunks_szad, node); + extent_tree_ad_remove(chunks_ad, node); + if (leadsize != 0) { + /* Insert the leading space as a smaller chunk. */ + node->size = leadsize; + extent_tree_szad_insert(chunks_szad, node); + extent_tree_ad_insert(chunks_ad, node); + node = NULL; + } + if (trailsize != 0) { + /* Insert the trailing space as a smaller chunk. */ + if (node == NULL) { + /* + * An additional node is required, but + * base_node_alloc() can cause a new base chunk to be + * allocated. Drop chunks_mtx in order to avoid + * deadlock, and if node allocation fails, deallocate + * the result before returning an error. + */ + malloc_mutex_unlock(&chunks_mtx); + node = base_node_alloc(); + if (node == NULL) { + chunk_dealloc(ret, size, true); + return (NULL); + } + malloc_mutex_lock(&chunks_mtx); + } + node->addr = (void *)((uintptr_t)(ret) + size); + node->size = trailsize; + node->zeroed = zeroed; + extent_tree_szad_insert(chunks_szad, node); + extent_tree_ad_insert(chunks_ad, node); + node = NULL; + } + malloc_mutex_unlock(&chunks_mtx); + + if (node != NULL) + base_node_dealloc(node); + if (*zero) { + if (zeroed == false) + memset(ret, 0, size); + else if (config_debug) { + size_t i; + size_t *p = (size_t *)(uintptr_t)ret; + + VALGRIND_MAKE_MEM_DEFINED(ret, size); + for (i = 0; i < size / sizeof(size_t); i++) + assert(p[i] == 0); + } + } + return (ret); +} /* * If the caller specifies (*zero == false), it is still possible to receive @@ -34,79 +142,175 @@ size_t arena_maxclass; /* Max size class for arenas. */ * advantage of them if they are returned. */ void * -chunk_alloc(size_t size, bool base, bool *zero) +chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, + dss_prec_t dss_prec) { void *ret; assert(size != 0); assert((size & chunksize_mask) == 0); + assert(alignment != 0); + assert((alignment & chunksize_mask) == 0); -#ifdef JEMALLOC_SWAP - if (swap_enabled) { - ret = chunk_alloc_swap(size, zero); - if (ret != NULL) - goto RETURN; + /* "primary" dss. */ + if (config_dss && dss_prec == dss_prec_primary) { + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, + alignment, base, zero)) != NULL) + goto label_return; + if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + goto label_return; } - - if (swap_enabled == false || opt_overcommit) { -#endif -#ifdef JEMALLOC_DSS - ret = chunk_alloc_dss(size, zero); - if (ret != NULL) - goto RETURN; -#endif - ret = chunk_alloc_mmap(size); - if (ret != NULL) { - *zero = true; - goto RETURN; - } -#ifdef JEMALLOC_SWAP + /* mmap. */ + if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size, + alignment, base, zero)) != NULL) + goto label_return; + if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) + goto label_return; + /* "secondary" dss. */ + if (config_dss && dss_prec == dss_prec_secondary) { + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, + alignment, base, zero)) != NULL) + goto label_return; + if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + goto label_return; } -#endif /* All strategies for allocation failed. */ ret = NULL; -RETURN: -#ifdef JEMALLOC_IVSALLOC - if (base == false && ret != NULL) { - if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) { - chunk_dealloc(ret, size, true); - return (NULL); - } - } -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) +label_return: if (ret != NULL) { -# ifdef JEMALLOC_PROF - bool gdump; -# endif - malloc_mutex_lock(&chunks_mtx); -# ifdef JEMALLOC_STATS - stats_chunks.nchunks += (size / chunksize); -# endif - stats_chunks.curchunks += (size / chunksize); - if (stats_chunks.curchunks > stats_chunks.highchunks) { - stats_chunks.highchunks = stats_chunks.curchunks; -# ifdef JEMALLOC_PROF - gdump = true; -# endif + if (config_ivsalloc && base == false) { + if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) { + chunk_dealloc(ret, size, true); + return (NULL); + } } -# ifdef JEMALLOC_PROF - else - gdump = false; -# endif - malloc_mutex_unlock(&chunks_mtx); -# ifdef JEMALLOC_PROF - if (opt_prof && opt_prof_gdump && gdump) - prof_gdump(); -# endif + if (config_stats || config_prof) { + bool gdump; + malloc_mutex_lock(&chunks_mtx); + if (config_stats) + stats_chunks.nchunks += (size / chunksize); + stats_chunks.curchunks += (size / chunksize); + if (stats_chunks.curchunks > stats_chunks.highchunks) { + stats_chunks.highchunks = + stats_chunks.curchunks; + if (config_prof) + gdump = true; + } else if (config_prof) + gdump = false; + malloc_mutex_unlock(&chunks_mtx); + if (config_prof && opt_prof && opt_prof_gdump && gdump) + prof_gdump(); + } + if (config_valgrind) + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } -#endif - assert(CHUNK_ADDR2BASE(ret) == ret); return (ret); } +static void +chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, + size_t size) +{ + bool unzeroed; + extent_node_t *xnode, *node, *prev, *xprev, key; + + unzeroed = pages_purge(chunk, size); + VALGRIND_MAKE_MEM_NOACCESS(chunk, size); + + /* + * Allocate a node before acquiring chunks_mtx even though it might not + * be needed, because base_node_alloc() may cause a new base chunk to + * be allocated, which could cause deadlock if chunks_mtx were already + * held. + */ + xnode = base_node_alloc(); + /* Use xprev to implement conditional deferred deallocation of prev. */ + xprev = NULL; + + malloc_mutex_lock(&chunks_mtx); + key.addr = (void *)((uintptr_t)chunk + size); + node = extent_tree_ad_nsearch(chunks_ad, &key); + /* Try to coalesce forward. */ + if (node != NULL && node->addr == key.addr) { + /* + * Coalesce chunk with the following address range. This does + * not change the position within chunks_ad, so only + * remove/insert from/into chunks_szad. + */ + extent_tree_szad_remove(chunks_szad, node); + node->addr = chunk; + node->size += size; + node->zeroed = (node->zeroed && (unzeroed == false)); + extent_tree_szad_insert(chunks_szad, node); + } else { + /* Coalescing forward failed, so insert a new node. */ + if (xnode == NULL) { + /* + * base_node_alloc() failed, which is an exceedingly + * unlikely failure. Leak chunk; its pages have + * already been purged, so this is only a virtual + * memory leak. + */ + goto label_return; + } + node = xnode; + xnode = NULL; /* Prevent deallocation below. */ + node->addr = chunk; + node->size = size; + node->zeroed = (unzeroed == false); + extent_tree_ad_insert(chunks_ad, node); + extent_tree_szad_insert(chunks_szad, node); + } + + /* Try to coalesce backward. */ + prev = extent_tree_ad_prev(chunks_ad, node); + if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == + chunk) { + /* + * Coalesce chunk with the previous address range. This does + * not change the position within chunks_ad, so only + * remove/insert node from/into chunks_szad. + */ + extent_tree_szad_remove(chunks_szad, prev); + extent_tree_ad_remove(chunks_ad, prev); + + extent_tree_szad_remove(chunks_szad, node); + node->addr = prev->addr; + node->size += prev->size; + node->zeroed = (node->zeroed && prev->zeroed); + extent_tree_szad_insert(chunks_szad, node); + + xprev = prev; + } + +label_return: + malloc_mutex_unlock(&chunks_mtx); + /* + * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to + * avoid potential deadlock. + */ + if (xnode != NULL) + base_node_dealloc(xnode); + if (xprev != NULL) + base_node_dealloc(xprev); +} + +void +chunk_unmap(void *chunk, size_t size) +{ + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + assert(size != 0); + assert((size & chunksize_mask) == 0); + + if (config_dss && chunk_in_dss(chunk)) + chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); + else if (chunk_dealloc_mmap(chunk, size)) + chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); +} + void chunk_dealloc(void *chunk, size_t size, bool unmap) { @@ -116,26 +320,17 @@ chunk_dealloc(void *chunk, size_t size, bool unmap) assert(size != 0); assert((size & chunksize_mask) == 0); -#ifdef JEMALLOC_IVSALLOC - rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - malloc_mutex_lock(&chunks_mtx); - stats_chunks.curchunks -= (size / chunksize); - malloc_mutex_unlock(&chunks_mtx); -#endif - - if (unmap) { -#ifdef JEMALLOC_SWAP - if (swap_enabled && chunk_dealloc_swap(chunk, size) == false) - return; -#endif -#ifdef JEMALLOC_DSS - if (chunk_dealloc_dss(chunk, size) == false) - return; -#endif - chunk_dealloc_mmap(chunk, size); + if (config_ivsalloc) + rtree_set(chunks_rtree, (uintptr_t)chunk, 0); + if (config_stats || config_prof) { + malloc_mutex_lock(&chunks_mtx); + assert(stats_chunks.curchunks >= (size / chunksize)); + stats_chunks.curchunks -= (size / chunksize); + malloc_mutex_unlock(&chunks_mtx); } + + if (unmap) + chunk_unmap(chunk, size); } bool @@ -144,30 +339,57 @@ chunk_boot(void) /* Set variables according to the value of opt_lg_chunk. */ chunksize = (ZU(1) << opt_lg_chunk); - assert(chunksize >= PAGE_SIZE); + assert(chunksize >= PAGE); chunksize_mask = chunksize - 1; - chunk_npages = (chunksize >> PAGE_SHIFT); + chunk_npages = (chunksize >> LG_PAGE); -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - if (malloc_mutex_init(&chunks_mtx)) - return (true); - memset(&stats_chunks, 0, sizeof(chunk_stats_t)); -#endif -#ifdef JEMALLOC_SWAP - if (chunk_swap_boot()) - return (true); -#endif - if (chunk_mmap_boot()) - return (true); -#ifdef JEMALLOC_DSS - if (chunk_dss_boot()) - return (true); -#endif -#ifdef JEMALLOC_IVSALLOC - chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk); - if (chunks_rtree == NULL) + if (config_stats || config_prof) { + if (malloc_mutex_init(&chunks_mtx)) + return (true); + memset(&stats_chunks, 0, sizeof(chunk_stats_t)); + } + if (config_dss && chunk_dss_boot()) return (true); -#endif + extent_tree_szad_new(&chunks_szad_mmap); + extent_tree_ad_new(&chunks_ad_mmap); + extent_tree_szad_new(&chunks_szad_dss); + extent_tree_ad_new(&chunks_ad_dss); + if (config_ivsalloc) { + chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - + opt_lg_chunk, base_alloc, NULL); + if (chunks_rtree == NULL) + return (true); + } return (false); } + +void +chunk_prefork(void) +{ + + malloc_mutex_prefork(&chunks_mtx); + if (config_ivsalloc) + rtree_prefork(chunks_rtree); + chunk_dss_prefork(); +} + +void +chunk_postfork_parent(void) +{ + + chunk_dss_postfork_parent(); + if (config_ivsalloc) + rtree_postfork_parent(chunks_rtree); + malloc_mutex_postfork_parent(&chunks_mtx); +} + +void +chunk_postfork_child(void) +{ + + chunk_dss_postfork_child(); + if (config_ivsalloc) + rtree_postfork_child(chunks_rtree); + malloc_mutex_postfork_child(&chunks_mtx); +} diff --git a/deps/jemalloc/src/chunk_dss.c b/deps/jemalloc/src/chunk_dss.c index 5c0e290e441..510bb8bee85 100644 --- a/deps/jemalloc/src/chunk_dss.c +++ b/deps/jemalloc/src/chunk_dss.c @@ -1,82 +1,78 @@ #define JEMALLOC_CHUNK_DSS_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_DSS /******************************************************************************/ /* Data. */ -malloc_mutex_t dss_mtx; +const char *dss_prec_names[] = { + "disabled", + "primary", + "secondary", + "N/A" +}; -/* Base address of the DSS. */ -static void *dss_base; -/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ -static void *dss_prev; -/* Current upper limit on DSS addresses. */ -static void *dss_max; +/* Current dss precedence default, used when creating new arenas. */ +static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT; /* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. + * Protects sbrk() calls. This avoids malloc races among threads, though it + * does not protect against races with threads that call sbrk() directly. */ -static extent_tree_t dss_chunks_szad; -static extent_tree_t dss_chunks_ad; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ +static malloc_mutex_t dss_mtx; -static void *chunk_recycle_dss(size_t size, bool *zero); -static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size); +/* Base address of the DSS. */ +static void *dss_base; +/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ +static void *dss_prev; +/* Current upper limit on DSS addresses. */ +static void *dss_max; /******************************************************************************/ static void * -chunk_recycle_dss(size_t size, bool *zero) +chunk_dss_sbrk(intptr_t increment) +{ + +#ifdef JEMALLOC_HAVE_SBRK + return (sbrk(increment)); +#else + not_implemented(); + return (NULL); +#endif +} + +dss_prec_t +chunk_dss_prec_get(void) { - extent_node_t *node, key; + dss_prec_t ret; - key.addr = NULL; - key.size = size; + if (config_dss == false) + return (dss_prec_disabled); malloc_mutex_lock(&dss_mtx); - node = extent_tree_szad_nsearch(&dss_chunks_szad, &key); - if (node != NULL) { - void *ret = node->addr; - - /* Remove node from the tree. */ - extent_tree_szad_remove(&dss_chunks_szad, node); - if (node->size == size) { - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } else { - /* - * Insert the remainder of node's address range as a - * smaller chunk. Its position within dss_chunks_ad - * does not change. - */ - assert(node->size > size); - node->addr = (void *)((uintptr_t)node->addr + size); - node->size -= size; - extent_tree_szad_insert(&dss_chunks_szad, node); - } - malloc_mutex_unlock(&dss_mtx); - - if (*zero) - memset(ret, 0, size); - return (ret); - } + ret = dss_prec_default; malloc_mutex_unlock(&dss_mtx); + return (ret); +} - return (NULL); +bool +chunk_dss_prec_set(dss_prec_t dss_prec) +{ + + if (config_dss == false) + return (true); + malloc_mutex_lock(&dss_mtx); + dss_prec_default = dss_prec; + malloc_mutex_unlock(&dss_mtx); + return (false); } void * -chunk_alloc_dss(size_t size, bool *zero) +chunk_alloc_dss(size_t size, size_t alignment, bool *zero) { void *ret; - ret = chunk_recycle_dss(size, zero); - if (ret != NULL) - return (ret); + cassert(config_dss); + assert(size > 0 && (size & chunksize_mask) == 0); + assert(alignment > 0 && (alignment & chunksize_mask) == 0); /* * sbrk() uses a signed increment argument, so take care not to @@ -87,6 +83,8 @@ chunk_alloc_dss(size_t size, bool *zero) malloc_mutex_lock(&dss_mtx); if (dss_prev != (void *)-1) { + size_t gap_size, cpad_size; + void *cpad, *dss_next; intptr_t incr; /* @@ -96,27 +94,41 @@ chunk_alloc_dss(size_t size, bool *zero) */ do { /* Get the current end of the DSS. */ - dss_max = sbrk(0); - + dss_max = chunk_dss_sbrk(0); /* * Calculate how much padding is necessary to * chunk-align the end of the DSS. */ - incr = (intptr_t)size - - (intptr_t)CHUNK_ADDR2OFFSET(dss_max); - if (incr == (intptr_t)size) - ret = dss_max; - else { - ret = (void *)((intptr_t)dss_max + incr); - incr += size; + gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & + chunksize_mask; + /* + * Compute how much chunk-aligned pad space (if any) is + * necessary to satisfy alignment. This space can be + * recycled for later use. + */ + cpad = (void *)((uintptr_t)dss_max + gap_size); + ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, + alignment); + cpad_size = (uintptr_t)ret - (uintptr_t)cpad; + dss_next = (void *)((uintptr_t)ret + size); + if ((uintptr_t)ret < (uintptr_t)dss_max || + (uintptr_t)dss_next < (uintptr_t)dss_max) { + /* Wrap-around. */ + malloc_mutex_unlock(&dss_mtx); + return (NULL); } - - dss_prev = sbrk(incr); + incr = gap_size + cpad_size + size; + dss_prev = chunk_dss_sbrk(incr); if (dss_prev == dss_max) { /* Success. */ - dss_max = (void *)((intptr_t)dss_prev + incr); + dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); - *zero = true; + if (cpad_size != 0) + chunk_unmap(cpad, cpad_size); + if (*zero) { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + memset(ret, 0, size); + } return (ret); } } while (dss_prev != (void *)-1); @@ -126,84 +138,13 @@ chunk_alloc_dss(size_t size, bool *zero) return (NULL); } -static extent_node_t * -chunk_dealloc_dss_record(void *chunk, size_t size) -{ - extent_node_t *xnode, *node, *prev, key; - - xnode = NULL; - while (true) { - key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&dss_chunks_ad, &key); - /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. - * This does not change the position within - * dss_chunks_ad, so only remove/insert from/into - * dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = chunk; - node->size += size; - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } else if (xnode == NULL) { - /* - * It is possible that base_node_alloc() will cause a - * new base chunk to be allocated, so take care not to - * deadlock on dss_mtx, and recover if another thread - * deallocates an adjacent chunk while this one is busy - * allocating xnode. - */ - malloc_mutex_unlock(&dss_mtx); - xnode = base_node_alloc(); - malloc_mutex_lock(&dss_mtx); - if (xnode == NULL) - return (NULL); - } else { - /* Coalescing forward failed, so insert a new node. */ - node = xnode; - xnode = NULL; - node->addr = chunk; - node->size = size; - extent_tree_ad_insert(&dss_chunks_ad, node); - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } - } - /* Discard xnode if it ended up unused do to a race. */ - if (xnode != NULL) - base_node_dealloc(xnode); - - /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&dss_chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within dss_chunks_ad, so only - * remove/insert node from/into dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, prev); - extent_tree_ad_remove(&dss_chunks_ad, prev); - - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - extent_tree_szad_insert(&dss_chunks_szad, node); - - base_node_dealloc(prev); - } - - return (node); -} - bool chunk_in_dss(void *chunk) { bool ret; + cassert(config_dss); + malloc_mutex_lock(&dss_mtx); if ((uintptr_t)chunk >= (uintptr_t)dss_base && (uintptr_t)chunk < (uintptr_t)dss_max) @@ -216,69 +157,42 @@ chunk_in_dss(void *chunk) } bool -chunk_dealloc_dss(void *chunk, size_t size) +chunk_dss_boot(void) { - bool ret; - malloc_mutex_lock(&dss_mtx); - if ((uintptr_t)chunk >= (uintptr_t)dss_base - && (uintptr_t)chunk < (uintptr_t)dss_max) { - extent_node_t *node; + cassert(config_dss); - /* Try to coalesce with other unused chunks. */ - node = chunk_dealloc_dss_record(chunk, size); - if (node != NULL) { - chunk = node->addr; - size = node->size; - } - - /* Get the current end of the DSS. */ - dss_max = sbrk(0); + if (malloc_mutex_init(&dss_mtx)) + return (true); + dss_base = chunk_dss_sbrk(0); + dss_prev = dss_base; + dss_max = dss_base; - /* - * Try to shrink the DSS if this chunk is at the end of the - * DSS. The sbrk() call here is subject to a race condition - * with threads that use brk(2) or sbrk(2) directly, but the - * alternative would be to leak memory for the sake of poorly - * designed multi-threaded programs. - */ - if ((void *)((uintptr_t)chunk + size) == dss_max - && (dss_prev = sbrk(-(intptr_t)size)) == dss_max) { - /* Success. */ - dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size); - - if (node != NULL) { - extent_tree_szad_remove(&dss_chunks_szad, node); - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } - } else - madvise(chunk, size, MADV_DONTNEED); + return (false); +} - ret = false; - goto RETURN; - } +void +chunk_dss_prefork(void) +{ - ret = true; -RETURN: - malloc_mutex_unlock(&dss_mtx); - return (ret); + if (config_dss) + malloc_mutex_prefork(&dss_mtx); } -bool -chunk_dss_boot(void) +void +chunk_dss_postfork_parent(void) { - if (malloc_mutex_init(&dss_mtx)) - return (true); - dss_base = sbrk(0); - dss_prev = dss_base; - dss_max = dss_base; - extent_tree_szad_new(&dss_chunks_szad); - extent_tree_ad_new(&dss_chunks_ad); + if (config_dss) + malloc_mutex_postfork_parent(&dss_mtx); +} - return (false); +void +chunk_dss_postfork_child(void) +{ + + if (config_dss) + malloc_mutex_postfork_child(&dss_mtx); } /******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc/src/chunk_mmap.c b/deps/jemalloc/src/chunk_mmap.c index 164e86e7b38..2056d793f05 100644 --- a/deps/jemalloc/src/chunk_mmap.c +++ b/deps/jemalloc/src/chunk_mmap.c @@ -1,54 +1,37 @@ #define JEMALLOC_CHUNK_MMAP_C_ #include "jemalloc/internal/jemalloc_internal.h" -/******************************************************************************/ -/* Data. */ - -/* - * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and - * potentially avoid some system calls. - */ -#ifndef NO_TLS -static __thread bool mmap_unaligned_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#define MMAP_UNALIGNED_GET() mmap_unaligned_tls -#define MMAP_UNALIGNED_SET(v) do { \ - mmap_unaligned_tls = (v); \ -} while (0) -#else -static pthread_key_t mmap_unaligned_tsd; -#define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd)) -#define MMAP_UNALIGNED_SET(v) do { \ - pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \ -} while (0) -#endif - /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static void *pages_map(void *addr, size_t size, bool noreserve); +static void *pages_map(void *addr, size_t size); static void pages_unmap(void *addr, size_t size); -static void *chunk_alloc_mmap_slow(size_t size, bool unaligned, - bool noreserve); -static void *chunk_alloc_mmap_internal(size_t size, bool noreserve); +static void *chunk_alloc_mmap_slow(size_t size, size_t alignment, + bool *zero); /******************************************************************************/ static void * -pages_map(void *addr, size_t size, bool noreserve) +pages_map(void *addr, size_t size) { void *ret; + assert(size != 0); + +#ifdef _WIN32 + /* + * If VirtualAlloc can't allocate at the given address when one is + * given, it fails and returns NULL. + */ + ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE); +#else /* * We don't use MAP_FIXED here, because it can cause the *replacement* * of existing mappings, and we only want to create new mappings. */ - int flags = MAP_PRIVATE | MAP_ANON; -#ifdef MAP_NORESERVE - if (noreserve) - flags |= MAP_NORESERVE; -#endif - ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, + -1, 0); assert(ret != NULL); if (ret == MAP_FAILED) @@ -60,16 +43,15 @@ pages_map(void *addr, size_t size, bool noreserve) if (munmap(ret, size) == -1) { char buf[BUFERROR_BUF]; - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf(": Error in " +#ifdef _WIN32 + "VirtualFree" +#else + "munmap" +#endif + "(): %s\n", buf); if (opt_abort) abort(); } } static void * -chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve) +pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size) { - void *ret; - size_t offset; - - /* Beware size_t wrap-around. */ - if (size + chunksize <= size) + void *ret = (void *)((uintptr_t)addr + leadsize); + + assert(alloc_size >= leadsize + size); +#ifdef _WIN32 + { + void *new_addr; + + pages_unmap(addr, alloc_size); + new_addr = pages_map(ret, size); + if (new_addr == ret) + return (ret); + if (new_addr) + pages_unmap(new_addr, size); return (NULL); + } +#else + { + size_t trailsize = alloc_size - leadsize - size; + + if (leadsize != 0) + pages_unmap(addr, leadsize); + if (trailsize != 0) + pages_unmap((void *)((uintptr_t)ret + size), trailsize); + return (ret); + } +#endif +} - ret = pages_map(NULL, size + chunksize, noreserve); - if (ret == NULL) - return (NULL); +bool +pages_purge(void *addr, size_t length) +{ + bool unzeroed; - /* Clean up unneeded leading/trailing space. */ - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - /* Note that mmap() returned an unaligned mapping. */ - unaligned = true; - - /* Leading space. */ - pages_unmap(ret, chunksize - offset); - - ret = (void *)((uintptr_t)ret + - (chunksize - offset)); - - /* Trailing space. */ - pages_unmap((void *)((uintptr_t)ret + size), - offset); - } else { - /* Trailing space only. */ - pages_unmap((void *)((uintptr_t)ret + size), - chunksize); - } +#ifdef _WIN32 + VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE); + unzeroed = true; +#else +# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED +# define JEMALLOC_MADV_PURGE MADV_DONTNEED +# define JEMALLOC_MADV_ZEROS true +# elif defined(JEMALLOC_PURGE_MADVISE_FREE) +# define JEMALLOC_MADV_PURGE MADV_FREE +# define JEMALLOC_MADV_ZEROS false +# else +# error "No method defined for purging unused dirty pages." +# endif + int err = madvise(addr, length, JEMALLOC_MADV_PURGE); + unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0); +# undef JEMALLOC_MADV_PURGE +# undef JEMALLOC_MADV_ZEROS +#endif + return (unzeroed); +} - /* - * If mmap() returned an aligned mapping, reset mmap_unaligned so that - * the next chunk_alloc_mmap() execution tries the fast allocation - * method. - */ - if (unaligned == false) - MMAP_UNALIGNED_SET(false); +static void * +chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero) +{ + void *ret, *pages; + size_t alloc_size, leadsize; + + alloc_size = size + alignment - PAGE; + /* Beware size_t wrap-around. */ + if (alloc_size < size) + return (NULL); + do { + pages = pages_map(NULL, alloc_size); + if (pages == NULL) + return (NULL); + leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) - + (uintptr_t)pages; + ret = pages_trim(pages, alloc_size, leadsize, size); + } while (ret == NULL); + assert(ret != NULL); + *zero = true; return (ret); } -static void * -chunk_alloc_mmap_internal(size_t size, bool noreserve) +void * +chunk_alloc_mmap(size_t size, size_t alignment, bool *zero) { void *ret; + size_t offset; /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the - * excess. However, that always results in at least one call to + * excess. However, that always results in one or two calls to * pages_unmap(). * - * A more optimistic approach is to try mapping precisely the right - * amount, then try to append another mapping if alignment is off. In - * practice, this works out well as long as the application is not - * interleaving mappings via direct mmap() calls. If we do run into a - * situation where there is an interleaved mapping and we are unable to - * extend an unaligned mapping, our best option is to switch to the - * slow method until mmap() returns another aligned mapping. This will - * tend to leave a gap in the memory map that is too small to cause - * later problems for the optimistic method. - * - * Another possible confounding factor is address space layout - * randomization (ASLR), which causes mmap(2) to disregard the - * requested address. mmap_unaligned tracks whether the previous - * chunk_alloc_mmap() execution received any unaligned or relocated - * mappings, and if so, the current execution will immediately fall - * back to the slow method. However, we keep track of whether the fast - * method would have succeeded, and if so, we make a note to try the - * fast method next time. + * Optimistically try mapping precisely the right amount before falling + * back to the slow method, with the expectation that the optimistic + * approach works most of the time. */ - if (MMAP_UNALIGNED_GET() == false) { - size_t offset; - - ret = pages_map(NULL, size, noreserve); - if (ret == NULL) - return (NULL); + assert(alignment != 0); + assert((alignment & chunksize_mask) == 0); - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - MMAP_UNALIGNED_SET(true); - /* Try to extend chunk boundary. */ - if (pages_map((void *)((uintptr_t)ret + size), - chunksize - offset, noreserve) == NULL) { - /* - * Extension failed. Clean up, then revert to - * the reliable-but-expensive method. - */ - pages_unmap(ret, size); - ret = chunk_alloc_mmap_slow(size, true, - noreserve); - } else { - /* Clean up unneeded leading space. */ - pages_unmap(ret, chunksize - offset); - ret = (void *)((uintptr_t)ret + (chunksize - - offset)); - } - } - } else - ret = chunk_alloc_mmap_slow(size, false, noreserve); + ret = pages_map(NULL, size); + if (ret == NULL) + return (NULL); + offset = ALIGNMENT_ADDR2OFFSET(ret, alignment); + if (offset != 0) { + pages_unmap(ret, size); + return (chunk_alloc_mmap_slow(size, alignment, zero)); + } + assert(ret != NULL); + *zero = true; return (ret); } -void * -chunk_alloc_mmap(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, false)); -} - -void * -chunk_alloc_mmap_noreserve(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, true)); -} - -void -chunk_dealloc_mmap(void *chunk, size_t size) -{ - - pages_unmap(chunk, size); -} - bool -chunk_mmap_boot(void) +chunk_dealloc_mmap(void *chunk, size_t size) { -#ifdef NO_TLS - if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) { - malloc_write(": Error in pthread_key_create()\n"); - return (true); - } -#endif + if (config_munmap) + pages_unmap(chunk, size); - return (false); + return (config_munmap == false); } diff --git a/deps/jemalloc/src/chunk_swap.c b/deps/jemalloc/src/chunk_swap.c deleted file mode 100644 index cb25ae0dde2..00000000000 --- a/deps/jemalloc/src/chunk_swap.c +++ /dev/null @@ -1,402 +0,0 @@ -#define JEMALLOC_CHUNK_SWAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_SWAP -/******************************************************************************/ -/* Data. */ - -malloc_mutex_t swap_mtx; -bool swap_enabled; -bool swap_prezeroed; -size_t swap_nfds; -int *swap_fds; -#ifdef JEMALLOC_STATS -size_t swap_avail; -#endif - -/* Base address of the mmap()ed file(s). */ -static void *swap_base; -/* Current end of the space in use (<= swap_max). */ -static void *swap_end; -/* Absolute upper limit on file-backed addresses. */ -static void *swap_max; - -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ -static extent_tree_t swap_chunks_szad; -static extent_tree_t swap_chunks_ad; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *chunk_recycle_swap(size_t size, bool *zero); -static extent_node_t *chunk_dealloc_swap_record(void *chunk, size_t size); - -/******************************************************************************/ - -static void * -chunk_recycle_swap(size_t size, bool *zero) -{ - extent_node_t *node, key; - - key.addr = NULL; - key.size = size; - malloc_mutex_lock(&swap_mtx); - node = extent_tree_szad_nsearch(&swap_chunks_szad, &key); - if (node != NULL) { - void *ret = node->addr; - - /* Remove node from the tree. */ - extent_tree_szad_remove(&swap_chunks_szad, node); - if (node->size == size) { - extent_tree_ad_remove(&swap_chunks_ad, node); - base_node_dealloc(node); - } else { - /* - * Insert the remainder of node's address range as a - * smaller chunk. Its position within swap_chunks_ad - * does not change. - */ - assert(node->size > size); - node->addr = (void *)((uintptr_t)node->addr + size); - node->size -= size; - extent_tree_szad_insert(&swap_chunks_szad, node); - } -#ifdef JEMALLOC_STATS - swap_avail -= size; -#endif - malloc_mutex_unlock(&swap_mtx); - - if (*zero) - memset(ret, 0, size); - return (ret); - } - malloc_mutex_unlock(&swap_mtx); - - return (NULL); -} - -void * -chunk_alloc_swap(size_t size, bool *zero) -{ - void *ret; - - assert(swap_enabled); - - ret = chunk_recycle_swap(size, zero); - if (ret != NULL) - return (ret); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)swap_end + size <= (uintptr_t)swap_max) { - ret = swap_end; - swap_end = (void *)((uintptr_t)swap_end + size); -#ifdef JEMALLOC_STATS - swap_avail -= size; -#endif - malloc_mutex_unlock(&swap_mtx); - - if (swap_prezeroed) - *zero = true; - else if (*zero) - memset(ret, 0, size); - } else { - malloc_mutex_unlock(&swap_mtx); - return (NULL); - } - - return (ret); -} - -static extent_node_t * -chunk_dealloc_swap_record(void *chunk, size_t size) -{ - extent_node_t *xnode, *node, *prev, key; - - xnode = NULL; - while (true) { - key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&swap_chunks_ad, &key); - /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. - * This does not change the position within - * swap_chunks_ad, so only remove/insert from/into - * swap_chunks_szad. - */ - extent_tree_szad_remove(&swap_chunks_szad, node); - node->addr = chunk; - node->size += size; - extent_tree_szad_insert(&swap_chunks_szad, node); - break; - } else if (xnode == NULL) { - /* - * It is possible that base_node_alloc() will cause a - * new base chunk to be allocated, so take care not to - * deadlock on swap_mtx, and recover if another thread - * deallocates an adjacent chunk while this one is busy - * allocating xnode. - */ - malloc_mutex_unlock(&swap_mtx); - xnode = base_node_alloc(); - malloc_mutex_lock(&swap_mtx); - if (xnode == NULL) - return (NULL); - } else { - /* Coalescing forward failed, so insert a new node. */ - node = xnode; - xnode = NULL; - node->addr = chunk; - node->size = size; - extent_tree_ad_insert(&swap_chunks_ad, node); - extent_tree_szad_insert(&swap_chunks_szad, node); - break; - } - } - /* Discard xnode if it ended up unused do to a race. */ - if (xnode != NULL) - base_node_dealloc(xnode); - - /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&swap_chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within swap_chunks_ad, so only - * remove/insert node from/into swap_chunks_szad. - */ - extent_tree_szad_remove(&swap_chunks_szad, prev); - extent_tree_ad_remove(&swap_chunks_ad, prev); - - extent_tree_szad_remove(&swap_chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - extent_tree_szad_insert(&swap_chunks_szad, node); - - base_node_dealloc(prev); - } - - return (node); -} - -bool -chunk_in_swap(void *chunk) -{ - bool ret; - - assert(swap_enabled); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)chunk >= (uintptr_t)swap_base - && (uintptr_t)chunk < (uintptr_t)swap_max) - ret = true; - else - ret = false; - malloc_mutex_unlock(&swap_mtx); - - return (ret); -} - -bool -chunk_dealloc_swap(void *chunk, size_t size) -{ - bool ret; - - assert(swap_enabled); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)chunk >= (uintptr_t)swap_base - && (uintptr_t)chunk < (uintptr_t)swap_max) { - extent_node_t *node; - - /* Try to coalesce with other unused chunks. */ - node = chunk_dealloc_swap_record(chunk, size); - if (node != NULL) { - chunk = node->addr; - size = node->size; - } - - /* - * Try to shrink the in-use memory if this chunk is at the end - * of the in-use memory. - */ - if ((void *)((uintptr_t)chunk + size) == swap_end) { - swap_end = (void *)((uintptr_t)swap_end - size); - - if (node != NULL) { - extent_tree_szad_remove(&swap_chunks_szad, - node); - extent_tree_ad_remove(&swap_chunks_ad, node); - base_node_dealloc(node); - } - } else - madvise(chunk, size, MADV_DONTNEED); - -#ifdef JEMALLOC_STATS - swap_avail += size; -#endif - ret = false; - goto RETURN; - } - - ret = true; -RETURN: - malloc_mutex_unlock(&swap_mtx); - return (ret); -} - -bool -chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed) -{ - bool ret; - unsigned i; - off_t off; - void *vaddr; - size_t cumsize, voff; - size_t sizes[nfds]; - - malloc_mutex_lock(&swap_mtx); - - /* Get file sizes. */ - for (i = 0, cumsize = 0; i < nfds; i++) { - off = lseek(fds[i], 0, SEEK_END); - if (off == ((off_t)-1)) { - ret = true; - goto RETURN; - } - if (PAGE_CEILING(off) != off) { - /* Truncate to a multiple of the page size. */ - off &= ~PAGE_MASK; - if (ftruncate(fds[i], off) != 0) { - ret = true; - goto RETURN; - } - } - sizes[i] = off; - if (cumsize + off < cumsize) { - /* - * Cumulative file size is greater than the total - * address space. Bail out while it's still obvious - * what the problem is. - */ - ret = true; - goto RETURN; - } - cumsize += off; - } - - /* Round down to a multiple of the chunk size. */ - cumsize &= ~chunksize_mask; - if (cumsize == 0) { - ret = true; - goto RETURN; - } - - /* - * Allocate a chunk-aligned region of anonymous memory, which will - * be the final location for the memory-mapped files. - */ - vaddr = chunk_alloc_mmap_noreserve(cumsize); - if (vaddr == NULL) { - ret = true; - goto RETURN; - } - - /* Overlay the files onto the anonymous mapping. */ - for (i = 0, voff = 0; i < nfds; i++) { - void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i], - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0); - if (addr == MAP_FAILED) { - char buf[BUFERROR_BUF]; - - - buferror(errno, buf, sizeof(buf)); - malloc_write( - ": Error in mmap(..., MAP_FIXED, ...): "); - malloc_write(buf); - malloc_write("\n"); - if (opt_abort) - abort(); - if (munmap(vaddr, voff) == -1) { - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); - } - ret = true; - goto RETURN; - } - assert(addr == (void *)((uintptr_t)vaddr + voff)); - - /* - * Tell the kernel that the mapping will be accessed randomly, - * and that it should not gratuitously sync pages to the - * filesystem. - */ -#ifdef MADV_RANDOM - madvise(addr, sizes[i], MADV_RANDOM); -#endif -#ifdef MADV_NOSYNC - madvise(addr, sizes[i], MADV_NOSYNC); -#endif - - voff += sizes[i]; - } - - swap_prezeroed = prezeroed; - swap_base = vaddr; - swap_end = swap_base; - swap_max = (void *)((uintptr_t)vaddr + cumsize); - - /* Copy the fds array for mallctl purposes. */ - swap_fds = (int *)base_alloc(nfds * sizeof(int)); - if (swap_fds == NULL) { - ret = true; - goto RETURN; - } - memcpy(swap_fds, fds, nfds * sizeof(int)); - swap_nfds = nfds; - -#ifdef JEMALLOC_STATS - swap_avail = cumsize; -#endif - - swap_enabled = true; - - ret = false; -RETURN: - malloc_mutex_unlock(&swap_mtx); - return (ret); -} - -bool -chunk_swap_boot(void) -{ - - if (malloc_mutex_init(&swap_mtx)) - return (true); - - swap_enabled = false; - swap_prezeroed = false; /* swap.* mallctl's depend on this. */ - swap_nfds = 0; - swap_fds = NULL; -#ifdef JEMALLOC_STATS - swap_avail = 0; -#endif - swap_base = NULL; - swap_end = NULL; - swap_max = NULL; - - extent_tree_szad_new(&swap_chunks_szad); - extent_tree_ad_new(&swap_chunks_ad); - - return (false); -} - -/******************************************************************************/ -#endif /* JEMALLOC_SWAP */ diff --git a/deps/jemalloc/src/ckh.c b/deps/jemalloc/src/ckh.c index 43fcc25239d..04c52966193 100644 --- a/deps/jemalloc/src/ckh.c +++ b/deps/jemalloc/src/ckh.c @@ -49,7 +49,7 @@ static void ckh_shrink(ckh_t *ckh); * Search bucket for key and return the cell number if found; SIZE_T_MAX * otherwise. */ -JEMALLOC_INLINE size_t +JEMALLOC_INLINE_C size_t ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) { ckhc_t *cell; @@ -67,29 +67,28 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) /* * Search table for key and return cell number if found; SIZE_T_MAX otherwise. */ -JEMALLOC_INLINE size_t +JEMALLOC_INLINE_C size_t ckh_isearch(ckh_t *ckh, const void *key) { - size_t hash1, hash2, bucket, cell; + size_t hashes[2], bucket, cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + ckh->hash(key, hashes); /* Search primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); if (cell != SIZE_T_MAX) return (cell); /* Search secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); return (cell); } -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, const void *data) { @@ -100,7 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * Cycle through the cells in the bucket, starting at a random position. * The randomness avoids worst-case search overhead as buckets fill up. */ - prn32(offset, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; @@ -121,13 +120,13 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * eviction/relocation procedure until either success or detection of an * eviction/relocation bucket cycle. */ -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, void const **argdata) { const void *key, *data, *tkey, *tdata; ckhc_t *cell; - size_t hash1, hash2, bucket, tbucket; + size_t hashes[2], bucket, tbucket; unsigned i; bucket = argbucket; @@ -142,7 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, * were an item for which both hashes indicated the same * bucket. */ - prn32(i, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; assert(cell->key != NULL); @@ -156,10 +155,11 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, #endif /* Find the alternate bucket for the evicted item. */ - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); - tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + ckh->hash(key, hashes); + tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (tbucket == bucket) { - tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) + - 1); /* * It may be that (tbucket == bucket) still, if the * item's hashes both indicate this bucket. However, @@ -190,22 +190,22 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, } } -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { - size_t hash1, hash2, bucket; + size_t hashes[2], bucket; const void *key = *argkey; const void *data = *argdata; - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + ckh->hash(key, hashes); /* Try to insert in primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) return (false); /* Try to insert in secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) return (false); @@ -219,7 +219,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) * Try to rebuild the hash table from scratch by inserting all items from the * old table into the new. */ -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { size_t count, i, nins; @@ -265,15 +265,15 @@ ckh_grow(ckh_t *ckh) size_t usize; lg_curcells++; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) { ret = true; - goto RETURN; + goto label_return; } tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); if (tab == NULL) { ret = true; - goto RETURN; + goto label_return; } /* Swap in new table. */ ttab = ckh->tab; @@ -293,7 +293,7 @@ ckh_grow(ckh_t *ckh) } ret = false; -RETURN: +label_return: return (ret); } @@ -310,7 +310,7 @@ ckh_shrink(ckh_t *ckh) */ lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) return; tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); @@ -362,7 +362,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) ckh->ninserts = 0; ckh->nrelocs = 0; #endif - ckh->prn_state = 42; /* Value doesn't really matter. */ + ckh->prng_state = 42; /* Value doesn't really matter. */ ckh->count = 0; /* @@ -383,23 +383,19 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) ckh->hash = hash; ckh->keycomp = keycomp; - usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); if (usize == 0) { ret = true; - goto RETURN; + goto label_return; } ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); if (ckh->tab == NULL) { ret = true; - goto RETURN; + goto label_return; } -#ifdef JEMALLOC_DEBUG - ckh->magic = CKH_MAGIC; -#endif - ret = false; -RETURN: +label_return: return (ret); } @@ -408,7 +404,6 @@ ckh_delete(ckh_t *ckh) { assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); #ifdef CKH_VERBOSE malloc_printf( @@ -423,9 +418,8 @@ ckh_delete(ckh_t *ckh) #endif idalloc(ckh->tab); -#ifdef JEMALLOC_DEBUG - memset(ckh, 0x5a, sizeof(ckh_t)); -#endif + if (config_debug) + memset(ckh, 0x5a, sizeof(ckh_t)); } size_t @@ -433,7 +427,6 @@ ckh_count(ckh_t *ckh) { assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); return (ckh->count); } @@ -464,7 +457,6 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data) bool ret; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); assert(ckh_search(ckh, key, NULL, NULL)); #ifdef CKH_COUNT @@ -474,12 +466,12 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data) while (ckh_try_insert(ckh, &key, &data)) { if (ckh_grow(ckh)) { ret = true; - goto RETURN; + goto label_return; } } ret = false; -RETURN: +label_return: return (ret); } @@ -489,7 +481,6 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) size_t cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { @@ -521,7 +512,6 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) size_t cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { @@ -536,31 +526,10 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) } void -ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +ckh_string_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; - - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - - h = hash(key, strlen((const char *)key), 0x94122f335b332aeaLLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(key, strlen((const char *)key), - 0x8432a476666bbc13LLU); - } - *hash1 = ret1; - *hash2 = ret2; + hash(key, strlen((const char *)key), 0x94122f33U, r_hash); } bool @@ -574,41 +543,16 @@ ckh_string_keycomp(const void *k1, const void *k2) } void -ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2) +ckh_pointer_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; union { const void *v; - uint64_t i; + size_t i; } u; - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - assert(sizeof(u.v) == sizeof(u.i)); -#if (LG_SIZEOF_PTR != LG_SIZEOF_INT) - u.i = 0; -#endif u.v = key; - h = hash(&u.i, sizeof(u.i), 0xd983396e68886082LLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - assert(SIZEOF_PTR == 8); - ret1 = h; - ret2 = hash(&u.i, sizeof(u.i), 0x5e2be9aff8709a5dLLU); - } - - *hash1 = ret1; - *hash2 = ret2; + hash(&u.i, sizeof(u.i), 0xd983396eU, r_hash); } bool diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index e5336d36949..cc2c5aef570 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -8,14 +8,38 @@ * ctl_mtx protects the following: * - ctl_stats.* * - opt_prof_active - * - swap_enabled - * - swap_prezeroed */ static malloc_mutex_t ctl_mtx; static bool ctl_initialized; static uint64_t ctl_epoch; static ctl_stats_t ctl_stats; +/******************************************************************************/ +/* Helpers for named and indexed nodes. */ + +static inline const ctl_named_node_t * +ctl_named_node(const ctl_node_t *node) +{ + + return ((node->named) ? (const ctl_named_node_t *)node : NULL); +} + +static inline const ctl_named_node_t * +ctl_named_children(const ctl_named_node_t *node, int index) +{ + const ctl_named_node_t *children = ctl_named_node(node->children); + + return (children ? &children[index] : NULL); +} + +static inline const ctl_indexed_node_t * +ctl_indexed_node(const ctl_node_t *node) +{ + + return ((node->named == false) ? (const ctl_indexed_node_t *)node : + NULL); +} + /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -24,20 +48,17 @@ static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ size_t *oldlenp, void *newp, size_t newlen); #define INDEX_PROTO(n) \ -const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ - size_t i); +static const ctl_named_node_t *n##_index(const size_t *mib, \ + size_t miblen, size_t i); -#ifdef JEMALLOC_STATS static bool ctl_arena_init(ctl_arena_stats_t *astats); -#endif static void ctl_arena_clear(ctl_arena_stats_t *astats); -#ifdef JEMALLOC_STATS static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena); static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats); -#endif static void ctl_arena_refresh(arena_t *arena, unsigned i); +static bool ctl_grow(void); static void ctl_refresh(void); static bool ctl_init(void); static int ctl_lookup(const char *name, ctl_node_t const **nodesp, @@ -45,67 +66,56 @@ static int ctl_lookup(const char *name, ctl_node_t const **nodesp, CTL_PROTO(version) CTL_PROTO(epoch) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(tcache_flush) -#endif +CTL_PROTO(thread_tcache_enabled) +CTL_PROTO(thread_tcache_flush) CTL_PROTO(thread_arena) -#ifdef JEMALLOC_STATS CTL_PROTO(thread_allocated) CTL_PROTO(thread_allocatedp) CTL_PROTO(thread_deallocated) CTL_PROTO(thread_deallocatedp) -#endif CTL_PROTO(config_debug) CTL_PROTO(config_dss) -CTL_PROTO(config_dynamic_page_shift) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) +CTL_PROTO(config_mremap) +CTL_PROTO(config_munmap) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) CTL_PROTO(config_stats) -CTL_PROTO(config_swap) -CTL_PROTO(config_sysv) CTL_PROTO(config_tcache) -CTL_PROTO(config_tiny) CTL_PROTO(config_tls) +CTL_PROTO(config_utrace) +CTL_PROTO(config_valgrind) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) -CTL_PROTO(opt_lg_qspace_max) -CTL_PROTO(opt_lg_cspace_max) +CTL_PROTO(opt_dss) CTL_PROTO(opt_lg_chunk) CTL_PROTO(opt_narenas) CTL_PROTO(opt_lg_dirty_mult) CTL_PROTO(opt_stats_print) -#ifdef JEMALLOC_FILL CTL_PROTO(opt_junk) CTL_PROTO(opt_zero) -#endif -#ifdef JEMALLOC_SYSV -CTL_PROTO(opt_sysv) -#endif -#ifdef JEMALLOC_XMALLOC +CTL_PROTO(opt_quarantine) +CTL_PROTO(opt_redzone) +CTL_PROTO(opt_utrace) +CTL_PROTO(opt_valgrind) CTL_PROTO(opt_xmalloc) -#endif -#ifdef JEMALLOC_TCACHE CTL_PROTO(opt_tcache) -CTL_PROTO(opt_lg_tcache_gc_sweep) -#endif -#ifdef JEMALLOC_PROF +CTL_PROTO(opt_lg_tcache_max) CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) -CTL_PROTO(opt_lg_prof_bt_max) CTL_PROTO(opt_lg_prof_sample) CTL_PROTO(opt_lg_prof_interval) CTL_PROTO(opt_prof_gdump) +CTL_PROTO(opt_prof_final) CTL_PROTO(opt_prof_leak) CTL_PROTO(opt_prof_accum) -CTL_PROTO(opt_lg_prof_tcmax) -#endif -#ifdef JEMALLOC_SWAP -CTL_PROTO(opt_overcommit) -#endif +CTL_PROTO(arena_i_purge) +static void arena_purge(unsigned arena_ind); +CTL_PROTO(arena_i_dss) +INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_run_size) @@ -115,39 +125,16 @@ INDEX_PROTO(arenas_lrun_i) CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_initialized) CTL_PROTO(arenas_quantum) -CTL_PROTO(arenas_cacheline) -CTL_PROTO(arenas_subpage) -CTL_PROTO(arenas_pagesize) -CTL_PROTO(arenas_chunksize) -#ifdef JEMALLOC_TINY -CTL_PROTO(arenas_tspace_min) -CTL_PROTO(arenas_tspace_max) -#endif -CTL_PROTO(arenas_qspace_min) -CTL_PROTO(arenas_qspace_max) -CTL_PROTO(arenas_cspace_min) -CTL_PROTO(arenas_cspace_max) -CTL_PROTO(arenas_sspace_min) -CTL_PROTO(arenas_sspace_max) -#ifdef JEMALLOC_TCACHE +CTL_PROTO(arenas_page) CTL_PROTO(arenas_tcache_max) -#endif -CTL_PROTO(arenas_ntbins) -CTL_PROTO(arenas_nqbins) -CTL_PROTO(arenas_ncbins) -CTL_PROTO(arenas_nsbins) CTL_PROTO(arenas_nbins) -#ifdef JEMALLOC_TCACHE CTL_PROTO(arenas_nhbins) -#endif CTL_PROTO(arenas_nlruns) CTL_PROTO(arenas_purge) -#ifdef JEMALLOC_PROF +CTL_PROTO(arenas_extend) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) CTL_PROTO(prof_interval) -#endif -#ifdef JEMALLOC_STATS CTL_PROTO(stats_chunks_current) CTL_PROTO(stats_chunks_total) CTL_PROTO(stats_chunks_high) @@ -166,46 +153,30 @@ CTL_PROTO(stats_arenas_i_bins_j_allocated) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) -#ifdef JEMALLOC_TCACHE CTL_PROTO(stats_arenas_i_bins_j_nfills) CTL_PROTO(stats_arenas_i_bins_j_nflushes) -#endif CTL_PROTO(stats_arenas_i_bins_j_nruns) CTL_PROTO(stats_arenas_i_bins_j_nreruns) -CTL_PROTO(stats_arenas_i_bins_j_highruns) CTL_PROTO(stats_arenas_i_bins_j_curruns) INDEX_PROTO(stats_arenas_i_bins_j) CTL_PROTO(stats_arenas_i_lruns_j_nmalloc) CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) CTL_PROTO(stats_arenas_i_lruns_j_nrequests) -CTL_PROTO(stats_arenas_i_lruns_j_highruns) CTL_PROTO(stats_arenas_i_lruns_j_curruns) INDEX_PROTO(stats_arenas_i_lruns_j) -#endif CTL_PROTO(stats_arenas_i_nthreads) +CTL_PROTO(stats_arenas_i_dss) CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pdirty) -#ifdef JEMALLOC_STATS CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_npurge) CTL_PROTO(stats_arenas_i_nmadvise) CTL_PROTO(stats_arenas_i_purged) -#endif INDEX_PROTO(stats_arenas_i) -#ifdef JEMALLOC_STATS CTL_PROTO(stats_cactive) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) CTL_PROTO(stats_mapped) -#endif -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_PROTO(swap_avail) -# endif -CTL_PROTO(swap_prezeroed) -CTL_PROTO(swap_nfds) -CTL_PROTO(swap_fds) -#endif /******************************************************************************/ /* mallctl tree. */ @@ -213,296 +184,239 @@ CTL_PROTO(swap_fds) /* Maximum tree depth. */ #define CTL_MAX_DEPTH 6 -#define NAME(n) true, {.named = {n -#define CHILD(c) sizeof(c##_node) / sizeof(ctl_node_t), c##_node}}, NULL -#define CTL(c) 0, NULL}}, c##_ctl +#define NAME(n) {true}, n +#define CHILD(t, c) \ + sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ + (ctl_node_t *)c##_node, \ + NULL +#define CTL(c) 0, NULL, c##_ctl /* * Only handles internal indexed nodes, since there are currently no external * ones. */ -#define INDEX(i) false, {.indexed = {i##_index}}, NULL +#define INDEX(i) {false}, i##_index -#ifdef JEMALLOC_TCACHE -static const ctl_node_t tcache_node[] = { - {NAME("flush"), CTL(tcache_flush)} +static const ctl_named_node_t tcache_node[] = { + {NAME("enabled"), CTL(thread_tcache_enabled)}, + {NAME("flush"), CTL(thread_tcache_flush)} }; -#endif -static const ctl_node_t thread_node[] = { - {NAME("arena"), CTL(thread_arena)} -#ifdef JEMALLOC_STATS - , +static const ctl_named_node_t thread_node[] = { + {NAME("arena"), CTL(thread_arena)}, {NAME("allocated"), CTL(thread_allocated)}, {NAME("allocatedp"), CTL(thread_allocatedp)}, {NAME("deallocated"), CTL(thread_deallocated)}, - {NAME("deallocatedp"), CTL(thread_deallocatedp)} -#endif + {NAME("deallocatedp"), CTL(thread_deallocatedp)}, + {NAME("tcache"), CHILD(named, tcache)} }; -static const ctl_node_t config_node[] = { +static const ctl_named_node_t config_node[] = { {NAME("debug"), CTL(config_debug)}, {NAME("dss"), CTL(config_dss)}, - {NAME("dynamic_page_shift"), CTL(config_dynamic_page_shift)}, {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, + {NAME("mremap"), CTL(config_mremap)}, + {NAME("munmap"), CTL(config_munmap)}, {NAME("prof"), CTL(config_prof)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, {NAME("stats"), CTL(config_stats)}, - {NAME("swap"), CTL(config_swap)}, - {NAME("sysv"), CTL(config_sysv)}, {NAME("tcache"), CTL(config_tcache)}, - {NAME("tiny"), CTL(config_tiny)}, {NAME("tls"), CTL(config_tls)}, + {NAME("utrace"), CTL(config_utrace)}, + {NAME("valgrind"), CTL(config_valgrind)}, {NAME("xmalloc"), CTL(config_xmalloc)} }; -static const ctl_node_t opt_node[] = { +static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, - {NAME("lg_qspace_max"), CTL(opt_lg_qspace_max)}, - {NAME("lg_cspace_max"), CTL(opt_lg_cspace_max)}, + {NAME("dss"), CTL(opt_dss)}, {NAME("lg_chunk"), CTL(opt_lg_chunk)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, - {NAME("stats_print"), CTL(opt_stats_print)} -#ifdef JEMALLOC_FILL - , + {NAME("stats_print"), CTL(opt_stats_print)}, {NAME("junk"), CTL(opt_junk)}, - {NAME("zero"), CTL(opt_zero)} -#endif -#ifdef JEMALLOC_SYSV - , - {NAME("sysv"), CTL(opt_sysv)} -#endif -#ifdef JEMALLOC_XMALLOC - , - {NAME("xmalloc"), CTL(opt_xmalloc)} -#endif -#ifdef JEMALLOC_TCACHE - , + {NAME("zero"), CTL(opt_zero)}, + {NAME("quarantine"), CTL(opt_quarantine)}, + {NAME("redzone"), CTL(opt_redzone)}, + {NAME("utrace"), CTL(opt_utrace)}, + {NAME("valgrind"), CTL(opt_valgrind)}, + {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, - {NAME("lg_tcache_gc_sweep"), CTL(opt_lg_tcache_gc_sweep)} -#endif -#ifdef JEMALLOC_PROF - , + {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, {NAME("prof_active"), CTL(opt_prof_active)}, - {NAME("lg_prof_bt_max"), CTL(opt_lg_prof_bt_max)}, {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, {NAME("prof_gdump"), CTL(opt_prof_gdump)}, + {NAME("prof_final"), CTL(opt_prof_final)}, {NAME("prof_leak"), CTL(opt_prof_leak)}, - {NAME("prof_accum"), CTL(opt_prof_accum)}, - {NAME("lg_prof_tcmax"), CTL(opt_lg_prof_tcmax)} -#endif -#ifdef JEMALLOC_SWAP - , - {NAME("overcommit"), CTL(opt_overcommit)} -#endif + {NAME("prof_accum"), CTL(opt_prof_accum)} +}; + +static const ctl_named_node_t arena_i_node[] = { + {NAME("purge"), CTL(arena_i_purge)}, + {NAME("dss"), CTL(arena_i_dss)} +}; +static const ctl_named_node_t super_arena_i_node[] = { + {NAME(""), CHILD(named, arena_i)} }; -static const ctl_node_t arenas_bin_i_node[] = { +static const ctl_indexed_node_t arena_node[] = { + {INDEX(arena_i)} +}; + +static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, {NAME("run_size"), CTL(arenas_bin_i_run_size)} }; -static const ctl_node_t super_arenas_bin_i_node[] = { - {NAME(""), CHILD(arenas_bin_i)} +static const ctl_named_node_t super_arenas_bin_i_node[] = { + {NAME(""), CHILD(named, arenas_bin_i)} }; -static const ctl_node_t arenas_bin_node[] = { +static const ctl_indexed_node_t arenas_bin_node[] = { {INDEX(arenas_bin_i)} }; -static const ctl_node_t arenas_lrun_i_node[] = { +static const ctl_named_node_t arenas_lrun_i_node[] = { {NAME("size"), CTL(arenas_lrun_i_size)} }; -static const ctl_node_t super_arenas_lrun_i_node[] = { - {NAME(""), CHILD(arenas_lrun_i)} +static const ctl_named_node_t super_arenas_lrun_i_node[] = { + {NAME(""), CHILD(named, arenas_lrun_i)} }; -static const ctl_node_t arenas_lrun_node[] = { +static const ctl_indexed_node_t arenas_lrun_node[] = { {INDEX(arenas_lrun_i)} }; -static const ctl_node_t arenas_node[] = { +static const ctl_named_node_t arenas_node[] = { {NAME("narenas"), CTL(arenas_narenas)}, {NAME("initialized"), CTL(arenas_initialized)}, {NAME("quantum"), CTL(arenas_quantum)}, - {NAME("cacheline"), CTL(arenas_cacheline)}, - {NAME("subpage"), CTL(arenas_subpage)}, - {NAME("pagesize"), CTL(arenas_pagesize)}, - {NAME("chunksize"), CTL(arenas_chunksize)}, -#ifdef JEMALLOC_TINY - {NAME("tspace_min"), CTL(arenas_tspace_min)}, - {NAME("tspace_max"), CTL(arenas_tspace_max)}, -#endif - {NAME("qspace_min"), CTL(arenas_qspace_min)}, - {NAME("qspace_max"), CTL(arenas_qspace_max)}, - {NAME("cspace_min"), CTL(arenas_cspace_min)}, - {NAME("cspace_max"), CTL(arenas_cspace_max)}, - {NAME("sspace_min"), CTL(arenas_sspace_min)}, - {NAME("sspace_max"), CTL(arenas_sspace_max)}, -#ifdef JEMALLOC_TCACHE + {NAME("page"), CTL(arenas_page)}, {NAME("tcache_max"), CTL(arenas_tcache_max)}, -#endif - {NAME("ntbins"), CTL(arenas_ntbins)}, - {NAME("nqbins"), CTL(arenas_nqbins)}, - {NAME("ncbins"), CTL(arenas_ncbins)}, - {NAME("nsbins"), CTL(arenas_nsbins)}, {NAME("nbins"), CTL(arenas_nbins)}, -#ifdef JEMALLOC_TCACHE {NAME("nhbins"), CTL(arenas_nhbins)}, -#endif - {NAME("bin"), CHILD(arenas_bin)}, + {NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("nlruns"), CTL(arenas_nlruns)}, - {NAME("lrun"), CHILD(arenas_lrun)}, - {NAME("purge"), CTL(arenas_purge)} + {NAME("lrun"), CHILD(indexed, arenas_lrun)}, + {NAME("purge"), CTL(arenas_purge)}, + {NAME("extend"), CTL(arenas_extend)} }; -#ifdef JEMALLOC_PROF -static const ctl_node_t prof_node[] = { +static const ctl_named_node_t prof_node[] = { {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, {NAME("interval"), CTL(prof_interval)} }; -#endif -#ifdef JEMALLOC_STATS -static const ctl_node_t stats_chunks_node[] = { +static const ctl_named_node_t stats_chunks_node[] = { {NAME("current"), CTL(stats_chunks_current)}, {NAME("total"), CTL(stats_chunks_total)}, {NAME("high"), CTL(stats_chunks_high)} }; -static const ctl_node_t stats_huge_node[] = { +static const ctl_named_node_t stats_huge_node[] = { {NAME("allocated"), CTL(stats_huge_allocated)}, {NAME("nmalloc"), CTL(stats_huge_nmalloc)}, {NAME("ndalloc"), CTL(stats_huge_ndalloc)} }; -static const ctl_node_t stats_arenas_i_small_node[] = { +static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} }; -static const ctl_node_t stats_arenas_i_large_node[] = { +static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; -static const ctl_node_t stats_arenas_i_bins_j_node[] = { +static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, -#ifdef JEMALLOC_TCACHE {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, -#endif {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, - {NAME("highruns"), CTL(stats_arenas_i_bins_j_highruns)}, {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} }; -static const ctl_node_t super_stats_arenas_i_bins_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_bins_j)} +static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_bins_j)} }; -static const ctl_node_t stats_arenas_i_bins_node[] = { +static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { {INDEX(stats_arenas_i_bins_j)} }; -static const ctl_node_t stats_arenas_i_lruns_j_node[] = { +static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = { {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, - {NAME("highruns"), CTL(stats_arenas_i_lruns_j_highruns)}, {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} }; -static const ctl_node_t super_stats_arenas_i_lruns_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_lruns_j)} +static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_lruns_j)} }; -static const ctl_node_t stats_arenas_i_lruns_node[] = { +static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = { {INDEX(stats_arenas_i_lruns_j)} }; -#endif -static const ctl_node_t stats_arenas_i_node[] = { +static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, + {NAME("dss"), CTL(stats_arenas_i_dss)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)}, - {NAME("pdirty"), CTL(stats_arenas_i_pdirty)} -#ifdef JEMALLOC_STATS - , + {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("purged"), CTL(stats_arenas_i_purged)}, - {NAME("small"), CHILD(stats_arenas_i_small)}, - {NAME("large"), CHILD(stats_arenas_i_large)}, - {NAME("bins"), CHILD(stats_arenas_i_bins)}, - {NAME("lruns"), CHILD(stats_arenas_i_lruns)} -#endif + {NAME("small"), CHILD(named, stats_arenas_i_small)}, + {NAME("large"), CHILD(named, stats_arenas_i_large)}, + {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, + {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)} }; -static const ctl_node_t super_stats_arenas_i_node[] = { - {NAME(""), CHILD(stats_arenas_i)} +static const ctl_named_node_t super_stats_arenas_i_node[] = { + {NAME(""), CHILD(named, stats_arenas_i)} }; -static const ctl_node_t stats_arenas_node[] = { +static const ctl_indexed_node_t stats_arenas_node[] = { {INDEX(stats_arenas_i)} }; -static const ctl_node_t stats_node[] = { -#ifdef JEMALLOC_STATS +static const ctl_named_node_t stats_node[] = { {NAME("cactive"), CTL(stats_cactive)}, {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, {NAME("mapped"), CTL(stats_mapped)}, - {NAME("chunks"), CHILD(stats_chunks)}, - {NAME("huge"), CHILD(stats_huge)}, -#endif - {NAME("arenas"), CHILD(stats_arenas)} + {NAME("chunks"), CHILD(named, stats_chunks)}, + {NAME("huge"), CHILD(named, stats_huge)}, + {NAME("arenas"), CHILD(indexed, stats_arenas)} }; -#ifdef JEMALLOC_SWAP -static const ctl_node_t swap_node[] = { -# ifdef JEMALLOC_STATS - {NAME("avail"), CTL(swap_avail)}, -# endif - {NAME("prezeroed"), CTL(swap_prezeroed)}, - {NAME("nfds"), CTL(swap_nfds)}, - {NAME("fds"), CTL(swap_fds)} -}; -#endif - -static const ctl_node_t root_node[] = { +static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, -#ifdef JEMALLOC_TCACHE - {NAME("tcache"), CHILD(tcache)}, -#endif - {NAME("thread"), CHILD(thread)}, - {NAME("config"), CHILD(config)}, - {NAME("opt"), CHILD(opt)}, - {NAME("arenas"), CHILD(arenas)}, -#ifdef JEMALLOC_PROF - {NAME("prof"), CHILD(prof)}, -#endif - {NAME("stats"), CHILD(stats)} -#ifdef JEMALLOC_SWAP - , - {NAME("swap"), CHILD(swap)} -#endif + {NAME("thread"), CHILD(named, thread)}, + {NAME("config"), CHILD(named, config)}, + {NAME("opt"), CHILD(named, opt)}, + {NAME("arena"), CHILD(indexed, arena)}, + {NAME("arenas"), CHILD(named, arenas)}, + {NAME("prof"), CHILD(named, prof)}, + {NAME("stats"), CHILD(named, stats)} }; -static const ctl_node_t super_root_node[] = { - {NAME(""), CHILD(root)} +static const ctl_named_node_t super_root_node[] = { + {NAME(""), CHILD(named, root)} }; #undef NAME @@ -512,17 +426,10 @@ static const ctl_node_t super_root_node[] = { /******************************************************************************/ -#ifdef JEMALLOC_STATS static bool ctl_arena_init(ctl_arena_stats_t *astats) { - if (astats->bstats == NULL) { - astats->bstats = (malloc_bin_stats_t *)base_alloc(nbins * - sizeof(malloc_bin_stats_t)); - if (astats->bstats == NULL) - return (true); - } if (astats->lstats == NULL) { astats->lstats = (malloc_large_stats_t *)base_alloc(nlclasses * sizeof(malloc_large_stats_t)); @@ -532,35 +439,35 @@ ctl_arena_init(ctl_arena_stats_t *astats) return (false); } -#endif static void ctl_arena_clear(ctl_arena_stats_t *astats) { + astats->dss = dss_prec_names[dss_prec_limit]; astats->pactive = 0; astats->pdirty = 0; -#ifdef JEMALLOC_STATS - memset(&astats->astats, 0, sizeof(arena_stats_t)); - astats->allocated_small = 0; - astats->nmalloc_small = 0; - astats->ndalloc_small = 0; - astats->nrequests_small = 0; - memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t)); - memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); -#endif + if (config_stats) { + memset(&astats->astats, 0, sizeof(arena_stats_t)); + astats->allocated_small = 0; + astats->nmalloc_small = 0; + astats->ndalloc_small = 0; + astats->nrequests_small = 0; + memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t)); + memset(astats->lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); + } } -#ifdef JEMALLOC_STATS static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) { unsigned i; - arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, - &cstats->astats, cstats->bstats, cstats->lstats); + arena_stats_merge(arena, &cstats->dss, &cstats->pactive, + &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats); - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { cstats->allocated_small += cstats->bstats[i].allocated; cstats->nmalloc_small += cstats->bstats[i].nmalloc; cstats->ndalloc_small += cstats->bstats[i].ndalloc; @@ -595,86 +502,153 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; sstats->lstats[i].nrequests += astats->lstats[i].nrequests; - sstats->lstats[i].highruns += astats->lstats[i].highruns; sstats->lstats[i].curruns += astats->lstats[i].curruns; } - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { sstats->bstats[i].allocated += astats->bstats[i].allocated; sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sstats->bstats[i].nrequests += astats->bstats[i].nrequests; -#ifdef JEMALLOC_TCACHE - sstats->bstats[i].nfills += astats->bstats[i].nfills; - sstats->bstats[i].nflushes += astats->bstats[i].nflushes; -#endif + if (config_tcache) { + sstats->bstats[i].nfills += astats->bstats[i].nfills; + sstats->bstats[i].nflushes += + astats->bstats[i].nflushes; + } sstats->bstats[i].nruns += astats->bstats[i].nruns; sstats->bstats[i].reruns += astats->bstats[i].reruns; - sstats->bstats[i].highruns += astats->bstats[i].highruns; sstats->bstats[i].curruns += astats->bstats[i].curruns; } } -#endif static void ctl_arena_refresh(arena_t *arena, unsigned i) { ctl_arena_stats_t *astats = &ctl_stats.arenas[i]; - ctl_arena_stats_t *sstats = &ctl_stats.arenas[narenas]; + ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas]; ctl_arena_clear(astats); sstats->nthreads += astats->nthreads; -#ifdef JEMALLOC_STATS - ctl_arena_stats_amerge(astats, arena); - /* Merge into sum stats as well. */ - ctl_arena_stats_smerge(sstats, astats); -#else - astats->pactive += arena->nactive; - astats->pdirty += arena->ndirty; - /* Merge into sum stats as well. */ - sstats->pactive += arena->nactive; - sstats->pdirty += arena->ndirty; -#endif + if (config_stats) { + ctl_arena_stats_amerge(astats, arena); + /* Merge into sum stats as well. */ + ctl_arena_stats_smerge(sstats, astats); + } else { + astats->pactive += arena->nactive; + astats->pdirty += arena->ndirty; + /* Merge into sum stats as well. */ + sstats->pactive += arena->nactive; + sstats->pdirty += arena->ndirty; + } +} + +static bool +ctl_grow(void) +{ + ctl_arena_stats_t *astats; + arena_t **tarenas; + + /* Allocate extended arena stats and arenas arrays. */ + astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) * + sizeof(ctl_arena_stats_t)); + if (astats == NULL) + return (true); + tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * + sizeof(arena_t *)); + if (tarenas == NULL) { + idalloc(astats); + return (true); + } + + /* Initialize the new astats element. */ + memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * + sizeof(ctl_arena_stats_t)); + memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); + if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { + idalloc(tarenas); + idalloc(astats); + return (true); + } + /* Swap merged stats to their new location. */ + { + ctl_arena_stats_t tstats; + memcpy(&tstats, &astats[ctl_stats.narenas], + sizeof(ctl_arena_stats_t)); + memcpy(&astats[ctl_stats.narenas], + &astats[ctl_stats.narenas + 1], sizeof(ctl_arena_stats_t)); + memcpy(&astats[ctl_stats.narenas + 1], &tstats, + sizeof(ctl_arena_stats_t)); + } + /* Initialize the new arenas element. */ + tarenas[ctl_stats.narenas] = NULL; + { + arena_t **arenas_old = arenas; + /* + * Swap extended arenas array into place. Although ctl_mtx + * protects this function from other threads extending the + * array, it does not protect from other threads mutating it + * (i.e. initializing arenas and setting array elements to + * point to them). Therefore, array copying must happen under + * the protection of arenas_lock. + */ + malloc_mutex_lock(&arenas_lock); + arenas = tarenas; + memcpy(arenas, arenas_old, ctl_stats.narenas * + sizeof(arena_t *)); + narenas_total++; + arenas_extend(narenas_total - 1); + malloc_mutex_unlock(&arenas_lock); + /* + * Deallocate arenas_old only if it came from imalloc() (not + * base_alloc()). + */ + if (ctl_stats.narenas != narenas_auto) + idalloc(arenas_old); + } + ctl_stats.arenas = astats; + ctl_stats.narenas++; + + return (false); } static void ctl_refresh(void) { unsigned i; - arena_t *tarenas[narenas]; - -#ifdef JEMALLOC_STATS - malloc_mutex_lock(&chunks_mtx); - ctl_stats.chunks.current = stats_chunks.curchunks; - ctl_stats.chunks.total = stats_chunks.nchunks; - ctl_stats.chunks.high = stats_chunks.highchunks; - malloc_mutex_unlock(&chunks_mtx); - - malloc_mutex_lock(&huge_mtx); - ctl_stats.huge.allocated = huge_allocated; - ctl_stats.huge.nmalloc = huge_nmalloc; - ctl_stats.huge.ndalloc = huge_ndalloc; - malloc_mutex_unlock(&huge_mtx); -#endif + VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); + + if (config_stats) { + malloc_mutex_lock(&chunks_mtx); + ctl_stats.chunks.current = stats_chunks.curchunks; + ctl_stats.chunks.total = stats_chunks.nchunks; + ctl_stats.chunks.high = stats_chunks.highchunks; + malloc_mutex_unlock(&chunks_mtx); + + malloc_mutex_lock(&huge_mtx); + ctl_stats.huge.allocated = huge_allocated; + ctl_stats.huge.nmalloc = huge_nmalloc; + ctl_stats.huge.ndalloc = huge_ndalloc; + malloc_mutex_unlock(&huge_mtx); + } /* * Clear sum stats, since they will be merged into by * ctl_arena_refresh(). */ - ctl_stats.arenas[narenas].nthreads = 0; - ctl_arena_clear(&ctl_stats.arenas[narenas]); + ctl_stats.arenas[ctl_stats.narenas].nthreads = 0; + ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]); malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - for (i = 0; i < narenas; i++) { + memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); + for (i = 0; i < ctl_stats.narenas; i++) { if (arenas[i] != NULL) ctl_stats.arenas[i].nthreads = arenas[i]->nthreads; else ctl_stats.arenas[i].nthreads = 0; } malloc_mutex_unlock(&arenas_lock); - for (i = 0; i < narenas; i++) { + for (i = 0; i < ctl_stats.narenas; i++) { bool initialized = (tarenas[i] != NULL); ctl_stats.arenas[i].initialized = initialized; @@ -682,20 +656,16 @@ ctl_refresh(void) ctl_arena_refresh(tarenas[i], i); } -#ifdef JEMALLOC_STATS - ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small - + ctl_stats.arenas[narenas].astats.allocated_large - + ctl_stats.huge.allocated; - ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT) - + ctl_stats.huge.allocated; - ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); - -# ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); - ctl_stats.swap_avail = swap_avail; - malloc_mutex_unlock(&swap_mtx); -# endif -#endif + if (config_stats) { + ctl_stats.allocated = + ctl_stats.arenas[ctl_stats.narenas].allocated_small + + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large + + ctl_stats.huge.allocated; + ctl_stats.active = + (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE) + + ctl_stats.huge.allocated; + ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); + } ctl_epoch++; } @@ -707,21 +677,19 @@ ctl_init(void) malloc_mutex_lock(&ctl_mtx); if (ctl_initialized == false) { -#ifdef JEMALLOC_STATS - unsigned i; -#endif - /* * Allocate space for one extra arena stats element, which * contains summed stats across all arenas. */ + assert(narenas_auto == narenas_total_get()); + ctl_stats.narenas = narenas_auto; ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc( - (narenas + 1) * sizeof(ctl_arena_stats_t)); + (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; - goto RETURN; + goto label_return; } - memset(ctl_stats.arenas, 0, (narenas + 1) * + memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); /* @@ -729,15 +697,16 @@ ctl_init(void) * ever get used. Lazy initialization would allow errors to * cause inconsistent state to be viewable by the application. */ -#ifdef JEMALLOC_STATS - for (i = 0; i <= narenas; i++) { - if (ctl_arena_init(&ctl_stats.arenas[i])) { - ret = true; - goto RETURN; + if (config_stats) { + unsigned i; + for (i = 0; i <= ctl_stats.narenas; i++) { + if (ctl_arena_init(&ctl_stats.arenas[i])) { + ret = true; + goto label_return; + } } } -#endif - ctl_stats.arenas[narenas].initialized = true; + ctl_stats.arenas[ctl_stats.narenas].initialized = true; ctl_epoch = 0; ctl_refresh(); @@ -745,7 +714,7 @@ ctl_init(void) } ret = false; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -757,7 +726,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, int ret; const char *elm, *tdot, *dot; size_t elen, i, j; - const ctl_node_t *node; + const ctl_named_node_t *node; elm = name; /* Equivalent to strchrnul(). */ @@ -765,54 +734,53 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); if (elen == 0) { ret = ENOENT; - goto RETURN; + goto label_return; } node = super_root_node; for (i = 0; i < *depthp; i++) { - assert(node->named); - assert(node->u.named.nchildren > 0); - if (node->u.named.children[0].named) { - const ctl_node_t *pnode = node; + assert(node); + assert(node->nchildren > 0); + if (ctl_named_node(node->children) != NULL) { + const ctl_named_node_t *pnode = node; /* Children are named. */ - for (j = 0; j < node->u.named.nchildren; j++) { - const ctl_node_t *child = - &node->u.named.children[j]; - if (strlen(child->u.named.name) == elen - && strncmp(elm, child->u.named.name, - elen) == 0) { + for (j = 0; j < node->nchildren; j++) { + const ctl_named_node_t *child = + ctl_named_children(node, j); + if (strlen(child->name) == elen && + strncmp(elm, child->name, elen) == 0) { node = child; if (nodesp != NULL) - nodesp[i] = node; + nodesp[i] = + (const ctl_node_t *)node; mibp[i] = j; break; } } if (node == pnode) { ret = ENOENT; - goto RETURN; + goto label_return; } } else { - unsigned long index; - const ctl_node_t *inode; + uintmax_t index; + const ctl_indexed_node_t *inode; /* Children are indexed. */ - index = strtoul(elm, NULL, 10); - if (index == ULONG_MAX) { + index = malloc_strtoumax(elm, NULL, 10); + if (index == UINTMAX_MAX || index > SIZE_T_MAX) { ret = ENOENT; - goto RETURN; + goto label_return; } - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mibp, *depthp, - index); + inode = ctl_indexed_node(node->children); + node = inode->index(mibp, *depthp, (size_t)index); if (node == NULL) { ret = ENOENT; - goto RETURN; + goto label_return; } if (nodesp != NULL) - nodesp[i] = node; + nodesp[i] = (const ctl_node_t *)node; mibp[i] = (size_t)index; } @@ -824,7 +792,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, * in this path through the tree. */ ret = ENOENT; - goto RETURN; + goto label_return; } /* Complete lookup successful. */ *depthp = i + 1; @@ -835,7 +803,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, if (*dot == '\0') { /* No more elements. */ ret = ENOENT; - goto RETURN; + goto label_return; } elm = &dot[1]; dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : @@ -844,7 +812,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, } ret = 0; -RETURN: +label_return: return (ret); } @@ -856,25 +824,27 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t depth; ctl_node_t const *nodes[CTL_MAX_DEPTH]; size_t mib[CTL_MAX_DEPTH]; + const ctl_named_node_t *node; if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } depth = CTL_MAX_DEPTH; ret = ctl_lookup(name, nodes, mib, &depth); if (ret != 0) - goto RETURN; + goto label_return; - if (nodes[depth-1]->ctl == NULL) { + node = ctl_named_node(nodes[depth-1]); + if (node != NULL && node->ctl) + ret = node->ctl(mib, depth, oldp, oldlenp, newp, newlen); + else { /* The name refers to a partial path through the ctl tree. */ ret = ENOENT; - goto RETURN; } - ret = nodes[depth-1]->ctl(mib, depth, oldp, oldlenp, newp, newlen); -RETURN: +label_return: return(ret); } @@ -885,11 +855,11 @@ ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } ret = ctl_lookup(name, NULL, mibp, miblenp); -RETURN: +label_return: return(ret); } @@ -898,46 +868,48 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - const ctl_node_t *node; + const ctl_named_node_t *node; size_t i; if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } /* Iterate down the tree. */ node = super_root_node; for (i = 0; i < miblen; i++) { - if (node->u.named.children[0].named) { + assert(node); + assert(node->nchildren > 0); + if (ctl_named_node(node->children) != NULL) { /* Children are named. */ - if (node->u.named.nchildren <= mib[i]) { + if (node->nchildren <= mib[i]) { ret = ENOENT; - goto RETURN; + goto label_return; } - node = &node->u.named.children[mib[i]]; + node = ctl_named_children(node, mib[i]); } else { - const ctl_node_t *inode; + const ctl_indexed_node_t *inode; /* Indexed element. */ - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mib, miblen, mib[i]); + inode = ctl_indexed_node(node->children); + node = inode->index(mib, miblen, mib[i]); if (node == NULL) { ret = ENOENT; - goto RETURN; + goto label_return; } } } /* Call the ctl function. */ - if (node->ctl == NULL) { + if (node && node->ctl) + ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); + else { /* Partial MIB. */ ret = ENOENT; - goto RETURN; } - ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); -RETURN: +label_return: return(ret); } @@ -953,38 +925,54 @@ ctl_boot(void) return (false); } +void +ctl_prefork(void) +{ + + malloc_mutex_prefork(&ctl_mtx); +} + +void +ctl_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&ctl_mtx); +} + +void +ctl_postfork_child(void) +{ + + malloc_mutex_postfork_child(&ctl_mtx); +} + /******************************************************************************/ /* *_ctl() functions. */ #define READONLY() do { \ if (newp != NULL || newlen != 0) { \ ret = EPERM; \ - goto RETURN; \ + goto label_return; \ } \ } while (0) #define WRITEONLY() do { \ if (oldp != NULL || oldlenp != NULL) { \ ret = EPERM; \ - goto RETURN; \ + goto label_return; \ } \ } while (0) -#define VOID() do { \ - READONLY(); \ - WRITEONLY(); \ -} while (0) - #define READ(v, t) do { \ if (oldp != NULL && oldlenp != NULL) { \ if (*oldlenp != sizeof(t)) { \ size_t copylen = (sizeof(t) <= *oldlenp) \ ? sizeof(t) : *oldlenp; \ - memcpy(oldp, (void *)&v, copylen); \ + memcpy(oldp, (void *)&(v), copylen); \ ret = EINVAL; \ - goto RETURN; \ + goto label_return; \ } else \ - *(t *)oldp = v; \ + *(t *)oldp = (v); \ } \ } while (0) @@ -992,12 +980,60 @@ ctl_boot(void) if (newp != NULL) { \ if (newlen != sizeof(t)) { \ ret = EINVAL; \ - goto RETURN; \ + goto label_return; \ } \ - v = *(t *)newp; \ + (v) = *(t *)newp; \ } \ } while (0) +/* + * There's a lot of code duplication in the following macros due to limitations + * in how nested cpp macros are expanded. + */ +#define CTL_RO_CLGEN(c, l, n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + if ((c) == false) \ + return (ENOENT); \ + if (l) \ + malloc_mutex_lock(&ctl_mtx); \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ +label_return: \ + if (l) \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ +} + +#define CTL_RO_CGEN(c, n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + if ((c) == false) \ + return (ENOENT); \ + malloc_mutex_lock(&ctl_mtx); \ + READONLY(); \ + oldval = (v); \ + READ(oldval, t); \ + \ + ret = 0; \ +label_return: \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ +} + #define CTL_RO_GEN(n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ @@ -1008,11 +1044,11 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ \ malloc_mutex_lock(&ctl_mtx); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ malloc_mutex_unlock(&ctl_mtx); \ return (ret); \ } @@ -1021,7 +1057,7 @@ RETURN: \ * ctl_mtx is not acquired, under the assumption that no pertinent data will * mutate during the call. */ -#define CTL_RO_NL_GEN(n, v, t) \ +#define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ @@ -1029,33 +1065,35 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ int ret; \ t oldval; \ \ + if ((c) == false) \ + return (ENOENT); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } -#define CTL_RO_TRUE_GEN(n) \ +#define CTL_RO_NL_GEN(n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ { \ int ret; \ - bool oldval; \ + t oldval; \ \ READONLY(); \ - oldval = true; \ - READ(oldval, bool); \ + oldval = (v); \ + READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } -#define CTL_RO_FALSE_GEN(n) \ +#define CTL_RO_BOOL_CONFIG_GEN(n) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ @@ -1064,14 +1102,16 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ bool oldval; \ \ READONLY(); \ - oldval = false; \ + oldval = n; \ READ(oldval, bool); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } +/******************************************************************************/ + CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int @@ -1079,44 +1119,66 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - uint64_t newval; + UNUSED uint64_t newval; malloc_mutex_lock(&ctl_mtx); - newval = 0; WRITE(newval, uint64_t); - if (newval != 0) + if (newp != NULL) ctl_refresh(); READ(ctl_epoch, uint64_t); ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } -#ifdef JEMALLOC_TCACHE -static int -tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - tcache_t *tcache; +/******************************************************************************/ - VOID(); +CTL_RO_BOOL_CONFIG_GEN(config_debug) +CTL_RO_BOOL_CONFIG_GEN(config_dss) +CTL_RO_BOOL_CONFIG_GEN(config_fill) +CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) +CTL_RO_BOOL_CONFIG_GEN(config_mremap) +CTL_RO_BOOL_CONFIG_GEN(config_munmap) +CTL_RO_BOOL_CONFIG_GEN(config_prof) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind) +CTL_RO_BOOL_CONFIG_GEN(config_stats) +CTL_RO_BOOL_CONFIG_GEN(config_tcache) +CTL_RO_BOOL_CONFIG_GEN(config_tls) +CTL_RO_BOOL_CONFIG_GEN(config_utrace) +CTL_RO_BOOL_CONFIG_GEN(config_valgrind) +CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) - tcache = TCACHE_GET(); - if (tcache == NULL) { - ret = 0; - goto RETURN; - } - tcache_destroy(tcache); - TCACHE_SET(NULL); +/******************************************************************************/ - ret = 0; -RETURN: - return (ret); -} -#endif +CTL_RO_NL_GEN(opt_abort, opt_abort, bool) +CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) +CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) +CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) +CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) +CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) +CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) +CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) +CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) +CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) +CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) +CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) +CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) +CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) +CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) +CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */ +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) +CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) + +/******************************************************************************/ static int thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, @@ -1125,209 +1187,236 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; unsigned newind, oldind; - newind = oldind = choose_arena()->ind; + malloc_mutex_lock(&ctl_mtx); + newind = oldind = choose_arena(NULL)->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { arena_t *arena; - if (newind >= narenas) { + if (newind >= ctl_stats.narenas) { /* New arena index is out of range. */ ret = EFAULT; - goto RETURN; + goto label_return; } /* Initialize arena if necessary. */ malloc_mutex_lock(&arenas_lock); - if ((arena = arenas[newind]) == NULL) - arena = arenas_extend(newind); + if ((arena = arenas[newind]) == NULL && (arena = + arenas_extend(newind)) == NULL) { + malloc_mutex_unlock(&arenas_lock); + ret = EAGAIN; + goto label_return; + } + assert(arena == arenas[newind]); arenas[oldind]->nthreads--; arenas[newind]->nthreads++; malloc_mutex_unlock(&arenas_lock); - if (arena == NULL) { - ret = EAGAIN; - goto RETURN; - } /* Set new arena association. */ - ARENA_SET(arena); -#ifdef JEMALLOC_TCACHE - { - tcache_t *tcache = TCACHE_GET(); - if (tcache != NULL) - tcache->arena = arena; + if (config_tcache) { + tcache_t *tcache; + if ((uintptr_t)(tcache = *tcache_tsd_get()) > + (uintptr_t)TCACHE_STATE_MAX) { + tcache_arena_dissociate(tcache); + tcache_arena_associate(tcache, arena); + } } -#endif + arenas_tsd_set(&arena); } ret = 0; -RETURN: +label_return: + malloc_mutex_unlock(&ctl_mtx); return (ret); } -#ifdef JEMALLOC_STATS -CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *); -CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *); -#endif +CTL_RO_NL_CGEN(config_stats, thread_allocated, + thread_allocated_tsd_get()->allocated, uint64_t) +CTL_RO_NL_CGEN(config_stats, thread_allocatedp, + &thread_allocated_tsd_get()->allocated, uint64_t *) +CTL_RO_NL_CGEN(config_stats, thread_deallocated, + thread_allocated_tsd_get()->deallocated, uint64_t) +CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, + &thread_allocated_tsd_get()->deallocated, uint64_t *) -/******************************************************************************/ +static int +thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + bool oldval; -#ifdef JEMALLOC_DEBUG -CTL_RO_TRUE_GEN(config_debug) -#else -CTL_RO_FALSE_GEN(config_debug) -#endif - -#ifdef JEMALLOC_DSS -CTL_RO_TRUE_GEN(config_dss) -#else -CTL_RO_FALSE_GEN(config_dss) -#endif - -#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT -CTL_RO_TRUE_GEN(config_dynamic_page_shift) -#else -CTL_RO_FALSE_GEN(config_dynamic_page_shift) -#endif - -#ifdef JEMALLOC_FILL -CTL_RO_TRUE_GEN(config_fill) -#else -CTL_RO_FALSE_GEN(config_fill) -#endif - -#ifdef JEMALLOC_LAZY_LOCK -CTL_RO_TRUE_GEN(config_lazy_lock) -#else -CTL_RO_FALSE_GEN(config_lazy_lock) -#endif - -#ifdef JEMALLOC_PROF -CTL_RO_TRUE_GEN(config_prof) -#else -CTL_RO_FALSE_GEN(config_prof) -#endif - -#ifdef JEMALLOC_PROF_LIBGCC -CTL_RO_TRUE_GEN(config_prof_libgcc) -#else -CTL_RO_FALSE_GEN(config_prof_libgcc) -#endif - -#ifdef JEMALLOC_PROF_LIBUNWIND -CTL_RO_TRUE_GEN(config_prof_libunwind) -#else -CTL_RO_FALSE_GEN(config_prof_libunwind) -#endif - -#ifdef JEMALLOC_STATS -CTL_RO_TRUE_GEN(config_stats) -#else -CTL_RO_FALSE_GEN(config_stats) -#endif - -#ifdef JEMALLOC_SWAP -CTL_RO_TRUE_GEN(config_swap) -#else -CTL_RO_FALSE_GEN(config_swap) -#endif - -#ifdef JEMALLOC_SYSV -CTL_RO_TRUE_GEN(config_sysv) -#else -CTL_RO_FALSE_GEN(config_sysv) -#endif - -#ifdef JEMALLOC_TCACHE -CTL_RO_TRUE_GEN(config_tcache) -#else -CTL_RO_FALSE_GEN(config_tcache) -#endif - -#ifdef JEMALLOC_TINY -CTL_RO_TRUE_GEN(config_tiny) -#else -CTL_RO_FALSE_GEN(config_tiny) -#endif - -#ifdef JEMALLOC_TLS -CTL_RO_TRUE_GEN(config_tls) -#else -CTL_RO_FALSE_GEN(config_tls) -#endif - -#ifdef JEMALLOC_XMALLOC -CTL_RO_TRUE_GEN(config_xmalloc) -#else -CTL_RO_FALSE_GEN(config_xmalloc) -#endif + if (config_tcache == false) + return (ENOENT); -/******************************************************************************/ + oldval = tcache_enabled_get(); + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + tcache_enabled_set(*(bool *)newp); + } + READ(oldval, bool); -CTL_RO_NL_GEN(opt_abort, opt_abort, bool) -CTL_RO_NL_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t) -CTL_RO_NL_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t) -CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) -CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) -CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) -CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) -#ifdef JEMALLOC_FILL -CTL_RO_NL_GEN(opt_junk, opt_junk, bool) -CTL_RO_NL_GEN(opt_zero, opt_zero, bool) -#endif -#ifdef JEMALLOC_SYSV -CTL_RO_NL_GEN(opt_sysv, opt_sysv, bool) -#endif -#ifdef JEMALLOC_XMALLOC -CTL_RO_NL_GEN(opt_xmalloc, opt_xmalloc, bool) -#endif -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) -CTL_RO_NL_GEN(opt_lg_tcache_gc_sweep, opt_lg_tcache_gc_sweep, ssize_t) -#endif -#ifdef JEMALLOC_PROF -CTL_RO_NL_GEN(opt_prof, opt_prof, bool) -CTL_RO_NL_GEN(opt_prof_prefix, opt_prof_prefix, const char *) -CTL_RO_GEN(opt_prof_active, opt_prof_active, bool) /* Mutable. */ -CTL_RO_NL_GEN(opt_lg_prof_bt_max, opt_lg_prof_bt_max, size_t) -CTL_RO_NL_GEN(opt_lg_prof_sample, opt_lg_prof_sample, size_t) -CTL_RO_NL_GEN(opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) -CTL_RO_NL_GEN(opt_prof_gdump, opt_prof_gdump, bool) -CTL_RO_NL_GEN(opt_prof_leak, opt_prof_leak, bool) -CTL_RO_NL_GEN(opt_prof_accum, opt_prof_accum, bool) -CTL_RO_NL_GEN(opt_lg_prof_tcmax, opt_lg_prof_tcmax, ssize_t) -#endif -#ifdef JEMALLOC_SWAP -CTL_RO_NL_GEN(opt_overcommit, opt_overcommit, bool) -#endif + ret = 0; +label_return: + return (ret); +} + +static int +thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + + if (config_tcache == false) + return (ENOENT); + + READONLY(); + WRITEONLY(); + + tcache_flush(); + + ret = 0; +label_return: + return (ret); +} /******************************************************************************/ -CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) -CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) -CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) -const ctl_node_t * -arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) +/* ctl_mutex must be held during execution of this function. */ +static void +arena_purge(unsigned arena_ind) { + VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); - if (i > nbins) - return (NULL); - return (super_arenas_bin_i_node); + malloc_mutex_lock(&arenas_lock); + memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); + malloc_mutex_unlock(&arenas_lock); + + if (arena_ind == ctl_stats.narenas) { + unsigned i; + for (i = 0; i < ctl_stats.narenas; i++) { + if (tarenas[i] != NULL) + arena_purge_all(tarenas[i]); + } + } else { + assert(arena_ind < ctl_stats.narenas); + if (tarenas[arena_ind] != NULL) + arena_purge_all(tarenas[arena_ind]); + } } -CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << PAGE_SHIFT), size_t) -const ctl_node_t * -arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) +static int +arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { + int ret; - if (i > nlclasses) - return (NULL); - return (super_arenas_lrun_i_node); + READONLY(); + WRITEONLY(); + malloc_mutex_lock(&ctl_mtx); + arena_purge(mib[1]); + malloc_mutex_unlock(&ctl_mtx); + + ret = 0; +label_return: + return (ret); } -CTL_RO_NL_GEN(arenas_narenas, narenas, unsigned) +static int +arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret, i; + bool match, err; + const char *dss; + unsigned arena_ind = mib[1]; + dss_prec_t dss_prec_old = dss_prec_limit; + dss_prec_t dss_prec = dss_prec_limit; + + malloc_mutex_lock(&ctl_mtx); + WRITE(dss, const char *); + match = false; + for (i = 0; i < dss_prec_limit; i++) { + if (strcmp(dss_prec_names[i], dss) == 0) { + dss_prec = i; + match = true; + break; + } + } + if (match == false) { + ret = EINVAL; + goto label_return; + } + + if (arena_ind < ctl_stats.narenas) { + arena_t *arena = arenas[arena_ind]; + if (arena != NULL) { + dss_prec_old = arena_dss_prec_get(arena); + arena_dss_prec_set(arena, dss_prec); + err = false; + } else + err = true; + } else { + dss_prec_old = chunk_dss_prec_get(); + err = chunk_dss_prec_set(dss_prec); + } + dss = dss_prec_names[dss_prec_old]; + READ(dss, const char *); + if (err) { + ret = EFAULT; + goto label_return; + } + + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static const ctl_named_node_t * +arena_i_index(const size_t *mib, size_t miblen, size_t i) +{ + const ctl_named_node_t * ret; + + malloc_mutex_lock(&ctl_mtx); + if (i > ctl_stats.narenas) { + ret = NULL; + goto label_return; + } + + ret = super_arena_i_node; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +/******************************************************************************/ + +static int +arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + unsigned narenas; + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (*oldlenp != sizeof(unsigned)) { + ret = EINVAL; + goto label_return; + } + narenas = ctl_stats.narenas; + READ(narenas, unsigned); + + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} static int arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, @@ -1338,92 +1427,100 @@ arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, malloc_mutex_lock(&ctl_mtx); READONLY(); - if (*oldlenp != narenas * sizeof(bool)) { + if (*oldlenp != ctl_stats.narenas * sizeof(bool)) { ret = EINVAL; - nread = (*oldlenp < narenas * sizeof(bool)) - ? (*oldlenp / sizeof(bool)) : narenas; + nread = (*oldlenp < ctl_stats.narenas * sizeof(bool)) + ? (*oldlenp / sizeof(bool)) : ctl_stats.narenas; } else { ret = 0; - nread = narenas; + nread = ctl_stats.narenas; } for (i = 0; i < nread; i++) ((bool *)oldp)[i] = ctl_stats.arenas[i].initialized; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) -CTL_RO_NL_GEN(arenas_cacheline, CACHELINE, size_t) -CTL_RO_NL_GEN(arenas_subpage, SUBPAGE, size_t) -CTL_RO_NL_GEN(arenas_pagesize, PAGE_SIZE, size_t) -CTL_RO_NL_GEN(arenas_chunksize, chunksize, size_t) -#ifdef JEMALLOC_TINY -CTL_RO_NL_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t) -CTL_RO_NL_GEN(arenas_tspace_max, (qspace_min >> 1), size_t) -#endif -CTL_RO_NL_GEN(arenas_qspace_min, qspace_min, size_t) -CTL_RO_NL_GEN(arenas_qspace_max, qspace_max, size_t) -CTL_RO_NL_GEN(arenas_cspace_min, cspace_min, size_t) -CTL_RO_NL_GEN(arenas_cspace_max, cspace_max, size_t) -CTL_RO_NL_GEN(arenas_sspace_min, sspace_min, size_t) -CTL_RO_NL_GEN(arenas_sspace_max, sspace_max, size_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -#endif -CTL_RO_NL_GEN(arenas_ntbins, ntbins, unsigned) -CTL_RO_NL_GEN(arenas_nqbins, nqbins, unsigned) -CTL_RO_NL_GEN(arenas_ncbins, ncbins, unsigned) -CTL_RO_NL_GEN(arenas_nsbins, nsbins, unsigned) -CTL_RO_NL_GEN(arenas_nbins, nbins, unsigned) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) -#endif +CTL_RO_NL_GEN(arenas_page, PAGE, size_t) +CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) +CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) +CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) +CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) +CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) +static const ctl_named_node_t * +arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > NBINS) + return (NULL); + return (super_arenas_bin_i_node); +} + CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) +CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) +static const ctl_named_node_t * +arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > nlclasses) + return (NULL); + return (super_arenas_lrun_i_node); +} static int arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - unsigned arena; + unsigned arena_ind; + malloc_mutex_lock(&ctl_mtx); WRITEONLY(); - arena = UINT_MAX; - WRITE(arena, unsigned); - if (newp != NULL && arena >= narenas) { + arena_ind = UINT_MAX; + WRITE(arena_ind, unsigned); + if (newp != NULL && arena_ind >= ctl_stats.narenas) ret = EFAULT; - goto RETURN; - } else { - arena_t *tarenas[narenas]; + else { + if (arena_ind == UINT_MAX) + arena_ind = ctl_stats.narenas; + arena_purge(arena_ind); + ret = 0; + } - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - malloc_mutex_unlock(&arenas_lock); +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} - if (arena == UINT_MAX) { - unsigned i; - for (i = 0; i < narenas; i++) { - if (tarenas[i] != NULL) - arena_purge_all(tarenas[i]); - } - } else { - assert(arena < narenas); - if (tarenas[arena] != NULL) - arena_purge_all(tarenas[arena]); - } +static int +arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + unsigned narenas; + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (ctl_grow()) { + ret = EAGAIN; + goto label_return; } + narenas = ctl_stats.narenas - 1; + READ(narenas, unsigned); ret = 0; -RETURN: +label_return: + malloc_mutex_unlock(&ctl_mtx); return (ret); } /******************************************************************************/ -#ifdef JEMALLOC_PROF static int prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) @@ -1431,6 +1528,9 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; bool oldval; + if (config_prof == false) + return (ENOENT); + malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ oldval = opt_prof_active; if (newp != NULL) { @@ -1445,7 +1545,7 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, READ(oldval, bool); ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -1457,92 +1557,107 @@ prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; const char *filename = NULL; + if (config_prof == false) + return (ENOENT); + WRITEONLY(); WRITE(filename, const char *); if (prof_mdump(filename)) { ret = EFAULT; - goto RETURN; + goto label_return; } ret = 0; -RETURN: +label_return: return (ret); } -CTL_RO_NL_GEN(prof_interval, prof_interval, uint64_t) -#endif +CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) /******************************************************************************/ -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_chunks_current, ctl_stats.chunks.current, size_t) -CTL_RO_GEN(stats_chunks_total, ctl_stats.chunks.total, uint64_t) -CTL_RO_GEN(stats_chunks_high, ctl_stats.chunks.high, size_t) -CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t) -CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t) -CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_allocated, +CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) +CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) +CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) +CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) + +CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, + size_t) +CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t) +CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t) +CTL_RO_CGEN(config_stats, stats_huge_allocated, huge_allocated, size_t) +CTL_RO_CGEN(config_stats, stats_huge_nmalloc, huge_nmalloc, uint64_t) +CTL_RO_CGEN(config_stats, stats_huge_ndalloc, huge_ndalloc, uint64_t) + +CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) +CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) +CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) +CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, + ctl_stats.arenas[mib[2]].astats.mapped, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, + ctl_stats.arenas[mib[2]].astats.npurge, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, + ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_purged, + ctl_stats.arenas[mib[2]].astats.purged, uint64_t) + +CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, ctl_stats.arenas[mib[2]].allocated_small, size_t) -CTL_RO_GEN(stats_arenas_i_small_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_allocated, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) -CTL_RO_GEN(stats_arenas_i_large_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_allocated, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_allocated, ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_GEN(stats_arenas_i_bins_j_nfills, +CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nflushes, +CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) -#endif -CTL_RO_GEN(stats_arenas_i_bins_j_nruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nreruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_highruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].highruns, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_curruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) -const ctl_node_t * +static const ctl_named_node_t * stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) { - if (j > nbins) + if (j > NBINS) return (NULL); return (super_stats_arenas_i_bins_j_node); } -CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nmalloc, ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_ndalloc, ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests, ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_curruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns, ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_highruns, - ctl_stats.arenas[mib[2]].lstats[mib[4]].highruns, size_t) -const ctl_node_t * +static const ctl_named_node_t * stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) { @@ -1551,120 +1666,19 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) return (super_stats_arenas_i_lruns_j_node); } -#endif -CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) -CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) -CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_arenas_i_mapped, ctl_stats.arenas[mib[2]].astats.mapped, - size_t) -CTL_RO_GEN(stats_arenas_i_npurge, ctl_stats.arenas[mib[2]].astats.npurge, - uint64_t) -CTL_RO_GEN(stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, - uint64_t) -CTL_RO_GEN(stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, - uint64_t) -#endif - -const ctl_node_t * +static const ctl_named_node_t * stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) { - const ctl_node_t * ret; + const ctl_named_node_t * ret; malloc_mutex_lock(&ctl_mtx); - if (ctl_stats.arenas[i].initialized == false) { + if (i > ctl_stats.narenas || ctl_stats.arenas[i].initialized == false) { ret = NULL; - goto RETURN; + goto label_return; } ret = super_stats_arenas_i_node; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_cactive, &stats_cactive, size_t *) -CTL_RO_GEN(stats_allocated, ctl_stats.allocated, size_t) -CTL_RO_GEN(stats_active, ctl_stats.active, size_t) -CTL_RO_GEN(stats_mapped, ctl_stats.mapped, size_t) -#endif - -/******************************************************************************/ - -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_RO_GEN(swap_avail, ctl_stats.swap_avail, size_t) -# endif - -static int -swap_prezeroed_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else { - /* - * swap_prezeroed isn't actually used by the swap code until it - * is set during a successful chunk_swap_enabled() call. We - * use it here to store the value that we'll pass to - * chunk_swap_enable() in a swap.fds mallctl(). This is not - * very clean, but the obvious alternatives are even worse. - */ - WRITE(swap_prezeroed, bool); - } - - READ(swap_prezeroed, bool); - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -CTL_RO_GEN(swap_nfds, swap_nfds, size_t) - -static int -swap_fds_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else if (newp != NULL) { - size_t nfds = newlen / sizeof(int); - - { - int fds[nfds]; - - memcpy(fds, newp, nfds * sizeof(int)); - if (chunk_swap_enable(fds, nfds, swap_prezeroed)) { - ret = EFAULT; - goto RETURN; - } - } - } - - if (oldp != NULL && oldlenp != NULL) { - if (*oldlenp != swap_nfds * sizeof(int)) { - size_t copylen = (swap_nfds * sizeof(int) <= *oldlenp) - ? swap_nfds * sizeof(int) : *oldlenp; - - memcpy(oldp, swap_fds, copylen); - ret = EINVAL; - goto RETURN; - } else - memcpy(oldp, swap_fds, *oldlenp); - } - - ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } -#endif diff --git a/deps/jemalloc/src/extent.c b/deps/jemalloc/src/extent.c index 3c04d3aa5d1..8c09b486ed8 100644 --- a/deps/jemalloc/src/extent.c +++ b/deps/jemalloc/src/extent.c @@ -3,7 +3,6 @@ /******************************************************************************/ -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) static inline int extent_szad_comp(extent_node_t *a, extent_node_t *b) { @@ -25,7 +24,6 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b) /* Generate red-black tree functions. */ rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, extent_szad_comp) -#endif static inline int extent_ad_comp(extent_node_t *a, extent_node_t *b) diff --git a/deps/jemalloc/src/huge.c b/deps/jemalloc/src/huge.c index a4f9b054ed5..d72f2135702 100644 --- a/deps/jemalloc/src/huge.c +++ b/deps/jemalloc/src/huge.c @@ -4,11 +4,9 @@ /******************************************************************************/ /* Data. */ -#ifdef JEMALLOC_STATS uint64_t huge_nmalloc; uint64_t huge_ndalloc; size_t huge_allocated; -#endif malloc_mutex_t huge_mtx; @@ -18,11 +16,19 @@ malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(size_t size, bool zero) +huge_malloc(size_t size, bool zero, dss_prec_t dss_prec) +{ + + return (huge_palloc(size, chunksize, zero, dss_prec)); +} + +void * +huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec) { void *ret; size_t csize; extent_node_t *node; + bool is_zeroed; /* Allocate one or more contiguous chunks for this request. */ @@ -37,7 +43,12 @@ huge_malloc(size_t size, bool zero) if (node == NULL) return (NULL); - ret = chunk_alloc(csize, false, &zero); + /* + * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that + * it is possible to make correct junk/zero fill decisions below. + */ + is_zeroed = zero; + ret = chunk_alloc(csize, alignment, false, &is_zeroed, dss_prec); if (ret == NULL) { base_node_dealloc(node); return (NULL); @@ -49,111 +60,24 @@ huge_malloc(size_t size, bool zero) malloc_mutex_lock(&huge_mtx); extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(csize); - huge_nmalloc++; - huge_allocated += csize; -#endif + if (config_stats) { + stats_cactive_add(csize); + huge_nmalloc++; + huge_allocated += csize; + } malloc_mutex_unlock(&huge_mtx); -#ifdef JEMALLOC_FILL - if (zero == false) { + if (config_fill && zero == false) { if (opt_junk) memset(ret, 0xa5, csize); - else if (opt_zero) + else if (opt_zero && is_zeroed == false) memset(ret, 0, csize); } -#endif return (ret); } -/* Only handles large allocations that require more than chunk alignment. */ -void * -huge_palloc(size_t size, size_t alignment, bool zero) -{ - void *ret; - size_t alloc_size, chunk_size, offset; - extent_node_t *node; - - /* - * This allocation requires alignment that is even larger than chunk - * alignment. This means that huge_malloc() isn't good enough. - * - * Allocate almost twice as many chunks as are demanded by the size or - * alignment, in order to assure the alignment can be achieved, then - * unmap leading and trailing chunks. - */ - assert(alignment > chunksize); - - chunk_size = CHUNK_CEILING(size); - - if (size >= alignment) - alloc_size = chunk_size + alignment - chunksize; - else - alloc_size = (alignment << 1) - chunksize; - - /* Allocate an extent node with which to track the chunk. */ - node = base_node_alloc(); - if (node == NULL) - return (NULL); - - ret = chunk_alloc(alloc_size, false, &zero); - if (ret == NULL) { - base_node_dealloc(node); - return (NULL); - } - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & chunksize_mask) == 0); - assert(offset < alloc_size); - if (offset == 0) { - /* Trim trailing space. */ - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size - - chunk_size, true); - } else { - size_t trailsize; - - /* Trim leading space. */ - chunk_dealloc(ret, alignment - offset, true); - - ret = (void *)((uintptr_t)ret + (alignment - offset)); - - trailsize = alloc_size - (alignment - offset) - chunk_size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), - trailsize, true); - } - } - - /* Insert node into huge. */ - node->addr = ret; - node->size = chunk_size; - - malloc_mutex_lock(&huge_mtx); - extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(chunk_size); - huge_nmalloc++; - huge_allocated += chunk_size; -#endif - malloc_mutex_unlock(&huge_mtx); - -#ifdef JEMALLOC_FILL - if (zero == false) { - if (opt_junk) - memset(ret, 0xa5, chunk_size); - else if (opt_zero) - memset(ret, 0, chunk_size); - } -#endif - - return (ret); -} - -void * +bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) { @@ -164,30 +88,23 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { assert(CHUNK_CEILING(oldsize) == oldsize); -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } -#endif - return (ptr); + return (false); } /* Reallocation would require a move. */ - return (NULL); + return (true); } void * huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) + size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec) { void *ret; size_t copysize; /* Try to avoid moving the allocation. */ - ret = huge_ralloc_no_move(ptr, oldsize, size, extra); - if (ret != NULL) - return (ret); + if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false) + return (ptr); /* * size and oldsize are different enough that we need to use a @@ -195,18 +112,18 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * space and copying. */ if (alignment > chunksize) - ret = huge_palloc(size + extra, alignment, zero); + ret = huge_palloc(size + extra, alignment, zero, dss_prec); else - ret = huge_malloc(size + extra, zero); + ret = huge_malloc(size + extra, zero, dss_prec); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment > chunksize) - ret = huge_palloc(size, alignment, zero); + ret = huge_palloc(size, alignment, zero, dss_prec); else - ret = huge_malloc(size, zero); + ret = huge_malloc(size, zero, dss_prec); if (ret == NULL) return (NULL); @@ -218,20 +135,13 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ copysize = (size < oldsize) ? size : oldsize; +#ifdef JEMALLOC_MREMAP /* * Use mremap(2) if this is a huge-->huge reallocation, and neither the - * source nor the destination are in swap or dss. + * source nor the destination are in dss. */ -#ifdef JEMALLOC_MREMAP_FIXED - if (oldsize >= chunksize -# ifdef JEMALLOC_SWAP - && (swap_enabled == false || (chunk_in_swap(ptr) == false && - chunk_in_swap(ret) == false)) -# endif -# ifdef JEMALLOC_DSS - && chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false -# endif - ) { + if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr) + == false && chunk_in_dss(ret) == false))) { size_t newsize = huge_salloc(ret); /* @@ -253,24 +163,56 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ char buf[BUFERROR_BUF]; - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in mremap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf(": Error in mremap(): %s\n", + buf); if (opt_abort) abort(); memcpy(ret, ptr, copysize); chunk_dealloc_mmap(ptr, oldsize); + } else if (config_fill && zero == false && opt_junk && oldsize + < newsize) { + /* + * mremap(2) clobbers the original mapping, so + * junk/zero filling is not preserved. There is no + * need to zero fill here, since any trailing + * uninititialized memory is demand-zeroed by the + * kernel, but junk filling must be redone. + */ + memset(ret + oldsize, 0xa5, newsize - oldsize); } } else #endif { memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloct(ptr, try_tcache_dalloc); } return (ret); } +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) +#endif +static void +huge_dalloc_junk(void *ptr, size_t usize) +{ + + if (config_fill && config_dss && opt_junk) { + /* + * Only bother junk filling if the chunk isn't about to be + * unmapped. + */ + if (config_munmap == false || (config_dss && chunk_in_dss(ptr))) + memset(ptr, 0x5a, usize); + } +} +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) +huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); +#endif + void huge_dalloc(void *ptr, bool unmap) { @@ -285,23 +227,16 @@ huge_dalloc(void *ptr, bool unmap) assert(node->addr == ptr); extent_tree_ad_remove(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_sub(node->size); - huge_ndalloc++; - huge_allocated -= node->size; -#endif + if (config_stats) { + stats_cactive_sub(node->size); + huge_ndalloc++; + huge_allocated -= node->size; + } malloc_mutex_unlock(&huge_mtx); - if (unmap) { - /* Unmap chunk. */ -#ifdef JEMALLOC_FILL -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) - if (opt_junk) - memset(node->addr, 0x5a, node->size); -#endif -#endif - } + if (unmap) + huge_dalloc_junk(node->addr, node->size); chunk_dealloc(node->addr, node->size, unmap); @@ -328,7 +263,13 @@ huge_salloc(const void *ptr) return (ret); } -#ifdef JEMALLOC_PROF +dss_prec_t +huge_dss_prec_get(arena_t *arena) +{ + + return (arena_dss_prec_get(choose_arena(arena))); +} + prof_ctx_t * huge_prof_ctx_get(const void *ptr) { @@ -365,7 +306,6 @@ huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) malloc_mutex_unlock(&huge_mtx); } -#endif bool huge_boot(void) @@ -376,11 +316,32 @@ huge_boot(void) return (true); extent_tree_ad_new(&huge); -#ifdef JEMALLOC_STATS - huge_nmalloc = 0; - huge_ndalloc = 0; - huge_allocated = 0; -#endif + if (config_stats) { + huge_nmalloc = 0; + huge_ndalloc = 0; + huge_allocated = 0; + } return (false); } + +void +huge_prefork(void) +{ + + malloc_mutex_prefork(&huge_mtx); +} + +void +huge_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&huge_mtx); +} + +void +huge_postfork_child(void) +{ + + malloc_mutex_postfork_child(&huge_mtx); +} diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index a161c2e26e1..204778bc89d 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -4,111 +4,108 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t arenas_lock; -arena_t **arenas; -unsigned narenas; +malloc_tsd_data(, arenas, arena_t *, NULL) +malloc_tsd_data(, thread_allocated, thread_allocated_t, + THREAD_ALLOCATED_INITIALIZER) -pthread_key_t arenas_tsd; -#ifndef NO_TLS -__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); +/* Runtime configuration options. */ +const char *je_malloc_conf; +bool opt_abort = +#ifdef JEMALLOC_DEBUG + true +#else + false #endif - -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -__thread thread_allocated_t thread_allocated_tls; -# else -pthread_key_t thread_allocated_tsd; -# endif + ; +bool opt_junk = +#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) + true +#else + false #endif + ; +size_t opt_quarantine = ZU(0); +bool opt_redzone = false; +bool opt_utrace = false; +bool opt_valgrind = false; +bool opt_xmalloc = false; +bool opt_zero = false; +size_t opt_narenas = 0; + +unsigned ncpus; + +malloc_mutex_t arenas_lock; +arena_t **arenas; +unsigned narenas_total; +unsigned narenas_auto; /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; +#ifdef JEMALLOC_THREADED_INIT /* Used to let the initializing thread recursively allocate. */ -static pthread_t malloc_initializer = (unsigned long)0; - -/* Used to avoid initialization races. */ -static malloc_mutex_t init_lock = -#ifdef JEMALLOC_OSSPIN - 0 +# define NO_INITIALIZER ((unsigned long)0) +# define INITIALIZER pthread_self() +# define IS_INITIALIZER (malloc_initializer == pthread_self()) +static pthread_t malloc_initializer = NO_INITIALIZER; #else - MALLOC_MUTEX_INITIALIZER +# define NO_INITIALIZER false +# define INITIALIZER true +# define IS_INITIALIZER malloc_initializer +static bool malloc_initializer = NO_INITIALIZER; #endif - ; -#ifdef DYNAMIC_PAGE_SHIFT -size_t pagesize; -size_t pagesize_mask; -size_t lg_pagesize; -#endif +/* Used to avoid initialization races. */ +#ifdef _WIN32 +static malloc_mutex_t init_lock; -unsigned ncpus; +JEMALLOC_ATTR(constructor) +static void WINAPI +_init_init_lock(void) +{ -/* Runtime configuration options. */ -const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); -#ifdef JEMALLOC_DEBUG -bool opt_abort = true; -# ifdef JEMALLOC_FILL -bool opt_junk = true; -# endif -#else -bool opt_abort = false; -# ifdef JEMALLOC_FILL -bool opt_junk = false; -# endif -#endif -#ifdef JEMALLOC_SYSV -bool opt_sysv = false; -#endif -#ifdef JEMALLOC_XMALLOC -bool opt_xmalloc = false; -#endif -#ifdef JEMALLOC_FILL -bool opt_zero = false; -#endif -size_t opt_narenas = 0; + malloc_mutex_init(&init_lock); +} -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void wrtmessage(void *cbopaque, const char *s); -static void stats_print_atexit(void); -static unsigned malloc_ncpus(void); -static void arenas_cleanup(void *arg); -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void thread_allocated_cleanup(void *arg); +#ifdef _MSC_VER +# pragma section(".CRT$XCU", read) +JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) +static const void (WINAPI *init_init_lock)(void) = _init_init_lock; #endif -static bool malloc_conf_next(char const **opts_p, char const **k_p, - size_t *klen_p, char const **v_p, size_t *vlen_p); -static void malloc_conf_error(const char *msg, const char *k, size_t klen, - const char *v, size_t vlen); -static void malloc_conf_init(void); -static bool malloc_init_hard(void); -static int imemalign(void **memptr, size_t alignment, size_t size); - -/******************************************************************************/ -/* malloc_message() setup. */ -#ifdef JEMALLOC_HAVE_ATTR -JEMALLOC_ATTR(visibility("hidden")) #else -static -#endif -void -wrtmessage(void *cbopaque, const char *s) -{ -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - write(STDERR_FILENO, s, strlen(s)); -#ifdef JEMALLOC_CC_SILENCE - if (result < 0) - result = errno; +static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; +#endif + +typedef struct { + void *p; /* Input pointer (as in realloc(p, s)). */ + size_t s; /* Request size. */ + void *r; /* Result pointer. */ +} malloc_utrace_t; + +#ifdef JEMALLOC_UTRACE +# define UTRACE(a, b, c) do { \ + if (opt_utrace) { \ + int utrace_serrno = errno; \ + malloc_utrace_t ut; \ + ut.p = (a); \ + ut.s = (b); \ + ut.r = (c); \ + utrace(&ut, sizeof(ut)); \ + errno = utrace_serrno; \ + } \ +} while (0) +#else +# define UTRACE(a, b, c) #endif -} -void (*JEMALLOC_P(malloc_message))(void *, const char *s) - JEMALLOC_ATTR(visibility("default")) = wrtmessage; +/******************************************************************************/ +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + +static bool malloc_init_hard(void); /******************************************************************************/ /* @@ -121,9 +118,7 @@ arenas_extend(unsigned ind) { arena_t *ret; - /* Allocate enough space for trailing bins. */ - ret = (arena_t *)base_alloc(offsetof(arena_t, bins) - + (sizeof(arena_bin_t) * nbins)); + ret = (arena_t *)base_alloc(sizeof(arena_t)); if (ret != NULL && arena_new(ret, ind) == false) { arenas[ind] = ret; return (ret); @@ -143,23 +138,20 @@ arenas_extend(unsigned ind) return (arenas[0]); } -/* - * Choose an arena based on a per-thread value (slow-path code only, called - * only by choose_arena()). - */ +/* Slow path, called only by choose_arena(). */ arena_t * choose_arena_hard(void) { arena_t *ret; - if (narenas > 1) { + if (narenas_auto > 1) { unsigned i, choose, first_null; choose = 0; - first_null = narenas; + first_null = narenas_auto; malloc_mutex_lock(&arenas_lock); assert(arenas[0] != NULL); - for (i = 1; i < narenas; i++) { + for (i = 1; i < narenas_auto; i++) { if (arenas[i] != NULL) { /* * Choose the first arena that has the lowest @@ -168,7 +160,7 @@ choose_arena_hard(void) if (arenas[i]->nthreads < arenas[choose]->nthreads) choose = i; - } else if (first_null == narenas) { + } else if (first_null == narenas_auto) { /* * Record the index of the first uninitialized * arena, in case all extant arenas are in use. @@ -182,7 +174,8 @@ choose_arena_hard(void) } } - if (arenas[choose] == 0 || first_null == narenas) { + if (arenas[choose]->nthreads == 0 + || first_null == narenas_auto) { /* * Use an unloaded arena, or the least loaded arena if * all arenas are already initialized. @@ -201,85 +194,46 @@ choose_arena_hard(void) malloc_mutex_unlock(&arenas_lock); } - ARENA_SET(ret); + arenas_tsd_set(&ret); return (ret); } -/* - * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so - * provide a wrapper. - */ -int -buferror(int errnum, char *buf, size_t buflen) -{ -#ifdef _GNU_SOURCE - char *b = strerror_r(errno, buf, buflen); - if (b != buf) { - strncpy(buf, b, buflen); - buf[buflen-1] = '\0'; - } - return (0); -#else - return (strerror_r(errno, buf, buflen)); -#endif -} - static void stats_print_atexit(void) { -#if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) - unsigned i; + if (config_tcache && config_stats) { + unsigned narenas, i; - /* - * Merge stats from extant threads. This is racy, since individual - * threads do not lock when recording tcache stats events. As a - * consequence, the final stats may be slightly out of date by the time - * they are reported, if other threads continue to allocate. - */ - for (i = 0; i < narenas; i++) { - arena_t *arena = arenas[i]; - if (arena != NULL) { - tcache_t *tcache; + /* + * Merge stats from extant threads. This is racy, since + * individual threads do not lock when recording tcache stats + * events. As a consequence, the final stats may be slightly + * out of date by the time they are reported, if other threads + * continue to allocate. + */ + for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { + arena_t *arena = arenas[i]; + if (arena != NULL) { + tcache_t *tcache; - /* - * tcache_stats_merge() locks bins, so if any code is - * introduced that acquires both arena and bin locks in - * the opposite order, deadlocks may result. - */ - malloc_mutex_lock(&arena->lock); - ql_foreach(tcache, &arena->tcache_ql, link) { - tcache_stats_merge(tcache, arena); + /* + * tcache_stats_merge() locks bins, so if any + * code is introduced that acquires both arena + * and bin locks in the opposite order, + * deadlocks may result. + */ + malloc_mutex_lock(&arena->lock); + ql_foreach(tcache, &arena->tcache_ql, link) { + tcache_stats_merge(tcache, arena); + } + malloc_mutex_unlock(&arena->lock); } - malloc_mutex_unlock(&arena->lock); } } -#endif - JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); -} - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t * -thread_allocated_get_hard(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - imalloc(sizeof(thread_allocated_t)); - if (thread_allocated == NULL) { - static thread_allocated_t static_thread_allocated = {0, 0}; - malloc_write(": Error allocating TSD;" - " mallctl(\"thread.{de,}allocated[p]\", ...)" - " will be inaccurate\n"); - if (opt_abort) - abort(); - return (&static_thread_allocated); - } - pthread_setspecific(thread_allocated_tsd, thread_allocated); - thread_allocated->allocated = 0; - thread_allocated->deallocated = 0; - return (thread_allocated); + je_malloc_stats_print(NULL, NULL, NULL); } -#endif /* * End miscellaneous support functions. @@ -292,51 +246,52 @@ thread_allocated_get_hard(void) static unsigned malloc_ncpus(void) { - unsigned ret; long result; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwNumberOfProcessors; +#else result = sysconf(_SC_NPROCESSORS_ONLN); - if (result == -1) { - /* Error. */ - ret = 1; - } - ret = (unsigned)result; - - return (ret); +#endif + return ((result == -1) ? 1 : (unsigned)result); } -static void +void arenas_cleanup(void *arg) { - arena_t *arena = (arena_t *)arg; + arena_t *arena = *(arena_t **)arg; malloc_mutex_lock(&arenas_lock); arena->nthreads--; malloc_mutex_unlock(&arenas_lock); } -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void -thread_allocated_cleanup(void *arg) +JEMALLOC_ALWAYS_INLINE_C void +malloc_thread_init(void) { - uint64_t *allocated = (uint64_t *)arg; - if (allocated != NULL) - idalloc(allocated); + /* + * TSD initialization can't be safely done as a side effect of + * deallocation, because it is possible for a thread to do nothing but + * deallocate its TLS data via free(), in which case writing to TLS + * would cause write-after-free memory corruption. The quarantine + * facility *only* gets used as a side effect of deallocation, so make + * a best effort attempt at initializing its TSD by hooking all + * allocation events. + */ + if (config_fill && opt_quarantine) + quarantine_alloc_hook(); } -#endif -/* - * FreeBSD's pthreads implementation calls malloc(3), so the malloc - * implementation has to take pains to avoid infinite recursion during - * initialization. - */ -static inline bool +JEMALLOC_ALWAYS_INLINE_C bool malloc_init(void) { - if (malloc_initialized == false) - return (malloc_init_hard()); + if (malloc_initialized == false && malloc_init_hard()) + return (true); + malloc_thread_init(); return (false); } @@ -352,68 +307,64 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, for (accept = false; accept == false;) { switch (*opts) { - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': case 'G': case 'H': case 'I': case 'J': - case 'K': case 'L': case 'M': case 'N': case 'O': - case 'P': case 'Q': case 'R': case 'S': case 'T': - case 'U': case 'V': case 'W': case 'X': case 'Y': - case 'Z': - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': case 'g': case 'h': case 'i': case 'j': - case 'k': case 'l': case 'm': case 'n': case 'o': - case 'p': case 'q': case 'r': case 's': case 't': - case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - case '_': - opts++; - break; - case ':': - opts++; - *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; - *v_p = opts; - accept = true; - break; - case '\0': - if (opts != *opts_p) { - malloc_write(": Conf string " - "ends with key\n"); - } - return (true); - default: - malloc_write(": Malformed conf " - "string\n"); - return (true); + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': + case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': + case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + case '_': + opts++; + break; + case ':': + opts++; + *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; + *v_p = opts; + accept = true; + break; + case '\0': + if (opts != *opts_p) { + malloc_write(": Conf string ends " + "with key\n"); + } + return (true); + default: + malloc_write(": Malformed conf string\n"); + return (true); } } for (accept = false; accept == false;) { switch (*opts) { - case ',': - opts++; - /* - * Look ahead one character here, because the - * next time this function is called, it will - * assume that end of input has been cleanly - * reached if no input remains, but we have - * optimistically already consumed the comma if - * one exists. - */ - if (*opts == '\0') { - malloc_write(": Conf string " - "ends with comma\n"); - } - *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; - accept = true; - break; - case '\0': - *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; - accept = true; - break; - default: - opts++; - break; + case ',': + opts++; + /* + * Look ahead one character here, because the next time + * this function is called, it will assume that end of + * input has been cleanly reached if no input remains, + * but we have optimistically already consumed the + * comma if one exists. + */ + if (*opts == '\0') { + malloc_write(": Conf string ends " + "with comma\n"); + } + *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; + accept = true; + break; + case '\0': + *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; + accept = true; + break; + default: + opts++; + break; } } @@ -425,17 +376,9 @@ static void malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, size_t vlen) { - char buf[PATH_MAX + 1]; - malloc_write(": "); - malloc_write(msg); - malloc_write(": "); - memcpy(buf, k, klen); - memcpy(&buf[klen], ":", 1); - memcpy(&buf[klen+1], v, vlen); - buf[klen+1+vlen] = '\0'; - malloc_write(buf); - malloc_write("\n"); + malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, + (int)vlen, v); } static void @@ -446,16 +389,32 @@ malloc_conf_init(void) const char *opts, *k, *v; size_t klen, vlen; + /* + * Automatically configure valgrind before processing options. The + * valgrind option remains in jemalloc 3.x for compatibility reasons. + */ + if (config_valgrind) { + opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; + if (config_fill && opt_valgrind) { + opt_junk = false; + assert(opt_zero == false); + opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; + opt_redzone = true; + } + if (config_tcache && opt_valgrind) + opt_tcache = false; + } + for (i = 0; i < 3; i++) { /* Get runtime configuration. */ switch (i) { case 0: - if (JEMALLOC_P(malloc_conf) != NULL) { + if (je_malloc_conf != NULL) { /* * Use options that were compiled into the * program. */ - opts = JEMALLOC_P(malloc_conf); + opts = je_malloc_conf; } else { /* No configuration specified. */ buf[0] = '\0'; @@ -463,31 +422,33 @@ malloc_conf_init(void) } break; case 1: { - int linklen; + int linklen = 0; +#ifndef _WIN32 + int saved_errno = errno; const char *linkname = -#ifdef JEMALLOC_PREFIX +# ifdef JEMALLOC_PREFIX "/etc/"JEMALLOC_PREFIX"malloc.conf" -#else +# else "/etc/malloc.conf" -#endif +# endif ; - if ((linklen = readlink(linkname, buf, - sizeof(buf) - 1)) != -1) { - /* - * Use the contents of the "/etc/malloc.conf" - * symbolic link's name. - */ - buf[linklen] = '\0'; - opts = buf; - } else { + /* + * Try to use the contents of the "/etc/malloc.conf" + * symbolic link's name. + */ + linklen = readlink(linkname, buf, sizeof(buf) - 1); + if (linklen == -1) { /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; + linklen = 0; + /* restore errno */ + set_errno(saved_errno); } +#endif + buf[linklen] = '\0'; + opts = buf; break; - } - case 2: { + } case 2: { const char *envname = #ifdef JEMALLOC_PREFIX JEMALLOC_CPREFIX"MALLOC_CONF" @@ -508,25 +469,23 @@ malloc_conf_init(void) opts = buf; } break; - } - default: - /* NOTREACHED */ - assert(false); + } default: + not_reached(); buf[0] = '\0'; opts = buf; } while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, &vlen) == false) { -#define CONF_HANDLE_BOOL(n) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_BOOL(o, n) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ if (strncmp("true", v, vlen) == 0 && \ vlen == sizeof("true")-1) \ - opt_##n = true; \ + o = true; \ else if (strncmp("false", v, vlen) == \ 0 && vlen == sizeof("false")-1) \ - opt_##n = false; \ + o = false; \ else { \ malloc_conf_error( \ "Invalid conf value", \ @@ -534,36 +493,47 @@ malloc_conf_init(void) } \ continue; \ } -#define CONF_HANDLE_SIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ - unsigned long ul; \ + uintmax_t um; \ char *end; \ \ - errno = 0; \ - ul = strtoul(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ + set_errno(0); \ + um = malloc_strtoumax(v, &end, 0); \ + if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ - } else if (ul < min || ul > max) { \ - malloc_conf_error( \ - "Out-of-range conf value", \ - k, klen, v, vlen); \ - } else \ - opt_##n = ul; \ + } else if (clip) { \ + if (min != 0 && um < min) \ + o = min; \ + else if (um > max) \ + o = max; \ + else \ + o = um; \ + } else { \ + if ((min != 0 && um < min) || \ + um > max) { \ + malloc_conf_error( \ + "Out-of-range " \ + "conf value", \ + k, klen, v, vlen); \ + } else \ + o = um; \ + } \ continue; \ } -#define CONF_HANDLE_SSIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ long l; \ char *end; \ \ - errno = 0; \ + set_errno(0); \ l = strtol(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ + if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ malloc_conf_error( \ "Invalid conf value", \ @@ -574,70 +544,98 @@ malloc_conf_init(void) "Out-of-range conf value", \ k, klen, v, vlen); \ } else \ - opt_##n = l; \ + o = l; \ continue; \ } -#define CONF_HANDLE_CHAR_P(n, d) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_CHAR_P(o, n, d) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ size_t cpylen = (vlen <= \ - sizeof(opt_##n)-1) ? vlen : \ - sizeof(opt_##n)-1; \ - strncpy(opt_##n, v, cpylen); \ - opt_##n[cpylen] = '\0'; \ + sizeof(o)-1) ? vlen : \ + sizeof(o)-1; \ + strncpy(o, v, cpylen); \ + o[cpylen] = '\0'; \ continue; \ } - CONF_HANDLE_BOOL(abort) - CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, - PAGE_SHIFT-1) - CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, - PAGE_SHIFT-1) + CONF_HANDLE_BOOL(opt_abort, "abort") /* - * Chunks always require at least one * header page, - * plus one data page. + * Chunks always require at least one header page, plus + * one data page in the absence of redzones, or three + * pages in the presence of redzones. In order to + * simplify options processing, fix the limit based on + * config_fill. */ - CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) - CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_BOOL(stats_print) -#ifdef JEMALLOC_FILL - CONF_HANDLE_BOOL(junk) - CONF_HANDLE_BOOL(zero) -#endif -#ifdef JEMALLOC_SYSV - CONF_HANDLE_BOOL(sysv) -#endif -#ifdef JEMALLOC_XMALLOC - CONF_HANDLE_BOOL(xmalloc) -#endif -#ifdef JEMALLOC_TCACHE - CONF_HANDLE_BOOL(tcache) - CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, - (sizeof(size_t) << 3) - 1) -#endif -#ifdef JEMALLOC_PROF - CONF_HANDLE_BOOL(prof) - CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") - CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) - CONF_HANDLE_BOOL(prof_active) - CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_accum) - CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_gdump) - CONF_HANDLE_BOOL(prof_leak) -#endif -#ifdef JEMALLOC_SWAP - CONF_HANDLE_BOOL(overcommit) -#endif + CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + + (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, + true) + if (strncmp("dss", k, klen) == 0) { + int i; + bool match = false; + for (i = 0; i < dss_prec_limit; i++) { + if (strncmp(dss_prec_names[i], v, vlen) + == 0) { + if (chunk_dss_prec_set(i)) { + malloc_conf_error( + "Error setting dss", + k, klen, v, vlen); + } else { + opt_dss = + dss_prec_names[i]; + match = true; + break; + } + } + } + if (match == false) { + malloc_conf_error("Invalid conf value", + k, klen, v, vlen); + } + continue; + } + CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, + SIZE_T_MAX, false) + CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", + -1, (sizeof(size_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_stats_print, "stats_print") + if (config_fill) { + CONF_HANDLE_BOOL(opt_junk, "junk") + CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", + 0, SIZE_T_MAX, false) + CONF_HANDLE_BOOL(opt_redzone, "redzone") + CONF_HANDLE_BOOL(opt_zero, "zero") + } + if (config_utrace) { + CONF_HANDLE_BOOL(opt_utrace, "utrace") + } + if (config_valgrind) { + CONF_HANDLE_BOOL(opt_valgrind, "valgrind") + } + if (config_xmalloc) { + CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") + } + if (config_tcache) { + CONF_HANDLE_BOOL(opt_tcache, "tcache") + CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, + "lg_tcache_max", -1, + (sizeof(size_t) << 3) - 1) + } + if (config_prof) { + CONF_HANDLE_BOOL(opt_prof, "prof") + CONF_HANDLE_CHAR_P(opt_prof_prefix, + "prof_prefix", "jeprof") + CONF_HANDLE_BOOL(opt_prof_active, "prof_active") + CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, + "lg_prof_sample", 0, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") + CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, + "lg_prof_interval", -1, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") + CONF_HANDLE_BOOL(opt_prof_final, "prof_final") + CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + } malloc_conf_error("Invalid conf pair", k, klen, v, vlen); #undef CONF_HANDLE_BOOL @@ -645,14 +643,6 @@ malloc_conf_init(void) #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P } - - /* Validate configuration of options that are inter-related. */ - if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { - malloc_write(": Invalid lg_[qc]space_max " - "relationship; restoring defaults\n"); - opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; - opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; - } } } @@ -662,7 +652,7 @@ malloc_init_hard(void) arena_t *init_arenas[1]; malloc_mutex_lock(&init_lock); - if (malloc_initialized || malloc_initializer == pthread_self()) { + if (malloc_initialized || IS_INITIALIZER) { /* * Another thread initialized the allocator before this one * acquired init_lock, or this thread is the initializing @@ -671,7 +661,8 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } - if (malloc_initializer != (unsigned long)0) { +#ifdef JEMALLOC_THREADED_INIT + if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { /* Busy-wait until the initializing thread completes. */ do { malloc_mutex_unlock(&init_lock); @@ -681,45 +672,15 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } - -#ifdef DYNAMIC_PAGE_SHIFT - /* Get page size. */ - { - long result; - - result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; - - /* - * We assume that pagesize is a power of 2 when calculating - * pagesize_mask and lg_pagesize. - */ - assert(((result - 1) & result) == 0); - pagesize_mask = result - 1; - lg_pagesize = ffs((int)result) - 1; - } #endif + malloc_initializer = INITIALIZER; -#ifdef JEMALLOC_PROF - prof_boot0(); -#endif + malloc_tsd_boot(); + if (config_prof) + prof_boot0(); malloc_conf_init(); - /* Register fork handlers. */ - if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, - jemalloc_postfork) != 0) { - malloc_write(": Error in pthread_atfork()\n"); - if (opt_abort) - abort(); - } - - if (ctl_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -729,50 +690,37 @@ malloc_init_hard(void) } } - if (chunk_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - if (base_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#ifdef JEMALLOC_PROF - prof_boot1(); -#endif - - if (arena_boot()) { + if (chunk_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#ifdef JEMALLOC_TCACHE - if (tcache_boot()) { + if (ctl_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif - if (huge_boot()) { + if (config_prof) + prof_boot1(); + + arena_boot(); + + if (config_tcache && tcache_boot0()) { malloc_mutex_unlock(&init_lock); return (true); } -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) - /* Initialize allocation counters before any allocations can occur. */ - if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) - != 0) { + if (huge_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif - - if (malloc_mutex_init(&arenas_lock)) - return (true); - if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { + if (malloc_mutex_init(&arenas_lock)) { malloc_mutex_unlock(&init_lock); return (true); } @@ -781,9 +729,9 @@ malloc_init_hard(void) * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ - narenas = 1; + narenas_total = narenas_auto = 1; arenas = init_arenas; - memset(arenas, 0, sizeof(arena_t *) * narenas); + memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -795,27 +743,58 @@ malloc_init_hard(void) return (true); } - /* - * Assign the initial arena to the initial thread, in order to avoid - * spurious creation of an extra arena if the application switches to - * threaded mode. - */ - ARENA_SET(arenas[0]); - arenas[0]->nthreads++; + /* Initialize allocation counters before any allocations can occur. */ + if (config_stats && thread_allocated_tsd_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } -#ifdef JEMALLOC_PROF - if (prof_boot2()) { + if (arenas_tsd_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_tcache && tcache_boot1()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_fill && quarantine_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_prof && prof_boot2()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif - /* Get number of CPUs. */ - malloc_initializer = pthread_self(); malloc_mutex_unlock(&init_lock); + /**********************************************************************/ + /* Recursive allocation may follow. */ + ncpus = malloc_ncpus(); + +#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ + && !defined(_WIN32)) + /* LinuxThreads's pthread_atfork() allocates. */ + if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, + jemalloc_postfork_child) != 0) { + malloc_write(": Error in pthread_atfork()\n"); + if (opt_abort) + abort(); + } +#endif + + /* Done recursively allocating. */ + /**********************************************************************/ malloc_mutex_lock(&init_lock); + if (mutex_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (opt_narenas == 0) { /* * For SMP systems, create more than one arena per CPU by @@ -826,23 +805,21 @@ malloc_init_hard(void) else opt_narenas = 1; } - narenas = opt_narenas; + narenas_auto = opt_narenas; /* * Make sure that the arenas array can be allocated. In practice, this * limit is enough to allow the allocator to function, but the ctl * machinery will fail to allocate memory at far lower limits. */ - if (narenas > chunksize / sizeof(arena_t *)) { - char buf[UMAX2S_BUFSIZE]; - - narenas = chunksize / sizeof(arena_t *); - malloc_write(": Reducing narenas to limit ("); - malloc_write(u2s(narenas, 10, buf)); - malloc_write(")\n"); + if (narenas_auto > chunksize / sizeof(arena_t *)) { + narenas_auto = chunksize / sizeof(arena_t *); + malloc_printf(": Reducing narenas to limit (%d)\n", + narenas_auto); } + narenas_total = narenas_auto; /* Allocate and initialize arenas. */ - arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); + arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); if (arenas == NULL) { malloc_mutex_unlock(&init_lock); return (true); @@ -851,37 +828,15 @@ malloc_init_hard(void) * Zero the array. In practice, this should always be pre-zeroed, * since it was just mmap()ed, but let's be sure. */ - memset(arenas, 0, sizeof(arena_t *) * narenas); + memset(arenas, 0, sizeof(arena_t *) * narenas_total); /* Copy the pointer to the one arena that was already initialized. */ arenas[0] = init_arenas[0]; -#ifdef JEMALLOC_ZONE - /* Register the custom zone. */ - malloc_zone_register(create_zone()); - - /* - * Convert the default szone to an "overlay zone" that is capable of - * deallocating szone-allocated objects, but allocating new objects - * from jemalloc. - */ - szone2ozone(malloc_default_zone()); -#endif - malloc_initialized = true; malloc_mutex_unlock(&init_lock); - return (false); -} -#ifdef JEMALLOC_ZONE -JEMALLOC_ATTR(constructor) -void -jemalloc_darwin_init(void) -{ - - if (malloc_init_hard()) - abort(); + return (false); } -#endif /* * End initialization functions. @@ -891,283 +846,294 @@ jemalloc_darwin_init(void) * Begin malloc(3)-compatible functions. */ -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(malloc)(size_t size) +static void * +imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) { - void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + void *p; - if (malloc_init()) { - ret = NULL; - goto OOM; - } + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = imalloc(SMALL_MAXCLASS+1); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = imalloc(usize); - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in malloc(): " - "invalid size 0\n"); - abort(); - } -# endif - ret = NULL; - goto RETURN; - } -#endif - } + return (p); +} -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto OOM; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = imalloc(size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif - ret = imalloc(size); - } +JEMALLOC_ALWAYS_INLINE_C void * +imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if ((uintptr_t)cnt != (uintptr_t)1U) + p = imalloc_prof_sample(usize, cnt); + else + p = imalloc(usize); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); +} + +/* + * MALLOC_BODY() is a macro rather than a function because its contents are in + * the fast path, but inlining would cause reliability issues when determining + * how many frames to discard from heap profiling backtraces. + */ +#define MALLOC_BODY(ret, size, usize) do { \ + if (malloc_init()) \ + ret = NULL; \ + else { \ + if (config_prof && opt_prof) { \ + prof_thr_cnt_t *cnt; \ + \ + usize = s2u(size); \ + /* \ + * Call PROF_ALLOC_PREP() here rather than in \ + * imalloc_prof() so that imalloc_prof() can be \ + * inlined without introducing uncertainty \ + * about the number of backtrace frames to \ + * ignore. imalloc_prof() is in the fast path \ + * when heap profiling is enabled, so inlining \ + * is critical to performance. (For \ + * consistency all callers of PROF_ALLOC_PREP() \ + * are structured similarly, even though e.g. \ + * realloc() isn't called enough for inlining \ + * to be critical.) \ + */ \ + PROF_ALLOC_PREP(1, usize, cnt); \ + ret = imalloc_prof(usize, cnt); \ + } else { \ + if (config_stats || (config_valgrind && \ + opt_valgrind)) \ + usize = s2u(size); \ + ret = imalloc(size); \ + } \ + } \ +} while (0) + +void * +je_malloc(size_t size) +{ + void *ret; + size_t usize JEMALLOC_CC_SILENCE_INIT(0); + + if (size == 0) + size = 1; + + MALLOC_BODY(ret, size, usize); -OOM: if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in malloc(): " "out of memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } - -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); + if (config_stats && ret != NULL) { + assert(usize == isalloc(ret, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif + UTRACE(0, size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); return (ret); } +static void * +imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); + p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, + false); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = ipalloc(usize, alignment, false); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if ((uintptr_t)cnt != (uintptr_t)1U) + p = imemalign_prof_sample(alignment, usize, cnt); + else + p = ipalloc(usize, alignment, false); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); +} + JEMALLOC_ATTR(nonnull(1)) #ifdef JEMALLOC_PROF /* - * Avoid any uncertainty as to how many backtrace frames to ignore in + * Avoid any uncertainty as to how many backtrace frames to ignore in * PROF_ALLOC_PREP(). */ -JEMALLOC_ATTR(noinline) +JEMALLOC_NOINLINE #endif static int -imemalign(void **memptr, size_t alignment, size_t size) +imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) { int ret; - size_t usize -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; + size_t usize; void *result; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif - if (malloc_init()) + assert(min_alignment != 0); + + if (malloc_init()) { result = NULL; - else { - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid size " - "0\n"); - abort(); - } -# endif - result = NULL; - *memptr = NULL; - ret = 0; - goto RETURN; - } -#endif - } + goto label_oom; + } else { + if (size == 0) + size = 1; /* Make sure that alignment is a large enough power of 2. */ if (((alignment - 1) & alignment) != 0 - || alignment < sizeof(void *)) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid alignment\n"); + || (alignment < min_alignment)) { + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error allocating " + "aligned memory: invalid alignment\n"); abort(); } -#endif result = NULL; ret = EINVAL; - goto RETURN; + goto label_return; } - usize = sa2u(size, alignment, NULL); + usize = sa2u(size, alignment); if (usize == 0) { result = NULL; - ret = ENOMEM; - goto RETURN; + goto label_oom; } -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + PROF_ALLOC_PREP(2, usize, cnt); - if (cnt == NULL) { - result = NULL; - ret = EINVAL; - } else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= small_maxclass) { - assert(sa2u(small_maxclass+1, - alignment, NULL) != 0); - result = ipalloc(sa2u(small_maxclass+1, - alignment, NULL), alignment, false); - if (result != NULL) { - arena_prof_promoted(result, - usize); - } - } else { - result = ipalloc(usize, alignment, - false); - } - } + result = imemalign_prof(alignment, usize, cnt); } else -#endif result = ipalloc(usize, alignment, false); - } - - if (result == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in posix_memalign(): " - "out of memory\n"); - abort(); - } -#endif - ret = ENOMEM; - goto RETURN; + if (result == NULL) + goto label_oom; } *memptr = result; ret = 0; - -RETURN: -#ifdef JEMALLOC_STATS - if (result != NULL) { - assert(usize == isalloc(result)); - ALLOCATED_ADD(usize, 0); +label_return: + if (config_stats && result != NULL) { + assert(usize == isalloc(result, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && result != NULL) - prof_malloc(result, usize, cnt); -#endif + UTRACE(0, size, result); return (ret); +label_oom: + assert(result == NULL); + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error allocating aligned memory: " + "out of memory\n"); + abort(); + } + ret = ENOMEM; + goto label_return; } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) +je_posix_memalign(void **memptr, size_t alignment, size_t size) +{ + int ret = imemalign(memptr, alignment, size, sizeof(void *)); + JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, + config_prof), false); + return (ret); +} + +void * +je_aligned_alloc(size_t alignment, size_t size) +{ + void *ret; + int err; + + if ((err = imemalign(&ret, alignment, size, 1)) != 0) { + ret = NULL; + set_errno(err); + } + JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), + false); + return (ret); +} + +static void * +icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = icalloc(SMALL_MAXCLASS+1); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = icalloc(usize); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) { + void *p; - return imemalign(memptr, alignment, size); + if ((uintptr_t)cnt != (uintptr_t)1U) + p = icalloc_prof_sample(usize, cnt); + else + p = icalloc(usize); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); } -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(calloc)(size_t num, size_t size) +je_calloc(size_t num, size_t size) { void *ret; size_t num_size; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + size_t usize JEMALLOC_CC_SILENCE_INIT(0); if (malloc_init()) { num_size = 0; ret = NULL; - goto RETURN; + goto label_return; } num_size = num * size; if (num_size == 0) { -#ifdef JEMALLOC_SYSV - if ((opt_sysv == false) && ((num == 0) || (size == 0))) -#endif + if (num == 0 || size == 0) num_size = 1; -#ifdef JEMALLOC_SYSV else { ret = NULL; - goto RETURN; + goto label_return; } -#endif /* * Try to avoid division here. We know that it isn't possible to * overflow during multiplication if neither operand uses any of the @@ -1177,264 +1143,169 @@ JEMALLOC_P(calloc)(size_t num, size_t size) && (num_size / size != num)) { /* size_t overflow. */ ret = NULL; - goto RETURN; + goto label_return; } -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + usize = s2u(num_size); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto RETURN; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize - <= small_maxclass) { - ret = icalloc(small_maxclass+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = icalloc(num_size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(num_size); -#endif + ret = icalloc_prof(usize, cnt); + } else { + if (config_stats || (config_valgrind && opt_valgrind)) + usize = s2u(num_size); ret = icalloc(num_size); } -RETURN: +label_return: if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in calloc(): out of " "memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } - -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); + if (config_stats && ret != NULL) { + assert(usize == isalloc(ret, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif + UTRACE(0, num_size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); return (ret); } -JEMALLOC_ATTR(visibility("default")) +static void * +irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = iralloc(oldptr, usize, 0, 0, false); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + prof_ctx_t *old_ctx; + + old_ctx = prof_ctx_get(oldptr); + if ((uintptr_t)cnt != (uintptr_t)1U) + p = irealloc_prof_sample(oldptr, usize, cnt); + else + p = iralloc(oldptr, usize, 0, 0, false); + if (p == NULL) + return (NULL); + prof_realloc(p, usize, cnt, old_usize, old_ctx); + + return (p); +} + +JEMALLOC_INLINE_C void +ifree(void *ptr) +{ + size_t usize; + UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + + assert(ptr != NULL); + assert(malloc_initialized || IS_INITIALIZER); + + if (config_prof && opt_prof) { + usize = isalloc(ptr, config_prof); + prof_free(ptr, usize); + } else if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); +} + void * -JEMALLOC_P(realloc)(void *ptr, size_t size) +je_realloc(void *ptr, size_t size) { void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; - size_t old_size = 0; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; - prof_ctx_t *old_ctx -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + size_t usize JEMALLOC_CC_SILENCE_INIT(0); + size_t old_usize = 0; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { - if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { - old_ctx = prof_ctx_get(ptr); - cnt = NULL; - } -#endif - idalloc(ptr); - } -#ifdef JEMALLOC_PROF - else if (opt_prof) { - old_ctx = NULL; - cnt = NULL; - } -#endif - ret = NULL; - goto RETURN; + if (ptr != NULL) { + /* realloc(ptr, 0) is equivalent to free(ptr). */ + UTRACE(ptr, 0, 0); + ifree(ptr); + return (NULL); } -#endif + size = 1; } if (ptr != NULL) { - assert(malloc_initialized || malloc_initializer == - pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && opt_valgrind)) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); + + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { usize = s2u(size); - old_ctx = prof_ctx_get(ptr); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - old_ctx = NULL; - ret = NULL; - goto OOM; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && - usize <= small_maxclass) { - ret = iralloc(ptr, small_maxclass+1, 0, 0, - false, false); - if (ret != NULL) - arena_prof_promoted(ret, usize); - else - old_ctx = NULL; - } else { - ret = iralloc(ptr, size, 0, 0, false, false); - if (ret == NULL) - old_ctx = NULL; - } - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif - ret = iralloc(ptr, size, 0, 0, false, false); - } - -#ifdef JEMALLOC_PROF -OOM: -#endif - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } -#endif - errno = ENOMEM; - } - } else { -#ifdef JEMALLOC_PROF - if (opt_prof) - old_ctx = NULL; -#endif - if (malloc_init()) { -#ifdef JEMALLOC_PROF - if (opt_prof) - cnt = NULL; -#endif - ret = NULL; + ret = irealloc_prof(ptr, old_usize, usize, cnt); } else { -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) - ret = NULL; - else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); - if (ret != NULL) { - arena_prof_promoted(ret, - usize); - } - } else - ret = imalloc(size); - } - } else -#endif - { -#ifdef JEMALLOC_STATS + if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(size); -#endif - ret = imalloc(size); - } + ret = iralloc(ptr, size, 0, 0, false); } + } else { + /* realloc(NULL, size) is equivalent to malloc(size). */ + MALLOC_BODY(ret, size, usize); + } - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } -#endif - errno = ENOMEM; + if (ret == NULL) { + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error in realloc(): " + "out of memory\n"); + abort(); } + set_errno(ENOMEM); } - -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) - prof_realloc(ret, usize, cnt, old_size, old_ctx); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, old_size); + if (config_stats && ret != NULL) { + thread_allocated_t *ta; + assert(usize == isalloc(ret, config_prof)); + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_usize; } -#endif + UTRACE(ptr, size, ret); + JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize, + false); return (ret); } -JEMALLOC_ATTR(visibility("default")) void -JEMALLOC_P(free)(void *ptr) +je_free(void *ptr) { - if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize; -#endif - - assert(malloc_initialized || malloc_initializer == - pthread_self()); - -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif - prof_free(ptr, usize); - } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); - } + UTRACE(ptr, 0, 0); + if (ptr != NULL) + ifree(ptr); } /* @@ -1443,51 +1314,55 @@ JEMALLOC_P(free)(void *ptr) /******************************************************************************/ /* * Begin non-standard override functions. - * - * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the - * entire point is to avoid accidental mixed allocator usage. */ -#ifndef JEMALLOC_PREFIX #ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(memalign)(size_t alignment, size_t size) +je_memalign(size_t alignment, size_t size) { - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, alignment, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif + void *ret JEMALLOC_CC_SILENCE_INIT(NULL); + imemalign(&ret, alignment, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif #ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(valloc)(size_t size) +je_valloc(size_t size) { - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, PAGE_SIZE, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif + void *ret JEMALLOC_CC_SILENCE_INIT(NULL); + imemalign(&ret, PAGE, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif -#endif /* JEMALLOC_PREFIX */ +/* + * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has + * #define je_malloc malloc + */ +#define malloc_is_malloc 1 +#define is_malloc_(a) malloc_is_ ## a +#define is_malloc(a) is_malloc_(a) + +#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) +/* + * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible + * to inconsistently reference libc's malloc(3)-compatible functions + * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). + * + * These definitions interpose hooks in glibc. The functions are actually + * passed an extra argument for the caller return address, which will be + * ignored. + */ +JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free; +JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc; +JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc; +JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = + je_memalign; +#endif + /* * End non-standard override functions. */ @@ -1496,386 +1371,741 @@ JEMALLOC_P(valloc)(size_t size) * Begin non-standard functions. */ -JEMALLOC_ATTR(visibility("default")) -size_t -JEMALLOC_P(malloc_usable_size)(const void *ptr) +JEMALLOC_ALWAYS_INLINE_C void * +imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { - size_t ret; - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, + alignment))); -#ifdef JEMALLOC_IVSALLOC - ret = ivsalloc(ptr); -#else - assert(ptr != NULL); - ret = isalloc(ptr); -#endif - - return (ret); + if (alignment != 0) + return (ipalloct(usize, alignment, zero, try_tcache, arena)); + else if (zero) + return (icalloct(usize, try_tcache, arena)); + else + return (imalloct(usize, try_tcache, arena)); } -JEMALLOC_ATTR(visibility("default")) -void -JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts) +static void * +imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; - stats_print(write_cb, cbopaque, opts); + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + size_t usize_promoted = (alignment == 0) ? + s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); + assert(usize_promoted != 0); + p = imallocx(usize_promoted, alignment, zero, try_tcache, + arena); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + + return (p); } -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) +JEMALLOC_ALWAYS_INLINE_C void * +imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; - if (malloc_init()) - return (EAGAIN); + if ((uintptr_t)cnt != (uintptr_t)1U) { + p = imallocx_prof_sample(usize, alignment, zero, try_tcache, + arena, cnt); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); - return (ctl_byname(name, oldp, oldlenp, newp, newlen)); + return (p); } -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) +void * +je_mallocx(size_t size, int flags) { + void *p; + size_t usize; + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + bool zero = flags & MALLOCX_ZERO; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + arena_t *arena; + bool try_tcache; + + assert(size != 0); if (malloc_init()) - return (EAGAIN); + goto label_oom; - return (ctl_nametomib(name, mibp, miblenp)); + if (arena_ind != UINT_MAX) { + arena = arenas[arena_ind]; + try_tcache = false; + } else { + arena = NULL; + try_tcache = true; + } + + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + assert(usize != 0); + + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + + PROF_ALLOC_PREP(1, usize, cnt); + p = imallocx_prof(usize, alignment, zero, try_tcache, arena, + cnt); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + if (p == NULL) + goto label_oom; + + if (config_stats) { + assert(usize == isalloc(p, config_prof)); + thread_allocated_tsd_get()->allocated += usize; + } + UTRACE(0, size, p); + JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); + return (p); +label_oom: + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error in mallocx(): out of memory\n"); + abort(); + } + UTRACE(0, size, 0); + return (NULL); } -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) +static void * +irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, + prof_thr_cnt_t *cnt) { + void *p; - if (malloc_init()) - return (EAGAIN); + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else { + p = iralloct(oldptr, size, 0, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + } - return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); + return (p); } -JEMALLOC_INLINE void * -iallocm(size_t usize, size_t alignment, bool zero) +JEMALLOC_ALWAYS_INLINE_C void * +irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, + size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; + prof_ctx_t *old_ctx; - assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, - NULL))); + old_ctx = prof_ctx_get(oldptr); + if ((uintptr_t)cnt != (uintptr_t)1U) + p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, + try_tcache_alloc, try_tcache_dalloc, arena, cnt); + else { + p = iralloct(oldptr, size, 0, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + } + if (p == NULL) + return (NULL); - if (alignment != 0) - return (ipalloc(usize, alignment, zero)); - else if (zero) - return (icalloc(usize)); - else - return (imalloc(usize)); + if (p == oldptr && alignment != 0) { + /* + * The allocation did not move, so it is possible that the size + * class is smaller than would guarantee the requested + * alignment, and that the alignment constraint was + * serendipitously satisfied. Additionally, old_usize may not + * be the same as the current usize because of in-place large + * reallocation. Therefore, query the actual value of usize. + */ + *usize = isalloc(p, config_prof); + } + prof_realloc(p, *usize, cnt, old_usize, old_ctx); + + return (p); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) +void * +je_rallocx(void *ptr, size_t size, int flags) { void *p; - size_t usize; - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + size_t usize, old_usize; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif + bool zero = flags & MALLOCX_ZERO; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + bool try_tcache_alloc, try_tcache_dalloc; + arena_t *arena; assert(ptr != NULL); assert(size != 0); + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if (arena_ind != UINT_MAX) { + arena_chunk_t *chunk; + try_tcache_alloc = false; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + try_tcache_dalloc = (chunk == ptr || chunk->arena != + arenas[arena_ind]); + arena = arenas[arena_ind]; + } else { + try_tcache_alloc = true; + try_tcache_dalloc = true; + arena = NULL; + } - if (malloc_init()) - goto OOM; + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && opt_valgrind)) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_usize); - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL); - if (usize == 0) - goto OOM; + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; -#ifdef JEMALLOC_PROF - if (opt_prof) { + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + assert(usize != 0); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) - goto OOM; - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - size_t usize_promoted = (alignment == 0) ? - s2u(small_maxclass+1) : sa2u(small_maxclass+1, - alignment, NULL); - assert(usize_promoted != 0); - p = iallocm(usize_promoted, alignment, zero); - if (p == NULL) - goto OOM; - arena_prof_promoted(p, usize); - } else { - p = iallocm(usize, alignment, zero); - if (p == NULL) - goto OOM; - } - prof_malloc(p, usize, cnt); - if (rsize != NULL) - *rsize = usize; - } else -#endif - { - p = iallocm(usize, alignment, zero); + p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, + try_tcache_alloc, try_tcache_dalloc, arena, cnt); if (p == NULL) - goto OOM; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; - } + goto label_oom; + } else { + p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, + try_tcache_dalloc, arena); + if (p == NULL) + goto label_oom; + if (config_stats || (config_valgrind && opt_valgrind)) + usize = isalloc(p, config_prof); } - *ptr = p; -#ifdef JEMALLOC_STATS - assert(usize == isalloc(p)); - ALLOCATED_ADD(usize, 0); -#endif - return (ALLOCM_SUCCESS); -OOM: -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in allocm(): " - "out of memory\n"); + if (config_stats) { + thread_allocated_t *ta; + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_usize; + } + UTRACE(ptr, size, p); + JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero); + return (p); +label_oom: + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error in rallocx(): out of memory\n"); abort(); } -#endif - *ptr = NULL; - return (ALLOCM_ERR_OOM); + UTRACE(ptr, size, 0); + return (NULL); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, - int flags) +JEMALLOC_ALWAYS_INLINE_C size_t +ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, bool zero, arena_t *arena) { - void *p, *q; size_t usize; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t old_size; -#endif - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + + if (ixalloc(ptr, size, extra, alignment, zero)) + return (old_usize); + usize = isalloc(ptr, config_prof); + + return (usize); +} + +static size_t +ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, size_t max_usize, bool zero, arena_t *arena, + prof_thr_cnt_t *cnt) +{ + size_t usize; + + if (cnt == NULL) + return (old_usize); + /* Use minimum usize to determine whether promotion may happen. */ + if (prof_promote && ((alignment == 0) ? s2u(size) : sa2u(size, + alignment)) <= SMALL_MAXCLASS) { + if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), + alignment, zero)) + return (old_usize); + usize = isalloc(ptr, config_prof); + if (max_usize < PAGE) + arena_prof_promoted(ptr, usize); + } else { + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); + } + + return (usize); +} + +JEMALLOC_ALWAYS_INLINE_C size_t +ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, size_t max_usize, bool zero, arena_t *arena, + prof_thr_cnt_t *cnt) +{ + size_t usize; + prof_ctx_t *old_ctx; + + old_ctx = prof_ctx_get(ptr); + if ((uintptr_t)cnt != (uintptr_t)1U) { + usize = ixallocx_prof_sample(ptr, old_usize, size, extra, + alignment, zero, max_usize, arena, cnt); + } else { + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); + } + if (usize == old_usize) + return (usize); + prof_realloc(ptr, usize, cnt, old_usize, old_ctx); + + return (usize); +} + +size_t +je_xallocx(void *ptr, size_t size, size_t extra, int flags) +{ + size_t usize, old_usize; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; - bool no_move = flags & ALLOCM_NO_MOVE; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif + bool zero = flags & MALLOCX_ZERO; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + arena_t *arena; assert(ptr != NULL); - assert(*ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); - p = *ptr; -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (arena_ind != UINT_MAX) + arena = arenas[arena_ind]; + else + arena = NULL; + + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_usize); + + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; /* - * usize isn't knowable before iralloc() returns when extra is + * usize isn't knowable before ixalloc() returns when extra is * non-zero. Therefore, compute its maximum possible value and * use that in PROF_ALLOC_PREP() to decide whether to capture a * backtrace. prof_realloc() will use the actual usize to * decide whether to sample. */ size_t max_usize = (alignment == 0) ? s2u(size+extra) : - sa2u(size+extra, alignment, NULL); - prof_ctx_t *old_ctx = prof_ctx_get(p); - old_size = isalloc(p); + sa2u(size+extra, alignment); PROF_ALLOC_PREP(1, max_usize, cnt); - if (cnt == NULL) - goto OOM; - /* - * Use minimum usize to determine whether promotion may happen. - */ - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U - && ((alignment == 0) ? s2u(size) : sa2u(size, - alignment, NULL)) <= small_maxclass) { - q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= - size+extra) ? 0 : size+extra - (small_maxclass+1), - alignment, zero, no_move); - if (q == NULL) - goto ERR; - if (max_usize < PAGE_SIZE) { - usize = max_usize; - arena_prof_promoted(q, usize); - } else - usize = isalloc(q); - } else { - q = iralloc(p, size, extra, alignment, zero, no_move); - if (q == NULL) - goto ERR; - usize = isalloc(q); - } - prof_realloc(q, usize, cnt, old_size, old_ctx); - if (rsize != NULL) - *rsize = usize; - } else -#endif - { -#ifdef JEMALLOC_STATS - old_size = isalloc(p); -#endif - q = iralloc(p, size, extra, alignment, zero, no_move); - if (q == NULL) - goto ERR; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { - usize = isalloc(q); -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; - } + usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, + max_usize, zero, arena, cnt); + } else { + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); + } + if (usize == old_usize) + goto label_not_resized; + + if (config_stats) { + thread_allocated_t *ta; + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_usize; } + JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero); +label_not_resized: + UTRACE(ptr, size, ptr); + return (usize); +} - *ptr = q; -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(usize, old_size); -#endif - return (ALLOCM_SUCCESS); -ERR: - if (no_move) - return (ALLOCM_ERR_NOT_MOVED); -#ifdef JEMALLOC_PROF -OOM: -#endif -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in rallocm(): " - "out of memory\n"); - abort(); +size_t +je_sallocx(const void *ptr, int flags) +{ + size_t usize; + + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if (config_ivsalloc) + usize = ivsalloc(ptr, config_prof); + else { + assert(ptr != NULL); + usize = isalloc(ptr, config_prof); } -#endif - return (ALLOCM_ERR_OOM); + + return (usize); +} + +void +je_dallocx(void *ptr, int flags) +{ + size_t usize; + UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + bool try_tcache; + + assert(ptr != NULL); + assert(malloc_initialized || IS_INITIALIZER); + + if (arena_ind != UINT_MAX) { + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + try_tcache = (chunk == ptr || chunk->arena != + arenas[arena_ind]); + } else + try_tcache = true; + + UTRACE(ptr, 0, 0); + if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); + if (config_prof && opt_prof) { + if (config_stats == false && config_valgrind == false) + usize = isalloc(ptr, config_prof); + prof_free(ptr, usize); + } + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloct(ptr, try_tcache); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); +} + +size_t +je_nallocx(size_t size, int flags) +{ + size_t usize; + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + + assert(size != 0); + + if (malloc_init()) + return (0); + + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + assert(usize != 0); + return (usize); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) +je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { - size_t sz; - assert(malloc_initialized || malloc_initializer == pthread_self()); + if (malloc_init()) + return (EAGAIN); + + return (ctl_byname(name, oldp, oldlenp, newp, newlen)); +} + +int +je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_nametomib(name, mibp, miblenp)); +} + +int +je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); +} + +void +je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, + const char *opts) +{ + + stats_print(write_cb, cbopaque, opts); +} + +size_t +je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) +{ + size_t ret; + + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if (config_ivsalloc) + ret = ivsalloc(ptr, config_prof); + else + ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; + + return (ret); +} + +/* + * End non-standard functions. + */ +/******************************************************************************/ +/* + * Begin experimental functions. + */ +#ifdef JEMALLOC_EXPERIMENTAL + +int +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) +{ + void *p; -#ifdef JEMALLOC_IVSALLOC - sz = ivsalloc(ptr); -#else assert(ptr != NULL); - sz = isalloc(ptr); -#endif - assert(rsize != NULL); - *rsize = sz; + p = je_mallocx(size, flags); + if (p == NULL) + return (ALLOCM_ERR_OOM); + if (rsize != NULL) + *rsize = isalloc(p, config_prof); + *ptr = p; return (ALLOCM_SUCCESS); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(dallocm)(void *ptr, int flags) +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize; -#endif + int ret; + bool no_move = flags & ALLOCM_NO_MOVE; assert(ptr != NULL); - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(*ptr != NULL); + assert(size != 0); + assert(SIZE_T_MAX - size >= extra); -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif - prof_free(ptr, usize); + if (no_move) { + size_t usize = je_xallocx(*ptr, size, extra, flags); + ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; + if (rsize != NULL) + *rsize = usize; + } else { + void *p = je_rallocx(*ptr, size+extra, flags); + if (p != NULL) { + *ptr = p; + ret = ALLOCM_SUCCESS; + } else + ret = ALLOCM_ERR_OOM; + if (rsize != NULL) + *rsize = isalloc(*ptr, config_prof); } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); + return (ret); +} + +int +je_sallocm(const void *ptr, size_t *rsize, int flags) +{ + + assert(rsize != NULL); + *rsize = je_sallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_dallocm(void *ptr, int flags) +{ + + je_dallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_nallocm(size_t *rsize, size_t size, int flags) +{ + size_t usize; + usize = je_nallocx(size, flags); + if (usize == 0) + return (ALLOCM_ERR_OOM); + if (rsize != NULL) + *rsize = usize; return (ALLOCM_SUCCESS); } +#endif /* - * End non-standard functions. + * End experimental functions. */ /******************************************************************************/ - /* * The following functions are used by threading libraries for protection of * malloc during fork(). */ +/* + * If an application creates a thread before doing any allocation in the main + * thread, then calls fork(2) in the main thread followed by memory allocation + * in the child process, a race can occur that results in deadlock within the + * child: the main thread may have forked while the created thread had + * partially initialized the allocator. Ordinarily jemalloc prevents + * fork/malloc races via the following functions it registers during + * initialization using pthread_atfork(), but of course that does no good if + * the allocator isn't fully initialized at fork time. The following library + * constructor is a partial solution to this problem. It may still possible to + * trigger the deadlock described above, but doing so would involve forking via + * a library constructor that runs before jemalloc's runs. + */ +JEMALLOC_ATTR(constructor) +static void +jemalloc_constructor(void) +{ + + malloc_init(); +} + +#ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_prefork(void) +#else +JEMALLOC_EXPORT void +_malloc_prefork(void) +#endif { unsigned i; - /* Acquire all mutexes in a safe order. */ +#ifdef JEMALLOC_MUTEX_INIT_CB + if (malloc_initialized == false) + return; +#endif + assert(malloc_initialized); - malloc_mutex_lock(&arenas_lock); - for (i = 0; i < narenas; i++) { + /* Acquire all mutexes in a safe order. */ + ctl_prefork(); + prof_prefork(); + malloc_mutex_prefork(&arenas_lock); + for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) - malloc_mutex_lock(&arenas[i]->lock); + arena_prefork(arenas[i]); } + chunk_prefork(); + base_prefork(); + huge_prefork(); +} - malloc_mutex_lock(&base_mtx); - - malloc_mutex_lock(&huge_mtx); - -#ifdef JEMALLOC_DSS - malloc_mutex_lock(&dss_mtx); +#ifndef JEMALLOC_MUTEX_INIT_CB +void +jemalloc_postfork_parent(void) +#else +JEMALLOC_EXPORT void +_malloc_postfork(void) #endif +{ + unsigned i; -#ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); +#ifdef JEMALLOC_MUTEX_INIT_CB + if (malloc_initialized == false) + return; #endif + assert(malloc_initialized); + + /* Release all mutexes, now that fork() has completed. */ + huge_postfork_parent(); + base_postfork_parent(); + chunk_postfork_parent(); + for (i = 0; i < narenas_total; i++) { + if (arenas[i] != NULL) + arena_postfork_parent(arenas[i]); + } + malloc_mutex_postfork_parent(&arenas_lock); + prof_postfork_parent(); + ctl_postfork_parent(); } void -jemalloc_postfork(void) +jemalloc_postfork_child(void) { unsigned i; + assert(malloc_initialized); + /* Release all mutexes, now that fork() has completed. */ + huge_postfork_child(); + base_postfork_child(); + chunk_postfork_child(); + for (i = 0; i < narenas_total; i++) { + if (arenas[i] != NULL) + arena_postfork_child(arenas[i]); + } + malloc_mutex_postfork_child(&arenas_lock); + prof_postfork_child(); + ctl_postfork_child(); +} -#ifdef JEMALLOC_SWAP - malloc_mutex_unlock(&swap_mtx); -#endif +/******************************************************************************/ +/* + * The following functions are used for TLS allocation/deallocation in static + * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() + * is that these avoid accessing TLS variables. + */ -#ifdef JEMALLOC_DSS - malloc_mutex_unlock(&dss_mtx); -#endif +static void * +a0alloc(size_t size, bool zero) +{ - malloc_mutex_unlock(&huge_mtx); + if (malloc_init()) + return (NULL); - malloc_mutex_unlock(&base_mtx); + if (size == 0) + size = 1; - for (i = 0; i < narenas; i++) { - if (arenas[i] != NULL) - malloc_mutex_unlock(&arenas[i]->lock); - } - malloc_mutex_unlock(&arenas_lock); + if (size <= arena_maxclass) + return (arena_malloc(arenas[0], size, zero, false)); + else + return (huge_malloc(size, zero, huge_dss_prec_get(arenas[0]))); +} + +void * +a0malloc(size_t size) +{ + + return (a0alloc(size, false)); +} + +void * +a0calloc(size_t num, size_t size) +{ + + return (a0alloc(num * size, true)); +} + +void +a0free(void *ptr) +{ + arena_chunk_t *chunk; + + if (ptr == NULL) + return; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) + arena_dalloc(chunk->arena, chunk, ptr, false); + else + huge_dalloc(ptr, true); } /******************************************************************************/ diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index ca89ef1c962..788eca38703 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -1,14 +1,26 @@ #define JEMALLOC_MUTEX_C_ #include "jemalloc/internal/jemalloc_internal.h" +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) +#include +#endif + +#ifndef _CRT_SPINCOUNT +#define _CRT_SPINCOUNT 4000 +#endif + /******************************************************************************/ /* Data. */ #ifdef JEMALLOC_LAZY_LOCK bool isthreaded = false; #endif +#ifdef JEMALLOC_MUTEX_INIT_CB +static bool postpone_init = true; +static malloc_mutex_t *postponed_mutexes = NULL; +#endif -#ifdef JEMALLOC_LAZY_LOCK +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) static void pthread_create_once(void); #endif @@ -18,7 +30,7 @@ static void pthread_create_once(void); * process goes multi-threaded. */ -#ifdef JEMALLOC_LAZY_LOCK +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); @@ -36,8 +48,7 @@ pthread_create_once(void) isthreaded = true; } -JEMALLOC_ATTR(visibility("default")) -int +JEMALLOC_EXPORT int pthread_create(pthread_t *__restrict thread, const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), void *__restrict arg) @@ -52,39 +63,87 @@ pthread_create(pthread_t *__restrict thread, /******************************************************************************/ +#ifdef JEMALLOC_MUTEX_INIT_CB +JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)); +#endif + bool malloc_mutex_init(malloc_mutex_t *mutex) { -#ifdef JEMALLOC_OSSPIN - *mutex = 0; + +#ifdef _WIN32 + if (!InitializeCriticalSectionAndSpinCount(&mutex->lock, + _CRT_SPINCOUNT)) + return (true); +#elif (defined(JEMALLOC_OSSPIN)) + mutex->lock = 0; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + if (postpone_init) { + mutex->postponed_next = postponed_mutexes; + postponed_mutexes = mutex; + } else { + if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) != + 0) + return (true); + } #else pthread_mutexattr_t attr; if (pthread_mutexattr_init(&attr) != 0) return (true); -#ifdef PTHREAD_MUTEX_ADAPTIVE_NP - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); -#else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); -#endif - if (pthread_mutex_init(mutex, &attr) != 0) { + pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); + if (pthread_mutex_init(&mutex->lock, &attr) != 0) { pthread_mutexattr_destroy(&attr); return (true); } pthread_mutexattr_destroy(&attr); - #endif return (false); } void -malloc_mutex_destroy(malloc_mutex_t *mutex) +malloc_mutex_prefork(malloc_mutex_t *mutex) { -#ifndef JEMALLOC_OSSPIN - if (pthread_mutex_destroy(mutex) != 0) { - malloc_write(": Error in pthread_mutex_destroy()\n"); - abort(); + malloc_mutex_lock(mutex); +} + +void +malloc_mutex_postfork_parent(malloc_mutex_t *mutex) +{ + + malloc_mutex_unlock(mutex); +} + +void +malloc_mutex_postfork_child(malloc_mutex_t *mutex) +{ + +#ifdef JEMALLOC_MUTEX_INIT_CB + malloc_mutex_unlock(mutex); +#else + if (malloc_mutex_init(mutex)) { + malloc_printf(": Error re-initializing mutex in " + "child\n"); + if (opt_abort) + abort(); } #endif } + +bool +mutex_boot(void) +{ + +#ifdef JEMALLOC_MUTEX_INIT_CB + postpone_init = false; + while (postponed_mutexes != NULL) { + if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, + base_calloc) != 0) + return (true); + postponed_mutexes = postponed_mutexes->postponed_next; + } +#endif + return (false); +} diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index 8a144b4e46c..7722b7b4373 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -1,6 +1,5 @@ #define JEMALLOC_PROF_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_PROF /******************************************************************************/ #ifdef JEMALLOC_PROF_LIBUNWIND @@ -15,27 +14,35 @@ /******************************************************************************/ /* Data. */ +malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL) + bool opt_prof = false; bool opt_prof_active = true; -size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; bool opt_prof_gdump = false; +bool opt_prof_final = true; bool opt_prof_leak = false; -bool opt_prof_accum = true; -ssize_t opt_lg_prof_tcmax = LG_PROF_TCMAX_DEFAULT; -char opt_prof_prefix[PATH_MAX + 1]; +bool opt_prof_accum = false; +char opt_prof_prefix[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; -uint64_t prof_interval; +uint64_t prof_interval = 0; bool prof_promote; -unsigned prof_bt_max; - -#ifndef NO_TLS -__thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#endif -pthread_key_t prof_tdata_tsd; +/* + * Table of mutexes that are shared among ctx's. These are leaf locks, so + * there is no problem with using them for more than one ctx at the same time. + * The primary motivation for this sharing though is that ctx's are ephemeral, + * and destroying mutexes causes complications for systems that allocate when + * creating/destroying mutexes. + */ +static malloc_mutex_t *ctx_locks; +static unsigned cum_ctxs; /* Atomic counter. */ /* * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data @@ -52,56 +59,31 @@ static uint64_t prof_dump_useq; /* * This buffer is rather large for stack allocation, so use a single buffer for - * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since - * it must be locked anyway during dumping. + * all profile dumps. */ -static char prof_dump_buf[PROF_DUMP_BUF_SIZE]; +static malloc_mutex_t prof_dump_mtx; +static char prof_dump_buf[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PROF_DUMP_BUFSIZE +#else + 1 +#endif +]; static unsigned prof_dump_buf_end; static int prof_dump_fd; /* Do not dump any profiles until bootstrapping is complete. */ static bool prof_booted = false; -static malloc_mutex_t enq_mtx; -static bool enq; -static bool enq_idump; -static bool enq_gdump; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static prof_bt_t *bt_dup(prof_bt_t *bt); -static void bt_destroy(prof_bt_t *bt); -#ifdef JEMALLOC_PROF_LIBGCC -static _Unwind_Reason_Code prof_unwind_init_callback( - struct _Unwind_Context *context, void *arg); -static _Unwind_Reason_Code prof_unwind_callback( - struct _Unwind_Context *context, void *arg); -#endif -static bool prof_flush(bool propagate_err); -static bool prof_write(const char *s, bool propagate_err); -static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, - size_t *leak_nctx); -static void prof_ctx_destroy(prof_ctx_t *ctx); -static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); -static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, - bool propagate_err); -static bool prof_dump_maps(bool propagate_err); -static bool prof_dump(const char *filename, bool leakcheck, - bool propagate_err); -static void prof_dump_filename(char *filename, char v, int64_t vseq); -static void prof_fdump(void); -static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); -static bool prof_bt_keycomp(const void *k1, const void *k2); -static void prof_tdata_cleanup(void *arg); - /******************************************************************************/ void bt_init(prof_bt_t *bt, void **vec) { + cassert(config_prof); + bt->vec = vec; bt->len = 0; } @@ -110,6 +92,8 @@ static void bt_destroy(prof_bt_t *bt) { + cassert(config_prof); + idalloc(bt); } @@ -118,6 +102,8 @@ bt_dup(prof_bt_t *bt) { prof_bt_t *ret; + cassert(config_prof); + /* * Create a single allocation that has space for vec immediately * following the prof_bt_t structure. The backtraces that get @@ -138,30 +124,32 @@ bt_dup(prof_bt_t *bt) } static inline void -prof_enter(void) +prof_enter(prof_tdata_t *prof_tdata) { - malloc_mutex_lock(&enq_mtx); - enq = true; - malloc_mutex_unlock(&enq_mtx); + cassert(config_prof); + + assert(prof_tdata->enq == false); + prof_tdata->enq = true; malloc_mutex_lock(&bt2ctx_mtx); } static inline void -prof_leave(void) +prof_leave(prof_tdata_t *prof_tdata) { bool idump, gdump; + cassert(config_prof); + malloc_mutex_unlock(&bt2ctx_mtx); - malloc_mutex_lock(&enq_mtx); - enq = false; - idump = enq_idump; - enq_idump = false; - gdump = enq_gdump; - enq_gdump = false; - malloc_mutex_unlock(&enq_mtx); + assert(prof_tdata->enq); + prof_tdata->enq = false; + idump = prof_tdata->enq_idump; + prof_tdata->enq_idump = false; + gdump = prof_tdata->enq_gdump; + prof_tdata->enq_gdump = false; if (idump) prof_idump(); @@ -171,16 +159,16 @@ prof_leave(void) #ifdef JEMALLOC_PROF_LIBUNWIND void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { unw_context_t uc; unw_cursor_t cursor; unsigned i; int err; + cassert(config_prof); assert(bt->len == 0); assert(bt->vec != NULL); - assert(max <= (1U << opt_lg_prof_bt_max)); unw_getcontext(&uc); unw_init_local(&cursor, &uc); @@ -196,7 +184,7 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) * Iterate over stack frames until there are no more, or until no space * remains in bt. */ - for (i = 0; i < max; i++) { + for (i = 0; i < PROF_BT_MAX; i++) { unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); bt->len++; err = unw_step(&cursor); @@ -204,12 +192,13 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) break; } } -#endif -#ifdef JEMALLOC_PROF_LIBGCC +#elif (defined(JEMALLOC_PROF_LIBGCC)) static _Unwind_Reason_Code prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) { + cassert(config_prof); + return (_URC_NO_REASON); } @@ -218,6 +207,8 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) { prof_unwind_data_t *data = (prof_unwind_data_t *)arg; + cassert(config_prof); + if (data->nignore > 0) data->nignore--; else { @@ -231,19 +222,20 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) } void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { - prof_unwind_data_t data = {bt, nignore, max}; + prof_unwind_data_t data = {bt, nignore, PROF_BT_MAX}; + + cassert(config_prof); _Unwind_Backtrace(prof_unwind_callback, &data); } -#endif -#ifdef JEMALLOC_PROF_GCC +#elif (defined(JEMALLOC_PROF_GCC)) void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { #define BT_FRAME(i) \ - if ((i) < nignore + max) { \ + if ((i) < nignore + PROF_BT_MAX) { \ void *p; \ if (__builtin_frame_address(i) == 0) \ return; \ @@ -257,8 +249,8 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) } else \ return; + cassert(config_prof); assert(nignore <= 3); - assert(max <= (1U << opt_lg_prof_bt_max)); BT_FRAME(0) BT_FRAME(1) @@ -407,8 +399,175 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) BT_FRAME(130) #undef BT_FRAME } +#else +void +prof_backtrace(prof_bt_t *bt, unsigned nignore) +{ + + cassert(config_prof); + not_reached(); +} #endif +static malloc_mutex_t * +prof_ctx_mutex_choose(void) +{ + unsigned nctxs = atomic_add_u(&cum_ctxs, 1); + + return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); +} + +static void +prof_ctx_init(prof_ctx_t *ctx, prof_bt_t *bt) +{ + + ctx->bt = bt; + ctx->lock = prof_ctx_mutex_choose(); + /* + * Set nlimbo to 1, in order to avoid a race condition with + * prof_ctx_merge()/prof_ctx_destroy(). + */ + ctx->nlimbo = 1; + ql_elm_new(ctx, dump_link); + memset(&ctx->cnt_merged, 0, sizeof(prof_cnt_t)); + ql_new(&ctx->cnts_ql); +} + +static void +prof_ctx_destroy(prof_ctx_t *ctx) +{ + prof_tdata_t *prof_tdata; + + cassert(config_prof); + + /* + * Check that ctx is still unused by any thread cache before destroying + * it. prof_lookup() increments ctx->nlimbo in order to avoid a race + * condition with this function, as does prof_ctx_merge() in order to + * avoid a race between the main body of prof_ctx_merge() and entry + * into this function. + */ + prof_tdata = prof_tdata_get(false); + assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); + prof_enter(prof_tdata); + malloc_mutex_lock(ctx->lock); + if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 && + ctx->nlimbo == 1) { + assert(ctx->cnt_merged.curbytes == 0); + assert(ctx->cnt_merged.accumobjs == 0); + assert(ctx->cnt_merged.accumbytes == 0); + /* Remove ctx from bt2ctx. */ + if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) + not_reached(); + prof_leave(prof_tdata); + /* Destroy ctx. */ + malloc_mutex_unlock(ctx->lock); + bt_destroy(ctx->bt); + idalloc(ctx); + } else { + /* + * Compensate for increment in prof_ctx_merge() or + * prof_lookup(). + */ + ctx->nlimbo--; + malloc_mutex_unlock(ctx->lock); + prof_leave(prof_tdata); + } +} + +static void +prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +{ + bool destroy; + + cassert(config_prof); + + /* Merge cnt stats and detach from ctx. */ + malloc_mutex_lock(ctx->lock); + ctx->cnt_merged.curobjs += cnt->cnts.curobjs; + ctx->cnt_merged.curbytes += cnt->cnts.curbytes; + ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; + ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; + ql_remove(&ctx->cnts_ql, cnt, cnts_link); + if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && + ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { + /* + * Increment ctx->nlimbo in order to keep another thread from + * winning the race to destroy ctx while this one has ctx->lock + * dropped. Without this, it would be possible for another + * thread to: + * + * 1) Sample an allocation associated with ctx. + * 2) Deallocate the sampled object. + * 3) Successfully prof_ctx_destroy(ctx). + * + * The result would be that ctx no longer exists by the time + * this thread accesses it in prof_ctx_destroy(). + */ + ctx->nlimbo++; + destroy = true; + } else + destroy = false; + malloc_mutex_unlock(ctx->lock); + if (destroy) + prof_ctx_destroy(ctx); +} + +static bool +prof_lookup_global(prof_bt_t *bt, prof_tdata_t *prof_tdata, void **p_btkey, + prof_ctx_t **p_ctx, bool *p_new_ctx) +{ + union { + prof_ctx_t *p; + void *v; + } ctx; + union { + prof_bt_t *p; + void *v; + } btkey; + bool new_ctx; + + prof_enter(prof_tdata); + if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { + /* bt has never been seen before. Insert it. */ + ctx.v = imalloc(sizeof(prof_ctx_t)); + if (ctx.v == NULL) { + prof_leave(prof_tdata); + return (true); + } + btkey.p = bt_dup(bt); + if (btkey.v == NULL) { + prof_leave(prof_tdata); + idalloc(ctx.v); + return (true); + } + prof_ctx_init(ctx.p, btkey.p); + if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { + /* OOM. */ + prof_leave(prof_tdata); + idalloc(btkey.v); + idalloc(ctx.v); + return (true); + } + new_ctx = true; + } else { + /* + * Increment nlimbo, in order to avoid a race condition with + * prof_ctx_merge()/prof_ctx_destroy(). + */ + malloc_mutex_lock(ctx.p->lock); + ctx.p->nlimbo++; + malloc_mutex_unlock(ctx.p->lock); + new_ctx = false; + } + prof_leave(prof_tdata); + + *p_btkey = btkey.v; + *p_ctx = ctx.p; + *p_new_ctx = new_ctx; + return (false); +} + prof_thr_cnt_t * prof_lookup(prof_bt_t *bt) { @@ -418,84 +577,26 @@ prof_lookup(prof_bt_t *bt) } ret; prof_tdata_t *prof_tdata; - prof_tdata = PROF_TCACHE_GET(); - if (prof_tdata == NULL) { - prof_tdata = prof_tdata_init(); - if (prof_tdata == NULL) - return (NULL); - } + cassert(config_prof); + + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (NULL); if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { - union { - prof_bt_t *p; - void *v; - } btkey; - union { - prof_ctx_t *p; - void *v; - } ctx; + void *btkey; + prof_ctx_t *ctx; bool new_ctx; /* * This thread's cache lacks bt. Look for it in the global * cache. */ - prof_enter(); - if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { - /* bt has never been seen before. Insert it. */ - ctx.v = imalloc(sizeof(prof_ctx_t)); - if (ctx.v == NULL) { - prof_leave(); - return (NULL); - } - btkey.p = bt_dup(bt); - if (btkey.v == NULL) { - prof_leave(); - idalloc(ctx.v); - return (NULL); - } - ctx.p->bt = btkey.p; - if (malloc_mutex_init(&ctx.p->lock)) { - prof_leave(); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } - memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); - ql_new(&ctx.p->cnts_ql); - if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { - /* OOM. */ - prof_leave(); - malloc_mutex_destroy(&ctx.p->lock); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } - /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). - * - * No locking is necessary for ctx here because no other - * threads have had the opportunity to fetch it from - * bt2ctx yet. - */ - ctx.p->cnt_merged.curobjs++; - new_ctx = true; - } else { - /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). - */ - malloc_mutex_lock(&ctx.p->lock); - ctx.p->cnt_merged.curobjs++; - malloc_mutex_unlock(&ctx.p->lock); - new_ctx = false; - } - prof_leave(); + if (prof_lookup_global(bt, prof_tdata, &btkey, &ctx, &new_ctx)) + return (NULL); /* Link a prof_thd_cnt_t into ctx for this thread. */ - if (opt_lg_prof_tcmax >= 0 && ckh_count(&prof_tdata->bt2cnt) - == (ZU(1) << opt_lg_prof_tcmax)) { + if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { assert(ckh_count(&prof_tdata->bt2cnt) > 0); /* * Flush the least recently used cnt in order to keep @@ -505,39 +606,37 @@ prof_lookup(prof_bt_t *bt) assert(ret.v != NULL); if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, NULL, NULL)) - assert(false); + not_reached(); ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); prof_ctx_merge(ret.p->ctx, ret.p); /* ret can now be re-used. */ } else { - assert(opt_lg_prof_tcmax < 0 || - ckh_count(&prof_tdata->bt2cnt) < (ZU(1) << - opt_lg_prof_tcmax)); + assert(ckh_count(&prof_tdata->bt2cnt) < PROF_TCMAX); /* Allocate and partially initialize a new cnt. */ ret.v = imalloc(sizeof(prof_thr_cnt_t)); if (ret.p == NULL) { if (new_ctx) - prof_ctx_destroy(ctx.p); + prof_ctx_destroy(ctx); return (NULL); } ql_elm_new(ret.p, cnts_link); ql_elm_new(ret.p, lru_link); } /* Finish initializing ret. */ - ret.p->ctx = ctx.p; + ret.p->ctx = ctx; ret.p->epoch = 0; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); - if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { + if (ckh_insert(&prof_tdata->bt2cnt, btkey, ret.v)) { if (new_ctx) - prof_ctx_destroy(ctx.p); + prof_ctx_destroy(ctx); idalloc(ret.v); return (NULL); } ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); - malloc_mutex_lock(&ctx.p->lock); - ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); - ctx.p->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx.p->lock); + malloc_mutex_lock(ctx->lock); + ql_tail_insert(&ctx->cnts_ql, ret.p, cnts_link); + ctx->nlimbo--; + malloc_mutex_unlock(ctx->lock); } else { /* Move ret to the front of the LRU. */ ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); @@ -547,12 +646,58 @@ prof_lookup(prof_bt_t *bt) return (ret.p); } +#ifdef JEMALLOC_JET +size_t +prof_bt_count(void) +{ + size_t bt_count; + prof_tdata_t *prof_tdata; + + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (0); + + prof_enter(prof_tdata); + bt_count = ckh_count(&bt2ctx); + prof_leave(prof_tdata); + + return (bt_count); +} +#endif + +#ifdef JEMALLOC_JET +#undef prof_dump_open +#define prof_dump_open JEMALLOC_N(prof_dump_open_impl) +#endif +static int +prof_dump_open(bool propagate_err, const char *filename) +{ + int fd; + + fd = creat(filename, 0644); + if (fd == -1 && propagate_err == false) { + malloc_printf(": creat(\"%s\"), 0644) failed\n", + filename); + if (opt_abort) + abort(); + } + + return (fd); +} +#ifdef JEMALLOC_JET +#undef prof_dump_open +#define prof_dump_open JEMALLOC_N(prof_dump_open) +prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); +#endif + static bool -prof_flush(bool propagate_err) +prof_dump_flush(bool propagate_err) { bool ret = false; ssize_t err; + cassert(config_prof); + err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); if (err == -1) { if (propagate_err == false) { @@ -569,24 +714,39 @@ prof_flush(bool propagate_err) } static bool -prof_write(const char *s, bool propagate_err) +prof_dump_close(bool propagate_err) +{ + bool ret; + + assert(prof_dump_fd != -1); + ret = prof_dump_flush(propagate_err); + close(prof_dump_fd); + prof_dump_fd = -1; + + return (ret); +} + +static bool +prof_dump_write(bool propagate_err, const char *s) { unsigned i, slen, n; + cassert(config_prof); + i = 0; slen = strlen(s); while (i < slen) { /* Flush the buffer if it is full. */ - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) - if (prof_flush(propagate_err) && propagate_err) + if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) + if (prof_dump_flush(propagate_err) && propagate_err) return (true); - if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) { + if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { /* Finish writing. */ n = slen - i; } else { /* Write as much of s as will fit. */ - n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end; + n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; } memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); prof_dump_buf_end += n; @@ -596,13 +756,40 @@ prof_write(const char *s, bool propagate_err) return (false); } +JEMALLOC_ATTR(format(printf, 2, 3)) +static bool +prof_dump_printf(bool propagate_err, const char *format, ...) +{ + bool ret; + va_list ap; + char buf[PROF_PRINTF_BUFSIZE]; + + va_start(ap, format); + malloc_vsnprintf(buf, sizeof(buf), format, ap); + va_end(ap); + ret = prof_dump_write(propagate_err, buf); + + return (ret); +} + static void -prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) +prof_dump_ctx_prep(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx, + prof_ctx_list_t *ctx_ql) { prof_thr_cnt_t *thr_cnt; prof_cnt_t tcnt; - malloc_mutex_lock(&ctx->lock); + cassert(config_prof); + + malloc_mutex_lock(ctx->lock); + + /* + * Increment nlimbo so that ctx won't go away before dump. + * Additionally, link ctx into the dump list so that it is included in + * prof_dump()'s second pass. + */ + ctx->nlimbo++; + ql_tail_insert(ctx_ql, ctx, dump_link); memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { @@ -641,337 +828,257 @@ prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) cnt_all->accumbytes += ctx->cnt_summed.accumbytes; } - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); } -static void -prof_ctx_destroy(prof_ctx_t *ctx) +static bool +prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) { - /* - * Check that ctx is still unused by any thread cache before destroying - * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in - * order to avoid a race condition with this function, as does - * prof_ctx_merge() in order to avoid a race between the main body of - * prof_ctx_merge() and entry into this function. - */ - prof_enter(); - malloc_mutex_lock(&ctx->lock); - if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { - assert(ctx->cnt_merged.curbytes == 0); - assert(ctx->cnt_merged.accumobjs == 0); - assert(ctx->cnt_merged.accumbytes == 0); - /* Remove ctx from bt2ctx. */ - if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) - assert(false); - prof_leave(); - /* Destroy ctx. */ - malloc_mutex_unlock(&ctx->lock); - bt_destroy(ctx->bt); - malloc_mutex_destroy(&ctx->lock); - idalloc(ctx); + if (opt_lg_prof_sample == 0) { + if (prof_dump_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heapprofile\n", + cnt_all->curobjs, cnt_all->curbytes, + cnt_all->accumobjs, cnt_all->accumbytes)) + return (true); } else { - /* - * Compensate for increment in prof_ctx_merge() or - * prof_lookup(). - */ - ctx->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx->lock); - prof_leave(); + if (prof_dump_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", + cnt_all->curobjs, cnt_all->curbytes, + cnt_all->accumobjs, cnt_all->accumbytes, + ((uint64_t)1U << opt_lg_prof_sample))) + return (true); } + + return (false); } static void -prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +prof_dump_ctx_cleanup_locked(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) { - bool destroy; - /* Merge cnt stats and detach from ctx. */ - malloc_mutex_lock(&ctx->lock); - ctx->cnt_merged.curobjs += cnt->cnts.curobjs; - ctx->cnt_merged.curbytes += cnt->cnts.curbytes; - ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; - ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; - ql_remove(&ctx->cnts_ql, cnt, cnts_link); - if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && - ctx->cnt_merged.curobjs == 0) { - /* - * Artificially raise ctx->cnt_merged.curobjs in order to keep - * another thread from winning the race to destroy ctx while - * this one has ctx->lock dropped. Without this, it would be - * possible for another thread to: - * - * 1) Sample an allocation associated with ctx. - * 2) Deallocate the sampled object. - * 3) Successfully prof_ctx_destroy(ctx). - * - * The result would be that ctx no longer exists by the time - * this thread accesses it in prof_ctx_destroy(). - */ - ctx->cnt_merged.curobjs++; - destroy = true; - } else - destroy = false; - malloc_mutex_unlock(&ctx->lock); - if (destroy) - prof_ctx_destroy(ctx); + ctx->nlimbo--; + ql_remove(ctx_ql, ctx, dump_link); +} + +static void +prof_dump_ctx_cleanup(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) +{ + + malloc_mutex_lock(ctx->lock); + prof_dump_ctx_cleanup_locked(ctx, ctx_ql); + malloc_mutex_unlock(ctx->lock); } static bool -prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err) +prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, + prof_ctx_list_t *ctx_ql) { - char buf[UMAX2S_BUFSIZE]; + bool ret; unsigned i; - if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { + cassert(config_prof); + + /* + * Current statistics can sum to 0 as a result of unmerged per thread + * statistics. Additionally, interval- and growth-triggered dumps can + * occur between the time a ctx is created and when its statistics are + * filled in. Avoid dumping any ctx that is an artifact of either + * implementation detail. + */ + malloc_mutex_lock(ctx->lock); + if ((opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) || + (opt_prof_accum && ctx->cnt_summed.accumobjs == 0)) { + assert(ctx->cnt_summed.curobjs == 0); assert(ctx->cnt_summed.curbytes == 0); assert(ctx->cnt_summed.accumobjs == 0); assert(ctx->cnt_summed.accumbytes == 0); - return (false); + ret = false; + goto label_return; } - if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf), - propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf), - propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf), - propagate_err) - || prof_write("] @", propagate_err)) - return (true); + if (prof_dump_printf(propagate_err, "%"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @", + ctx->cnt_summed.curobjs, ctx->cnt_summed.curbytes, + ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) { + ret = true; + goto label_return; + } for (i = 0; i < bt->len; i++) { - if (prof_write(" 0x", propagate_err) - || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf), - propagate_err)) - return (true); + if (prof_dump_printf(propagate_err, " %#"PRIxPTR, + (uintptr_t)bt->vec[i])) { + ret = true; + goto label_return; + } } - if (prof_write("\n", propagate_err)) - return (true); + if (prof_dump_write(propagate_err, "\n")) { + ret = true; + goto label_return; + } - return (false); + ret = false; +label_return: + prof_dump_ctx_cleanup_locked(ctx, ctx_ql); + malloc_mutex_unlock(ctx->lock); + return (ret); } static bool prof_dump_maps(bool propagate_err) { + bool ret; int mfd; - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - /* /proc//maps\0 */ - char mpath[6 + UMAX2S_BUFSIZE - + 5 + 1]; - - i = 0; - - s = "/proc/"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = "/maps"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - mpath[i] = '\0'; - - mfd = open(mpath, O_RDONLY); + char filename[PATH_MAX + 1]; + + cassert(config_prof); +#ifdef __FreeBSD__ + malloc_snprintf(filename, sizeof(filename), "/proc/curproc/map"); +#else + malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps", + (int)getpid()); +#endif + mfd = open(filename, O_RDONLY); if (mfd != -1) { ssize_t nread; - if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) && - propagate_err) - return (true); + if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && + propagate_err) { + ret = true; + goto label_return; + } nread = 0; do { prof_dump_buf_end += nread; - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) { + if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { /* Make space in prof_dump_buf before read(). */ - if (prof_flush(propagate_err) && propagate_err) - return (true); + if (prof_dump_flush(propagate_err) && + propagate_err) { + ret = true; + goto label_return; + } } nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], - PROF_DUMP_BUF_SIZE - prof_dump_buf_end); + PROF_DUMP_BUFSIZE - prof_dump_buf_end); } while (nread > 0); + } else { + ret = true; + goto label_return; + } + + ret = false; +label_return: + if (mfd != -1) close(mfd); - } else - return (true); + return (ret); +} - return (false); +static void +prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_nctx, + const char *filename) +{ + + if (cnt_all->curbytes != 0) { + malloc_printf(": Leak summary: %"PRId64" byte%s, %" + PRId64" object%s, %zu context%s\n", + cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "", + cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "", + leak_nctx, (leak_nctx != 1) ? "s" : ""); + malloc_printf( + ": Run pprof on \"%s\" for leak detail\n", + filename); + } } static bool -prof_dump(const char *filename, bool leakcheck, bool propagate_err) +prof_dump(bool propagate_err, const char *filename, bool leakcheck) { + prof_tdata_t *prof_tdata; prof_cnt_t cnt_all; size_t tabind; - union { - prof_bt_t *p; - void *v; - } bt; union { prof_ctx_t *p; void *v; } ctx; - char buf[UMAX2S_BUFSIZE]; size_t leak_nctx; + prof_ctx_list_t ctx_ql; - prof_enter(); - prof_dump_fd = creat(filename, 0644); - if (prof_dump_fd == -1) { - if (propagate_err == false) { - malloc_write(": creat(\""); - malloc_write(filename); - malloc_write("\", 0644) failed\n"); - if (opt_abort) - abort(); - } - goto ERROR; - } + cassert(config_prof); + + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (true); + + malloc_mutex_lock(&prof_dump_mtx); /* Merge per thread profile stats, and sum them in cnt_all. */ memset(&cnt_all, 0, sizeof(prof_cnt_t)); leak_nctx = 0; + ql_new(&ctx_ql); + prof_enter(prof_tdata); for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) - prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); + prof_dump_ctx_prep(ctx.p, &cnt_all, &leak_nctx, &ctx_ql); + prof_leave(prof_tdata); - /* Dump profile header. */ - if (prof_write("heap profile: ", propagate_err) - || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err)) - goto ERROR; + /* Create dump file. */ + if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) + goto label_open_close_error; - if (opt_lg_prof_sample == 0) { - if (prof_write("] @ heapprofile\n", propagate_err)) - goto ERROR; - } else { - if (prof_write("] @ heap_v2/", propagate_err) - || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10, - buf), propagate_err) - || prof_write("\n", propagate_err)) - goto ERROR; - } + /* Dump profile header. */ + if (prof_dump_header(propagate_err, &cnt_all)) + goto label_write_error; - /* Dump per ctx profile stats. */ - for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) - == false;) { - if (prof_dump_ctx(ctx.p, bt.p, propagate_err)) - goto ERROR; + /* Dump per ctx profile stats. */ + while ((ctx.p = ql_first(&ctx_ql)) != NULL) { + if (prof_dump_ctx(propagate_err, ctx.p, ctx.p->bt, &ctx_ql)) + goto label_write_error; } /* Dump /proc//maps if possible. */ if (prof_dump_maps(propagate_err)) - goto ERROR; + goto label_write_error; - if (prof_flush(propagate_err)) - goto ERROR; - close(prof_dump_fd); - prof_leave(); - - if (leakcheck && cnt_all.curbytes != 0) { - malloc_write(": Leak summary: "); - malloc_write(u2s(cnt_all.curbytes, 10, buf)); - malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, "); - malloc_write(u2s(cnt_all.curobjs, 10, buf)); - malloc_write((cnt_all.curobjs != 1) ? " objects, " : - " object, "); - malloc_write(u2s(leak_nctx, 10, buf)); - malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n"); - malloc_write(": Run pprof on \""); - malloc_write(filename); - malloc_write("\" for leak detail\n"); - } + if (prof_dump_close(propagate_err)) + goto label_open_close_error; + + malloc_mutex_unlock(&prof_dump_mtx); + + if (leakcheck) + prof_leakcheck(&cnt_all, leak_nctx, filename); return (false); -ERROR: - prof_leave(); +label_write_error: + prof_dump_close(propagate_err); +label_open_close_error: + while ((ctx.p = ql_first(&ctx_ql)) != NULL) + prof_dump_ctx_cleanup(ctx.p, &ctx_ql); + malloc_mutex_unlock(&prof_dump_mtx); return (true); } -#define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \ - + 1 \ - + UMAX2S_BUFSIZE \ - + 2 \ - + UMAX2S_BUFSIZE \ - + 5 + 1) +#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) +#define VSEQ_INVALID UINT64_C(0xffffffffffffffff) static void prof_dump_filename(char *filename, char v, int64_t vseq) { - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - - /* - * Construct a filename of the form: - * - * ...v.heap\0 - */ - - i = 0; - - s = opt_prof_prefix; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = u2s(prof_dump_seq, 10, buf); - prof_dump_seq++; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; + cassert(config_prof); - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = v; - i++; - - if (vseq != 0xffffffffffffffffLLU) { - s = u2s(vseq, 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; + if (vseq != VSEQ_INVALID) { + /* "...v.heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%"PRIu64".%c%"PRId64".heap", + opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); + } else { + /* "....heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%"PRIu64".%c.heap", + opt_prof_prefix, (int)getpid(), prof_dump_seq, v); } - - s = ".heap"; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = '\0'; + prof_dump_seq++; } static void @@ -979,38 +1086,43 @@ prof_fdump(void) { char filename[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (prof_booted == false) return; - if (opt_prof_prefix[0] != '\0') { + if (opt_prof_final && opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU); + prof_dump_filename(filename, 'f', VSEQ_INVALID); malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, opt_prof_leak, false); + prof_dump(false, filename, opt_prof_leak); } } void prof_idump(void) { - char filename[DUMP_FILENAME_BUFSIZE]; + prof_tdata_t *prof_tdata; + char filename[PATH_MAX + 1]; + + cassert(config_prof); if (prof_booted == false) return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_idump = true; - malloc_mutex_unlock(&enq_mtx); + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return; + if (prof_tdata->enq) { + prof_tdata->enq_idump = true; return; } - malloc_mutex_unlock(&enq_mtx); if (opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'i', prof_dump_iseq); prof_dump_iseq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); + prof_dump(false, filename, false); } } @@ -1019,6 +1131,8 @@ prof_mdump(const char *filename) { char filename_buf[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (opt_prof == false || prof_booted == false) return (true); @@ -1032,60 +1146,44 @@ prof_mdump(const char *filename) malloc_mutex_unlock(&prof_dump_seq_mtx); filename = filename_buf; } - return (prof_dump(filename, false, true)); + return (prof_dump(true, filename, false)); } void prof_gdump(void) { + prof_tdata_t *prof_tdata; char filename[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (prof_booted == false) return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_gdump = true; - malloc_mutex_unlock(&enq_mtx); + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return; + if (prof_tdata->enq) { + prof_tdata->enq_gdump = true; return; } - malloc_mutex_unlock(&enq_mtx); if (opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'u', prof_dump_useq); prof_dump_useq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); + prof_dump(false, filename, false); } } static void -prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +prof_bt_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; prof_bt_t *bt = (prof_bt_t *)key; - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); + cassert(config_prof); - h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(bt->vec, bt->len * sizeof(void *), - 0x8432a476666bbc13LLU); - } - - *hash1 = ret1; - *hash2 = ret2; + hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); } static bool @@ -1094,6 +1192,8 @@ prof_bt_keycomp(const void *k1, const void *k2) const prof_bt_t *bt1 = (prof_bt_t *)k1; const prof_bt_t *bt2 = (prof_bt_t *)k2; + cassert(config_prof); + if (bt1->len != bt2->len) return (false); return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); @@ -1104,6 +1204,8 @@ prof_tdata_init(void) { prof_tdata_t *prof_tdata; + cassert(config_prof); + /* Initialize an empty cache for this thread. */ prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); if (prof_tdata == NULL) @@ -1116,51 +1218,77 @@ prof_tdata_init(void) } ql_new(&prof_tdata->lru_ql); - prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max); + prof_tdata->vec = imalloc(sizeof(void *) * PROF_BT_MAX); if (prof_tdata->vec == NULL) { ckh_delete(&prof_tdata->bt2cnt); idalloc(prof_tdata); return (NULL); } - prof_tdata->prn_state = 0; + prof_tdata->prng_state = 0; prof_tdata->threshold = 0; prof_tdata->accum = 0; - PROF_TCACHE_SET(prof_tdata); + prof_tdata->enq = false; + prof_tdata->enq_idump = false; + prof_tdata->enq_gdump = false; + + prof_tdata_tsd_set(&prof_tdata); return (prof_tdata); } -static void +void prof_tdata_cleanup(void *arg) { prof_thr_cnt_t *cnt; - prof_tdata_t *prof_tdata = (prof_tdata_t *)arg; + prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg; - /* - * Delete the hash table. All of its contents can still be iterated - * over via the LRU. - */ - ckh_delete(&prof_tdata->bt2cnt); + cassert(config_prof); - /* Iteratively merge cnt's into the global stats and delete them. */ - while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { - ql_remove(&prof_tdata->lru_ql, cnt, lru_link); - prof_ctx_merge(cnt->ctx, cnt); - idalloc(cnt); + if (prof_tdata == PROF_TDATA_STATE_REINCARNATED) { + /* + * Another destructor deallocated memory after this destructor + * was called. Reset prof_tdata to PROF_TDATA_STATE_PURGATORY + * in order to receive another callback. + */ + prof_tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&prof_tdata); + } else if (prof_tdata == PROF_TDATA_STATE_PURGATORY) { + /* + * The previous time this destructor was called, we set the key + * to PROF_TDATA_STATE_PURGATORY so that other destructors + * wouldn't cause re-creation of the prof_tdata. This time, do + * nothing, so that the destructor will not be called again. + */ + } else if (prof_tdata != NULL) { + /* + * Delete the hash table. All of its contents can still be + * iterated over via the LRU. + */ + ckh_delete(&prof_tdata->bt2cnt); + /* + * Iteratively merge cnt's into the global stats and delete + * them. + */ + while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { + ql_remove(&prof_tdata->lru_ql, cnt, lru_link); + prof_ctx_merge(cnt->ctx, cnt); + idalloc(cnt); + } + idalloc(prof_tdata->vec); + idalloc(prof_tdata); + prof_tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&prof_tdata); } - - idalloc(prof_tdata->vec); - - idalloc(prof_tdata); - PROF_TCACHE_SET(NULL); } void prof_boot0(void) { + cassert(config_prof); + memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, sizeof(PROF_PREFIX_DEFAULT)); } @@ -1169,6 +1297,8 @@ void prof_boot1(void) { + cassert(config_prof); + /* * opt_prof and prof_promote must be in their final state before any * arenas are initialized, so this function must be executed early. @@ -1181,50 +1311,55 @@ prof_boot1(void) */ opt_prof = true; opt_prof_gdump = false; - prof_interval = 0; } else if (opt_prof) { if (opt_lg_prof_interval >= 0) { prof_interval = (((uint64_t)1U) << opt_lg_prof_interval); - } else - prof_interval = 0; + } } - prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT); + prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE); } bool prof_boot2(void) { + cassert(config_prof); + if (opt_prof) { + unsigned i; + if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) return (true); if (malloc_mutex_init(&bt2ctx_mtx)) return (true); - if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup) - != 0) { + if (prof_tdata_tsd_boot()) { malloc_write( ": Error in pthread_key_create()\n"); abort(); } - prof_bt_max = (1U << opt_lg_prof_bt_max); if (malloc_mutex_init(&prof_dump_seq_mtx)) return (true); - - if (malloc_mutex_init(&enq_mtx)) + if (malloc_mutex_init(&prof_dump_mtx)) return (true); - enq = false; - enq_idump = false; - enq_gdump = false; if (atexit(prof_fdump) != 0) { malloc_write(": Error in atexit()\n"); if (opt_abort) abort(); } + + ctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * + sizeof(malloc_mutex_t)); + if (ctx_locks == NULL) + return (true); + for (i = 0; i < PROF_NCTX_LOCKS; i++) { + if (malloc_mutex_init(&ctx_locks[i])) + return (true); + } } #ifdef JEMALLOC_PROF_LIBGCC @@ -1240,5 +1375,46 @@ prof_boot2(void) return (false); } +void +prof_prefork(void) +{ + + if (opt_prof) { + unsigned i; + + malloc_mutex_prefork(&bt2ctx_mtx); + malloc_mutex_prefork(&prof_dump_seq_mtx); + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_prefork(&ctx_locks[i]); + } +} + +void +prof_postfork_parent(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_parent(&ctx_locks[i]); + malloc_mutex_postfork_parent(&prof_dump_seq_mtx); + malloc_mutex_postfork_parent(&bt2ctx_mtx); + } +} + +void +prof_postfork_child(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_child(&ctx_locks[i]); + malloc_mutex_postfork_child(&prof_dump_seq_mtx); + malloc_mutex_postfork_child(&bt2ctx_mtx); + } +} + /******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc/src/quarantine.c b/deps/jemalloc/src/quarantine.c new file mode 100644 index 00000000000..5431511640a --- /dev/null +++ b/deps/jemalloc/src/quarantine.c @@ -0,0 +1,199 @@ +#define JEMALLOC_QUARANTINE_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/* + * quarantine pointers close to NULL are used to encode state information that + * is used for cleaning up during thread shutdown. + */ +#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) +#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) +#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY + +/******************************************************************************/ +/* Data. */ + +malloc_tsd_data(, quarantine, quarantine_t *, NULL) + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static quarantine_t *quarantine_grow(quarantine_t *quarantine); +static void quarantine_drain_one(quarantine_t *quarantine); +static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); + +/******************************************************************************/ + +quarantine_t * +quarantine_init(size_t lg_maxobjs) +{ + quarantine_t *quarantine; + + quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); + if (quarantine == NULL) + return (NULL); + quarantine->curbytes = 0; + quarantine->curobjs = 0; + quarantine->first = 0; + quarantine->lg_maxobjs = lg_maxobjs; + + quarantine_tsd_set(&quarantine); + + return (quarantine); +} + +static quarantine_t * +quarantine_grow(quarantine_t *quarantine) +{ + quarantine_t *ret; + + ret = quarantine_init(quarantine->lg_maxobjs + 1); + if (ret == NULL) { + quarantine_drain_one(quarantine); + return (quarantine); + } + + ret->curbytes = quarantine->curbytes; + ret->curobjs = quarantine->curobjs; + if (quarantine->first + quarantine->curobjs <= (ZU(1) << + quarantine->lg_maxobjs)) { + /* objs ring buffer data are contiguous. */ + memcpy(ret->objs, &quarantine->objs[quarantine->first], + quarantine->curobjs * sizeof(quarantine_obj_t)); + } else { + /* objs ring buffer data wrap around. */ + size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - + quarantine->first; + size_t ncopy_b = quarantine->curobjs - ncopy_a; + + memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a + * sizeof(quarantine_obj_t)); + memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * + sizeof(quarantine_obj_t)); + } + idalloc(quarantine); + + return (ret); +} + +static void +quarantine_drain_one(quarantine_t *quarantine) +{ + quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; + assert(obj->usize == isalloc(obj->ptr, config_prof)); + idalloc(obj->ptr); + quarantine->curbytes -= obj->usize; + quarantine->curobjs--; + quarantine->first = (quarantine->first + 1) & ((ZU(1) << + quarantine->lg_maxobjs) - 1); +} + +static void +quarantine_drain(quarantine_t *quarantine, size_t upper_bound) +{ + + while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) + quarantine_drain_one(quarantine); +} + +void +quarantine(void *ptr) +{ + quarantine_t *quarantine; + size_t usize = isalloc(ptr, config_prof); + + cassert(config_fill); + assert(opt_quarantine); + + quarantine = *quarantine_tsd_get(); + if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { + if (quarantine == QUARANTINE_STATE_PURGATORY) { + /* + * Make a note that quarantine() was called after + * quarantine_cleanup() was called. + */ + quarantine = QUARANTINE_STATE_REINCARNATED; + quarantine_tsd_set(&quarantine); + } + idalloc(ptr); + return; + } + /* + * Drain one or more objects if the quarantine size limit would be + * exceeded by appending ptr. + */ + if (quarantine->curbytes + usize > opt_quarantine) { + size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine + - usize : 0; + quarantine_drain(quarantine, upper_bound); + } + /* Grow the quarantine ring buffer if it's full. */ + if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) + quarantine = quarantine_grow(quarantine); + /* quarantine_grow() must free a slot if it fails to grow. */ + assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); + /* Append ptr if its size doesn't exceed the quarantine size. */ + if (quarantine->curbytes + usize <= opt_quarantine) { + size_t offset = (quarantine->first + quarantine->curobjs) & + ((ZU(1) << quarantine->lg_maxobjs) - 1); + quarantine_obj_t *obj = &quarantine->objs[offset]; + obj->ptr = ptr; + obj->usize = usize; + quarantine->curbytes += usize; + quarantine->curobjs++; + if (config_fill && opt_junk) { + /* + * Only do redzone validation if Valgrind isn't in + * operation. + */ + if ((config_valgrind == false || opt_valgrind == false) + && usize <= SMALL_MAXCLASS) + arena_quarantine_junk_small(ptr, usize); + else + memset(ptr, 0x5a, usize); + } + } else { + assert(quarantine->curbytes == 0); + idalloc(ptr); + } +} + +void +quarantine_cleanup(void *arg) +{ + quarantine_t *quarantine = *(quarantine_t **)arg; + + if (quarantine == QUARANTINE_STATE_REINCARNATED) { + /* + * Another destructor deallocated memory after this destructor + * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY + * in order to receive another callback. + */ + quarantine = QUARANTINE_STATE_PURGATORY; + quarantine_tsd_set(&quarantine); + } else if (quarantine == QUARANTINE_STATE_PURGATORY) { + /* + * The previous time this destructor was called, we set the key + * to QUARANTINE_STATE_PURGATORY so that other destructors + * wouldn't cause re-creation of the quarantine. This time, do + * nothing, so that the destructor will not be called again. + */ + } else if (quarantine != NULL) { + quarantine_drain(quarantine, 0); + idalloc(quarantine); + quarantine = QUARANTINE_STATE_PURGATORY; + quarantine_tsd_set(&quarantine); + } +} + +bool +quarantine_boot(void) +{ + + cassert(config_fill); + + if (quarantine_tsd_boot()) + return (true); + + return (false); +} diff --git a/deps/jemalloc/src/rtree.c b/deps/jemalloc/src/rtree.c index eb0ff1e24af..205957ac4e1 100644 --- a/deps/jemalloc/src/rtree.c +++ b/deps/jemalloc/src/rtree.c @@ -2,45 +2,104 @@ #include "jemalloc/internal/jemalloc_internal.h" rtree_t * -rtree_new(unsigned bits) +rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc) { rtree_t *ret; - unsigned bits_per_level, height, i; + unsigned bits_per_level, bits_in_leaf, height, i; + + assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3)); bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; - height = bits / bits_per_level; - if (height * bits_per_level != bits) - height++; - assert(height * bits_per_level >= bits); + bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1; + if (bits > bits_in_leaf) { + height = 1 + (bits - bits_in_leaf) / bits_per_level; + if ((height-1) * bits_per_level + bits_in_leaf != bits) + height++; + } else { + height = 1; + } + assert((height-1) * bits_per_level + bits_in_leaf >= bits); - ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) + + ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) + (sizeof(unsigned) * height)); if (ret == NULL) return (NULL); memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) * height)); + ret->alloc = alloc; + ret->dalloc = dalloc; if (malloc_mutex_init(&ret->mutex)) { - /* Leak the rtree. */ + if (dalloc != NULL) + dalloc(ret); return (NULL); } ret->height = height; - if (bits_per_level * height > bits) - ret->level2bits[0] = bits % bits_per_level; - else - ret->level2bits[0] = bits_per_level; - for (i = 1; i < height; i++) - ret->level2bits[i] = bits_per_level; - - ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]); + if (height > 1) { + if ((height-1) * bits_per_level + bits_in_leaf > bits) { + ret->level2bits[0] = (bits - bits_in_leaf) % + bits_per_level; + } else + ret->level2bits[0] = bits_per_level; + for (i = 1; i < height-1; i++) + ret->level2bits[i] = bits_per_level; + ret->level2bits[height-1] = bits_in_leaf; + } else + ret->level2bits[0] = bits; + + ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]); if (ret->root == NULL) { - /* - * We leak the rtree here, since there's no generic base - * deallocation. - */ + if (dalloc != NULL) + dalloc(ret); return (NULL); } memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]); return (ret); } + +static void +rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level) +{ + + if (level < rtree->height - 1) { + size_t nchildren, i; + + nchildren = ZU(1) << rtree->level2bits[level]; + for (i = 0; i < nchildren; i++) { + void **child = (void **)node[i]; + if (child != NULL) + rtree_delete_subtree(rtree, child, level + 1); + } + } + rtree->dalloc(node); +} + +void +rtree_delete(rtree_t *rtree) +{ + + rtree_delete_subtree(rtree, rtree->root, 0); + rtree->dalloc(rtree); +} + +void +rtree_prefork(rtree_t *rtree) +{ + + malloc_mutex_prefork(&rtree->mutex); +} + +void +rtree_postfork_parent(rtree_t *rtree) +{ + + malloc_mutex_postfork_parent(&rtree->mutex); +} + +void +rtree_postfork_child(rtree_t *rtree) +{ + + malloc_mutex_postfork_child(&rtree->mutex); +} diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index dc172e425c0..bef2ab33cd4 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -39,140 +39,40 @@ bool opt_stats_print = false; -#ifdef JEMALLOC_STATS size_t stats_cactive = 0; -#endif /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -#ifdef JEMALLOC_STATS -static void malloc_vcprintf(void (*write_cb)(void *, const char *), - void *cbopaque, const char *format, va_list ap); static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i); -#endif + void *cbopaque, unsigned i, bool bins, bool large); /******************************************************************************/ -/* - * We don't want to depend on vsnprintf() for production builds, since that can - * cause unnecessary bloat for static binaries. u2s() provides minimal integer - * printing functionality, so that malloc_printf() use can be limited to - * JEMALLOC_STATS code. - */ -char * -u2s(uint64_t x, unsigned base, char *s) -{ - unsigned i; - - i = UMAX2S_BUFSIZE - 1; - s[i] = '\0'; - switch (base) { - case 10: - do { - i--; - s[i] = "0123456789"[x % (uint64_t)10]; - x /= (uint64_t)10; - } while (x > 0); - break; - case 16: - do { - i--; - s[i] = "0123456789abcdef"[x & 0xf]; - x >>= 4; - } while (x > 0); - break; - default: - do { - i--; - s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % - (uint64_t)base]; - x /= (uint64_t)base; - } while (x > 0); - } - - return (&s[i]); -} - -#ifdef JEMALLOC_STATS -static void -malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, va_list ap) -{ - char buf[4096]; - - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - - vsnprintf(buf, sizeof(buf), format, ap); - write_cb(cbopaque, buf); -} - -/* - * Print to a callback function in such a way as to (hopefully) avoid memory - * allocation. - */ -JEMALLOC_ATTR(format(printf, 3, 4)) -void -malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(write_cb, cbopaque, format, ap); - va_end(ap); -} - -/* - * Print to stderr in such a way as to (hopefully) avoid memory allocation. - */ -JEMALLOC_ATTR(format(printf, 1, 2)) -void -malloc_printf(const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(NULL, NULL, format, ap); - va_end(ap); -} -#endif - -#ifdef JEMALLOC_STATS static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { - size_t pagesize; + size_t page; bool config_tcache; unsigned nbins, j, gap_start; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); CTL_GET("config.tcache", &config_tcache, bool); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" + "bins: bin size regs pgs allocated nmalloc" " ndalloc nrequests nfills nflushes" - " newruns reruns maxruns curruns\n"); + " newruns reruns curruns\n"); } else { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc newruns reruns maxruns" - " curruns\n"); + "bins: bin size regs pgs allocated nmalloc" + " ndalloc newruns reruns curruns\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { @@ -183,12 +83,11 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, if (gap_start == UINT_MAX) gap_start = j; } else { - unsigned ntbins_, nqbins, ncbins, nsbins; size_t reg_size, run_size, allocated; uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t reruns; - size_t highruns, curruns; + size_t curruns; if (gap_start != UINT_MAX) { if (j > gap_start + 1) { @@ -203,10 +102,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } gap_start = UINT_MAX; } - CTL_GET("arenas.ntbins", &ntbins_, unsigned); - CTL_GET("arenas.nqbins", &nqbins, unsigned); - CTL_GET("arenas.ncbins", &ncbins, unsigned); - CTL_GET("arenas.nsbins", &nsbins, unsigned); CTL_J_GET("arenas.bin.0.size", ®_size, size_t); CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); @@ -226,36 +121,25 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.highruns", &highruns, - size_t); CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, size_t); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + "%13u %5zu %4u %3zu %12zu %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, + " %12zu\n", + j, reg_size, nregs, run_size / page, allocated, nmalloc, ndalloc, nrequests, - nfills, nflushes, nruns, reruns, highruns, - curruns); + nfills, nflushes, nruns, reruns, curruns); } else { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + "%13u %5zu %4u %3zu %12zu %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, + " %12zu\n", + j, reg_size, nregs, run_size / page, allocated, nmalloc, ndalloc, nruns, reruns, - highruns, curruns); + curruns); } } } @@ -275,18 +159,18 @@ static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { - size_t pagesize, nlruns, j; + size_t page, nlruns, j; ssize_t gap_start; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); malloc_cprintf(write_cb, cbopaque, "large: size pages nmalloc ndalloc nrequests" - " maxruns curruns\n"); + " curruns\n"); CTL_GET("arenas.nlruns", &nlruns, size_t); for (j = 0, gap_start = -1; j < nlruns; j++) { uint64_t nmalloc, ndalloc, nrequests; - size_t run_size, highruns, curruns; + size_t run_size, curruns; CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc, uint64_t); @@ -299,8 +183,6 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, gap_start = j; } else { CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.highruns", &highruns, - size_t); CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, size_t); if (gap_start != -1) { @@ -310,9 +192,9 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, } malloc_cprintf(write_cb, cbopaque, "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - run_size, run_size / pagesize, nmalloc, ndalloc, - nrequests, highruns, curruns); + " %12zu\n", + run_size, run_size / page, nmalloc, ndalloc, + nrequests, curruns); } } if (gap_start != -1) @@ -321,21 +203,25 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, static void stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i) + unsigned i, bool bins, bool large) { unsigned nthreads; - size_t pagesize, pactive, pdirty, mapped; + const char *dss; + size_t page, pactive, pdirty, mapped; uint64_t npurge, nmadvise, purged; size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t large_allocated; uint64_t large_nmalloc, large_ndalloc, large_nrequests; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); malloc_cprintf(write_cb, cbopaque, "assigned threads: %u\n", nthreads); + CTL_I_GET("stats.arenas.0.dss", &dss, const char *); + malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n", + dss); CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); @@ -369,15 +255,15 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, small_nmalloc + large_nmalloc, small_ndalloc + large_ndalloc, small_nrequests + large_nrequests); - malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", - pactive * pagesize ); + malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * page); CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); - stats_arena_bins_print(write_cb, cbopaque, i); - stats_arena_lruns_print(write_cb, cbopaque, i); + if (bins) + stats_arena_bins_print(write_cb, cbopaque, i); + if (large) + stats_arena_lruns_print(write_cb, cbopaque, i); } -#endif void stats_print(void (*write_cb)(void *, const char *), void *cbopaque, @@ -386,7 +272,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, int err; uint64_t epoch; size_t u64sz; - char s[UMAX2S_BUFSIZE]; bool general = true; bool merged = true; bool unmerged = true; @@ -402,8 +287,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, * */ epoch = 1; u64sz = sizeof(uint64_t); - err = JEMALLOC_P(mallctl)("epoch", &epoch, &u64sz, &epoch, - sizeof(uint64_t)); + err = je_mallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t)); if (err != 0) { if (err == EAGAIN) { malloc_write(": Memory allocation failure in " @@ -415,42 +299,33 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, abort(); } - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - if (opts != NULL) { unsigned i; for (i = 0; opts[i] != '\0'; i++) { switch (opts[i]) { - case 'g': - general = false; - break; - case 'm': - merged = false; - break; - case 'a': - unmerged = false; - break; - case 'b': - bins = false; - break; - case 'l': - large = false; - break; - default:; + case 'g': + general = false; + break; + case 'm': + merged = false; + break; + case 'a': + unmerged = false; + break; + case 'b': + bins = false; + break; + case 'l': + large = false; + break; + default:; } } } - write_cb(cbopaque, "___ Begin jemalloc statistics ___\n"); + malloc_cprintf(write_cb, cbopaque, + "___ Begin jemalloc statistics ___\n"); if (general) { int err; const char *cpv; @@ -465,229 +340,127 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cpsz = sizeof(const char *); CTL_GET("version", &cpv, const char *); - write_cb(cbopaque, "Version: "); - write_cb(cbopaque, cpv); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Version: %s\n", cpv); CTL_GET("config.debug", &bv, bool); - write_cb(cbopaque, "Assertions "); - write_cb(cbopaque, bv ? "enabled" : "disabled"); - write_cb(cbopaque, "\n"); - -#define OPT_WRITE_BOOL(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &bv, &bsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, bv ? "true" : "false"); \ - write_cb(cbopaque, "\n"); \ + malloc_cprintf(write_cb, cbopaque, "Assertions %s\n", + bv ? "enabled" : "disabled"); + +#define OPT_WRITE_BOOL(n) \ + if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %s\n", bv ? "true" : "false"); \ } -#define OPT_WRITE_SIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &sv, &ssz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(sv, 10, s)); \ - write_cb(cbopaque, "\n"); \ +#define OPT_WRITE_SIZE_T(n) \ + if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %zu\n", sv); \ } -#define OPT_WRITE_SSIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &ssv, &sssz, \ - NULL, 0)) == 0) { \ - if (ssv >= 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(ssv, 10, s)); \ - } else { \ - write_cb(cbopaque, " opt."#n": -"); \ - write_cb(cbopaque, u2s(-ssv, 10, s)); \ - } \ - write_cb(cbopaque, "\n"); \ +#define OPT_WRITE_SSIZE_T(n) \ + if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %zd\n", ssv); \ } -#define OPT_WRITE_CHAR_P(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &cpv, &cpsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": \""); \ - write_cb(cbopaque, cpv); \ - write_cb(cbopaque, "\"\n"); \ +#define OPT_WRITE_CHAR_P(n) \ + if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": \"%s\"\n", cpv); \ } - write_cb(cbopaque, "Run-time option settings:\n"); + malloc_cprintf(write_cb, cbopaque, + "Run-time option settings:\n"); OPT_WRITE_BOOL(abort) - OPT_WRITE_SIZE_T(lg_qspace_max) - OPT_WRITE_SIZE_T(lg_cspace_max) OPT_WRITE_SIZE_T(lg_chunk) + OPT_WRITE_CHAR_P(dss) OPT_WRITE_SIZE_T(narenas) OPT_WRITE_SSIZE_T(lg_dirty_mult) OPT_WRITE_BOOL(stats_print) OPT_WRITE_BOOL(junk) + OPT_WRITE_SIZE_T(quarantine) + OPT_WRITE_BOOL(redzone) OPT_WRITE_BOOL(zero) - OPT_WRITE_BOOL(sysv) + OPT_WRITE_BOOL(utrace) + OPT_WRITE_BOOL(valgrind) OPT_WRITE_BOOL(xmalloc) OPT_WRITE_BOOL(tcache) - OPT_WRITE_SSIZE_T(lg_tcache_gc_sweep) OPT_WRITE_SSIZE_T(lg_tcache_max) OPT_WRITE_BOOL(prof) OPT_WRITE_CHAR_P(prof_prefix) - OPT_WRITE_SIZE_T(lg_prof_bt_max) OPT_WRITE_BOOL(prof_active) OPT_WRITE_SSIZE_T(lg_prof_sample) OPT_WRITE_BOOL(prof_accum) - OPT_WRITE_SSIZE_T(lg_prof_tcmax) OPT_WRITE_SSIZE_T(lg_prof_interval) OPT_WRITE_BOOL(prof_gdump) + OPT_WRITE_BOOL(prof_final) OPT_WRITE_BOOL(prof_leak) - OPT_WRITE_BOOL(overcommit) #undef OPT_WRITE_BOOL #undef OPT_WRITE_SIZE_T #undef OPT_WRITE_SSIZE_T #undef OPT_WRITE_CHAR_P - write_cb(cbopaque, "CPUs: "); - write_cb(cbopaque, u2s(ncpus, 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "CPUs: %u\n", ncpus); CTL_GET("arenas.narenas", &uv, unsigned); - write_cb(cbopaque, "Max arenas: "); - write_cb(cbopaque, u2s(uv, 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv); - write_cb(cbopaque, "Pointer size: "); - write_cb(cbopaque, u2s(sizeof(void *), 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Pointer size: %zu\n", + sizeof(void *)); CTL_GET("arenas.quantum", &sv, size_t); - write_cb(cbopaque, "Quantum size: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.cacheline", &sv, size_t); - write_cb(cbopaque, "Cacheline size (assumed): "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.subpage", &sv, size_t); - write_cb(cbopaque, "Subpage spacing: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz, - NULL, 0)) == 0) { - write_cb(cbopaque, "Tiny 2^n-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - - CTL_GET("arenas.tspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - } + malloc_cprintf(write_cb, cbopaque, "Quantum size: %zu\n", sv); - CTL_GET("arenas.qspace_min", &sv, size_t); - write_cb(cbopaque, "Quantum-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.qspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.cspace_min", &sv, size_t); - write_cb(cbopaque, "Cacheline-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.cspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.sspace_min", &sv, size_t); - write_cb(cbopaque, "Subpage-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.sspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); + CTL_GET("arenas.page", &sv, size_t); + malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv); CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t); if (ssv >= 0) { - write_cb(cbopaque, - "Min active:dirty page ratio per arena: "); - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, ":1\n"); + malloc_cprintf(write_cb, cbopaque, + "Min active:dirty page ratio per arena: %u:1\n", + (1U << ssv)); } else { - write_cb(cbopaque, + malloc_cprintf(write_cb, cbopaque, "Min active:dirty page ratio per arena: N/A\n"); } - if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv, - &ssz, NULL, 0)) == 0) { - write_cb(cbopaque, - "Maximum thread-cached size class: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - } - if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv, - &ssz, NULL, 0)) == 0) { - size_t tcache_gc_sweep = (1U << ssv); - bool tcache_enabled; - CTL_GET("opt.tcache", &tcache_enabled, bool); - write_cb(cbopaque, "Thread cache GC sweep interval: "); - write_cb(cbopaque, tcache_enabled && ssv >= 0 ? - u2s(tcache_gc_sweep, 10, s) : "N/A"); - write_cb(cbopaque, "\n"); + if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0)) + == 0) { + malloc_cprintf(write_cb, cbopaque, + "Maximum thread-cached size class: %zu\n", sv); } - if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0)) - == 0 && bv) { - CTL_GET("opt.lg_prof_bt_max", &sv, size_t); - write_cb(cbopaque, "Maximum profile backtrace depth: "); - write_cb(cbopaque, u2s((1U << sv), 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("opt.lg_prof_tcmax", &ssv, ssize_t); - write_cb(cbopaque, - "Maximum per thread backtrace cache: "); - if (ssv >= 0) { - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); - + if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 && + bv) { CTL_GET("opt.lg_prof_sample", &sv, size_t); - write_cb(cbopaque, "Average profile sample interval: "); - write_cb(cbopaque, u2s((((uint64_t)1U) << sv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); + malloc_cprintf(write_cb, cbopaque, + "Average profile sample interval: %"PRIu64 + " (2^%zu)\n", (((uint64_t)1U) << sv), sv); CTL_GET("opt.lg_prof_interval", &ssv, ssize_t); - write_cb(cbopaque, "Average profile dump interval: "); if (ssv >= 0) { - write_cb(cbopaque, u2s((((uint64_t)1U) << ssv), - 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); + malloc_cprintf(write_cb, cbopaque, + "Average profile dump interval: %"PRIu64 + " (2^%zd)\n", + (((uint64_t)1U) << ssv), ssv); + } else { + malloc_cprintf(write_cb, cbopaque, + "Average profile dump interval: N/A\n"); + } } - CTL_GET("arenas.chunksize", &sv, size_t); - write_cb(cbopaque, "Chunk size: "); - write_cb(cbopaque, u2s(sv, 10, s)); CTL_GET("opt.lg_chunk", &sv, size_t); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); + malloc_cprintf(write_cb, cbopaque, "Chunk size: %zu (2^%zu)\n", + (ZU(1) << sv), sv); } -#ifdef JEMALLOC_STATS - { - int err; - size_t sszp, ssz; + if (config_stats) { size_t *cactive; size_t allocated, active, mapped; - size_t chunks_current, chunks_high, swap_avail; + size_t chunks_current, chunks_high; uint64_t chunks_total; size_t huge_allocated; uint64_t huge_nmalloc, huge_ndalloc; - sszp = sizeof(size_t *); - ssz = sizeof(size_t); - CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); @@ -702,24 +475,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("stats.chunks.total", &chunks_total, uint64_t); CTL_GET("stats.chunks.high", &chunks_high, size_t); CTL_GET("stats.chunks.current", &chunks_current, size_t); - if ((err = JEMALLOC_P(mallctl)("swap.avail", &swap_avail, &ssz, - NULL, 0)) == 0) { - size_t lg_chunk; - - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks swap_avail\n"); - CTL_GET("opt.lg_chunk", &lg_chunk, size_t); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu%13zu\n", - chunks_total, chunks_high, chunks_current, - swap_avail << lg_chunk); - } else { - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks\n"); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu\n", - chunks_total, chunks_high, chunks_current); - } + malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " + "highchunks curchunks\n"); + malloc_cprintf(write_cb, cbopaque, + " %13"PRIu64" %12zu %12zu\n", + chunks_total, chunks_high, chunks_current); /* Print huge stats. */ CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t); @@ -736,11 +496,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.narenas", &narenas, unsigned); { - bool initialized[narenas]; + VARIABLE_ARRAY(bool, initialized, narenas); size_t isz; unsigned i, ninitialized; - isz = sizeof(initialized); + isz = sizeof(bool) * narenas; xmallctl("arenas.initialized", initialized, &isz, NULL, 0); for (i = ninitialized = 0; i < narenas; i++) { @@ -753,7 +513,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\nMerged arenas stats:\n"); stats_arena_print(write_cb, cbopaque, - narenas); + narenas, bins, large); } } } @@ -765,11 +525,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.narenas", &narenas, unsigned); { - bool initialized[narenas]; + VARIABLE_ARRAY(bool, initialized, narenas); size_t isz; unsigned i; - isz = sizeof(initialized); + isz = sizeof(bool) * narenas; xmallctl("arenas.initialized", initialized, &isz, NULL, 0); @@ -779,12 +539,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque, "\narenas[%u]:\n", i); stats_arena_print(write_cb, - cbopaque, i); + cbopaque, i, bins, large); } } } } } -#endif /* #ifdef JEMALLOC_STATS */ - write_cb(cbopaque, "--- End jemalloc statistics ---\n"); + malloc_cprintf(write_cb, cbopaque, "--- End jemalloc statistics ---\n"); } diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index 31c329e1613..6de92960b2d 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -1,70 +1,92 @@ #define JEMALLOC_TCACHE_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_TCACHE + /******************************************************************************/ /* Data. */ +malloc_tsd_data(, tcache, tcache_t *, NULL) +malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default) + bool opt_tcache = true; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; -ssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; tcache_bin_info_t *tcache_bin_info; static unsigned stack_nelms; /* Total stack elms per tcache. */ -/* Map of thread-specific caches. */ -#ifndef NO_TLS -__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); -#endif +size_t nhbins; +size_t tcache_maxclass; -/* - * Same contents as tcache, but initialized such that the TSD destructor is - * called when a thread exits, so that the cache can be cleaned up. - */ -pthread_key_t tcache_tsd; +/******************************************************************************/ -size_t nhbins; -size_t tcache_maxclass; -unsigned tcache_gc_incr; +size_t tcache_salloc(const void *ptr) +{ -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ + return (arena_salloc(ptr, false)); +} -static void tcache_thread_cleanup(void *arg); +void +tcache_event_hard(tcache_t *tcache) +{ + size_t binind = tcache->next_gc_bin; + tcache_bin_t *tbin = &tcache->tbins[binind]; + tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; -/******************************************************************************/ + if (tbin->low_water > 0) { + /* + * Flush (ceiling) 3/4 of the objects below the low water mark. + */ + if (binind < NBINS) { + tcache_bin_flush_small(tbin, binind, tbin->ncached - + tbin->low_water + (tbin->low_water >> 2), tcache); + } else { + tcache_bin_flush_large(tbin, binind, tbin->ncached - + tbin->low_water + (tbin->low_water >> 2), tcache); + } + /* + * Reduce fill count by 2X. Limit lg_fill_div such that the + * fill count is always at least 1. + */ + if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1) + tbin->lg_fill_div++; + } else if (tbin->low_water < 0) { + /* + * Increase fill count by 2X. Make sure lg_fill_div stays + * greater than 0. + */ + if (tbin->lg_fill_div > 1) + tbin->lg_fill_div--; + } + tbin->low_water = tbin->ncached; + + tcache->next_gc_bin++; + if (tcache->next_gc_bin == nhbins) + tcache->next_gc_bin = 0; + tcache->ev_cnt = 0; +} void * tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) { void *ret; - arena_tcache_fill_small(tcache->arena, tbin, binind -#ifdef JEMALLOC_PROF - , tcache->prof_accumbytes -#endif - ); -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes = 0; -#endif + arena_tcache_fill_small(tcache->arena, tbin, binind, + config_prof ? tcache->prof_accumbytes : 0); + if (config_prof) + tcache->prof_accumbytes = 0; ret = tcache_alloc_easy(tbin); return (ret); } void -tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) +tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache) { void *ptr; unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS bool merged_stats = false; -#endif - assert(binind < nbins); + assert(binind < NBINS); assert(rem <= tbin->ncached); for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { @@ -74,25 +96,20 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem arena_t *arena = chunk->arena; arena_bin_t *bin = &arena->bins[binind]; -#ifdef JEMALLOC_PROF - if (arena == tcache->arena) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&arena->lock); + if (config_prof && arena == tcache->arena) { + if (arena_prof_accum(arena, tcache->prof_accumbytes)) + prof_idump(); tcache->prof_accumbytes = 0; } -#endif malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - if (arena == tcache->arena) { + if (config_stats && arena == tcache->arena) { assert(merged_stats == false); merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; } -#endif ndeferred = 0; for (i = 0; i < nflush; i++) { ptr = tbin->avail[i]; @@ -100,10 +117,15 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk->arena == arena) { size_t pageind = ((uintptr_t)ptr - - (uintptr_t)chunk) >> PAGE_SHIFT; + (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = - &chunk->map[pageind-map_bias]; - arena_dalloc_bin(arena, chunk, ptr, mapelm); + arena_mapp_get(chunk, pageind); + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, + &arena_bin_info[binind], true); + } + arena_dalloc_bin_locked(arena, chunk, ptr, + mapelm); } else { /* * This object was allocated via a different @@ -117,8 +139,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem } malloc_mutex_unlock(&bin->lock); } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { + if (config_stats && merged_stats == false) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -130,7 +151,6 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem tbin->tstats.nrequests = 0; malloc_mutex_unlock(&bin->lock); } -#endif memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], rem * sizeof(void *)); @@ -140,17 +160,12 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem } void -tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) +tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache) { void *ptr; unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS bool merged_stats = false; -#endif assert(binind < nhbins); assert(rem <= tbin->ncached); @@ -160,32 +175,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); arena_t *arena = chunk->arena; + UNUSED bool idump; + if (config_prof) + idump = false; malloc_mutex_lock(&arena->lock); -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - if (arena == tcache->arena) { -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, tcache->prof_accumbytes); - tcache->prof_accumbytes = 0; -#endif -#ifdef JEMALLOC_STATS - merged_stats = true; - arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += - tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; -#endif -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + if ((config_prof || config_stats) && arena == tcache->arena) { + if (config_prof) { + idump = arena_prof_accum_locked(arena, + tcache->prof_accumbytes); + tcache->prof_accumbytes = 0; + } + if (config_stats) { + merged_stats = true; + arena->stats.nrequests_large += + tbin->tstats.nrequests; + arena->stats.lstats[binind - NBINS].nrequests += + tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + } } -#endif ndeferred = 0; for (i = 0; i < nflush; i++) { ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk->arena == arena) - arena_dalloc_large(arena, chunk, ptr); + arena_dalloc_large_locked(arena, chunk, ptr); else { /* * This object was allocated via a different @@ -198,9 +214,10 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem } } malloc_mutex_unlock(&arena->lock); + if (config_prof && idump) + prof_idump(); } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { + if (config_stats && merged_stats == false) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -208,12 +225,11 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += + arena->stats.lstats[binind - NBINS].nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; malloc_mutex_unlock(&arena->lock); } -#endif memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], rem * sizeof(void *)); @@ -222,6 +238,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem tbin->low_water = tbin->ncached; } +void +tcache_arena_associate(tcache_t *tcache, arena_t *arena) +{ + + if (config_stats) { + /* Link into list of extant tcaches. */ + malloc_mutex_lock(&arena->lock); + ql_elm_new(tcache, link); + ql_tail_insert(&arena->tcache_ql, tcache, link); + malloc_mutex_unlock(&arena->lock); + } + tcache->arena = arena; +} + +void +tcache_arena_dissociate(tcache_t *tcache) +{ + + if (config_stats) { + /* Unlink from list of extant tcaches. */ + malloc_mutex_lock(&tcache->arena->lock); + ql_remove(&tcache->arena->tcache_ql, tcache, link); + tcache_stats_merge(tcache, tcache->arena); + malloc_mutex_unlock(&tcache->arena->lock); + } +} + tcache_t * tcache_create(arena_t *arena) { @@ -244,25 +287,18 @@ tcache_create(arena_t *arena) */ size = (size + CACHELINE_MASK) & (-CACHELINE); - if (size <= small_maxclass) + if (size <= SMALL_MAXCLASS) tcache = (tcache_t *)arena_malloc_small(arena, size, true); else if (size <= tcache_maxclass) tcache = (tcache_t *)arena_malloc_large(arena, size, true); else - tcache = (tcache_t *)icalloc(size); + tcache = (tcache_t *)icalloct(size, false, arena); if (tcache == NULL) return (NULL); -#ifdef JEMALLOC_STATS - /* Link into list of extant tcaches. */ - malloc_mutex_lock(&arena->lock); - ql_elm_new(tcache, link); - ql_tail_insert(&arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&arena->lock); -#endif + tcache_arena_associate(tcache, arena); - tcache->arena = arena; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); for (i = 0; i < nhbins; i++) { tcache->tbins[i].lg_fill_div = 1; @@ -271,7 +307,7 @@ tcache_create(arena_t *arena) stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); } - TCACHE_SET(tcache); + tcache_tsd_set(&tcache); return (tcache); } @@ -282,121 +318,97 @@ tcache_destroy(tcache_t *tcache) unsigned i; size_t tcache_size; -#ifdef JEMALLOC_STATS - /* Unlink from list of extant tcaches. */ - malloc_mutex_lock(&tcache->arena->lock); - ql_remove(&tcache->arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&tcache->arena->lock); - tcache_stats_merge(tcache, tcache->arena); -#endif + tcache_arena_dissociate(tcache); - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_small(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { + tcache_bin_flush_small(tbin, i, 0, tcache); + + if (config_stats && tbin->tstats.nrequests != 0) { arena_t *arena = tcache->arena; arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(&bin->lock); } -#endif } for (; i < nhbins; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_large(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { + tcache_bin_flush_large(tbin, i, 0, tcache); + + if (config_stats && tbin->tstats.nrequests != 0) { arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[i - nbins].nrequests += + arena->stats.lstats[i - NBINS].nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(&arena->lock); } -#endif } -#ifdef JEMALLOC_PROF - if (tcache->prof_accumbytes > 0) { - malloc_mutex_lock(&tcache->arena->lock); - arena_prof_accum(tcache->arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&tcache->arena->lock); - } -#endif + if (config_prof && tcache->prof_accumbytes > 0 && + arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) + prof_idump(); - tcache_size = arena_salloc(tcache); - if (tcache_size <= small_maxclass) { + tcache_size = arena_salloc(tcache, false); + if (tcache_size <= SMALL_MAXCLASS) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> - PAGE_SHIFT; - arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; + LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, tcache, mapelm); - malloc_mutex_unlock(&bin->lock); + arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm); } else if (tcache_size <= tcache_maxclass) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; - malloc_mutex_lock(&arena->lock); arena_dalloc_large(arena, chunk, tcache); - malloc_mutex_unlock(&arena->lock); } else - idalloc(tcache); + idalloct(tcache, false); } -static void +void tcache_thread_cleanup(void *arg) { - tcache_t *tcache = (tcache_t *)arg; + tcache_t *tcache = *(tcache_t **)arg; - if (tcache == (void *)(uintptr_t)1) { + if (tcache == TCACHE_STATE_DISABLED) { + /* Do nothing. */ + } else if (tcache == TCACHE_STATE_REINCARNATED) { /* - * The previous time this destructor was called, we set the key - * to 1 so that other destructors wouldn't cause re-creation of - * the tcache. This time, do nothing, so that the destructor - * will not be called again. + * Another destructor called an allocator function after this + * destructor was called. Reset tcache to + * TCACHE_STATE_PURGATORY in order to receive another callback. */ - } else if (tcache == (void *)(uintptr_t)2) { + tcache = TCACHE_STATE_PURGATORY; + tcache_tsd_set(&tcache); + } else if (tcache == TCACHE_STATE_PURGATORY) { /* - * Another destructor called an allocator function after this - * destructor was called. Reset tcache to 1 in order to - * receive another callback. + * The previous time this destructor was called, we set the key + * to TCACHE_STATE_PURGATORY so that other destructors wouldn't + * cause re-creation of the tcache. This time, do nothing, so + * that the destructor will not be called again. */ - TCACHE_SET((uintptr_t)1); } else if (tcache != NULL) { - assert(tcache != (void *)(uintptr_t)1); + assert(tcache != TCACHE_STATE_PURGATORY); tcache_destroy(tcache); - TCACHE_SET((uintptr_t)1); + tcache = TCACHE_STATE_PURGATORY; + tcache_tsd_set(&tcache); } } -#ifdef JEMALLOC_STATS +/* Caller must own arena->lock. */ void tcache_stats_merge(tcache_t *tcache, arena_t *arena) { unsigned i; + cassert(config_stats); + /* Merge and reset tcache stats. */ - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; tcache_bin_t *tbin = &tcache->tbins[i]; malloc_mutex_lock(&bin->lock); @@ -406,75 +418,62 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena) } for (; i < nhbins; i++) { - malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; + malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS]; tcache_bin_t *tbin = &tcache->tbins[i]; arena->stats.nrequests_large += tbin->tstats.nrequests; lstats->nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; } } -#endif bool -tcache_boot(void) +tcache_boot0(void) { + unsigned i; - if (opt_tcache) { - unsigned i; - - /* - * If necessary, clamp opt_lg_tcache_max, now that - * small_maxclass and arena_maxclass are known. - */ - if (opt_lg_tcache_max < 0 || (1U << - opt_lg_tcache_max) < small_maxclass) - tcache_maxclass = small_maxclass; - else if ((1U << opt_lg_tcache_max) > arena_maxclass) - tcache_maxclass = arena_maxclass; - else - tcache_maxclass = (1U << opt_lg_tcache_max); - - nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); - - /* Initialize tcache_bin_info. */ - tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * - sizeof(tcache_bin_info_t)); - if (tcache_bin_info == NULL) - return (true); - stack_nelms = 0; - for (i = 0; i < nbins; i++) { - if ((arena_bin_info[i].nregs << 1) <= - TCACHE_NSLOTS_SMALL_MAX) { - tcache_bin_info[i].ncached_max = - (arena_bin_info[i].nregs << 1); - } else { - tcache_bin_info[i].ncached_max = - TCACHE_NSLOTS_SMALL_MAX; - } - stack_nelms += tcache_bin_info[i].ncached_max; - } - for (; i < nhbins; i++) { - tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; - stack_nelms += tcache_bin_info[i].ncached_max; - } - - /* Compute incremental GC event threshold. */ - if (opt_lg_tcache_gc_sweep >= 0) { - tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / - nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == - 0) ? 0 : 1); - } else - tcache_gc_incr = 0; - - if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != - 0) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); + /* + * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is + * known. + */ + if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS) + tcache_maxclass = SMALL_MAXCLASS; + else if ((1U << opt_lg_tcache_max) > arena_maxclass) + tcache_maxclass = arena_maxclass; + else + tcache_maxclass = (1U << opt_lg_tcache_max); + + nhbins = NBINS + (tcache_maxclass >> LG_PAGE); + + /* Initialize tcache_bin_info. */ + tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * + sizeof(tcache_bin_info_t)); + if (tcache_bin_info == NULL) + return (true); + stack_nelms = 0; + for (i = 0; i < NBINS; i++) { + if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { + tcache_bin_info[i].ncached_max = + (arena_bin_info[i].nregs << 1); + } else { + tcache_bin_info[i].ncached_max = + TCACHE_NSLOTS_SMALL_MAX; } + stack_nelms += tcache_bin_info[i].ncached_max; + } + for (; i < nhbins; i++) { + tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; + stack_nelms += tcache_bin_info[i].ncached_max; } return (false); } -/******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ + +bool +tcache_boot1(void) +{ + + if (tcache_tsd_boot() || tcache_enabled_tsd_boot()) + return (true); + + return (false); +} diff --git a/deps/jemalloc/src/tsd.c b/deps/jemalloc/src/tsd.c new file mode 100644 index 00000000000..700caabfe47 --- /dev/null +++ b/deps/jemalloc/src/tsd.c @@ -0,0 +1,141 @@ +#define JEMALLOC_TSD_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +static unsigned ncleanups; +static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; + +/******************************************************************************/ + +void * +malloc_tsd_malloc(size_t size) +{ + + /* Avoid choose_arena() in order to dodge bootstrapping issues. */ + return (arena_malloc(arenas[0], size, false, false)); +} + +void +malloc_tsd_dalloc(void *wrapper) +{ + + idalloct(wrapper, false); +} + +void +malloc_tsd_no_cleanup(void *arg) +{ + + not_reached(); +} + +#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) +#ifndef _WIN32 +JEMALLOC_EXPORT +#endif +void +_malloc_thread_cleanup(void) +{ + bool pending[MALLOC_TSD_CLEANUPS_MAX], again; + unsigned i; + + for (i = 0; i < ncleanups; i++) + pending[i] = true; + + do { + again = false; + for (i = 0; i < ncleanups; i++) { + if (pending[i]) { + pending[i] = cleanups[i](); + if (pending[i]) + again = true; + } + } + } while (again); +} +#endif + +void +malloc_tsd_cleanup_register(bool (*f)(void)) +{ + + assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); + cleanups[ncleanups] = f; + ncleanups++; +} + +void +malloc_tsd_boot(void) +{ + + ncleanups = 0; +} + +#ifdef _WIN32 +static BOOL WINAPI +_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) +{ + + switch (fdwReason) { +#ifdef JEMALLOC_LAZY_LOCK + case DLL_THREAD_ATTACH: + isthreaded = true; + break; +#endif + case DLL_THREAD_DETACH: + _malloc_thread_cleanup(); + break; + default: + break; + } + return (true); +} + +#ifdef _MSC_VER +# ifdef _M_IX86 +# pragma comment(linker, "/INCLUDE:__tls_used") +# else +# pragma comment(linker, "/INCLUDE:_tls_used") +# endif +# pragma section(".CRT$XLY",long,read) +#endif +JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) +static const BOOL (WINAPI *tls_callback)(HINSTANCE hinstDLL, + DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; +#endif + +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +void * +tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) +{ + pthread_t self = pthread_self(); + tsd_init_block_t *iter; + + /* Check whether this thread has already inserted into the list. */ + malloc_mutex_lock(&head->lock); + ql_foreach(iter, &head->blocks, link) { + if (iter->thread == self) { + malloc_mutex_unlock(&head->lock); + return (iter->data); + } + } + /* Insert block into list. */ + ql_elm_new(block, link); + block->thread = self; + ql_tail_insert(&head->blocks, block, link); + malloc_mutex_unlock(&head->lock); + return (NULL); +} + +void +tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) +{ + + malloc_mutex_lock(&head->lock); + ql_remove(&head->blocks, block, link); + malloc_mutex_unlock(&head->lock); +} +#endif diff --git a/deps/jemalloc/src/util.c b/deps/jemalloc/src/util.c new file mode 100644 index 00000000000..93a19fd16f7 --- /dev/null +++ b/deps/jemalloc/src/util.c @@ -0,0 +1,648 @@ +#define assert(e) do { \ + if (config_debug && !(e)) { \ + malloc_write(": Failed assertion\n"); \ + abort(); \ + } \ +} while (0) + +#define not_reached() do { \ + if (config_debug) { \ + malloc_write(": Unreachable code reached\n"); \ + abort(); \ + } \ +} while (0) + +#define not_implemented() do { \ + if (config_debug) { \ + malloc_write(": Not implemented\n"); \ + abort(); \ + } \ +} while (0) + +#define JEMALLOC_UTIL_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void wrtmessage(void *cbopaque, const char *s); +#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) +static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, + size_t *slen_p); +#define D2S_BUFSIZE (1 + U2S_BUFSIZE) +static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); +#define O2S_BUFSIZE (1 + U2S_BUFSIZE) +static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); +#define X2S_BUFSIZE (2 + U2S_BUFSIZE) +static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, + size_t *slen_p); + +/******************************************************************************/ + +/* malloc_message() setup. */ +static void +wrtmessage(void *cbopaque, const char *s) +{ + +#ifdef SYS_write + /* + * Use syscall(2) rather than write(2) when possible in order to avoid + * the possibility of memory allocation within libc. This is necessary + * on FreeBSD; most operating systems do not have this problem though. + */ + UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s)); +#else + UNUSED int result = write(STDERR_FILENO, s, strlen(s)); +#endif +} + +JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); + +/* + * Wrapper around malloc_message() that avoids the need for + * je_malloc_message(...) throughout the code. + */ +void +malloc_write(const char *s) +{ + + if (je_malloc_message != NULL) + je_malloc_message(NULL, s); + else + wrtmessage(NULL, s); +} + +/* + * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so + * provide a wrapper. + */ +int +buferror(int err, char *buf, size_t buflen) +{ + +#ifdef _WIN32 + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0, + (LPSTR)buf, buflen, NULL); + return (0); +#elif defined(_GNU_SOURCE) + char *b = strerror_r(err, buf, buflen); + if (b != buf) { + strncpy(buf, b, buflen); + buf[buflen-1] = '\0'; + } + return (0); +#else + return (strerror_r(err, buf, buflen)); +#endif +} + +uintmax_t +malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) +{ + uintmax_t ret, digit; + int b; + bool neg; + const char *p, *ns; + + p = nptr; + if (base < 0 || base == 1 || base > 36) { + ns = p; + set_errno(EINVAL); + ret = UINTMAX_MAX; + goto label_return; + } + b = base; + + /* Swallow leading whitespace and get sign, if any. */ + neg = false; + while (true) { + switch (*p) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + p++; + break; + case '-': + neg = true; + /* Fall through. */ + case '+': + p++; + /* Fall through. */ + default: + goto label_prefix; + } + } + + /* Get prefix, if any. */ + label_prefix: + /* + * Note where the first non-whitespace/sign character is so that it is + * possible to tell whether any digits are consumed (e.g., " 0" vs. + * " -x"). + */ + ns = p; + if (*p == '0') { + switch (p[1]) { + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': + if (b == 0) + b = 8; + if (b == 8) + p++; + break; + case 'X': case 'x': + switch (p[2]) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + if (b == 0) + b = 16; + if (b == 16) + p += 2; + break; + default: + break; + } + break; + default: + p++; + ret = 0; + goto label_return; + } + } + if (b == 0) + b = 10; + + /* Convert. */ + ret = 0; + while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) + || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) + || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { + uintmax_t pret = ret; + ret *= b; + ret += digit; + if (ret < pret) { + /* Overflow. */ + set_errno(ERANGE); + ret = UINTMAX_MAX; + goto label_return; + } + p++; + } + if (neg) + ret = -ret; + + if (p == ns) { + /* No conversion performed. */ + set_errno(EINVAL); + ret = UINTMAX_MAX; + goto label_return; + } + +label_return: + if (endptr != NULL) { + if (p == ns) { + /* No characters were converted. */ + *endptr = (char *)nptr; + } else + *endptr = (char *)p; + } + return (ret); +} + +static char * +u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) +{ + unsigned i; + + i = U2S_BUFSIZE - 1; + s[i] = '\0'; + switch (base) { + case 10: + do { + i--; + s[i] = "0123456789"[x % (uint64_t)10]; + x /= (uint64_t)10; + } while (x > 0); + break; + case 16: { + const char *digits = (uppercase) + ? "0123456789ABCDEF" + : "0123456789abcdef"; + + do { + i--; + s[i] = digits[x & 0xf]; + x >>= 4; + } while (x > 0); + break; + } default: { + const char *digits = (uppercase) + ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + : "0123456789abcdefghijklmnopqrstuvwxyz"; + + assert(base >= 2 && base <= 36); + do { + i--; + s[i] = digits[x % (uint64_t)base]; + x /= (uint64_t)base; + } while (x > 0); + }} + + *slen_p = U2S_BUFSIZE - 1 - i; + return (&s[i]); +} + +static char * +d2s(intmax_t x, char sign, char *s, size_t *slen_p) +{ + bool neg; + + if ((neg = (x < 0))) + x = -x; + s = u2s(x, 10, false, s, slen_p); + if (neg) + sign = '-'; + switch (sign) { + case '-': + if (neg == false) + break; + /* Fall through. */ + case ' ': + case '+': + s--; + (*slen_p)++; + *s = sign; + break; + default: not_reached(); + } + return (s); +} + +static char * +o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) +{ + + s = u2s(x, 8, false, s, slen_p); + if (alt_form && *s != '0') { + s--; + (*slen_p)++; + *s = '0'; + } + return (s); +} + +static char * +x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) +{ + + s = u2s(x, 16, uppercase, s, slen_p); + if (alt_form) { + s -= 2; + (*slen_p) += 2; + memcpy(s, uppercase ? "0X" : "0x", 2); + } + return (s); +} + +int +malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) +{ + int ret; + size_t i; + const char *f; + +#define APPEND_C(c) do { \ + if (i < size) \ + str[i] = (c); \ + i++; \ +} while (0) +#define APPEND_S(s, slen) do { \ + if (i < size) { \ + size_t cpylen = (slen <= size - i) ? slen : size - i; \ + memcpy(&str[i], s, cpylen); \ + } \ + i += slen; \ +} while (0) +#define APPEND_PADDED_S(s, slen, width, left_justify) do { \ + /* Left padding. */ \ + size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ + (size_t)width - slen : 0); \ + if (left_justify == false && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) \ + APPEND_C(' '); \ + } \ + /* Value. */ \ + APPEND_S(s, slen); \ + /* Right padding. */ \ + if (left_justify && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) \ + APPEND_C(' '); \ + } \ +} while (0) +#define GET_ARG_NUMERIC(val, len) do { \ + switch (len) { \ + case '?': \ + val = va_arg(ap, int); \ + break; \ + case '?' | 0x80: \ + val = va_arg(ap, unsigned int); \ + break; \ + case 'l': \ + val = va_arg(ap, long); \ + break; \ + case 'l' | 0x80: \ + val = va_arg(ap, unsigned long); \ + break; \ + case 'q': \ + val = va_arg(ap, long long); \ + break; \ + case 'q' | 0x80: \ + val = va_arg(ap, unsigned long long); \ + break; \ + case 'j': \ + val = va_arg(ap, intmax_t); \ + break; \ + case 'j' | 0x80: \ + val = va_arg(ap, uintmax_t); \ + break; \ + case 't': \ + val = va_arg(ap, ptrdiff_t); \ + break; \ + case 'z': \ + val = va_arg(ap, ssize_t); \ + break; \ + case 'z' | 0x80: \ + val = va_arg(ap, size_t); \ + break; \ + case 'p': /* Synthetic; used for %p. */ \ + val = va_arg(ap, uintptr_t); \ + break; \ + default: not_reached(); \ + } \ +} while (0) + + i = 0; + f = format; + while (true) { + switch (*f) { + case '\0': goto label_out; + case '%': { + bool alt_form = false; + bool left_justify = false; + bool plus_space = false; + bool plus_plus = false; + int prec = -1; + int width = -1; + unsigned char len = '?'; + + f++; + /* Flags. */ + while (true) { + switch (*f) { + case '#': + assert(alt_form == false); + alt_form = true; + break; + case '-': + assert(left_justify == false); + left_justify = true; + break; + case ' ': + assert(plus_space == false); + plus_space = true; + break; + case '+': + assert(plus_plus == false); + plus_plus = true; + break; + default: goto label_width; + } + f++; + } + /* Width. */ + label_width: + switch (*f) { + case '*': + width = va_arg(ap, int); + f++; + if (width < 0) { + left_justify = true; + width = -width; + } + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + uintmax_t uwidth; + set_errno(0); + uwidth = malloc_strtoumax(f, (char **)&f, 10); + assert(uwidth != UINTMAX_MAX || get_errno() != + ERANGE); + width = (int)uwidth; + break; + } default: + break; + } + /* Width/precision separator. */ + if (*f == '.') + f++; + else + goto label_length; + /* Precision. */ + switch (*f) { + case '*': + prec = va_arg(ap, int); + f++; + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + uintmax_t uprec; + set_errno(0); + uprec = malloc_strtoumax(f, (char **)&f, 10); + assert(uprec != UINTMAX_MAX || get_errno() != + ERANGE); + prec = (int)uprec; + break; + } + default: break; + } + /* Length. */ + label_length: + switch (*f) { + case 'l': + f++; + if (*f == 'l') { + len = 'q'; + f++; + } else + len = 'l'; + break; + case 'q': case 'j': case 't': case 'z': + len = *f; + f++; + break; + default: break; + } + /* Conversion specifier. */ + switch (*f) { + char *s; + size_t slen; + case '%': + /* %% */ + APPEND_C(*f); + f++; + break; + case 'd': case 'i': { + intmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[D2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len); + s = d2s(val, (plus_plus ? '+' : (plus_space ? + ' ' : '-')), buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'o': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[O2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = o2s(val, alt_form, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'u': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[U2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = u2s(val, 10, false, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'x': case 'X': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[X2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = x2s(val, alt_form, *f == 'X', buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'c': { + unsigned char val; + char buf[2]; + + assert(len == '?' || len == 'l'); + assert_not_implemented(len != 'l'); + val = va_arg(ap, int); + buf[0] = val; + buf[1] = '\0'; + APPEND_PADDED_S(buf, 1, width, left_justify); + f++; + break; + } case 's': + assert(len == '?' || len == 'l'); + assert_not_implemented(len != 'l'); + s = va_arg(ap, char *); + slen = (prec < 0) ? strlen(s) : prec; + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + case 'p': { + uintmax_t val; + char buf[X2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, 'p'); + s = x2s(val, true, false, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } default: not_reached(); + } + break; + } default: { + APPEND_C(*f); + f++; + break; + }} + } + label_out: + if (i < size) + str[i] = '\0'; + else + str[size - 1] = '\0'; + ret = i; + +#undef APPEND_C +#undef APPEND_S +#undef APPEND_PADDED_S +#undef GET_ARG_NUMERIC + return (ret); +} + +JEMALLOC_ATTR(format(printf, 3, 4)) +int +malloc_snprintf(char *str, size_t size, const char *format, ...) +{ + int ret; + va_list ap; + + va_start(ap, format); + ret = malloc_vsnprintf(str, size, format, ap); + va_end(ap); + + return (ret); +} + +void +malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, va_list ap) +{ + char buf[MALLOC_PRINTF_BUFSIZE]; + + if (write_cb == NULL) { + /* + * The caller did not provide an alternate write_cb callback + * function, so use the default one. malloc_write() is an + * inline function, so use malloc_message() directly here. + */ + write_cb = (je_malloc_message != NULL) ? je_malloc_message : + wrtmessage; + cbopaque = NULL; + } + + malloc_vsnprintf(buf, sizeof(buf), format, ap); + write_cb(cbopaque, buf); +} + +/* + * Print to a callback function in such a way as to (hopefully) avoid memory + * allocation. + */ +JEMALLOC_ATTR(format(printf, 3, 4)) +void +malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(write_cb, cbopaque, format, ap); + va_end(ap); +} + +/* Print to stderr in such a way as to avoid memory allocation. */ +JEMALLOC_ATTR(format(printf, 1, 2)) +void +malloc_printf(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); +} diff --git a/deps/jemalloc/src/zone.c b/deps/jemalloc/src/zone.c index 2c1b2318ef4..e0302ef4edc 100644 --- a/deps/jemalloc/src/zone.c +++ b/deps/jemalloc/src/zone.c @@ -3,11 +3,18 @@ # error "This source file is for zones on Darwin (OS X)." #endif +/* + * The malloc_default_purgeable_zone function is only available on >= 10.6. + * We need to check whether it is present at runtime, thus the weak_import. + */ +extern malloc_zone_t *malloc_default_purgeable_zone(void) +JEMALLOC_ATTR(weak_import); + /******************************************************************************/ /* Data. */ -static malloc_zone_t zone, szone; -static struct malloc_introspection_t zone_introspect, ozone_introspect; +static malloc_zone_t zone; +static struct malloc_introspection_t zone_introspect; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -18,8 +25,10 @@ static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); static void *zone_valloc(malloc_zone_t *zone, size_t size); static void zone_free(malloc_zone_t *zone, void *ptr); static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) static void *zone_memalign(malloc_zone_t *zone, size_t alignment, +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) size_t size); static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size); @@ -28,19 +37,6 @@ static void *zone_destroy(malloc_zone_t *zone); static size_t zone_good_size(malloc_zone_t *zone, size_t size); static void zone_force_lock(malloc_zone_t *zone); static void zone_force_unlock(malloc_zone_t *zone); -static size_t ozone_size(malloc_zone_t *zone, void *ptr); -static void ozone_free(malloc_zone_t *zone, void *ptr); -static void *ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -static unsigned ozone_batch_malloc(malloc_zone_t *zone, size_t size, - void **results, unsigned num_requested); -static void ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, - unsigned num); -#if (JEMALLOC_ZONE_VERSION >= 6) -static void ozone_free_definite_size(malloc_zone_t *zone, void *ptr, - size_t size); -#endif -static void ozone_force_lock(malloc_zone_t *zone); -static void ozone_force_unlock(malloc_zone_t *zone); /******************************************************************************/ /* @@ -60,21 +56,21 @@ zone_size(malloc_zone_t *zone, void *ptr) * not work in practice, we must check all pointers to assure that they * reside within a mapped chunk before determining size. */ - return (ivsalloc(ptr)); + return (ivsalloc(ptr, config_prof)); } static void * zone_malloc(malloc_zone_t *zone, size_t size) { - return (JEMALLOC_P(malloc)(size)); + return (je_malloc(size)); } static void * zone_calloc(malloc_zone_t *zone, size_t num, size_t size) { - return (JEMALLOC_P(calloc)(num, size)); + return (je_calloc(num, size)); } static void * @@ -82,7 +78,7 @@ zone_valloc(malloc_zone_t *zone, size_t size) { void *ret = NULL; /* Assignment avoids useless compiler warning. */ - JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size); + je_posix_memalign(&ret, PAGE, size); return (ret); } @@ -91,33 +87,48 @@ static void zone_free(malloc_zone_t *zone, void *ptr) { - JEMALLOC_P(free)(ptr); + if (ivsalloc(ptr, config_prof) != 0) { + je_free(ptr); + return; + } + + free(ptr); } static void * zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { - return (JEMALLOC_P(realloc)(ptr, size)); + if (ivsalloc(ptr, config_prof) != 0) + return (je_realloc(ptr, size)); + + return (realloc(ptr, size)); } -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) static void * zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) { void *ret = NULL; /* Assignment avoids useless compiler warning. */ - JEMALLOC_P(posix_memalign)(&ret, alignment, size); + je_posix_memalign(&ret, alignment, size); return (ret); } +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); + if (ivsalloc(ptr, config_prof) != 0) { + assert(ivsalloc(ptr, config_prof) == size); + je_free(ptr); + return; + } + + free(ptr); } #endif @@ -126,29 +137,17 @@ zone_destroy(malloc_zone_t *zone) { /* This function should never be called. */ - assert(false); + not_reached(); return (NULL); } static size_t zone_good_size(malloc_zone_t *zone, size_t size) { - size_t ret; - void *p; - /* - * Actually create an object of the appropriate size, then find out - * how large it could have been without moving up to the next size - * class. - */ - p = JEMALLOC_P(malloc)(size); - if (p != NULL) { - ret = isalloc(p); - JEMALLOC_P(free)(p); - } else - ret = size; - - return (ret); + if (size == 0) + size = 1; + return (s2u(size)); } static void @@ -164,13 +163,24 @@ zone_force_unlock(malloc_zone_t *zone) { if (isthreaded) - jemalloc_postfork(); + jemalloc_postfork_parent(); } -malloc_zone_t * -create_zone(void) +JEMALLOC_ATTR(constructor) +void +register_zone(void) { + /* + * If something else replaced the system default zone allocator, don't + * register jemalloc's. + */ + malloc_zone_t *default_zone = malloc_default_zone(); + if (!default_zone->zone_name || + strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) { + return; + } + zone.size = (void *)zone_size; zone.malloc = (void *)zone_malloc; zone.calloc = (void *)zone_calloc; @@ -183,10 +193,15 @@ create_zone(void) zone.batch_free = NULL; zone.introspect = &zone_introspect; zone.version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) zone.memalign = zone_memalign; +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) zone.free_definite_size = zone_free_definite_size; #endif +#if (JEMALLOC_ZONE_VERSION >= 8) + zone.pressure_relief = NULL; +#endif zone_introspect.enumerator = NULL; zone_introspect.good_size = (void *)zone_good_size; @@ -199,156 +214,45 @@ create_zone(void) #if (JEMALLOC_ZONE_VERSION >= 6) zone_introspect.zone_locked = NULL; #endif - - return (&zone); -} - -static size_t -ozone_size(malloc_zone_t *zone, void *ptr) -{ - size_t ret; - - ret = ivsalloc(ptr); - if (ret == 0) - ret = szone.size(zone, ptr); - - return (ret); -} - -static void -ozone_free(malloc_zone_t *zone, void *ptr) -{ - - if (ivsalloc(ptr) != 0) - JEMALLOC_P(free)(ptr); - else { - size_t size = szone.size(zone, ptr); - if (size != 0) - (szone.free)(zone, ptr); - } -} - -static void * -ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size) -{ - size_t oldsize; - - if (ptr == NULL) - return (JEMALLOC_P(malloc)(size)); - - oldsize = ivsalloc(ptr); - if (oldsize != 0) - return (JEMALLOC_P(realloc)(ptr, size)); - else { - oldsize = szone.size(zone, ptr); - if (oldsize == 0) - return (JEMALLOC_P(malloc)(size)); - else { - void *ret = JEMALLOC_P(malloc)(size); - if (ret != NULL) { - memcpy(ret, ptr, (oldsize < size) ? oldsize : - size); - (szone.free)(zone, ptr); - } - return (ret); - } - } -} - -static unsigned -ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, - unsigned num_requested) -{ - - /* Don't bother implementing this interface, since it isn't required. */ - return (0); -} - -static void -ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) -{ - unsigned i; - - for (i = 0; i < num; i++) - ozone_free(zone, to_be_freed[i]); -} - -#if (JEMALLOC_ZONE_VERSION >= 6) -static void -ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) -{ - - if (ivsalloc(ptr) != 0) { - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); - } else { - assert(size == szone.size(zone, ptr)); - szone.free_definite_size(zone, ptr, size); - } -} +#if (JEMALLOC_ZONE_VERSION >= 7) + zone_introspect.enable_discharge_checking = NULL; + zone_introspect.disable_discharge_checking = NULL; + zone_introspect.discharge = NULL; +#ifdef __BLOCKS__ + zone_introspect.enumerate_discharged_pointers = NULL; +#else + zone_introspect.enumerate_unavailable_without_blocks = NULL; +#endif #endif -static void -ozone_force_lock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_lock(zone); -} - -static void -ozone_force_unlock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_unlock(zone); -} + /* + * The default purgeable zone is created lazily by OSX's libc. It uses + * the default zone when it is created for "small" allocations + * (< 15 KiB), but assumes the default zone is a scalable_zone. This + * obviously fails when the default zone is the jemalloc zone, so + * malloc_default_purgeable_zone is called beforehand so that the + * default purgeable zone is created when the default zone is still + * a scalable_zone. As purgeable zones only exist on >= 10.6, we need + * to check for the existence of malloc_default_purgeable_zone() at + * run time. + */ + if (malloc_default_purgeable_zone != NULL) + malloc_default_purgeable_zone(); -/* - * Overlay the default scalable zone (szone) such that existing allocations are - * drained, and further allocations come from jemalloc. This is necessary - * because Core Foundation directly accesses and uses the szone before the - * jemalloc library is even loaded. - */ -void -szone2ozone(malloc_zone_t *zone) -{ + /* Register the custom zone. At this point it won't be the default. */ + malloc_zone_register(&zone); /* - * Stash a copy of the original szone so that we can call its - * functions as needed. Note that the internally, the szone stores its - * bookkeeping data structures immediately following the malloc_zone_t - * header, so when calling szone functions, we need to pass a pointer - * to the original zone structure. + * Unregister and reregister the default zone. On OSX >= 10.6, + * unregistering takes the last registered zone and places it at the + * location of the specified zone. Unregistering the default zone thus + * makes the last registered one the default. On OSX < 10.6, + * unregistering shifts all registered zones. The first registered zone + * then becomes the default. */ - memcpy(&szone, zone, sizeof(malloc_zone_t)); - - zone->size = (void *)ozone_size; - zone->malloc = (void *)zone_malloc; - zone->calloc = (void *)zone_calloc; - zone->valloc = (void *)zone_valloc; - zone->free = (void *)ozone_free; - zone->realloc = (void *)ozone_realloc; - zone->destroy = (void *)zone_destroy; - zone->zone_name = "jemalloc_ozone"; - zone->batch_malloc = ozone_batch_malloc; - zone->batch_free = ozone_batch_free; - zone->introspect = &ozone_introspect; - zone->version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) - zone->memalign = zone_memalign; - zone->free_definite_size = ozone_free_definite_size; -#endif - - ozone_introspect.enumerator = NULL; - ozone_introspect.good_size = (void *)zone_good_size; - ozone_introspect.check = NULL; - ozone_introspect.print = NULL; - ozone_introspect.log = NULL; - ozone_introspect.force_lock = (void *)ozone_force_lock; - ozone_introspect.force_unlock = (void *)ozone_force_unlock; - ozone_introspect.statistics = NULL; -#if (JEMALLOC_ZONE_VERSION >= 6) - ozone_introspect.zone_locked = NULL; -#endif + do { + default_zone = malloc_default_zone(); + malloc_zone_unregister(default_zone); + malloc_zone_register(default_zone); + } while (malloc_default_zone() != &zone); } diff --git a/deps/jemalloc/test/allocated.c b/deps/jemalloc/test/allocated.c deleted file mode 100644 index b1e40e4720e..00000000000 --- a/deps/jemalloc/test/allocated.c +++ /dev/null @@ -1,142 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -void * -thread_start(void *arg) -{ - int err; - void *p; - uint64_t a0, a1, d0, d1; - uint64_t *ap0, *ap1, *dp0, *dp1; - size_t sz, usize; - - sz = sizeof(a0); - if ((err = JEMALLOC_P(mallctl)("thread.allocated", &a0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(ap0); - if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*ap0 == a0); - - sz = sizeof(d0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(dp0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*dp0 == d0); - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - exit(1); - } - - sz = sizeof(a1); - JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0); - sz = sizeof(ap1); - JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0); - assert(*ap1 == a1); - assert(ap0 == ap1); - - usize = JEMALLOC_P(malloc_usable_size)(p); - assert(a0 + usize <= a1); - - JEMALLOC_P(free)(p); - - sz = sizeof(d1); - JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0); - sz = sizeof(dp1); - JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0); - assert(*dp1 == d1); - assert(dp0 == dp1); - - assert(d0 + usize <= d1); - -RETURN: - return (NULL); -} - -int -main(void) -{ - int ret = 0; - pthread_t thread; - - fprintf(stderr, "Test begin\n"); - - thread_start(NULL); - - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); - - thread_start(NULL); - - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); - - thread_start(NULL); - -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/allocated.exp b/deps/jemalloc/test/allocated.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/allocated.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/allocm.c b/deps/jemalloc/test/allocm.c deleted file mode 100644 index 59d0002e7be..00000000000 --- a/deps/jemalloc/test/allocm.c +++ /dev/null @@ -1,133 +0,0 @@ -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) -#define NITER 4 - -int -main(void) -{ - int r; - void *p; - size_t sz, alignment, total, tsz; - unsigned i; - void *ps[NITER]; - - fprintf(stderr, "Test begin\n"); - - sz = 0; - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (sz < 42) - fprintf(stderr, "Real size smaller than expected\n"); - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - - r = JEMALLOC_P(allocm)(&p, NULL, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - - r = JEMALLOC_P(allocm)(&p, NULL, 42, ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - -#if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - sz = 0x8000000000000000LLU; -#else - alignment = 0x80000000LU; - sz = 0x80000000LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - sz = 0x8400000000000001LLU; -#else - alignment = 0x40000000LU; - sz = 0x84000001LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - - alignment = 0x10LLU; -#if LG_SIZEOF_PTR == 3 - sz = 0xfffffffffffffff0LLU; -#else - sz = 0xfffffff0LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); - for (sz = 1; - sz < 3 * alignment && sz < (1U << 31); - sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - r = JEMALLOC_P(allocm)(&ps[i], NULL, sz, - ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, - "Error for size %zu (0x%zx): %d\n", - sz, sz, r); - exit(1); - } - if ((uintptr_t)p & (alignment-1)) { - fprintf(stderr, - "%p inadequately aligned for" - " alignment: %zu\n", p, alignment); - } - JEMALLOC_P(sallocm)(ps[i], &tsz, 0); - total += tsz; - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - JEMALLOC_P(dallocm)(ps[i], 0); - ps[i] = NULL; - } - } - } - } - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/allocm.exp b/deps/jemalloc/test/allocm.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc/test/allocm.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc/test/bitmap.c b/deps/jemalloc/test/bitmap.c deleted file mode 100644 index adfaacfe52b..00000000000 --- a/deps/jemalloc/test/bitmap.c +++ /dev/null @@ -1,157 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -/* - * Avoid using the assert() from jemalloc_internal.h, since it requires - * internal libjemalloc functionality. - * */ -#include - -/* - * Directly include the bitmap code, since it isn't exposed outside - * libjemalloc. - */ -#include "../src/bitmap.c" - -#if (LG_BITMAP_MAXBITS > 12) -# define MAXBITS 4500 -#else -# define MAXBITS (1U << LG_BITMAP_MAXBITS) -#endif - -static void -test_bitmap_size(void) -{ - size_t i, prev_size; - - prev_size = 0; - for (i = 1; i <= MAXBITS; i++) { - size_t size = bitmap_size(i); - assert(size >= prev_size); - prev_size = size; - } -} - -static void -test_bitmap_init(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - assert(bitmap_get(bitmap, &binfo, j) == false); - - } - } -} - -static void -test_bitmap_set(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -static void -test_bitmap_unset(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - for (j = 0; j < i; j++) - bitmap_unset(bitmap, &binfo, j); - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -static void -test_bitmap_sfu(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - ssize_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - /* Iteratively set bits starting at the beginning. */ - for (j = 0; j < i; j++) - assert(bitmap_sfu(bitmap, &binfo) == j); - assert(bitmap_full(bitmap, &binfo)); - - /* - * Iteratively unset bits starting at the end, and - * verify that bitmap_sfu() reaches the unset bits. - */ - for (j = i - 1; j >= 0; j--) { - bitmap_unset(bitmap, &binfo, j); - assert(bitmap_sfu(bitmap, &binfo) == j); - bitmap_unset(bitmap, &binfo, j); - } - assert(bitmap_get(bitmap, &binfo, 0) == false); - - /* - * Iteratively set bits starting at the beginning, and - * verify that bitmap_sfu() looks past them. - */ - for (j = 1; j < i; j++) { - bitmap_set(bitmap, &binfo, j - 1); - assert(bitmap_sfu(bitmap, &binfo) == j); - bitmap_unset(bitmap, &binfo, j); - } - assert(bitmap_sfu(bitmap, &binfo) == i - 1); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -int -main(void) -{ - fprintf(stderr, "Test begin\n"); - - test_bitmap_size(); - test_bitmap_init(); - test_bitmap_set(); - test_bitmap_unset(); - test_bitmap_sfu(); - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/bitmap.exp b/deps/jemalloc/test/bitmap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/bitmap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/include/test/SFMT-alti.h b/deps/jemalloc/test/include/test/SFMT-alti.h new file mode 100644 index 00000000000..0005df6b484 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-alti.h @@ -0,0 +1,186 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT-alti.h + * + * @brief SIMD oriented Fast Mersenne Twister(SFMT) + * pseudorandom number generator + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software. + * see LICENSE.txt + */ + +#ifndef SFMT_ALTI_H +#define SFMT_ALTI_H + +/** + * This function represents the recursion formula in AltiVec and BIG ENDIAN. + * @param a a 128-bit part of the interal state array + * @param b a 128-bit part of the interal state array + * @param c a 128-bit part of the interal state array + * @param d a 128-bit part of the interal state array + * @return output + */ +JEMALLOC_ALWAYS_INLINE +vector unsigned int vec_recursion(vector unsigned int a, + vector unsigned int b, + vector unsigned int c, + vector unsigned int d) { + + const vector unsigned int sl1 = ALTI_SL1; + const vector unsigned int sr1 = ALTI_SR1; +#ifdef ONLY64 + const vector unsigned int mask = ALTI_MSK64; + const vector unsigned char perm_sl = ALTI_SL2_PERM64; + const vector unsigned char perm_sr = ALTI_SR2_PERM64; +#else + const vector unsigned int mask = ALTI_MSK; + const vector unsigned char perm_sl = ALTI_SL2_PERM; + const vector unsigned char perm_sr = ALTI_SR2_PERM; +#endif + vector unsigned int v, w, x, y, z; + x = vec_perm(a, (vector unsigned int)perm_sl, perm_sl); + v = a; + y = vec_sr(b, sr1); + z = vec_perm(c, (vector unsigned int)perm_sr, perm_sr); + w = vec_sl(d, sl1); + z = vec_xor(z, w); + y = vec_and(y, mask); + v = vec_xor(v, x); + z = vec_xor(z, y); + z = vec_xor(z, v); + return z; +} + +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) { + int i; + vector unsigned int r, r1, r2; + + r1 = ctx->sfmt[N - 2].s; + r2 = ctx->sfmt[N - 1].s; + for (i = 0; i < N - POS1; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2); + ctx->sfmt[i].s = r; + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1 - N].s, r1, r2); + ctx->sfmt[i].s = r; + r1 = r2; + r2 = r; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pesudorandom numbers to be generated. + */ +JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + vector unsigned int r, r1, r2; + + r1 = ctx->sfmt[N - 2].s; + r2 = ctx->sfmt[N - 1].s; + for (i = 0; i < N - POS1; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = vec_recursion(ctx->sfmt[i].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + /* main loop */ + for (; i < size - N; i++) { + r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + for (j = 0; j < 2 * N - size; j++) { + ctx->sfmt[j].s = array[j + size - N].s; + } + for (; i < size; i++) { + r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + ctx->sfmt[j++].s = r; + r1 = r2; + r2 = r; + } +} + +#ifndef ONLY64 +#if defined(__APPLE__) +#define ALTI_SWAP (vector unsigned char) \ + (4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11) +#else +#define ALTI_SWAP {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11} +#endif +/** + * This function swaps high and low 32-bit of 64-bit integers in user + * specified array. + * + * @param array an 128-bit array to be swaped. + * @param size size of 128-bit array. + */ +JEMALLOC_INLINE void swap(w128_t *array, int size) { + int i; + const vector unsigned char perm = ALTI_SWAP; + + for (i = 0; i < size; i++) { + array[i].s = vec_perm(array[i].s, (vector unsigned int)perm, perm); + } +} +#endif + +#endif diff --git a/deps/jemalloc/test/include/test/SFMT-params.h b/deps/jemalloc/test/include/test/SFMT-params.h new file mode 100644 index 00000000000..ade6622206d --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params.h @@ -0,0 +1,132 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS_H +#define SFMT_PARAMS_H + +#if !defined(MEXP) +#ifdef __GNUC__ + #warning "MEXP is not defined. I assume MEXP is 19937." +#endif + #define MEXP 19937 +#endif +/*----------------- + BASIC DEFINITIONS + -----------------*/ +/** Mersenne Exponent. The period of the sequence + * is a multiple of 2^MEXP-1. + * #define MEXP 19937 */ +/** SFMT generator has an internal state array of 128-bit integers, + * and N is its size. */ +#define N (MEXP / 128 + 1) +/** N32 is the size of internal state array when regarded as an array + * of 32-bit integers.*/ +#define N32 (N * 4) +/** N64 is the size of internal state array when regarded as an array + * of 64-bit integers.*/ +#define N64 (N * 2) + +/*---------------------- + the parameters of SFMT + following definitions are in paramsXXXX.h file. + ----------------------*/ +/** the pick up position of the array. +#define POS1 122 +*/ + +/** the parameter of shift left as four 32-bit registers. +#define SL1 18 + */ + +/** the parameter of shift left as one 128-bit register. + * The 128-bit integer is shifted by (SL2 * 8) bits. +#define SL2 1 +*/ + +/** the parameter of shift right as four 32-bit registers. +#define SR1 11 +*/ + +/** the parameter of shift right as one 128-bit register. + * The 128-bit integer is shifted by (SL2 * 8) bits. +#define SR2 1 +*/ + +/** A bitmask, used in the recursion. These parameters are introduced + * to break symmetry of SIMD. +#define MSK1 0xdfffffefU +#define MSK2 0xddfecb7fU +#define MSK3 0xbffaffffU +#define MSK4 0xbffffff6U +*/ + +/** These definitions are part of a 128-bit period certification vector. +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0xc98e126aU +*/ + +#if MEXP == 607 + #include "test/SFMT-params607.h" +#elif MEXP == 1279 + #include "test/SFMT-params1279.h" +#elif MEXP == 2281 + #include "test/SFMT-params2281.h" +#elif MEXP == 4253 + #include "test/SFMT-params4253.h" +#elif MEXP == 11213 + #include "test/SFMT-params11213.h" +#elif MEXP == 19937 + #include "test/SFMT-params19937.h" +#elif MEXP == 44497 + #include "test/SFMT-params44497.h" +#elif MEXP == 86243 + #include "test/SFMT-params86243.h" +#elif MEXP == 132049 + #include "test/SFMT-params132049.h" +#elif MEXP == 216091 + #include "test/SFMT-params216091.h" +#else +#ifdef __GNUC__ + #error "MEXP is not valid." + #undef MEXP +#else + #undef MEXP +#endif + +#endif + +#endif /* SFMT_PARAMS_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params11213.h b/deps/jemalloc/test/include/test/SFMT-params11213.h new file mode 100644 index 00000000000..2994bd21da2 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params11213.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS11213_H +#define SFMT_PARAMS11213_H + +#define POS1 68 +#define SL1 14 +#define SL2 3 +#define SR1 7 +#define SR2 3 +#define MSK1 0xeffff7fbU +#define MSK2 0xffffffefU +#define MSK3 0xdfdfbfffU +#define MSK4 0x7fffdbfdU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xe8148000U +#define PARITY4 0xd0c7afa3U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-11213:68-14-3-7-3:effff7fb-ffffffef-dfdfbfff-7fffdbfd" + +#endif /* SFMT_PARAMS11213_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params1279.h b/deps/jemalloc/test/include/test/SFMT-params1279.h new file mode 100644 index 00000000000..d7959f98089 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params1279.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS1279_H +#define SFMT_PARAMS1279_H + +#define POS1 7 +#define SL1 14 +#define SL2 3 +#define SR1 5 +#define SR2 1 +#define MSK1 0xf7fefffdU +#define MSK2 0x7fefcfffU +#define MSK3 0xaff3ef3fU +#define MSK4 0xb5ffff7fU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x20000000U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-1279:7-14-3-5-1:f7fefffd-7fefcfff-aff3ef3f-b5ffff7f" + +#endif /* SFMT_PARAMS1279_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params132049.h b/deps/jemalloc/test/include/test/SFMT-params132049.h new file mode 100644 index 00000000000..a1dcec39267 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params132049.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS132049_H +#define SFMT_PARAMS132049_H + +#define POS1 110 +#define SL1 19 +#define SL2 1 +#define SR1 21 +#define SR2 1 +#define MSK1 0xffffbb5fU +#define MSK2 0xfb6ebf95U +#define MSK3 0xfffefffaU +#define MSK4 0xcff77fffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xcb520000U +#define PARITY4 0xc7e91c7dU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-132049:110-19-1-21-1:ffffbb5f-fb6ebf95-fffefffa-cff77fff" + +#endif /* SFMT_PARAMS132049_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params19937.h b/deps/jemalloc/test/include/test/SFMT-params19937.h new file mode 100644 index 00000000000..fb92b4c9b09 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params19937.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS19937_H +#define SFMT_PARAMS19937_H + +#define POS1 122 +#define SL1 18 +#define SL2 1 +#define SR1 11 +#define SR2 1 +#define MSK1 0xdfffffefU +#define MSK2 0xddfecb7fU +#define MSK3 0xbffaffffU +#define MSK4 0xbffffff6U +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x13c9e684U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-19937:122-18-1-11-1:dfffffef-ddfecb7f-bffaffff-bffffff6" + +#endif /* SFMT_PARAMS19937_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params216091.h b/deps/jemalloc/test/include/test/SFMT-params216091.h new file mode 100644 index 00000000000..125ce282048 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params216091.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS216091_H +#define SFMT_PARAMS216091_H + +#define POS1 627 +#define SL1 11 +#define SL2 3 +#define SR1 10 +#define SR2 1 +#define MSK1 0xbff7bff7U +#define MSK2 0xbfffffffU +#define MSK3 0xbffffa7fU +#define MSK4 0xffddfbfbU +#define PARITY1 0xf8000001U +#define PARITY2 0x89e80709U +#define PARITY3 0x3bd2b64bU +#define PARITY4 0x0c64b1e4U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-216091:627-11-3-10-1:bff7bff7-bfffffff-bffffa7f-ffddfbfb" + +#endif /* SFMT_PARAMS216091_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params2281.h b/deps/jemalloc/test/include/test/SFMT-params2281.h new file mode 100644 index 00000000000..0ef85c40701 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params2281.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS2281_H +#define SFMT_PARAMS2281_H + +#define POS1 12 +#define SL1 19 +#define SL2 1 +#define SR1 5 +#define SR2 1 +#define MSK1 0xbff7ffbfU +#define MSK2 0xfdfffffeU +#define MSK3 0xf7ffef7fU +#define MSK4 0xf2f7cbbfU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x41dfa600U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-2281:12-19-1-5-1:bff7ffbf-fdfffffe-f7ffef7f-f2f7cbbf" + +#endif /* SFMT_PARAMS2281_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params4253.h b/deps/jemalloc/test/include/test/SFMT-params4253.h new file mode 100644 index 00000000000..9f07bc67e1a --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params4253.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS4253_H +#define SFMT_PARAMS4253_H + +#define POS1 17 +#define SL1 20 +#define SL2 1 +#define SR1 7 +#define SR2 1 +#define MSK1 0x9f7bffffU +#define MSK2 0x9fffff5fU +#define MSK3 0x3efffffbU +#define MSK4 0xfffff7bbU +#define PARITY1 0xa8000001U +#define PARITY2 0xaf5390a3U +#define PARITY3 0xb740b3f8U +#define PARITY4 0x6c11486dU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-4253:17-20-1-7-1:9f7bffff-9fffff5f-3efffffb-fffff7bb" + +#endif /* SFMT_PARAMS4253_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params44497.h b/deps/jemalloc/test/include/test/SFMT-params44497.h new file mode 100644 index 00000000000..85598fed519 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params44497.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS44497_H +#define SFMT_PARAMS44497_H + +#define POS1 330 +#define SL1 5 +#define SL2 3 +#define SR1 9 +#define SR2 3 +#define MSK1 0xeffffffbU +#define MSK2 0xdfbebfffU +#define MSK3 0xbfbf7befU +#define MSK4 0x9ffd7bffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xa3ac4000U +#define PARITY4 0xecc1327aU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-44497:330-5-3-9-3:effffffb-dfbebfff-bfbf7bef-9ffd7bff" + +#endif /* SFMT_PARAMS44497_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params607.h b/deps/jemalloc/test/include/test/SFMT-params607.h new file mode 100644 index 00000000000..bc76485f8b6 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params607.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS607_H +#define SFMT_PARAMS607_H + +#define POS1 2 +#define SL1 15 +#define SL2 3 +#define SR1 13 +#define SR2 3 +#define MSK1 0xfdff37ffU +#define MSK2 0xef7f3f7dU +#define MSK3 0xff777b7dU +#define MSK4 0x7ff7fb2fU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x5986f054U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-607:2-15-3-13-3:fdff37ff-ef7f3f7d-ff777b7d-7ff7fb2f" + +#endif /* SFMT_PARAMS607_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params86243.h b/deps/jemalloc/test/include/test/SFMT-params86243.h new file mode 100644 index 00000000000..5e4d783c5d7 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params86243.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS86243_H +#define SFMT_PARAMS86243_H + +#define POS1 366 +#define SL1 6 +#define SL2 7 +#define SR1 19 +#define SR2 1 +#define MSK1 0xfdbffbffU +#define MSK2 0xbff7ff3fU +#define MSK3 0xfd77efffU +#define MSK4 0xbf9ff3ffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0xe9528d85U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6} + #define ALTI_SL2_PERM64 {7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-86243:366-6-7-19-1:fdbffbff-bff7ff3f-fd77efff-bf9ff3ff" + +#endif /* SFMT_PARAMS86243_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-sse2.h b/deps/jemalloc/test/include/test/SFMT-sse2.h new file mode 100644 index 00000000000..0314a163d58 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-sse2.h @@ -0,0 +1,157 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT-sse2.h + * @brief SIMD oriented Fast Mersenne Twister(SFMT) for Intel SSE2 + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * @note We assume LITTLE ENDIAN in this file + * + * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software, see LICENSE.txt + */ + +#ifndef SFMT_SSE2_H +#define SFMT_SSE2_H + +/** + * This function represents the recursion formula. + * @param a a 128-bit part of the interal state array + * @param b a 128-bit part of the interal state array + * @param c a 128-bit part of the interal state array + * @param d a 128-bit part of the interal state array + * @param mask 128-bit mask + * @return output + */ +JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b, + __m128i c, __m128i d, __m128i mask) { + __m128i v, x, y, z; + + x = _mm_load_si128(a); + y = _mm_srli_epi32(*b, SR1); + z = _mm_srli_si128(c, SR2); + v = _mm_slli_epi32(d, SL1); + z = _mm_xor_si128(z, x); + z = _mm_xor_si128(z, v); + x = _mm_slli_si128(x, SL2); + y = _mm_and_si128(y, mask); + z = _mm_xor_si128(z, x); + z = _mm_xor_si128(z, y); + return z; +} + +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) { + int i; + __m128i r, r1, r2, mask; + mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1); + + r1 = _mm_load_si128(&ctx->sfmt[N - 2].si); + r2 = _mm_load_si128(&ctx->sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2, + mask); + _mm_store_si128(&ctx->sfmt[i].si, r); + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&ctx->sfmt[i].si, r); + r1 = r2; + r2 = r; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pesudorandom numbers to be generated. + */ +JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + __m128i r, r1, r2, mask; + mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1); + + r1 = _mm_load_si128(&ctx->sfmt[N - 2].si); + r2 = _mm_load_si128(&ctx->sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + /* main loop */ + for (; i < size - N; i++) { + r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + for (j = 0; j < 2 * N - size; j++) { + r = _mm_load_si128(&array[j + size - N].si); + _mm_store_si128(&ctx->sfmt[j].si, r); + } + for (; i < size; i++) { + r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + _mm_store_si128(&ctx->sfmt[j++].si, r); + r1 = r2; + r2 = r; + } +} + +#endif diff --git a/deps/jemalloc/test/include/test/SFMT.h b/deps/jemalloc/test/include/test/SFMT.h new file mode 100644 index 00000000000..09c1607dd4c --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT.h @@ -0,0 +1,171 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT.h + * + * @brief SIMD oriented Fast Mersenne Twister(SFMT) pseudorandom + * number generator + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software. + * see LICENSE.txt + * + * @note We assume that your system has inttypes.h. If your system + * doesn't have inttypes.h, you have to typedef uint32_t and uint64_t, + * and you have to define PRIu64 and PRIx64 in this file as follows: + * @verbatim + typedef unsigned int uint32_t + typedef unsigned long long uint64_t + #define PRIu64 "llu" + #define PRIx64 "llx" +@endverbatim + * uint32_t must be exactly 32-bit unsigned integer type (no more, no + * less), and uint64_t must be exactly 64-bit unsigned integer type. + * PRIu64 and PRIx64 are used for printf function to print 64-bit + * unsigned int and 64-bit unsigned int in hexadecimal format. + */ + +#ifndef SFMT_H +#define SFMT_H + +typedef struct sfmt_s sfmt_t; + +uint32_t gen_rand32(sfmt_t *ctx); +uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit); +uint64_t gen_rand64(sfmt_t *ctx); +uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit); +void fill_array32(sfmt_t *ctx, uint32_t *array, int size); +void fill_array64(sfmt_t *ctx, uint64_t *array, int size); +sfmt_t *init_gen_rand(uint32_t seed); +sfmt_t *init_by_array(uint32_t *init_key, int key_length); +void fini_gen_rand(sfmt_t *ctx); +const char *get_idstring(void); +int get_min_array_size32(void); +int get_min_array_size64(void); + +#ifndef JEMALLOC_ENABLE_INLINE +double to_real1(uint32_t v); +double genrand_real1(sfmt_t *ctx); +double to_real2(uint32_t v); +double genrand_real2(sfmt_t *ctx); +double to_real3(uint32_t v); +double genrand_real3(sfmt_t *ctx); +double to_res53(uint64_t v); +double to_res53_mix(uint32_t x, uint32_t y); +double genrand_res53(sfmt_t *ctx); +double genrand_res53_mix(sfmt_t *ctx); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(SFMT_C_)) +/* These real versions are due to Isaku Wada */ +/** generates a random number on [0,1]-real-interval */ +JEMALLOC_INLINE double to_real1(uint32_t v) +{ + return v * (1.0/4294967295.0); + /* divided by 2^32-1 */ +} + +/** generates a random number on [0,1]-real-interval */ +JEMALLOC_INLINE double genrand_real1(sfmt_t *ctx) +{ + return to_real1(gen_rand32(ctx)); +} + +/** generates a random number on [0,1)-real-interval */ +JEMALLOC_INLINE double to_real2(uint32_t v) +{ + return v * (1.0/4294967296.0); + /* divided by 2^32 */ +} + +/** generates a random number on [0,1)-real-interval */ +JEMALLOC_INLINE double genrand_real2(sfmt_t *ctx) +{ + return to_real2(gen_rand32(ctx)); +} + +/** generates a random number on (0,1)-real-interval */ +JEMALLOC_INLINE double to_real3(uint32_t v) +{ + return (((double)v) + 0.5)*(1.0/4294967296.0); + /* divided by 2^32 */ +} + +/** generates a random number on (0,1)-real-interval */ +JEMALLOC_INLINE double genrand_real3(sfmt_t *ctx) +{ + return to_real3(gen_rand32(ctx)); +} +/** These real versions are due to Isaku Wada */ + +/** generates a random number on [0,1) with 53-bit resolution*/ +JEMALLOC_INLINE double to_res53(uint64_t v) +{ + return v * (1.0/18446744073709551616.0L); +} + +/** generates a random number on [0,1) with 53-bit resolution from two + * 32 bit integers */ +JEMALLOC_INLINE double to_res53_mix(uint32_t x, uint32_t y) +{ + return to_res53(x | ((uint64_t)y << 32)); +} + +/** generates a random number on [0,1) with 53-bit resolution + */ +JEMALLOC_INLINE double genrand_res53(sfmt_t *ctx) +{ + return to_res53(gen_rand64(ctx)); +} + +/** generates a random number on [0,1) with 53-bit resolution + using 32bit integer. + */ +JEMALLOC_INLINE double genrand_res53_mix(sfmt_t *ctx) +{ + uint32_t x, y; + + x = gen_rand32(ctx); + y = gen_rand32(ctx); + return to_res53_mix(x, y); +} +#endif +#endif diff --git a/deps/jemalloc/test/include/test/jemalloc_test.h.in b/deps/jemalloc/test/include/test/jemalloc_test.h.in new file mode 100644 index 00000000000..730a55dba23 --- /dev/null +++ b/deps/jemalloc/test/include/test/jemalloc_test.h.in @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +#else +# include +#endif + +/******************************************************************************/ +/* + * Define always-enabled assertion macros, so that test assertions execute even + * if assertions are disabled in the library code. These definitions must + * exist prior to including "jemalloc/internal/util.h". + */ +#define assert(e) do { \ + if (!(e)) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ +} while (0) + +#define not_reached() do { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define not_implemented() do { \ + malloc_printf(": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define assert_not_implemented(e) do { \ + if (!(e)) \ + not_implemented(); \ +} while (0) + +#include "test/jemalloc_test_defs.h" + +#ifdef JEMALLOC_OSSPIN +# include +#endif + +#if defined(HAVE_ALTIVEC) && !defined(__APPLE__) +# include +#endif +#ifdef HAVE_SSE2 +# include +#endif + +/******************************************************************************/ +/* + * For unit tests, expose all public and private interfaces. + */ +#ifdef JEMALLOC_UNIT_TEST +# define JEMALLOC_JET +# define JEMALLOC_MANGLE +# include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* + * For integration tests, expose the public jemalloc interfaces, but only + * expose the minimum necessary internal utility code (to avoid re-implementing + * essentially identical code within the test infrastructure). + */ +#elif defined(JEMALLOC_INTEGRATION_TEST) +# define JEMALLOC_MANGLE +# include "jemalloc/jemalloc@install_suffix@.h" +# include "jemalloc/internal/jemalloc_internal_defs.h" +# include "jemalloc/internal/jemalloc_internal_macros.h" + +# define JEMALLOC_N(n) @private_namespace@##n +# include "jemalloc/internal/private_namespace.h" + +# define JEMALLOC_H_TYPES +# define JEMALLOC_H_STRUCTS +# define JEMALLOC_H_EXTERNS +# define JEMALLOC_H_INLINES +# include "jemalloc/internal/util.h" +# include "jemalloc/internal/qr.h" +# include "jemalloc/internal/ql.h" +# undef JEMALLOC_H_TYPES +# undef JEMALLOC_H_STRUCTS +# undef JEMALLOC_H_EXTERNS +# undef JEMALLOC_H_INLINES + +/******************************************************************************/ +/* + * For stress tests, expose the public jemalloc interfaces with name mangling + * so that they can be tested as e.g. malloc() and free(). Also expose the + * public jemalloc interfaces with jet_ prefixes, so that stress tests can use + * a separate allocator for their internal data structures. + */ +#elif defined(JEMALLOC_STRESS_TEST) +# include "jemalloc/jemalloc@install_suffix@.h" + +# include "jemalloc/jemalloc_protos_jet.h" + +# define JEMALLOC_JET +# include "jemalloc/internal/jemalloc_internal.h" +# include "jemalloc/internal/public_unnamespace.h" +# undef JEMALLOC_JET + +# include "jemalloc/jemalloc_rename.h" +# define JEMALLOC_MANGLE +# ifdef JEMALLOC_STRESS_TESTLIB +# include "jemalloc/jemalloc_mangle_jet.h" +# else +# include "jemalloc/jemalloc_mangle.h" +# endif + +/******************************************************************************/ +/* + * This header does dangerous things, the effects of which only test code + * should be subject to. + */ +#else +# error "This header cannot be included outside a testing context" +#endif + +/******************************************************************************/ +/* + * Common test utilities. + */ +#include "test/math.h" +#include "test/mtx.h" +#include "test/mq.h" +#include "test/test.h" +#include "test/thd.h" +#define MEXP 19937 +#include "test/SFMT.h" diff --git a/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in b/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in new file mode 100644 index 00000000000..18a9773d705 --- /dev/null +++ b/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in @@ -0,0 +1,5 @@ +#include "jemalloc/internal/jemalloc_internal_defs.h" + +/* For use by SFMT. */ +#undef HAVE_SSE2 +#undef HAVE_ALTIVEC diff --git a/deps/jemalloc/test/include/test/math.h b/deps/jemalloc/test/include/test/math.h new file mode 100644 index 00000000000..a862ed7db24 --- /dev/null +++ b/deps/jemalloc/test/include/test/math.h @@ -0,0 +1,311 @@ +#ifndef JEMALLOC_ENABLE_INLINE +double ln_gamma(double x); +double i_gamma(double x, double p, double ln_gamma_p); +double pt_norm(double p); +double pt_chi2(double p, double df, double ln_gamma_df_2); +double pt_gamma(double p, double shape, double scale, double ln_gamma_shape); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(MATH_C_)) +/* + * Compute the natural log of Gamma(x), accurate to 10 decimal places. + * + * This implementation is based on: + * + * Pike, M.C., I.D. Hill (1966) Algorithm 291: Logarithm of Gamma function + * [S14]. Communications of the ACM 9(9):684. + */ +JEMALLOC_INLINE double +ln_gamma(double x) +{ + double f, z; + + assert(x > 0.0); + + if (x < 7.0) { + f = 1.0; + z = x; + while (z < 7.0) { + f *= z; + z += 1.0; + } + x = z; + f = -log(f); + } else + f = 0.0; + + z = 1.0 / (x * x); + + return (f + (x-0.5) * log(x) - x + 0.918938533204673 + + (((-0.000595238095238 * z + 0.000793650793651) * z - + 0.002777777777778) * z + 0.083333333333333) / x); +} + +/* + * Compute the incomplete Gamma ratio for [0..x], where p is the shape + * parameter, and ln_gamma_p is ln_gamma(p). + * + * This implementation is based on: + * + * Bhattacharjee, G.P. (1970) Algorithm AS 32: The incomplete Gamma integral. + * Applied Statistics 19:285-287. + */ +JEMALLOC_INLINE double +i_gamma(double x, double p, double ln_gamma_p) +{ + double acu, factor, oflo, gin, term, rn, a, b, an, dif; + double pn[6]; + unsigned i; + + assert(p > 0.0); + assert(x >= 0.0); + + if (x == 0.0) + return (0.0); + + acu = 1.0e-10; + oflo = 1.0e30; + gin = 0.0; + factor = exp(p * log(x) - x - ln_gamma_p); + + if (x <= 1.0 || x < p) { + /* Calculation by series expansion. */ + gin = 1.0; + term = 1.0; + rn = p; + + while (true) { + rn += 1.0; + term *= x / rn; + gin += term; + if (term <= acu) { + gin *= factor / p; + return (gin); + } + } + } else { + /* Calculation by continued fraction. */ + a = 1.0 - p; + b = a + x + 1.0; + term = 0.0; + pn[0] = 1.0; + pn[1] = x; + pn[2] = x + 1.0; + pn[3] = x * b; + gin = pn[2] / pn[3]; + + while (true) { + a += 1.0; + b += 2.0; + term += 1.0; + an = a * term; + for (i = 0; i < 2; i++) + pn[i+4] = b * pn[i+2] - an * pn[i]; + if (pn[5] != 0.0) { + rn = pn[4] / pn[5]; + dif = fabs(gin - rn); + if (dif <= acu && dif <= acu * rn) { + gin = 1.0 - factor * gin; + return (gin); + } + gin = rn; + } + for (i = 0; i < 4; i++) + pn[i] = pn[i+2]; + + if (fabs(pn[4]) >= oflo) { + for (i = 0; i < 4; i++) + pn[i] /= oflo; + } + } + } +} + +/* + * Given a value p in [0..1] of the lower tail area of the normal distribution, + * compute the limit on the definite integral from [-inf..z] that satisfies p, + * accurate to 16 decimal places. + * + * This implementation is based on: + * + * Wichura, M.J. (1988) Algorithm AS 241: The percentage points of the normal + * distribution. Applied Statistics 37(3):477-484. + */ +JEMALLOC_INLINE double +pt_norm(double p) +{ + double q, r, ret; + + assert(p > 0.0 && p < 1.0); + + q = p - 0.5; + if (fabs(q) <= 0.425) { + /* p close to 1/2. */ + r = 0.180625 - q * q; + return (q * (((((((2.5090809287301226727e3 * r + + 3.3430575583588128105e4) * r + 6.7265770927008700853e4) * r + + 4.5921953931549871457e4) * r + 1.3731693765509461125e4) * + r + 1.9715909503065514427e3) * r + 1.3314166789178437745e2) + * r + 3.3871328727963666080e0) / + (((((((5.2264952788528545610e3 * r + + 2.8729085735721942674e4) * r + 3.9307895800092710610e4) * r + + 2.1213794301586595867e4) * r + 5.3941960214247511077e3) * + r + 6.8718700749205790830e2) * r + 4.2313330701600911252e1) + * r + 1.0)); + } else { + if (q < 0.0) + r = p; + else + r = 1.0 - p; + assert(r > 0.0); + + r = sqrt(-log(r)); + if (r <= 5.0) { + /* p neither close to 1/2 nor 0 or 1. */ + r -= 1.6; + ret = ((((((((7.74545014278341407640e-4 * r + + 2.27238449892691845833e-2) * r + + 2.41780725177450611770e-1) * r + + 1.27045825245236838258e0) * r + + 3.64784832476320460504e0) * r + + 5.76949722146069140550e0) * r + + 4.63033784615654529590e0) * r + + 1.42343711074968357734e0) / + (((((((1.05075007164441684324e-9 * r + + 5.47593808499534494600e-4) * r + + 1.51986665636164571966e-2) + * r + 1.48103976427480074590e-1) * r + + 6.89767334985100004550e-1) * r + + 1.67638483018380384940e0) * r + + 2.05319162663775882187e0) * r + 1.0)); + } else { + /* p near 0 or 1. */ + r -= 5.0; + ret = ((((((((2.01033439929228813265e-7 * r + + 2.71155556874348757815e-5) * r + + 1.24266094738807843860e-3) * r + + 2.65321895265761230930e-2) * r + + 2.96560571828504891230e-1) * r + + 1.78482653991729133580e0) * r + + 5.46378491116411436990e0) * r + + 6.65790464350110377720e0) / + (((((((2.04426310338993978564e-15 * r + + 1.42151175831644588870e-7) * r + + 1.84631831751005468180e-5) * r + + 7.86869131145613259100e-4) * r + + 1.48753612908506148525e-2) * r + + 1.36929880922735805310e-1) * r + + 5.99832206555887937690e-1) + * r + 1.0)); + } + if (q < 0.0) + ret = -ret; + return (ret); + } +} + +/* + * Given a value p in [0..1] of the lower tail area of the Chi^2 distribution + * with df degrees of freedom, where ln_gamma_df_2 is ln_gamma(df/2.0), compute + * the upper limit on the definite integral from [0..z] that satisfies p, + * accurate to 12 decimal places. + * + * This implementation is based on: + * + * Best, D.J., D.E. Roberts (1975) Algorithm AS 91: The percentage points of + * the Chi^2 distribution. Applied Statistics 24(3):385-388. + * + * Shea, B.L. (1991) Algorithm AS R85: A remark on AS 91: The percentage + * points of the Chi^2 distribution. Applied Statistics 40(1):233-235. + */ +JEMALLOC_INLINE double +pt_chi2(double p, double df, double ln_gamma_df_2) +{ + double e, aa, xx, c, ch, a, q, p1, p2, t, x, b, s1, s2, s3, s4, s5, s6; + unsigned i; + + assert(p >= 0.0 && p < 1.0); + assert(df > 0.0); + + e = 5.0e-7; + aa = 0.6931471805; + + xx = 0.5 * df; + c = xx - 1.0; + + if (df < -1.24 * log(p)) { + /* Starting approximation for small Chi^2. */ + ch = pow(p * xx * exp(ln_gamma_df_2 + xx * aa), 1.0 / xx); + if (ch - e < 0.0) + return (ch); + } else { + if (df > 0.32) { + x = pt_norm(p); + /* + * Starting approximation using Wilson and Hilferty + * estimate. + */ + p1 = 0.222222 / df; + ch = df * pow(x * sqrt(p1) + 1.0 - p1, 3.0); + /* Starting approximation for p tending to 1. */ + if (ch > 2.2 * df + 6.0) { + ch = -2.0 * (log(1.0 - p) - c * log(0.5 * ch) + + ln_gamma_df_2); + } + } else { + ch = 0.4; + a = log(1.0 - p); + while (true) { + q = ch; + p1 = 1.0 + ch * (4.67 + ch); + p2 = ch * (6.73 + ch * (6.66 + ch)); + t = -0.5 + (4.67 + 2.0 * ch) / p1 - (6.73 + ch + * (13.32 + 3.0 * ch)) / p2; + ch -= (1.0 - exp(a + ln_gamma_df_2 + 0.5 * ch + + c * aa) * p2 / p1) / t; + if (fabs(q / ch - 1.0) - 0.01 <= 0.0) + break; + } + } + } + + for (i = 0; i < 20; i++) { + /* Calculation of seven-term Taylor series. */ + q = ch; + p1 = 0.5 * ch; + if (p1 < 0.0) + return (-1.0); + p2 = p - i_gamma(p1, xx, ln_gamma_df_2); + t = p2 * exp(xx * aa + ln_gamma_df_2 + p1 - c * log(ch)); + b = t / ch; + a = 0.5 * t - b * c; + s1 = (210.0 + a * (140.0 + a * (105.0 + a * (84.0 + a * (70.0 + + 60.0 * a))))) / 420.0; + s2 = (420.0 + a * (735.0 + a * (966.0 + a * (1141.0 + 1278.0 * + a)))) / 2520.0; + s3 = (210.0 + a * (462.0 + a * (707.0 + 932.0 * a))) / 2520.0; + s4 = (252.0 + a * (672.0 + 1182.0 * a) + c * (294.0 + a * + (889.0 + 1740.0 * a))) / 5040.0; + s5 = (84.0 + 264.0 * a + c * (175.0 + 606.0 * a)) / 2520.0; + s6 = (120.0 + c * (346.0 + 127.0 * c)) / 5040.0; + ch += t * (1.0 + 0.5 * t * s1 - b * c * (s1 - b * (s2 - b * (s3 + - b * (s4 - b * (s5 - b * s6)))))); + if (fabs(q / ch - 1.0) <= e) + break; + } + + return (ch); +} + +/* + * Given a value p in [0..1] and Gamma distribution shape and scale parameters, + * compute the upper limit on the definite integeral from [0..z] that satisfies + * p. + */ +JEMALLOC_INLINE double +pt_gamma(double p, double shape, double scale, double ln_gamma_shape) +{ + + return (pt_chi2(p, shape * 2.0, ln_gamma_shape) * 0.5 * scale); +} +#endif diff --git a/deps/jemalloc/test/include/test/mq.h b/deps/jemalloc/test/include/test/mq.h new file mode 100644 index 00000000000..11188653c65 --- /dev/null +++ b/deps/jemalloc/test/include/test/mq.h @@ -0,0 +1,110 @@ +/* + * Simple templated message queue implementation that relies on only mutexes for + * synchronization (which reduces portability issues). Given the following + * setup: + * + * typedef struct mq_msg_s mq_msg_t; + * struct mq_msg_s { + * mq_msg(mq_msg_t) link; + * [message data] + * }; + * mq_gen(, mq_, mq_t, mq_msg_t, link) + * + * The API is as follows: + * + * bool mq_init(mq_t *mq); + * void mq_fini(mq_t *mq); + * unsigned mq_count(mq_t *mq); + * mq_msg_t *mq_tryget(mq_t *mq); + * mq_msg_t *mq_get(mq_t *mq); + * void mq_put(mq_t *mq, mq_msg_t *msg); + * + * The message queue linkage embedded in each message is to be treated as + * externally opaque (no need to initialize or clean up externally). mq_fini() + * does not perform any cleanup of messages, since it knows nothing of their + * payloads. + */ +#define mq_msg(a_mq_msg_type) ql_elm(a_mq_msg_type) + +#define mq_gen(a_attr, a_prefix, a_mq_type, a_mq_msg_type, a_field) \ +typedef struct { \ + mtx_t lock; \ + ql_head(a_mq_msg_type) msgs; \ + unsigned count; \ +} a_mq_type; \ +a_attr bool \ +a_prefix##init(a_mq_type *mq) { \ + \ + if (mtx_init(&mq->lock)) \ + return (true); \ + ql_new(&mq->msgs); \ + mq->count = 0; \ + return (false); \ +} \ +a_attr void \ +a_prefix##fini(a_mq_type *mq) \ +{ \ + \ + mtx_fini(&mq->lock); \ +} \ +a_attr unsigned \ +a_prefix##count(a_mq_type *mq) \ +{ \ + unsigned count; \ + \ + mtx_lock(&mq->lock); \ + count = mq->count; \ + mtx_unlock(&mq->lock); \ + return (count); \ +} \ +a_attr a_mq_msg_type * \ +a_prefix##tryget(a_mq_type *mq) \ +{ \ + a_mq_msg_type *msg; \ + \ + mtx_lock(&mq->lock); \ + msg = ql_first(&mq->msgs); \ + if (msg != NULL) { \ + ql_head_remove(&mq->msgs, a_mq_msg_type, a_field); \ + mq->count--; \ + } \ + mtx_unlock(&mq->lock); \ + return (msg); \ +} \ +a_attr a_mq_msg_type * \ +a_prefix##get(a_mq_type *mq) \ +{ \ + a_mq_msg_type *msg; \ + struct timespec timeout; \ + \ + msg = a_prefix##tryget(mq); \ + if (msg != NULL) \ + return (msg); \ + \ + timeout.tv_sec = 0; \ + timeout.tv_nsec = 1; \ + while (true) { \ + nanosleep(&timeout, NULL); \ + msg = a_prefix##tryget(mq); \ + if (msg != NULL) \ + return (msg); \ + if (timeout.tv_sec == 0) { \ + /* Double sleep time, up to max 1 second. */ \ + timeout.tv_nsec <<= 1; \ + if (timeout.tv_nsec >= 1000*1000*1000) { \ + timeout.tv_sec = 1; \ + timeout.tv_nsec = 0; \ + } \ + } \ + } \ +} \ +a_attr void \ +a_prefix##put(a_mq_type *mq, a_mq_msg_type *msg) \ +{ \ + \ + mtx_lock(&mq->lock); \ + ql_elm_new(msg, a_field); \ + ql_tail_insert(&mq->msgs, msg, a_field); \ + mq->count++; \ + mtx_unlock(&mq->lock); \ +} diff --git a/deps/jemalloc/test/include/test/mtx.h b/deps/jemalloc/test/include/test/mtx.h new file mode 100644 index 00000000000..bbe822f54b3 --- /dev/null +++ b/deps/jemalloc/test/include/test/mtx.h @@ -0,0 +1,21 @@ +/* + * mtx is a slightly simplified version of malloc_mutex. This code duplication + * is unfortunate, but there are allocator bootstrapping considerations that + * would leak into the test infrastructure if malloc_mutex were used directly + * in tests. + */ + +typedef struct { +#ifdef _WIN32 + CRITICAL_SECTION lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; +#else + pthread_mutex_t lock; +#endif +} mtx_t; + +bool mtx_init(mtx_t *mtx); +void mtx_fini(mtx_t *mtx); +void mtx_lock(mtx_t *mtx); +void mtx_unlock(mtx_t *mtx); diff --git a/deps/jemalloc/test/include/test/test.h b/deps/jemalloc/test/include/test/test.h new file mode 100644 index 00000000000..a32ec07c4c5 --- /dev/null +++ b/deps/jemalloc/test/include/test/test.h @@ -0,0 +1,329 @@ +#define ASSERT_BUFSIZE 256 + +#define assert_cmp(t, a, b, cmp, neg_cmp, pri, fmt...) do { \ + t a_ = (a); \ + t b_ = (b); \ + if (!(a_ cmp b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) "#cmp" (%s) --> " \ + "%"pri" "#neg_cmp" %"pri": ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_, b_); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) + +#define assert_ptr_eq(a, b, fmt...) assert_cmp(void *, a, b, ==, \ + !=, "p", fmt) +#define assert_ptr_ne(a, b, fmt...) assert_cmp(void *, a, b, !=, \ + ==, "p", fmt) +#define assert_ptr_null(a, fmt...) assert_cmp(void *, a, NULL, ==, \ + !=, "p", fmt) +#define assert_ptr_not_null(a, fmt...) assert_cmp(void *, a, NULL, !=, \ + ==, "p", fmt) + +#define assert_c_eq(a, b, fmt...) assert_cmp(char, a, b, ==, !=, "c", fmt) +#define assert_c_ne(a, b, fmt...) assert_cmp(char, a, b, !=, ==, "c", fmt) +#define assert_c_lt(a, b, fmt...) assert_cmp(char, a, b, <, >=, "c", fmt) +#define assert_c_le(a, b, fmt...) assert_cmp(char, a, b, <=, >, "c", fmt) +#define assert_c_ge(a, b, fmt...) assert_cmp(char, a, b, >=, <, "c", fmt) +#define assert_c_gt(a, b, fmt...) assert_cmp(char, a, b, >, <=, "c", fmt) + +#define assert_x_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "#x", fmt) +#define assert_x_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "#x", fmt) +#define assert_x_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "#x", fmt) +#define assert_x_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "#x", fmt) +#define assert_x_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "#x", fmt) +#define assert_x_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "#x", fmt) + +#define assert_d_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "d", fmt) +#define assert_d_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "d", fmt) +#define assert_d_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "d", fmt) +#define assert_d_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "d", fmt) +#define assert_d_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "d", fmt) +#define assert_d_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "d", fmt) + +#define assert_u_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "u", fmt) +#define assert_u_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "u", fmt) +#define assert_u_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "u", fmt) +#define assert_u_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "u", fmt) +#define assert_u_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "u", fmt) +#define assert_u_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "u", fmt) + +#define assert_ld_eq(a, b, fmt...) assert_cmp(long, a, b, ==, \ + !=, "ld", fmt) +#define assert_ld_ne(a, b, fmt...) assert_cmp(long, a, b, !=, \ + ==, "ld", fmt) +#define assert_ld_lt(a, b, fmt...) assert_cmp(long, a, b, <, \ + >=, "ld", fmt) +#define assert_ld_le(a, b, fmt...) assert_cmp(long, a, b, <=, \ + >, "ld", fmt) +#define assert_ld_ge(a, b, fmt...) assert_cmp(long, a, b, >=, \ + <, "ld", fmt) +#define assert_ld_gt(a, b, fmt...) assert_cmp(long, a, b, >, \ + <=, "ld", fmt) + +#define assert_lu_eq(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, ==, !=, "lu", fmt) +#define assert_lu_ne(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, !=, ==, "lu", fmt) +#define assert_lu_lt(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, <, >=, "lu", fmt) +#define assert_lu_le(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, <=, >, "lu", fmt) +#define assert_lu_ge(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, >=, <, "lu", fmt) +#define assert_lu_gt(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, >, <=, "lu", fmt) + +#define assert_qd_eq(a, b, fmt...) assert_cmp(long long, a, b, ==, \ + !=, "qd", fmt) +#define assert_qd_ne(a, b, fmt...) assert_cmp(long long, a, b, !=, \ + ==, "qd", fmt) +#define assert_qd_lt(a, b, fmt...) assert_cmp(long long, a, b, <, \ + >=, "qd", fmt) +#define assert_qd_le(a, b, fmt...) assert_cmp(long long, a, b, <=, \ + >, "qd", fmt) +#define assert_qd_ge(a, b, fmt...) assert_cmp(long long, a, b, >=, \ + <, "qd", fmt) +#define assert_qd_gt(a, b, fmt...) assert_cmp(long long, a, b, >, \ + <=, "qd", fmt) + +#define assert_qu_eq(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, ==, !=, "qu", fmt) +#define assert_qu_ne(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, !=, ==, "qu", fmt) +#define assert_qu_lt(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, <, >=, "qu", fmt) +#define assert_qu_le(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, <=, >, "qu", fmt) +#define assert_qu_ge(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, >=, <, "qu", fmt) +#define assert_qu_gt(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, >, <=, "qu", fmt) + +#define assert_jd_eq(a, b, fmt...) assert_cmp(intmax_t, a, b, ==, \ + !=, "jd", fmt) +#define assert_jd_ne(a, b, fmt...) assert_cmp(intmax_t, a, b, !=, \ + ==, "jd", fmt) +#define assert_jd_lt(a, b, fmt...) assert_cmp(intmax_t, a, b, <, \ + >=, "jd", fmt) +#define assert_jd_le(a, b, fmt...) assert_cmp(intmax_t, a, b, <=, \ + >, "jd", fmt) +#define assert_jd_ge(a, b, fmt...) assert_cmp(intmax_t, a, b, >=, \ + <, "jd", fmt) +#define assert_jd_gt(a, b, fmt...) assert_cmp(intmax_t, a, b, >, \ + <=, "jd", fmt) + +#define assert_ju_eq(a, b, fmt...) assert_cmp(uintmax_t, a, b, ==, \ + !=, "ju", fmt) +#define assert_ju_ne(a, b, fmt...) assert_cmp(uintmax_t, a, b, !=, \ + ==, "ju", fmt) +#define assert_ju_lt(a, b, fmt...) assert_cmp(uintmax_t, a, b, <, \ + >=, "ju", fmt) +#define assert_ju_le(a, b, fmt...) assert_cmp(uintmax_t, a, b, <=, \ + >, "ju", fmt) +#define assert_ju_ge(a, b, fmt...) assert_cmp(uintmax_t, a, b, >=, \ + <, "ju", fmt) +#define assert_ju_gt(a, b, fmt...) assert_cmp(uintmax_t, a, b, >, \ + <=, "ju", fmt) + +#define assert_zd_eq(a, b, fmt...) assert_cmp(ssize_t, a, b, ==, \ + !=, "zd", fmt) +#define assert_zd_ne(a, b, fmt...) assert_cmp(ssize_t, a, b, !=, \ + ==, "zd", fmt) +#define assert_zd_lt(a, b, fmt...) assert_cmp(ssize_t, a, b, <, \ + >=, "zd", fmt) +#define assert_zd_le(a, b, fmt...) assert_cmp(ssize_t, a, b, <=, \ + >, "zd", fmt) +#define assert_zd_ge(a, b, fmt...) assert_cmp(ssize_t, a, b, >=, \ + <, "zd", fmt) +#define assert_zd_gt(a, b, fmt...) assert_cmp(ssize_t, a, b, >, \ + <=, "zd", fmt) + +#define assert_zu_eq(a, b, fmt...) assert_cmp(size_t, a, b, ==, \ + !=, "zu", fmt) +#define assert_zu_ne(a, b, fmt...) assert_cmp(size_t, a, b, !=, \ + ==, "zu", fmt) +#define assert_zu_lt(a, b, fmt...) assert_cmp(size_t, a, b, <, \ + >=, "zu", fmt) +#define assert_zu_le(a, b, fmt...) assert_cmp(size_t, a, b, <=, \ + >, "zu", fmt) +#define assert_zu_ge(a, b, fmt...) assert_cmp(size_t, a, b, >=, \ + <, "zu", fmt) +#define assert_zu_gt(a, b, fmt...) assert_cmp(size_t, a, b, >, \ + <=, "zu", fmt) + +#define assert_d32_eq(a, b, fmt...) assert_cmp(int32_t, a, b, ==, \ + !=, PRId32, fmt) +#define assert_d32_ne(a, b, fmt...) assert_cmp(int32_t, a, b, !=, \ + ==, PRId32, fmt) +#define assert_d32_lt(a, b, fmt...) assert_cmp(int32_t, a, b, <, \ + >=, PRId32, fmt) +#define assert_d32_le(a, b, fmt...) assert_cmp(int32_t, a, b, <=, \ + >, PRId32, fmt) +#define assert_d32_ge(a, b, fmt...) assert_cmp(int32_t, a, b, >=, \ + <, PRId32, fmt) +#define assert_d32_gt(a, b, fmt...) assert_cmp(int32_t, a, b, >, \ + <=, PRId32, fmt) + +#define assert_u32_eq(a, b, fmt...) assert_cmp(uint32_t, a, b, ==, \ + !=, PRIu32, fmt) +#define assert_u32_ne(a, b, fmt...) assert_cmp(uint32_t, a, b, !=, \ + ==, PRIu32, fmt) +#define assert_u32_lt(a, b, fmt...) assert_cmp(uint32_t, a, b, <, \ + >=, PRIu32, fmt) +#define assert_u32_le(a, b, fmt...) assert_cmp(uint32_t, a, b, <=, \ + >, PRIu32, fmt) +#define assert_u32_ge(a, b, fmt...) assert_cmp(uint32_t, a, b, >=, \ + <, PRIu32, fmt) +#define assert_u32_gt(a, b, fmt...) assert_cmp(uint32_t, a, b, >, \ + <=, PRIu32, fmt) + +#define assert_d64_eq(a, b, fmt...) assert_cmp(int64_t, a, b, ==, \ + !=, PRId64, fmt) +#define assert_d64_ne(a, b, fmt...) assert_cmp(int64_t, a, b, !=, \ + ==, PRId64, fmt) +#define assert_d64_lt(a, b, fmt...) assert_cmp(int64_t, a, b, <, \ + >=, PRId64, fmt) +#define assert_d64_le(a, b, fmt...) assert_cmp(int64_t, a, b, <=, \ + >, PRId64, fmt) +#define assert_d64_ge(a, b, fmt...) assert_cmp(int64_t, a, b, >=, \ + <, PRId64, fmt) +#define assert_d64_gt(a, b, fmt...) assert_cmp(int64_t, a, b, >, \ + <=, PRId64, fmt) + +#define assert_u64_eq(a, b, fmt...) assert_cmp(uint64_t, a, b, ==, \ + !=, PRIu64, fmt) +#define assert_u64_ne(a, b, fmt...) assert_cmp(uint64_t, a, b, !=, \ + ==, PRIu64, fmt) +#define assert_u64_lt(a, b, fmt...) assert_cmp(uint64_t, a, b, <, \ + >=, PRIu64, fmt) +#define assert_u64_le(a, b, fmt...) assert_cmp(uint64_t, a, b, <=, \ + >, PRIu64, fmt) +#define assert_u64_ge(a, b, fmt...) assert_cmp(uint64_t, a, b, >=, \ + <, PRIu64, fmt) +#define assert_u64_gt(a, b, fmt...) assert_cmp(uint64_t, a, b, >, \ + <=, PRIu64, fmt) + +#define assert_b_eq(a, b, fmt...) do { \ + bool a_ = (a); \ + bool b_ = (b); \ + if (!(a_ == b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) == (%s) --> %s != %s: ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_ ? "true" : "false", \ + b_ ? "true" : "false"); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_b_ne(a, b, fmt...) do { \ + bool a_ = (a); \ + bool b_ = (b); \ + if (!(a_ != b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) != (%s) --> %s == %s: ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_ ? "true" : "false", \ + b_ ? "true" : "false"); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_true(a, fmt...) assert_b_eq(a, true, fmt) +#define assert_false(a, fmt...) assert_b_eq(a, false, fmt) + +#define assert_str_eq(a, b, fmt...) do { \ + if (strcmp((a), (b))) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) same as (%s) --> " \ + "\"%s\" differs from \"%s\": ", \ + __func__, __FILE__, __LINE__, #a, #b, a, b); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_str_ne(a, b, fmt...) do { \ + if (!strcmp((a), (b))) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) differs from (%s) --> " \ + "\"%s\" same as \"%s\": ", \ + __func__, __FILE__, __LINE__, #a, #b, a, b); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) + +#define assert_not_reached(fmt...) do { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Unreachable code reached: ", \ + __func__, __FILE__, __LINE__); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ +} while (0) + +/* + * If this enum changes, corresponding changes in test/test.sh.in are also + * necessary. + */ +typedef enum { + test_status_pass = 0, + test_status_skip = 1, + test_status_fail = 2, + + test_status_count = 3 +} test_status_t; + +typedef void (test_t)(void); + +#define TEST_BEGIN(f) \ +static void \ +f(void) \ +{ \ + p_test_init(#f); + +#define TEST_END \ + goto label_test_end; \ +label_test_end: \ + p_test_fini(); \ +} + +#define test(tests...) \ + p_test(tests, NULL) + +#define test_skip_if(e) do { \ + if (e) { \ + test_skip("%s:%s:%d: Test skipped: (%s)", \ + __func__, __FILE__, __LINE__, #e); \ + goto label_test_end; \ + } \ +} while (0) + +void test_skip(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); +void test_fail(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); + +/* For private use by macros. */ +test_status_t p_test(test_t* t, ...); +void p_test_init(const char *name); +void p_test_fini(void); +void p_test_fail(const char *prefix, const char *message); diff --git a/deps/jemalloc/test/include/test/thd.h b/deps/jemalloc/test/include/test/thd.h new file mode 100644 index 00000000000..f941d7a752f --- /dev/null +++ b/deps/jemalloc/test/include/test/thd.h @@ -0,0 +1,9 @@ +/* Abstraction layer for threading in tests */ +#ifdef _WIN32 +typedef HANDLE thd_t; +#else +typedef pthread_t thd_t; +#endif + +void thd_create(thd_t *thd, void *(*proc)(void *), void *arg); +void thd_join(thd_t thd, void **ret); diff --git a/deps/jemalloc/test/integration/MALLOCX_ARENA.c b/deps/jemalloc/test/integration/MALLOCX_ARENA.c new file mode 100644 index 00000000000..71cf6f255e8 --- /dev/null +++ b/deps/jemalloc/test/integration/MALLOCX_ARENA.c @@ -0,0 +1,58 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 10 + +void * +thd_start(void *arg) +{ + unsigned thread_ind = (unsigned)(uintptr_t)arg; + unsigned arena_ind; + void *p; + size_t sz; + + sz = sizeof(arena_ind); + assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0, + "Error in arenas.extend"); + + if (thread_ind % 4 != 3) { + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + const char *dss_precs[] = {"disabled", "primary", "secondary"}; + const char *dss = dss_precs[thread_ind % + (sizeof(dss_precs)/sizeof(char*))]; + assert_d_eq(mallctlnametomib("arena.0.dss", mib, &miblen), 0, + "Error in mallctlnametomib()"); + mib[1] = arena_ind; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss, + sizeof(const char *)), 0, "Error in mallctlbymib()"); + } + + p = mallocx(1, MALLOCX_ARENA(arena_ind)); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + dallocx(p, 0); + + return (NULL); +} + +TEST_BEGIN(test_ALLOCM_ARENA) +{ + thd_t thds[NTHREADS]; + unsigned i; + + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, + (void *)(uintptr_t)i); + } + + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ALLOCM_ARENA)); +} diff --git a/deps/jemalloc/test/integration/aligned_alloc.c b/deps/jemalloc/test/integration/aligned_alloc.c new file mode 100644 index 00000000000..609001487b8 --- /dev/null +++ b/deps/jemalloc/test/integration/aligned_alloc.c @@ -0,0 +1,125 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) +#define NITER 4 + +TEST_BEGIN(test_alignment_errors) +{ + size_t alignment; + void *p; + + alignment = 0; + set_errno(0); + p = aligned_alloc(alignment, 1); + assert_false(p != NULL || get_errno() != EINVAL, + "Expected error for invalid alignment %zu", alignment); + + for (alignment = sizeof(size_t); alignment < MAXALIGN; + alignment <<= 1) { + set_errno(0); + p = aligned_alloc(alignment + 1, 1); + assert_false(p != NULL || get_errno() != EINVAL, + "Expected error for invalid alignment %zu", + alignment + 1); + } +} +TEST_END + +TEST_BEGIN(test_oom_errors) +{ + size_t alignment, size; + void *p; + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x8000000000000000); + size = UINT64_C(0x8000000000000000); +#else + alignment = 0x80000000LU; + size = 0x80000000LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(%zu, %zu)", + alignment, size); + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x4000000000000000); + size = UINT64_C(0xc000000000000001); +#else + alignment = 0x40000000LU; + size = 0xc0000001LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(%zu, %zu)", + alignment, size); + + alignment = 0x10LU; +#if LG_SIZEOF_PTR == 3 + size = UINT64_C(0xfffffffffffffff0); +#else + size = 0xfffffff0LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(&p, %zu, %zu)", + alignment, size); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t alignment, size, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (size = 1; + size < 3 * alignment && size < (1U << 31); + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + ps[i] = aligned_alloc(alignment, size); + if (ps[i] == NULL) { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + test_fail( + "Error for alignment=%zu, " + "size=%zu (%#zx): %s", + alignment, size, size, buf); + } + total += malloc_usable_size(ps[i]); + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + free(ps[i]); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_alignment_errors, + test_oom_errors, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/allocated.c b/deps/jemalloc/test/integration/allocated.c new file mode 100644 index 00000000000..3630e80ce25 --- /dev/null +++ b/deps/jemalloc/test/integration/allocated.c @@ -0,0 +1,125 @@ +#include "test/jemalloc_test.h" + +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; + +void * +thd_start(void *arg) +{ + int err; + void *p; + uint64_t a0, a1, d0, d1; + uint64_t *ap0, *ap1, *dp0, *dp1; + size_t sz, usize; + + sz = sizeof(a0); + if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + sz = sizeof(ap0); + if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + assert_u64_eq(*ap0, a0, + "\"thread.allocatedp\" should provide a pointer to internal " + "storage"); + + sz = sizeof(d0); + if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + sz = sizeof(dp0); + if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + assert_u64_eq(*dp0, d0, + "\"thread.deallocatedp\" should provide a pointer to internal " + "storage"); + + p = malloc(1); + assert_ptr_not_null(p, "Unexpected malloc() error"); + + sz = sizeof(a1); + mallctl("thread.allocated", &a1, &sz, NULL, 0); + sz = sizeof(ap1); + mallctl("thread.allocatedp", &ap1, &sz, NULL, 0); + assert_u64_eq(*ap1, a1, + "Dereferenced \"thread.allocatedp\" value should equal " + "\"thread.allocated\" value"); + assert_ptr_eq(ap0, ap1, + "Pointer returned by \"thread.allocatedp\" should not change"); + + usize = malloc_usable_size(p); + assert_u64_le(a0 + usize, a1, + "Allocated memory counter should increase by at least the amount " + "explicitly allocated"); + + free(p); + + sz = sizeof(d1); + mallctl("thread.deallocated", &d1, &sz, NULL, 0); + sz = sizeof(dp1); + mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0); + assert_u64_eq(*dp1, d1, + "Dereferenced \"thread.deallocatedp\" value should equal " + "\"thread.deallocated\" value"); + assert_ptr_eq(dp0, dp1, + "Pointer returned by \"thread.deallocatedp\" should not change"); + + assert_u64_le(d0 + usize, d1, + "Deallocated memory counter should increase by at least the amount " + "explicitly deallocated"); + + return (NULL); +label_ENOENT: + assert_false(config_stats, + "ENOENT should only be returned if stats are disabled"); + test_skip("\"thread.allocated\" mallctl not available"); + return (NULL); +} + +TEST_BEGIN(test_main_thread) +{ + + thd_start(NULL); +} +TEST_END + +TEST_BEGIN(test_subthread) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} +TEST_END + +int +main(void) +{ + + /* Run tests multiple times to check for bad interactions. */ + return (test( + test_main_thread, + test_subthread, + test_main_thread, + test_subthread, + test_main_thread)); +} diff --git a/deps/jemalloc/test/integration/allocm.c b/deps/jemalloc/test/integration/allocm.c new file mode 100644 index 00000000000..7b4ea0c2c65 --- /dev/null +++ b/deps/jemalloc/test/integration/allocm.c @@ -0,0 +1,107 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +#define MAXALIGN (((size_t)1) << 25) +#define NITER 4 + +TEST_BEGIN(test_basic) +{ + size_t nsz, rsz, sz; + void *p; + + sz = 42; + nsz = 0; + assert_d_eq(nallocm(&nsz, sz, 0), ALLOCM_SUCCESS, + "Unexpected nallocm() error"); + rsz = 0; + assert_d_eq(allocm(&p, &rsz, sz, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); + + assert_d_eq(allocm(&p, NULL, sz, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); + + nsz = 0; + assert_d_eq(nallocm(&nsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, + "Unexpected nallocm() error"); + rsz = 0; + assert_d_eq(allocm(&p, &rsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + int r; + size_t nsz, rsz, sz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment) | + ALLOCM_ZERO); + assert_d_eq(r, ALLOCM_SUCCESS, + "nallocm() error for alignment=%zu, " + "size=%zu (%#zx): %d", + alignment, sz, sz, r); + rsz = 0; + r = allocm(&ps[i], &rsz, sz, + ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); + assert_d_eq(r, ALLOCM_SUCCESS, + "allocm() error for alignment=%zu, " + "size=%zu (%#zx): %d", + alignment, sz, sz, r); + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocm()/allocm() rsize mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + sallocm(ps[i], &rsz, 0); + total += rsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocm(ps[i], 0); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_basic, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/mallocx.c b/deps/jemalloc/test/integration/mallocx.c new file mode 100644 index 00000000000..123e041fa33 --- /dev/null +++ b/deps/jemalloc/test/integration/mallocx.c @@ -0,0 +1,97 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +#define MAXALIGN (((size_t)1) << 25) +#define NITER 4 + +TEST_BEGIN(test_basic) +{ + size_t nsz, rsz, sz; + void *p; + + sz = 42; + nsz = nallocx(sz, 0); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); + dallocx(p, 0); + + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + dallocx(p, 0); + + nsz = nallocx(sz, MALLOCX_ZERO); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t nsz, rsz, sz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_zu_ne(nsz, 0, + "nallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_ptr_not_null(ps[i], + "mallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + rsz = sallocx(ps[i], 0); + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocx()/sallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + total += rsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocx(ps[i], 0); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_basic, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/mremap.c b/deps/jemalloc/test/integration/mremap.c new file mode 100644 index 00000000000..a7fb7ef0abf --- /dev/null +++ b/deps/jemalloc/test/integration/mremap.c @@ -0,0 +1,45 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_mremap) +{ + int err; + size_t sz, lg_chunk, chunksize, i; + char *p, *q; + + sz = sizeof(lg_chunk); + err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0); + assert_d_eq(err, 0, "Error in mallctl(): %s", strerror(err)); + chunksize = ((size_t)1U) << lg_chunk; + + p = (char *)malloc(chunksize); + assert_ptr_not_null(p, "malloc(%zu) --> %p", chunksize, p); + memset(p, 'a', chunksize); + + q = (char *)realloc(p, chunksize * 2); + assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize * 2, + q); + for (i = 0; i < chunksize; i++) { + assert_c_eq(q[i], 'a', + "realloc() should preserve existing bytes across copies"); + } + + p = q; + + q = (char *)realloc(p, chunksize); + assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize, q); + for (i = 0; i < chunksize; i++) { + assert_c_eq(q[i], 'a', + "realloc() should preserve existing bytes across copies"); + } + + free(q); +} +TEST_END + +int +main(void) +{ + + return (test( + test_mremap)); +} diff --git a/deps/jemalloc/test/integration/posix_memalign.c b/deps/jemalloc/test/integration/posix_memalign.c new file mode 100644 index 00000000000..19741c6cb50 --- /dev/null +++ b/deps/jemalloc/test/integration/posix_memalign.c @@ -0,0 +1,119 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) +#define NITER 4 + +TEST_BEGIN(test_alignment_errors) +{ + size_t alignment; + void *p; + + for (alignment = 0; alignment < sizeof(void *); alignment++) { + assert_d_eq(posix_memalign(&p, alignment, 1), EINVAL, + "Expected error for invalid alignment %zu", + alignment); + } + + for (alignment = sizeof(size_t); alignment < MAXALIGN; + alignment <<= 1) { + assert_d_ne(posix_memalign(&p, alignment + 1, 1), 0, + "Expected error for invalid alignment %zu", + alignment + 1); + } +} +TEST_END + +TEST_BEGIN(test_oom_errors) +{ + size_t alignment, size; + void *p; + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x8000000000000000); + size = UINT64_C(0x8000000000000000); +#else + alignment = 0x80000000LU; + size = 0x80000000LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x4000000000000000); + size = UINT64_C(0xc000000000000001); +#else + alignment = 0x40000000LU; + size = 0xc0000001LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); + + alignment = 0x10LU; +#if LG_SIZEOF_PTR == 3 + size = UINT64_C(0xfffffffffffffff0); +#else + size = 0xfffffff0LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t alignment, size, total; + unsigned i; + int err; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (size = 1; + size < 3 * alignment && size < (1U << 31); + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + err = posix_memalign(&ps[i], + alignment, size); + if (err) { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + test_fail( + "Error for alignment=%zu, " + "size=%zu (%#zx): %s", + alignment, size, size, buf); + } + total += malloc_usable_size(ps[i]); + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + free(ps[i]); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_alignment_errors, + test_oom_errors, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/rallocm.c b/deps/jemalloc/test/integration/rallocm.c new file mode 100644 index 00000000000..33c11bb7cf6 --- /dev/null +++ b/deps/jemalloc/test/integration/rallocm.c @@ -0,0 +1,111 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_same_size) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE), ALLOCM_SUCCESS, + "Unexpected rallocm() error"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_extra_no_move) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz, sz-42, ALLOCM_NO_MOVE), + ALLOCM_SUCCESS, "Unexpected rallocm() error"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_no_move_fail) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE), + ALLOCM_ERR_NOT_MOVED, "Unexpected rallocm() result"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_grow_and_shrink) +{ + void *p, *q; + size_t tsz; +#define NCYCLES 3 + unsigned i, j; +#define NSZS 2500 + size_t szs[NSZS]; +#define MAXSZ ZU(12 * 1024 * 1024) + + assert_d_eq(allocm(&p, &szs[0], 1, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + for (i = 0; i < NCYCLES; i++) { + for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) { + q = p; + assert_d_eq(rallocm(&q, &szs[j], szs[j-1]+1, 0, 0), + ALLOCM_SUCCESS, + "Unexpected rallocm() error for size=%zu-->%zu", + szs[j-1], szs[j-1]+1); + assert_zu_ne(szs[j], szs[j-1]+1, + "Expected size to at least: %zu", szs[j-1]+1); + p = q; + } + + for (j--; j > 0; j--) { + q = p; + assert_d_eq(rallocm(&q, &tsz, szs[j-1], 0, 0), + ALLOCM_SUCCESS, + "Unexpected rallocm() error for size=%zu-->%zu", + szs[j], szs[j-1]); + assert_zu_eq(tsz, szs[j-1], + "Expected size=%zu, got size=%zu", szs[j-1], tsz); + p = q; + } + } + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_same_size, + test_extra_no_move, + test_no_move_fail, + test_grow_and_shrink)); +} diff --git a/deps/jemalloc/test/integration/rallocx.c b/deps/jemalloc/test/integration/rallocx.c new file mode 100644 index 00000000000..ee21aedff70 --- /dev/null +++ b/deps/jemalloc/test/integration/rallocx.c @@ -0,0 +1,182 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_grow_and_shrink) +{ + void *p, *q; + size_t tsz; +#define NCYCLES 3 + unsigned i, j; +#define NSZS 2500 + size_t szs[NSZS]; +#define MAXSZ ZU(12 * 1024 * 1024) + + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + szs[0] = sallocx(p, 0); + + for (i = 0; i < NCYCLES; i++) { + for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) { + q = rallocx(p, szs[j-1]+1, 0); + assert_ptr_not_null(q, + "Unexpected rallocx() error for size=%zu-->%zu", + szs[j-1], szs[j-1]+1); + szs[j] = sallocx(q, 0); + assert_zu_ne(szs[j], szs[j-1]+1, + "Expected size to at least: %zu", szs[j-1]+1); + p = q; + } + + for (j--; j > 0; j--) { + q = rallocx(p, szs[j-1], 0); + assert_ptr_not_null(q, + "Unexpected rallocx() error for size=%zu-->%zu", + szs[j], szs[j-1]); + tsz = sallocx(q, 0); + assert_zu_eq(tsz, szs[j-1], + "Expected size=%zu, got size=%zu", szs[j-1], tsz); + p = q; + } + } + + dallocx(p, 0); +#undef MAXSZ +#undef NSZS +#undef NCYCLES +} +TEST_END + +static bool +validate_fill(const void *p, uint8_t c, size_t offset, size_t len) +{ + bool ret = false; + const uint8_t *buf = (const uint8_t *)p; + size_t i; + + for (i = 0; i < len; i++) { + uint8_t b = buf[offset+i]; + if (b != c) { + test_fail("Allocation at %p contains %#x rather than " + "%#x at offset %zu", p, b, c, offset+i); + ret = true; + } + } + + return (ret); +} + +TEST_BEGIN(test_zero) +{ + void *p, *q; + size_t psz, qsz, i, j; + size_t start_sizes[] = {1, 3*1024, 63*1024, 4095*1024}; +#define FILL_BYTE 0xaaU +#define RANGE 2048 + + for (i = 0; i < sizeof(start_sizes)/sizeof(size_t); i++) { + size_t start_size = start_sizes[i]; + p = mallocx(start_size, MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + psz = sallocx(p, 0); + + assert_false(validate_fill(p, 0, 0, psz), + "Expected zeroed memory"); + memset(p, FILL_BYTE, psz); + assert_false(validate_fill(p, FILL_BYTE, 0, psz), + "Expected filled memory"); + + for (j = 1; j < RANGE; j++) { + q = rallocx(p, start_size+j, MALLOCX_ZERO); + assert_ptr_not_null(q, "Unexpected rallocx() error"); + qsz = sallocx(q, 0); + if (q != p || qsz != psz) { + assert_false(validate_fill(q, FILL_BYTE, 0, + psz), "Expected filled memory"); + assert_false(validate_fill(q, 0, psz, qsz-psz), + "Expected zeroed memory"); + } + if (psz != qsz) { + memset(q+psz, FILL_BYTE, qsz-psz); + psz = qsz; + } + p = q; + } + assert_false(validate_fill(p, FILL_BYTE, 0, psz), + "Expected filled memory"); + dallocx(p, 0); + } +#undef FILL_BYTE +} +TEST_END + +TEST_BEGIN(test_align) +{ + void *p, *q; + size_t align; +#define MAX_ALIGN (ZU(1) << 25) + + align = ZU(1); + p = mallocx(1, MALLOCX_ALIGN(align)); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + for (align <<= 1; align <= MAX_ALIGN; align <<= 1) { + q = rallocx(p, 1, MALLOCX_ALIGN(align)); + assert_ptr_not_null(q, + "Unexpected rallocx() error for align=%zu", align); + assert_ptr_null( + (void *)((uintptr_t)q & (align-1)), + "%p inadequately aligned for align=%zu", + q, align); + p = q; + } + dallocx(p, 0); +#undef MAX_ALIGN +} +TEST_END + +TEST_BEGIN(test_lg_align_and_zero) +{ + void *p, *q; + size_t lg_align, sz; +#define MAX_LG_ALIGN 25 +#define MAX_VALIDATE (ZU(1) << 22) + + lg_align = ZU(0); + p = mallocx(1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + for (lg_align++; lg_align <= MAX_LG_ALIGN; lg_align++) { + q = rallocx(p, 1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO); + assert_ptr_not_null(q, + "Unexpected rallocx() error for lg_align=%zu", lg_align); + assert_ptr_null( + (void *)((uintptr_t)q & ((ZU(1) << lg_align)-1)), + "%p inadequately aligned for lg_align=%zu", + q, lg_align); + sz = sallocx(q, 0); + if ((sz << 1) <= MAX_VALIDATE) { + assert_false(validate_fill(q, 0, 0, sz), + "Expected zeroed memory"); + } else { + assert_false(validate_fill(q, 0, 0, MAX_VALIDATE), + "Expected zeroed memory"); + assert_false(validate_fill(q+sz-MAX_VALIDATE, 0, 0, + MAX_VALIDATE), "Expected zeroed memory"); + } + p = q; + } + dallocx(p, 0); +#undef MAX_VALIDATE +#undef MAX_LG_ALIGN +} +TEST_END + +int +main(void) +{ + + return (test( + test_grow_and_shrink, + test_zero, + test_align, + test_lg_align_and_zero)); +} diff --git a/deps/jemalloc/test/integration/thread_arena.c b/deps/jemalloc/test/integration/thread_arena.c new file mode 100644 index 00000000000..67be5351335 --- /dev/null +++ b/deps/jemalloc/test/integration/thread_arena.c @@ -0,0 +1,79 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 10 + +void * +thd_start(void *arg) +{ + unsigned main_arena_ind = *(unsigned *)arg; + void *p; + unsigned arena_ind; + size_t size; + int err; + + p = malloc(1); + assert_ptr_not_null(p, "Error in malloc()"); + free(p); + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind, + sizeof(main_arena_ind)))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + assert_u_eq(arena_ind, main_arena_ind, + "Arena index should be same as for main thread"); + + return (NULL); +} + +TEST_BEGIN(test_thread_arena) +{ + void *p; + unsigned arena_ind; + size_t size; + int err; + thd_t thds[NTHREADS]; + unsigned i; + + p = malloc(1); + assert_ptr_not_null(p, "Error in malloc()"); + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, + (void *)&arena_ind); + } + + for (i = 0; i < NTHREADS; i++) { + intptr_t join_ret; + thd_join(thds[i], (void *)&join_ret); + assert_zd_eq(join_ret, 0, "Unexpected thread join error"); + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_thread_arena)); +} diff --git a/deps/jemalloc/test/integration/thread_tcache_enabled.c b/deps/jemalloc/test/integration/thread_tcache_enabled.c new file mode 100644 index 00000000000..f4e89c682a7 --- /dev/null +++ b/deps/jemalloc/test/integration/thread_tcache_enabled.c @@ -0,0 +1,113 @@ +#include "test/jemalloc_test.h" + +static const bool config_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; + +void * +thd_start(void *arg) +{ + int err; + size_t sz; + bool e0, e1; + + sz = sizeof(bool); + if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) { + if (err == ENOENT) { + assert_false(config_tcache, + "ENOENT should only be returned if tcache is " + "disabled"); + } + goto label_ENOENT; + } + + if (e0) { + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), + 0, "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + } + + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + free(malloc(1)); + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + free(malloc(1)); + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + return (NULL); +label_ENOENT: + test_skip("\"thread.tcache.enabled\" mallctl not available"); + return (NULL); +} + +TEST_BEGIN(test_main_thread) +{ + + thd_start(NULL); +} +TEST_END + +TEST_BEGIN(test_subthread) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} +TEST_END + +int +main(void) +{ + + /* Run tests multiple times to check for bad interactions. */ + return (test( + test_main_thread, + test_subthread, + test_main_thread, + test_subthread, + test_main_thread)); +} diff --git a/deps/jemalloc/test/integration/xallocx.c b/deps/jemalloc/test/integration/xallocx.c new file mode 100644 index 00000000000..ab4cf945e54 --- /dev/null +++ b/deps/jemalloc/test/integration/xallocx.c @@ -0,0 +1,59 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_same_size) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz, 0, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_extra_no_move) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz, sz-42, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_no_move_fail) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz + 5, 0, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_same_size, + test_extra_no_move, + test_no_move_fail)); +} diff --git a/deps/jemalloc/test/jemalloc_test.h.in b/deps/jemalloc/test/jemalloc_test.h.in deleted file mode 100644 index 0c48895e724..00000000000 --- a/deps/jemalloc/test/jemalloc_test.h.in +++ /dev/null @@ -1,6 +0,0 @@ -/* - * This header should be included by tests, rather than directly including - * jemalloc/jemalloc.h, because --with-install-suffix may cause the header to - * have a different name. - */ -#include "jemalloc/jemalloc@install_suffix@.h" diff --git a/deps/jemalloc/test/mremap.c b/deps/jemalloc/test/mremap.c deleted file mode 100644 index 146c66f4212..00000000000 --- a/deps/jemalloc/test/mremap.c +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - int ret, err; - size_t sz, lg_chunk, chunksize, i; - char *p, *q; - - fprintf(stderr, "Test begin\n"); - - sz = sizeof(lg_chunk); - if ((err = JEMALLOC_P(mallctl)("opt.lg_chunk", &lg_chunk, &sz, NULL, - 0))) { - assert(err != ENOENT); - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto RETURN; - } - chunksize = ((size_t)1U) << lg_chunk; - - p = (char *)malloc(chunksize); - if (p == NULL) { - fprintf(stderr, "malloc(%zu) --> %p\n", chunksize, p); - ret = 1; - goto RETURN; - } - memset(p, 'a', chunksize); - - q = (char *)realloc(p, chunksize * 2); - if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize * 2, - q); - ret = 1; - goto RETURN; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - p = q; - - q = (char *)realloc(p, chunksize); - if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize, q); - ret = 1; - goto RETURN; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - free(q); - - ret = 0; -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/mremap.exp b/deps/jemalloc/test/mremap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/mremap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/posix_memalign.c b/deps/jemalloc/test/posix_memalign.c deleted file mode 100644 index 3e306c01dd2..00000000000 --- a/deps/jemalloc/test/posix_memalign.c +++ /dev/null @@ -1,121 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) -#define NITER 4 - -int -main(void) -{ - size_t alignment, size, total; - unsigned i; - int err; - void *p, *ps[NITER]; - - fprintf(stderr, "Test begin\n"); - - /* Test error conditions. */ - for (alignment = 0; alignment < sizeof(void *); alignment++) { - err = JEMALLOC_P(posix_memalign)(&p, alignment, 1); - if (err != EINVAL) { - fprintf(stderr, - "Expected error for invalid alignment %zu\n", - alignment); - } - } - - for (alignment = sizeof(size_t); alignment < MAXALIGN; - alignment <<= 1) { - err = JEMALLOC_P(posix_memalign)(&p, alignment + 1, 1); - if (err == 0) { - fprintf(stderr, - "Expected error for invalid alignment %zu\n", - alignment + 1); - } - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - size = 0x8000000000000000LLU; -#else - alignment = 0x80000000LU; - size = 0x80000000LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - size = 0x8400000000000001LLU; -#else - alignment = 0x40000000LU; - size = 0x84000001LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - alignment = 0x10LLU; -#if LG_SIZEOF_PTR == 3 - size = 0xfffffffffffffff0LLU; -#else - size = 0xfffffff0LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); - for (size = 1; - size < 3 * alignment && size < (1U << 31); - size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - err = JEMALLOC_P(posix_memalign)(&ps[i], - alignment, size); - if (err) { - fprintf(stderr, - "Error for size %zu (0x%zx): %s\n", - size, size, strerror(err)); - exit(1); - } - total += JEMALLOC_P(malloc_usable_size)(ps[i]); - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - JEMALLOC_P(free)(ps[i]); - ps[i] = NULL; - } - } - } - } - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/posix_memalign.exp b/deps/jemalloc/test/posix_memalign.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc/test/posix_memalign.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc/test/rallocm.c b/deps/jemalloc/test/rallocm.c deleted file mode 100644 index ccf326bb523..00000000000 --- a/deps/jemalloc/test/rallocm.c +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - size_t pagesize; - void *p, *q; - size_t sz, tsz; - int r; - - fprintf(stderr, "Test begin\n"); - - /* Get page size. */ - { - long result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; - } - - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_ERR_NOT_MOVED) - fprintf(stderr, "Unexpected rallocm() result\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q == p) - fprintf(stderr, "Expected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q == p) - fprintf(stderr, "Expected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - JEMALLOC_P(dallocm)(p, 0); - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/rallocm.exp b/deps/jemalloc/test/rallocm.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/rallocm.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/src/SFMT.c b/deps/jemalloc/test/src/SFMT.c new file mode 100644 index 00000000000..e6f8deecb37 --- /dev/null +++ b/deps/jemalloc/test/src/SFMT.c @@ -0,0 +1,719 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT.c + * @brief SIMD oriented Fast Mersenne Twister(SFMT) + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software, see LICENSE.txt + */ +#define SFMT_C_ +#include "test/jemalloc_test.h" +#include "test/SFMT-params.h" + +#if defined(JEMALLOC_BIG_ENDIAN) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(__BIG_ENDIAN__) && !defined(__amd64) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(HAVE_ALTIVEC) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(ONLY64) && !defined(BIG_ENDIAN64) + #if defined(__GNUC__) + #error "-DONLY64 must be specified with -DBIG_ENDIAN64" + #endif +#undef ONLY64 +#endif +/*------------------------------------------------------ + 128-bit SIMD data type for Altivec, SSE2 or standard C + ------------------------------------------------------*/ +#if defined(HAVE_ALTIVEC) +/** 128-bit data structure */ +union W128_T { + vector unsigned int s; + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef union W128_T w128_t; + +#elif defined(HAVE_SSE2) +/** 128-bit data structure */ +union W128_T { + __m128i si; + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef union W128_T w128_t; + +#else + +/** 128-bit data structure */ +struct W128_T { + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef struct W128_T w128_t; + +#endif + +struct sfmt_s { + /** the 128-bit internal state array */ + w128_t sfmt[N]; + /** index counter to the 32-bit internal state array */ + int idx; + /** a flag: it is 0 if and only if the internal state is not yet + * initialized. */ + int initialized; +}; + +/*-------------------------------------- + FILE GLOBAL VARIABLES + internal state, index counter and flag + --------------------------------------*/ + +/** a parity check vector which certificate the period of 2^{MEXP} */ +static uint32_t parity[4] = {PARITY1, PARITY2, PARITY3, PARITY4}; + +/*---------------- + STATIC FUNCTIONS + ----------------*/ +JEMALLOC_INLINE_C int idxof(int i); +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift); +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift); +#endif +JEMALLOC_INLINE_C void gen_rand_all(sfmt_t *ctx); +JEMALLOC_INLINE_C void gen_rand_array(sfmt_t *ctx, w128_t *array, int size); +JEMALLOC_INLINE_C uint32_t func1(uint32_t x); +JEMALLOC_INLINE_C uint32_t func2(uint32_t x); +static void period_certification(sfmt_t *ctx); +#if defined(BIG_ENDIAN64) && !defined(ONLY64) +JEMALLOC_INLINE_C void swap(w128_t *array, int size); +#endif + +#if defined(HAVE_ALTIVEC) + #include "test/SFMT-alti.h" +#elif defined(HAVE_SSE2) + #include "test/SFMT-sse2.h" +#endif + +/** + * This function simulate a 64-bit index of LITTLE ENDIAN + * in BIG ENDIAN machine. + */ +#ifdef ONLY64 +JEMALLOC_INLINE_C int idxof(int i) { + return i ^ 1; +} +#else +JEMALLOC_INLINE_C int idxof(int i) { + return i; +} +#endif +/** + * This function simulates SIMD 128-bit right shift by the standard C. + * The 128-bit integer given in in is shifted by (shift * 8) bits. + * This function simulates the LITTLE ENDIAN SIMD. + * @param out the output of this function + * @param in the 128-bit data to be shifted + * @param shift the shift value + */ +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +#ifdef ONLY64 +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[2] << 32) | ((uint64_t)in->u[3]); + tl = ((uint64_t)in->u[0] << 32) | ((uint64_t)in->u[1]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[0] = (uint32_t)(ol >> 32); + out->u[1] = (uint32_t)ol; + out->u[2] = (uint32_t)(oh >> 32); + out->u[3] = (uint32_t)oh; +} +#else +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[3] << 32) | ((uint64_t)in->u[2]); + tl = ((uint64_t)in->u[1] << 32) | ((uint64_t)in->u[0]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[1] = (uint32_t)(ol >> 32); + out->u[0] = (uint32_t)ol; + out->u[3] = (uint32_t)(oh >> 32); + out->u[2] = (uint32_t)oh; +} +#endif +/** + * This function simulates SIMD 128-bit left shift by the standard C. + * The 128-bit integer given in in is shifted by (shift * 8) bits. + * This function simulates the LITTLE ENDIAN SIMD. + * @param out the output of this function + * @param in the 128-bit data to be shifted + * @param shift the shift value + */ +#ifdef ONLY64 +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[2] << 32) | ((uint64_t)in->u[3]); + tl = ((uint64_t)in->u[0] << 32) | ((uint64_t)in->u[1]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[0] = (uint32_t)(ol >> 32); + out->u[1] = (uint32_t)ol; + out->u[2] = (uint32_t)(oh >> 32); + out->u[3] = (uint32_t)oh; +} +#else +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[3] << 32) | ((uint64_t)in->u[2]); + tl = ((uint64_t)in->u[1] << 32) | ((uint64_t)in->u[0]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[1] = (uint32_t)(ol >> 32); + out->u[0] = (uint32_t)ol; + out->u[3] = (uint32_t)(oh >> 32); + out->u[2] = (uint32_t)oh; +} +#endif +#endif + +/** + * This function represents the recursion formula. + * @param r output + * @param a a 128-bit part of the internal state array + * @param b a 128-bit part of the internal state array + * @param c a 128-bit part of the internal state array + * @param d a 128-bit part of the internal state array + */ +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +#ifdef ONLY64 +JEMALLOC_INLINE_C void do_recursion(w128_t *r, w128_t *a, w128_t *b, w128_t *c, + w128_t *d) { + w128_t x; + w128_t y; + + lshift128(&x, a, SL2); + rshift128(&y, c, SR2); + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK2) ^ y.u[0] + ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK1) ^ y.u[1] + ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK4) ^ y.u[2] + ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK3) ^ y.u[3] + ^ (d->u[3] << SL1); +} +#else +JEMALLOC_INLINE_C void do_recursion(w128_t *r, w128_t *a, w128_t *b, w128_t *c, + w128_t *d) { + w128_t x; + w128_t y; + + lshift128(&x, a, SL2); + rshift128(&y, c, SR2); + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK1) ^ y.u[0] + ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK2) ^ y.u[1] + ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK3) ^ y.u[2] + ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK4) ^ y.u[3] + ^ (d->u[3] << SL1); +} +#endif +#endif + +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE_C void gen_rand_all(sfmt_t *ctx) { + int i; + w128_t *r1, *r2; + + r1 = &ctx->sfmt[N - 2]; + r2 = &ctx->sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { + do_recursion(&ctx->sfmt[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1], r1, + r2); + r1 = r2; + r2 = &ctx->sfmt[i]; + } + for (; i < N; i++) { + do_recursion(&ctx->sfmt[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1 - N], r1, + r2); + r1 = r2; + r2 = &ctx->sfmt[i]; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pseudorandom numbers to be generated. + */ +JEMALLOC_INLINE_C void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + w128_t *r1, *r2; + + r1 = &ctx->sfmt[N - 2]; + r2 = &ctx->sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { + do_recursion(&array[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (; i < N; i++) { + do_recursion(&array[i], &ctx->sfmt[i], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (; i < size - N; i++) { + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (j = 0; j < 2 * N - size; j++) { + ctx->sfmt[j] = array[j + size - N]; + } + for (; i < size; i++, j++) { + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + ctx->sfmt[j] = array[i]; + } +} +#endif + +#if defined(BIG_ENDIAN64) && !defined(ONLY64) && !defined(HAVE_ALTIVEC) +JEMALLOC_INLINE_C void swap(w128_t *array, int size) { + int i; + uint32_t x, y; + + for (i = 0; i < size; i++) { + x = array[i].u[0]; + y = array[i].u[2]; + array[i].u[0] = array[i].u[1]; + array[i].u[2] = array[i].u[3]; + array[i].u[1] = x; + array[i].u[3] = y; + } +} +#endif +/** + * This function represents a function used in the initialization + * by init_by_array + * @param x 32-bit integer + * @return 32-bit integer + */ +static uint32_t func1(uint32_t x) { + return (x ^ (x >> 27)) * (uint32_t)1664525UL; +} + +/** + * This function represents a function used in the initialization + * by init_by_array + * @param x 32-bit integer + * @return 32-bit integer + */ +static uint32_t func2(uint32_t x) { + return (x ^ (x >> 27)) * (uint32_t)1566083941UL; +} + +/** + * This function certificate the period of 2^{MEXP} + */ +static void period_certification(sfmt_t *ctx) { + int inner = 0; + int i, j; + uint32_t work; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; + + for (i = 0; i < 4; i++) + inner ^= psfmt32[idxof(i)] & parity[i]; + for (i = 16; i > 0; i >>= 1) + inner ^= inner >> i; + inner &= 1; + /* check OK */ + if (inner == 1) { + return; + } + /* check NG, and modification */ + for (i = 0; i < 4; i++) { + work = 1; + for (j = 0; j < 32; j++) { + if ((work & parity[i]) != 0) { + psfmt32[idxof(i)] ^= work; + return; + } + work = work << 1; + } + } +} + +/*---------------- + PUBLIC FUNCTIONS + ----------------*/ +/** + * This function returns the identification string. + * The string shows the word size, the Mersenne exponent, + * and all parameters of this generator. + */ +const char *get_idstring(void) { + return IDSTR; +} + +/** + * This function returns the minimum size of array used for \b + * fill_array32() function. + * @return minimum size of array used for fill_array32() function. + */ +int get_min_array_size32(void) { + return N32; +} + +/** + * This function returns the minimum size of array used for \b + * fill_array64() function. + * @return minimum size of array used for fill_array64() function. + */ +int get_min_array_size64(void) { + return N64; +} + +#ifndef ONLY64 +/** + * This function generates and returns 32-bit pseudorandom number. + * init_gen_rand or init_by_array must be called before this function. + * @return 32-bit pseudorandom number + */ +uint32_t gen_rand32(sfmt_t *ctx) { + uint32_t r; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; + + assert(ctx->initialized); + if (ctx->idx >= N32) { + gen_rand_all(ctx); + ctx->idx = 0; + } + r = psfmt32[ctx->idx++]; + return r; +} + +/* Generate a random integer in [0..limit). */ +uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit) { + uint32_t ret, above; + + above = 0xffffffffU - (0xffffffffU % limit); + while (1) { + ret = gen_rand32(ctx); + if (ret < above) { + ret %= limit; + break; + } + } + return ret; +} +#endif +/** + * This function generates and returns 64-bit pseudorandom number. + * init_gen_rand or init_by_array must be called before this function. + * The function gen_rand64 should not be called after gen_rand32, + * unless an initialization is again executed. + * @return 64-bit pseudorandom number + */ +uint64_t gen_rand64(sfmt_t *ctx) { +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + uint32_t r1, r2; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; +#else + uint64_t r; + uint64_t *psfmt64 = (uint64_t *)&ctx->sfmt[0].u[0]; +#endif + + assert(ctx->initialized); + assert(ctx->idx % 2 == 0); + + if (ctx->idx >= N32) { + gen_rand_all(ctx); + ctx->idx = 0; + } +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + r1 = psfmt32[ctx->idx]; + r2 = psfmt32[ctx->idx + 1]; + ctx->idx += 2; + return ((uint64_t)r2 << 32) | r1; +#else + r = psfmt64[ctx->idx / 2]; + ctx->idx += 2; + return r; +#endif +} + +/* Generate a random integer in [0..limit). */ +uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit) { + uint64_t ret, above; + + above = 0xffffffffffffffffLLU - (0xffffffffffffffffLLU % limit); + while (1) { + ret = gen_rand64(ctx); + if (ret < above) { + ret %= limit; + break; + } + } + return ret; +} + +#ifndef ONLY64 +/** + * This function generates pseudorandom 32-bit integers in the + * specified array[] by one call. The number of pseudorandom integers + * is specified by the argument size, which must be at least 624 and a + * multiple of four. The generation by this function is much faster + * than the following gen_rand function. + * + * For initialization, init_gen_rand or init_by_array must be called + * before the first call of this function. This function can not be + * used after calling gen_rand function, without initialization. + * + * @param array an array where pseudorandom 32-bit integers are filled + * by this function. The pointer to the array must be \b "aligned" + * (namely, must be a multiple of 16) in the SIMD version, since it + * refers to the address of a 128-bit integer. In the standard C + * version, the pointer is arbitrary. + * + * @param size the number of 32-bit pseudorandom integers to be + * generated. size must be a multiple of 4, and greater than or equal + * to (MEXP / 128 + 1) * 4. + * + * @note \b memalign or \b posix_memalign is available to get aligned + * memory. Mac OSX doesn't have these functions, but \b malloc of OSX + * returns the pointer to the aligned memory block. + */ +void fill_array32(sfmt_t *ctx, uint32_t *array, int size) { + assert(ctx->initialized); + assert(ctx->idx == N32); + assert(size % 4 == 0); + assert(size >= N32); + + gen_rand_array(ctx, (w128_t *)array, size / 4); + ctx->idx = N32; +} +#endif + +/** + * This function generates pseudorandom 64-bit integers in the + * specified array[] by one call. The number of pseudorandom integers + * is specified by the argument size, which must be at least 312 and a + * multiple of two. The generation by this function is much faster + * than the following gen_rand function. + * + * For initialization, init_gen_rand or init_by_array must be called + * before the first call of this function. This function can not be + * used after calling gen_rand function, without initialization. + * + * @param array an array where pseudorandom 64-bit integers are filled + * by this function. The pointer to the array must be "aligned" + * (namely, must be a multiple of 16) in the SIMD version, since it + * refers to the address of a 128-bit integer. In the standard C + * version, the pointer is arbitrary. + * + * @param size the number of 64-bit pseudorandom integers to be + * generated. size must be a multiple of 2, and greater than or equal + * to (MEXP / 128 + 1) * 2 + * + * @note \b memalign or \b posix_memalign is available to get aligned + * memory. Mac OSX doesn't have these functions, but \b malloc of OSX + * returns the pointer to the aligned memory block. + */ +void fill_array64(sfmt_t *ctx, uint64_t *array, int size) { + assert(ctx->initialized); + assert(ctx->idx == N32); + assert(size % 2 == 0); + assert(size >= N64); + + gen_rand_array(ctx, (w128_t *)array, size / 2); + ctx->idx = N32; + +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + swap((w128_t *)array, size /2); +#endif +} + +/** + * This function initializes the internal state array with a 32-bit + * integer seed. + * + * @param seed a 32-bit integer used as the seed. + */ +sfmt_t *init_gen_rand(uint32_t seed) { + void *p; + sfmt_t *ctx; + int i; + uint32_t *psfmt32; + + if (posix_memalign(&p, sizeof(w128_t), sizeof(sfmt_t)) != 0) { + return NULL; + } + ctx = (sfmt_t *)p; + psfmt32 = &ctx->sfmt[0].u[0]; + + psfmt32[idxof(0)] = seed; + for (i = 1; i < N32; i++) { + psfmt32[idxof(i)] = 1812433253UL * (psfmt32[idxof(i - 1)] + ^ (psfmt32[idxof(i - 1)] >> 30)) + + i; + } + ctx->idx = N32; + period_certification(ctx); + ctx->initialized = 1; + + return ctx; +} + +/** + * This function initializes the internal state array, + * with an array of 32-bit integers used as the seeds + * @param init_key the array of 32-bit integers, used as a seed. + * @param key_length the length of init_key. + */ +sfmt_t *init_by_array(uint32_t *init_key, int key_length) { + void *p; + sfmt_t *ctx; + int i, j, count; + uint32_t r; + int lag; + int mid; + int size = N * 4; + uint32_t *psfmt32; + + if (posix_memalign(&p, sizeof(w128_t), sizeof(sfmt_t)) != 0) { + return NULL; + } + ctx = (sfmt_t *)p; + psfmt32 = &ctx->sfmt[0].u[0]; + + if (size >= 623) { + lag = 11; + } else if (size >= 68) { + lag = 7; + } else if (size >= 39) { + lag = 5; + } else { + lag = 3; + } + mid = (size - lag) / 2; + + memset(ctx->sfmt, 0x8b, sizeof(ctx->sfmt)); + if (key_length + 1 > N32) { + count = key_length + 1; + } else { + count = N32; + } + r = func1(psfmt32[idxof(0)] ^ psfmt32[idxof(mid)] + ^ psfmt32[idxof(N32 - 1)]); + psfmt32[idxof(mid)] += r; + r += key_length; + psfmt32[idxof(mid + lag)] += r; + psfmt32[idxof(0)] = r; + + count--; + for (i = 1, j = 0; (j < count) && (j < key_length); j++) { + r = func1(psfmt32[idxof(i)] ^ psfmt32[idxof((i + mid) % N32)] + ^ psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] += r; + r += init_key[j] + i; + psfmt32[idxof((i + mid + lag) % N32)] += r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + for (; j < count; j++) { + r = func1(psfmt32[idxof(i)] ^ psfmt32[idxof((i + mid) % N32)] + ^ psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] += r; + r += i; + psfmt32[idxof((i + mid + lag) % N32)] += r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + for (j = 0; j < N32; j++) { + r = func2(psfmt32[idxof(i)] + psfmt32[idxof((i + mid) % N32)] + + psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] ^= r; + r -= i; + psfmt32[idxof((i + mid + lag) % N32)] ^= r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + + ctx->idx = N32; + period_certification(ctx); + ctx->initialized = 1; + + return ctx; +} + +void fini_gen_rand(sfmt_t *ctx) { + assert(ctx != NULL); + + ctx->initialized = 0; + free(ctx); +} diff --git a/deps/jemalloc/test/src/math.c b/deps/jemalloc/test/src/math.c new file mode 100644 index 00000000000..887a36390e4 --- /dev/null +++ b/deps/jemalloc/test/src/math.c @@ -0,0 +1,2 @@ +#define MATH_C_ +#include "test/jemalloc_test.h" diff --git a/deps/jemalloc/test/src/mtx.c b/deps/jemalloc/test/src/mtx.c new file mode 100644 index 00000000000..41b95d59de8 --- /dev/null +++ b/deps/jemalloc/test/src/mtx.c @@ -0,0 +1,62 @@ +#include "test/jemalloc_test.h" + +bool +mtx_init(mtx_t *mtx) +{ + +#ifdef _WIN32 + if (!InitializeCriticalSectionAndSpinCount(&mtx->lock, _CRT_SPINCOUNT)) + return (true); +#elif (defined(JEMALLOC_OSSPIN)) + mtx->lock = 0; +#else + pthread_mutexattr_t attr; + + if (pthread_mutexattr_init(&attr) != 0) + return (true); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); + if (pthread_mutex_init(&mtx->lock, &attr) != 0) { + pthread_mutexattr_destroy(&attr); + return (true); + } + pthread_mutexattr_destroy(&attr); +#endif + return (false); +} + +void +mtx_fini(mtx_t *mtx) +{ + +#ifdef _WIN32 +#elif (defined(JEMALLOC_OSSPIN)) +#else + pthread_mutex_destroy(&mtx->lock); +#endif +} + +void +mtx_lock(mtx_t *mtx) +{ + +#ifdef _WIN32 + EnterCriticalSection(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockLock(&mtx->lock); +#else + pthread_mutex_lock(&mtx->lock); +#endif +} + +void +mtx_unlock(mtx_t *mtx) +{ + +#ifdef _WIN32 + LeaveCriticalSection(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockUnlock(&mtx->lock); +#else + pthread_mutex_unlock(&mtx->lock); +#endif +} diff --git a/deps/jemalloc/test/src/test.c b/deps/jemalloc/test/src/test.c new file mode 100644 index 00000000000..528d858317a --- /dev/null +++ b/deps/jemalloc/test/src/test.c @@ -0,0 +1,94 @@ +#include "test/jemalloc_test.h" + +static unsigned test_count = 0; +static test_status_t test_counts[test_status_count] = {0, 0, 0}; +static test_status_t test_status = test_status_pass; +static const char * test_name = ""; + +JEMALLOC_ATTR(format(printf, 1, 2)) +void +test_skip(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); + malloc_printf("\n"); + test_status = test_status_skip; +} + +JEMALLOC_ATTR(format(printf, 1, 2)) +void +test_fail(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); + malloc_printf("\n"); + test_status = test_status_fail; +} + +static const char * +test_status_string(test_status_t test_status) +{ + + switch (test_status) { + case test_status_pass: return "pass"; + case test_status_skip: return "skip"; + case test_status_fail: return "fail"; + default: not_reached(); + } +} + +void +p_test_init(const char *name) +{ + + test_count++; + test_status = test_status_pass; + test_name = name; +} + +void +p_test_fini(void) +{ + + test_counts[test_status]++; + malloc_printf("%s: %s\n", test_name, test_status_string(test_status)); +} + +test_status_t +p_test(test_t* t, ...) +{ + test_status_t ret = test_status_pass; + va_list ap; + + va_start(ap, t); + for (; t != NULL; t = va_arg(ap, test_t*)) { + t(); + if (test_status > ret) + ret = test_status; + } + va_end(ap); + + malloc_printf("--- %s: %u/%u, %s: %u/%u, %s: %u/%u ---\n", + test_status_string(test_status_pass), + test_counts[test_status_pass], test_count, + test_status_string(test_status_skip), + test_counts[test_status_skip], test_count, + test_status_string(test_status_fail), + test_counts[test_status_fail], test_count); + + return (ret); +} + +void +p_test_fail(const char *prefix, const char *message) +{ + + malloc_cprintf(NULL, NULL, "%s%s\n", prefix, message); + test_status = test_status_fail; +} diff --git a/deps/jemalloc/test/src/thd.c b/deps/jemalloc/test/src/thd.c new file mode 100644 index 00000000000..233242a161e --- /dev/null +++ b/deps/jemalloc/test/src/thd.c @@ -0,0 +1,35 @@ +#include "test/jemalloc_test.h" + +#ifdef _WIN32 +void +thd_create(thd_t *thd, void *(*proc)(void *), void *arg) +{ + LPTHREAD_START_ROUTINE routine = (LPTHREAD_START_ROUTINE)proc; + *thd = CreateThread(NULL, 0, routine, arg, 0, NULL); + if (*thd == NULL) + test_fail("Error in CreateThread()\n"); +} + +void +thd_join(thd_t thd, void **ret) +{ + + WaitForSingleObject(thd, INFINITE); +} + +#else +void +thd_create(thd_t *thd, void *(*proc)(void *), void *arg) +{ + + if (pthread_create(thd, NULL, proc, arg) != 0) + test_fail("Error in pthread_create()\n"); +} + +void +thd_join(thd_t thd, void **ret) +{ + + pthread_join(thd, ret); +} +#endif diff --git a/deps/jemalloc/test/test.sh.in b/deps/jemalloc/test/test.sh.in new file mode 100644 index 00000000000..a39f99f6b54 --- /dev/null +++ b/deps/jemalloc/test/test.sh.in @@ -0,0 +1,53 @@ +#!/bin/sh + +case @abi@ in + macho) + export DYLD_FALLBACK_LIBRARY_PATH="@objroot@lib" + ;; + pecoff) + export PATH="${PATH}:@objroot@lib" + ;; + *) + ;; +esac + +# Corresponds to test_status_t. +pass_code=0 +skip_code=1 +fail_code=2 + +pass_count=0 +skip_count=0 +fail_count=0 +for t in $@; do + if [ $pass_count -ne 0 -o $skip_count -ne 0 -o $fail_count != 0 ] ; then + echo + fi + echo "=== ${t} ===" + ${t}@exe@ @abs_srcroot@ @abs_objroot@ + result_code=$? + case ${result_code} in + ${pass_code}) + pass_count=$((pass_count+1)) + ;; + ${skip_code}) + skip_count=$((skip_count+1)) + ;; + ${fail_code}) + fail_count=$((fail_count+1)) + ;; + *) + echo "Test harness error" 1>&2 + exit 1 + esac +done + +total_count=`expr ${pass_count} + ${skip_count} + ${fail_count}` +echo +echo "Test suite summary: pass: ${pass_count}/${total_count}, skip: ${skip_count}/${total_count}, fail: ${fail_count}/${total_count}" + +if [ ${fail_count} -eq 0 ] ; then + exit 0 +else + exit 1 +fi diff --git a/deps/jemalloc/test/thread_arena.c b/deps/jemalloc/test/thread_arena.c deleted file mode 100644 index ef8d6817500..00000000000 --- a/deps/jemalloc/test/thread_arena.c +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define NTHREADS 10 - -void * -thread_start(void *arg) -{ - unsigned main_arena_ind = *(unsigned *)arg; - void *p; - unsigned arena_ind; - size_t size; - int err; - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, - &main_arena_ind, sizeof(main_arena_ind)))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, - 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - assert(arena_ind == main_arena_ind); - - return (NULL); -} - -int -main(void) -{ - int ret = 0; - void *p; - unsigned arena_ind; - size_t size; - int err; - pthread_t threads[NTHREADS]; - unsigned i; - - fprintf(stderr, "Test begin\n"); - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - ret = 1; - goto RETURN; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, - 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto RETURN; - } - - for (i = 0; i < NTHREADS; i++) { - if (pthread_create(&threads[i], NULL, thread_start, - (void *)&arena_ind) != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", - __func__); - ret = 1; - goto RETURN; - } - } - - for (i = 0; i < NTHREADS; i++) - pthread_join(threads[i], (void *)&ret); - -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/thread_arena.exp b/deps/jemalloc/test/thread_arena.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/thread_arena.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/unit/SFMT.c b/deps/jemalloc/test/unit/SFMT.c new file mode 100644 index 00000000000..c57bd68df3d --- /dev/null +++ b/deps/jemalloc/test/unit/SFMT.c @@ -0,0 +1,1605 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test/jemalloc_test.h" + +#define BLOCK_SIZE 10000 +#define BLOCK_SIZE64 (BLOCK_SIZE / 2) +#define COUNT_1 1000 +#define COUNT_2 700 + +static const uint32_t init_gen_rand_32_expected[] = { + 3440181298U, 1564997079U, 1510669302U, 2930277156U, 1452439940U, + 3796268453U, 423124208U, 2143818589U, 3827219408U, 2987036003U, + 2674978610U, 1536842514U, 2027035537U, 2534897563U, 1686527725U, + 545368292U, 1489013321U, 1370534252U, 4231012796U, 3994803019U, + 1764869045U, 824597505U, 862581900U, 2469764249U, 812862514U, + 359318673U, 116957936U, 3367389672U, 2327178354U, 1898245200U, + 3206507879U, 2378925033U, 1040214787U, 2524778605U, 3088428700U, + 1417665896U, 964324147U, 2282797708U, 2456269299U, 313400376U, + 2245093271U, 1015729427U, 2694465011U, 3246975184U, 1992793635U, + 463679346U, 3721104591U, 3475064196U, 856141236U, 1499559719U, + 3522818941U, 3721533109U, 1954826617U, 1282044024U, 1543279136U, + 1301863085U, 2669145051U, 4221477354U, 3896016841U, 3392740262U, + 462466863U, 1037679449U, 1228140306U, 922298197U, 1205109853U, + 1872938061U, 3102547608U, 2742766808U, 1888626088U, 4028039414U, + 157593879U, 1136901695U, 4038377686U, 3572517236U, 4231706728U, + 2997311961U, 1189931652U, 3981543765U, 2826166703U, 87159245U, + 1721379072U, 3897926942U, 1790395498U, 2569178939U, 1047368729U, + 2340259131U, 3144212906U, 2301169789U, 2442885464U, 3034046771U, + 3667880593U, 3935928400U, 2372805237U, 1666397115U, 2460584504U, + 513866770U, 3810869743U, 2147400037U, 2792078025U, 2941761810U, + 3212265810U, 984692259U, 346590253U, 1804179199U, 3298543443U, + 750108141U, 2880257022U, 243310542U, 1869036465U, 1588062513U, + 2983949551U, 1931450364U, 4034505847U, 2735030199U, 1628461061U, + 2539522841U, 127965585U, 3992448871U, 913388237U, 559130076U, + 1202933193U, 4087643167U, 2590021067U, 2256240196U, 1746697293U, + 1013913783U, 1155864921U, 2715773730U, 915061862U, 1948766573U, + 2322882854U, 3761119102U, 1343405684U, 3078711943U, 3067431651U, + 3245156316U, 3588354584U, 3484623306U, 3899621563U, 4156689741U, + 3237090058U, 3880063844U, 862416318U, 4039923869U, 2303788317U, + 3073590536U, 701653667U, 2131530884U, 3169309950U, 2028486980U, + 747196777U, 3620218225U, 432016035U, 1449580595U, 2772266392U, + 444224948U, 1662832057U, 3184055582U, 3028331792U, 1861686254U, + 1104864179U, 342430307U, 1350510923U, 3024656237U, 1028417492U, + 2870772950U, 290847558U, 3675663500U, 508431529U, 4264340390U, + 2263569913U, 1669302976U, 519511383U, 2706411211U, 3764615828U, + 3883162495U, 4051445305U, 2412729798U, 3299405164U, 3991911166U, + 2348767304U, 2664054906U, 3763609282U, 593943581U, 3757090046U, + 2075338894U, 2020550814U, 4287452920U, 4290140003U, 1422957317U, + 2512716667U, 2003485045U, 2307520103U, 2288472169U, 3940751663U, + 4204638664U, 2892583423U, 1710068300U, 3904755993U, 2363243951U, + 3038334120U, 547099465U, 771105860U, 3199983734U, 4282046461U, + 2298388363U, 934810218U, 2837827901U, 3952500708U, 2095130248U, + 3083335297U, 26885281U, 3932155283U, 1531751116U, 1425227133U, + 495654159U, 3279634176U, 3855562207U, 3957195338U, 4159985527U, + 893375062U, 1875515536U, 1327247422U, 3754140693U, 1028923197U, + 1729880440U, 805571298U, 448971099U, 2726757106U, 2749436461U, + 2485987104U, 175337042U, 3235477922U, 3882114302U, 2020970972U, + 943926109U, 2762587195U, 1904195558U, 3452650564U, 108432281U, + 3893463573U, 3977583081U, 2636504348U, 1110673525U, 3548479841U, + 4258854744U, 980047703U, 4057175418U, 3890008292U, 145653646U, + 3141868989U, 3293216228U, 1194331837U, 1254570642U, 3049934521U, + 2868313360U, 2886032750U, 1110873820U, 279553524U, 3007258565U, + 1104807822U, 3186961098U, 315764646U, 2163680838U, 3574508994U, + 3099755655U, 191957684U, 3642656737U, 3317946149U, 3522087636U, + 444526410U, 779157624U, 1088229627U, 1092460223U, 1856013765U, + 3659877367U, 368270451U, 503570716U, 3000984671U, 2742789647U, + 928097709U, 2914109539U, 308843566U, 2816161253U, 3667192079U, + 2762679057U, 3395240989U, 2928925038U, 1491465914U, 3458702834U, + 3787782576U, 2894104823U, 1296880455U, 1253636503U, 989959407U, + 2291560361U, 2776790436U, 1913178042U, 1584677829U, 689637520U, + 1898406878U, 688391508U, 3385234998U, 845493284U, 1943591856U, + 2720472050U, 222695101U, 1653320868U, 2904632120U, 4084936008U, + 1080720688U, 3938032556U, 387896427U, 2650839632U, 99042991U, + 1720913794U, 1047186003U, 1877048040U, 2090457659U, 517087501U, + 4172014665U, 2129713163U, 2413533132U, 2760285054U, 4129272496U, + 1317737175U, 2309566414U, 2228873332U, 3889671280U, 1110864630U, + 3576797776U, 2074552772U, 832002644U, 3097122623U, 2464859298U, + 2679603822U, 1667489885U, 3237652716U, 1478413938U, 1719340335U, + 2306631119U, 639727358U, 3369698270U, 226902796U, 2099920751U, + 1892289957U, 2201594097U, 3508197013U, 3495811856U, 3900381493U, + 841660320U, 3974501451U, 3360949056U, 1676829340U, 728899254U, + 2047809627U, 2390948962U, 670165943U, 3412951831U, 4189320049U, + 1911595255U, 2055363086U, 507170575U, 418219594U, 4141495280U, + 2692088692U, 4203630654U, 3540093932U, 791986533U, 2237921051U, + 2526864324U, 2956616642U, 1394958700U, 1983768223U, 1893373266U, + 591653646U, 228432437U, 1611046598U, 3007736357U, 1040040725U, + 2726180733U, 2789804360U, 4263568405U, 829098158U, 3847722805U, + 1123578029U, 1804276347U, 997971319U, 4203797076U, 4185199713U, + 2811733626U, 2343642194U, 2985262313U, 1417930827U, 3759587724U, + 1967077982U, 1585223204U, 1097475516U, 1903944948U, 740382444U, + 1114142065U, 1541796065U, 1718384172U, 1544076191U, 1134682254U, + 3519754455U, 2866243923U, 341865437U, 645498576U, 2690735853U, + 1046963033U, 2493178460U, 1187604696U, 1619577821U, 488503634U, + 3255768161U, 2306666149U, 1630514044U, 2377698367U, 2751503746U, + 3794467088U, 1796415981U, 3657173746U, 409136296U, 1387122342U, + 1297726519U, 219544855U, 4270285558U, 437578827U, 1444698679U, + 2258519491U, 963109892U, 3982244073U, 3351535275U, 385328496U, + 1804784013U, 698059346U, 3920535147U, 708331212U, 784338163U, + 785678147U, 1238376158U, 1557298846U, 2037809321U, 271576218U, + 4145155269U, 1913481602U, 2763691931U, 588981080U, 1201098051U, + 3717640232U, 1509206239U, 662536967U, 3180523616U, 1133105435U, + 2963500837U, 2253971215U, 3153642623U, 1066925709U, 2582781958U, + 3034720222U, 1090798544U, 2942170004U, 4036187520U, 686972531U, + 2610990302U, 2641437026U, 1837562420U, 722096247U, 1315333033U, + 2102231203U, 3402389208U, 3403698140U, 1312402831U, 2898426558U, + 814384596U, 385649582U, 1916643285U, 1924625106U, 2512905582U, + 2501170304U, 4275223366U, 2841225246U, 1467663688U, 3563567847U, + 2969208552U, 884750901U, 102992576U, 227844301U, 3681442994U, + 3502881894U, 4034693299U, 1166727018U, 1697460687U, 1737778332U, + 1787161139U, 1053003655U, 1215024478U, 2791616766U, 2525841204U, + 1629323443U, 3233815U, 2003823032U, 3083834263U, 2379264872U, + 3752392312U, 1287475550U, 3770904171U, 3004244617U, 1502117784U, + 918698423U, 2419857538U, 3864502062U, 1751322107U, 2188775056U, + 4018728324U, 983712955U, 440071928U, 3710838677U, 2001027698U, + 3994702151U, 22493119U, 3584400918U, 3446253670U, 4254789085U, + 1405447860U, 1240245579U, 1800644159U, 1661363424U, 3278326132U, + 3403623451U, 67092802U, 2609352193U, 3914150340U, 1814842761U, + 3610830847U, 591531412U, 3880232807U, 1673505890U, 2585326991U, + 1678544474U, 3148435887U, 3457217359U, 1193226330U, 2816576908U, + 154025329U, 121678860U, 1164915738U, 973873761U, 269116100U, + 52087970U, 744015362U, 498556057U, 94298882U, 1563271621U, + 2383059628U, 4197367290U, 3958472990U, 2592083636U, 2906408439U, + 1097742433U, 3924840517U, 264557272U, 2292287003U, 3203307984U, + 4047038857U, 3820609705U, 2333416067U, 1839206046U, 3600944252U, + 3412254904U, 583538222U, 2390557166U, 4140459427U, 2810357445U, + 226777499U, 2496151295U, 2207301712U, 3283683112U, 611630281U, + 1933218215U, 3315610954U, 3889441987U, 3719454256U, 3957190521U, + 1313998161U, 2365383016U, 3146941060U, 1801206260U, 796124080U, + 2076248581U, 1747472464U, 3254365145U, 595543130U, 3573909503U, + 3758250204U, 2020768540U, 2439254210U, 93368951U, 3155792250U, + 2600232980U, 3709198295U, 3894900440U, 2971850836U, 1578909644U, + 1443493395U, 2581621665U, 3086506297U, 2443465861U, 558107211U, + 1519367835U, 249149686U, 908102264U, 2588765675U, 1232743965U, + 1001330373U, 3561331654U, 2259301289U, 1564977624U, 3835077093U, + 727244906U, 4255738067U, 1214133513U, 2570786021U, 3899704621U, + 1633861986U, 1636979509U, 1438500431U, 58463278U, 2823485629U, + 2297430187U, 2926781924U, 3371352948U, 1864009023U, 2722267973U, + 1444292075U, 437703973U, 1060414512U, 189705863U, 910018135U, + 4077357964U, 884213423U, 2644986052U, 3973488374U, 1187906116U, + 2331207875U, 780463700U, 3713351662U, 3854611290U, 412805574U, + 2978462572U, 2176222820U, 829424696U, 2790788332U, 2750819108U, + 1594611657U, 3899878394U, 3032870364U, 1702887682U, 1948167778U, + 14130042U, 192292500U, 947227076U, 90719497U, 3854230320U, + 784028434U, 2142399787U, 1563449646U, 2844400217U, 819143172U, + 2883302356U, 2328055304U, 1328532246U, 2603885363U, 3375188924U, + 933941291U, 3627039714U, 2129697284U, 2167253953U, 2506905438U, + 1412424497U, 2981395985U, 1418359660U, 2925902456U, 52752784U, + 3713667988U, 3924669405U, 648975707U, 1145520213U, 4018650664U, + 3805915440U, 2380542088U, 2013260958U, 3262572197U, 2465078101U, + 1114540067U, 3728768081U, 2396958768U, 590672271U, 904818725U, + 4263660715U, 700754408U, 1042601829U, 4094111823U, 4274838909U, + 2512692617U, 2774300207U, 2057306915U, 3470942453U, 99333088U, + 1142661026U, 2889931380U, 14316674U, 2201179167U, 415289459U, + 448265759U, 3515142743U, 3254903683U, 246633281U, 1184307224U, + 2418347830U, 2092967314U, 2682072314U, 2558750234U, 2000352263U, + 1544150531U, 399010405U, 1513946097U, 499682937U, 461167460U, + 3045570638U, 1633669705U, 851492362U, 4052801922U, 2055266765U, + 635556996U, 368266356U, 2385737383U, 3218202352U, 2603772408U, + 349178792U, 226482567U, 3102426060U, 3575998268U, 2103001871U, + 3243137071U, 225500688U, 1634718593U, 4283311431U, 4292122923U, + 3842802787U, 811735523U, 105712518U, 663434053U, 1855889273U, + 2847972595U, 1196355421U, 2552150115U, 4254510614U, 3752181265U, + 3430721819U, 3828705396U, 3436287905U, 3441964937U, 4123670631U, + 353001539U, 459496439U, 3799690868U, 1293777660U, 2761079737U, + 498096339U, 3398433374U, 4080378380U, 2304691596U, 2995729055U, + 4134660419U, 3903444024U, 3576494993U, 203682175U, 3321164857U, + 2747963611U, 79749085U, 2992890370U, 1240278549U, 1772175713U, + 2111331972U, 2655023449U, 1683896345U, 2836027212U, 3482868021U, + 2489884874U, 756853961U, 2298874501U, 4013448667U, 4143996022U, + 2948306858U, 4132920035U, 1283299272U, 995592228U, 3450508595U, + 1027845759U, 1766942720U, 3861411826U, 1446861231U, 95974993U, + 3502263554U, 1487532194U, 601502472U, 4129619129U, 250131773U, + 2050079547U, 3198903947U, 3105589778U, 4066481316U, 3026383978U, + 2276901713U, 365637751U, 2260718426U, 1394775634U, 1791172338U, + 2690503163U, 2952737846U, 1568710462U, 732623190U, 2980358000U, + 1053631832U, 1432426951U, 3229149635U, 1854113985U, 3719733532U, + 3204031934U, 735775531U, 107468620U, 3734611984U, 631009402U, + 3083622457U, 4109580626U, 159373458U, 1301970201U, 4132389302U, + 1293255004U, 847182752U, 4170022737U, 96712900U, 2641406755U, + 1381727755U, 405608287U, 4287919625U, 1703554290U, 3589580244U, + 2911403488U, 2166565U, 2647306451U, 2330535117U, 1200815358U, + 1165916754U, 245060911U, 4040679071U, 3684908771U, 2452834126U, + 2486872773U, 2318678365U, 2940627908U, 1837837240U, 3447897409U, + 4270484676U, 1495388728U, 3754288477U, 4204167884U, 1386977705U, + 2692224733U, 3076249689U, 4109568048U, 4170955115U, 4167531356U, + 4020189950U, 4261855038U, 3036907575U, 3410399885U, 3076395737U, + 1046178638U, 144496770U, 230725846U, 3349637149U, 17065717U, + 2809932048U, 2054581785U, 3608424964U, 3259628808U, 134897388U, + 3743067463U, 257685904U, 3795656590U, 1562468719U, 3589103904U, + 3120404710U, 254684547U, 2653661580U, 3663904795U, 2631942758U, + 1063234347U, 2609732900U, 2332080715U, 3521125233U, 1180599599U, + 1935868586U, 4110970440U, 296706371U, 2128666368U, 1319875791U, + 1570900197U, 3096025483U, 1799882517U, 1928302007U, 1163707758U, + 1244491489U, 3533770203U, 567496053U, 2757924305U, 2781639343U, + 2818420107U, 560404889U, 2619609724U, 4176035430U, 2511289753U, + 2521842019U, 3910553502U, 2926149387U, 3302078172U, 4237118867U, + 330725126U, 367400677U, 888239854U, 545570454U, 4259590525U, + 134343617U, 1102169784U, 1647463719U, 3260979784U, 1518840883U, + 3631537963U, 3342671457U, 1301549147U, 2083739356U, 146593792U, + 3217959080U, 652755743U, 2032187193U, 3898758414U, 1021358093U, + 4037409230U, 2176407931U, 3427391950U, 2883553603U, 985613827U, + 3105265092U, 3423168427U, 3387507672U, 467170288U, 2141266163U, + 3723870208U, 916410914U, 1293987799U, 2652584950U, 769160137U, + 3205292896U, 1561287359U, 1684510084U, 3136055621U, 3765171391U, + 639683232U, 2639569327U, 1218546948U, 4263586685U, 3058215773U, + 2352279820U, 401870217U, 2625822463U, 1529125296U, 2981801895U, + 1191285226U, 4027725437U, 3432700217U, 4098835661U, 971182783U, + 2443861173U, 3881457123U, 3874386651U, 457276199U, 2638294160U, + 4002809368U, 421169044U, 1112642589U, 3076213779U, 3387033971U, + 2499610950U, 3057240914U, 1662679783U, 461224431U, 1168395933U +}; +static const uint32_t init_by_array_32_expected[] = { + 2920711183U, 3885745737U, 3501893680U, 856470934U, 1421864068U, + 277361036U, 1518638004U, 2328404353U, 3355513634U, 64329189U, + 1624587673U, 3508467182U, 2481792141U, 3706480799U, 1925859037U, + 2913275699U, 882658412U, 384641219U, 422202002U, 1873384891U, + 2006084383U, 3924929912U, 1636718106U, 3108838742U, 1245465724U, + 4195470535U, 779207191U, 1577721373U, 1390469554U, 2928648150U, + 121399709U, 3170839019U, 4044347501U, 953953814U, 3821710850U, + 3085591323U, 3666535579U, 3577837737U, 2012008410U, 3565417471U, + 4044408017U, 433600965U, 1637785608U, 1798509764U, 860770589U, + 3081466273U, 3982393409U, 2451928325U, 3437124742U, 4093828739U, + 3357389386U, 2154596123U, 496568176U, 2650035164U, 2472361850U, + 3438299U, 2150366101U, 1577256676U, 3802546413U, 1787774626U, + 4078331588U, 3706103141U, 170391138U, 3806085154U, 1680970100U, + 1961637521U, 3316029766U, 890610272U, 1453751581U, 1430283664U, + 3051057411U, 3597003186U, 542563954U, 3796490244U, 1690016688U, + 3448752238U, 440702173U, 347290497U, 1121336647U, 2540588620U, + 280881896U, 2495136428U, 213707396U, 15104824U, 2946180358U, + 659000016U, 566379385U, 2614030979U, 2855760170U, 334526548U, + 2315569495U, 2729518615U, 564745877U, 1263517638U, 3157185798U, + 1604852056U, 1011639885U, 2950579535U, 2524219188U, 312951012U, + 1528896652U, 1327861054U, 2846910138U, 3966855905U, 2536721582U, + 855353911U, 1685434729U, 3303978929U, 1624872055U, 4020329649U, + 3164802143U, 1642802700U, 1957727869U, 1792352426U, 3334618929U, + 2631577923U, 3027156164U, 842334259U, 3353446843U, 1226432104U, + 1742801369U, 3552852535U, 3471698828U, 1653910186U, 3380330939U, + 2313782701U, 3351007196U, 2129839995U, 1800682418U, 4085884420U, + 1625156629U, 3669701987U, 615211810U, 3294791649U, 4131143784U, + 2590843588U, 3207422808U, 3275066464U, 561592872U, 3957205738U, + 3396578098U, 48410678U, 3505556445U, 1005764855U, 3920606528U, + 2936980473U, 2378918600U, 2404449845U, 1649515163U, 701203563U, + 3705256349U, 83714199U, 3586854132U, 922978446U, 2863406304U, + 3523398907U, 2606864832U, 2385399361U, 3171757816U, 4262841009U, + 3645837721U, 1169579486U, 3666433897U, 3174689479U, 1457866976U, + 3803895110U, 3346639145U, 1907224409U, 1978473712U, 1036712794U, + 980754888U, 1302782359U, 1765252468U, 459245755U, 3728923860U, + 1512894209U, 2046491914U, 207860527U, 514188684U, 2288713615U, + 1597354672U, 3349636117U, 2357291114U, 3995796221U, 945364213U, + 1893326518U, 3770814016U, 1691552714U, 2397527410U, 967486361U, + 776416472U, 4197661421U, 951150819U, 1852770983U, 4044624181U, + 1399439738U, 4194455275U, 2284037669U, 1550734958U, 3321078108U, + 1865235926U, 2912129961U, 2664980877U, 1357572033U, 2600196436U, + 2486728200U, 2372668724U, 1567316966U, 2374111491U, 1839843570U, + 20815612U, 3727008608U, 3871996229U, 824061249U, 1932503978U, + 3404541726U, 758428924U, 2609331364U, 1223966026U, 1299179808U, + 648499352U, 2180134401U, 880821170U, 3781130950U, 113491270U, + 1032413764U, 4185884695U, 2490396037U, 1201932817U, 4060951446U, + 4165586898U, 1629813212U, 2887821158U, 415045333U, 628926856U, + 2193466079U, 3391843445U, 2227540681U, 1907099846U, 2848448395U, + 1717828221U, 1372704537U, 1707549841U, 2294058813U, 2101214437U, + 2052479531U, 1695809164U, 3176587306U, 2632770465U, 81634404U, + 1603220563U, 644238487U, 302857763U, 897352968U, 2613146653U, + 1391730149U, 4245717312U, 4191828749U, 1948492526U, 2618174230U, + 3992984522U, 2178852787U, 3596044509U, 3445573503U, 2026614616U, + 915763564U, 3415689334U, 2532153403U, 3879661562U, 2215027417U, + 3111154986U, 2929478371U, 668346391U, 1152241381U, 2632029711U, + 3004150659U, 2135025926U, 948690501U, 2799119116U, 4228829406U, + 1981197489U, 4209064138U, 684318751U, 3459397845U, 201790843U, + 4022541136U, 3043635877U, 492509624U, 3263466772U, 1509148086U, + 921459029U, 3198857146U, 705479721U, 3835966910U, 3603356465U, + 576159741U, 1742849431U, 594214882U, 2055294343U, 3634861861U, + 449571793U, 3246390646U, 3868232151U, 1479156585U, 2900125656U, + 2464815318U, 3960178104U, 1784261920U, 18311476U, 3627135050U, + 644609697U, 424968996U, 919890700U, 2986824110U, 816423214U, + 4003562844U, 1392714305U, 1757384428U, 2569030598U, 995949559U, + 3875659880U, 2933807823U, 2752536860U, 2993858466U, 4030558899U, + 2770783427U, 2775406005U, 2777781742U, 1931292655U, 472147933U, + 3865853827U, 2726470545U, 2668412860U, 2887008249U, 408979190U, + 3578063323U, 3242082049U, 1778193530U, 27981909U, 2362826515U, + 389875677U, 1043878156U, 581653903U, 3830568952U, 389535942U, + 3713523185U, 2768373359U, 2526101582U, 1998618197U, 1160859704U, + 3951172488U, 1098005003U, 906275699U, 3446228002U, 2220677963U, + 2059306445U, 132199571U, 476838790U, 1868039399U, 3097344807U, + 857300945U, 396345050U, 2835919916U, 1782168828U, 1419519470U, + 4288137521U, 819087232U, 596301494U, 872823172U, 1526888217U, + 805161465U, 1116186205U, 2829002754U, 2352620120U, 620121516U, + 354159268U, 3601949785U, 209568138U, 1352371732U, 2145977349U, + 4236871834U, 1539414078U, 3558126206U, 3224857093U, 4164166682U, + 3817553440U, 3301780278U, 2682696837U, 3734994768U, 1370950260U, + 1477421202U, 2521315749U, 1330148125U, 1261554731U, 2769143688U, + 3554756293U, 4235882678U, 3254686059U, 3530579953U, 1215452615U, + 3574970923U, 4057131421U, 589224178U, 1000098193U, 171190718U, + 2521852045U, 2351447494U, 2284441580U, 2646685513U, 3486933563U, + 3789864960U, 1190528160U, 1702536782U, 1534105589U, 4262946827U, + 2726686826U, 3584544841U, 2348270128U, 2145092281U, 2502718509U, + 1027832411U, 3571171153U, 1287361161U, 4011474411U, 3241215351U, + 2419700818U, 971242709U, 1361975763U, 1096842482U, 3271045537U, + 81165449U, 612438025U, 3912966678U, 1356929810U, 733545735U, + 537003843U, 1282953084U, 884458241U, 588930090U, 3930269801U, + 2961472450U, 1219535534U, 3632251943U, 268183903U, 1441240533U, + 3653903360U, 3854473319U, 2259087390U, 2548293048U, 2022641195U, + 2105543911U, 1764085217U, 3246183186U, 482438805U, 888317895U, + 2628314765U, 2466219854U, 717546004U, 2322237039U, 416725234U, + 1544049923U, 1797944973U, 3398652364U, 3111909456U, 485742908U, + 2277491072U, 1056355088U, 3181001278U, 129695079U, 2693624550U, + 1764438564U, 3797785470U, 195503713U, 3266519725U, 2053389444U, + 1961527818U, 3400226523U, 3777903038U, 2597274307U, 4235851091U, + 4094406648U, 2171410785U, 1781151386U, 1378577117U, 654643266U, + 3424024173U, 3385813322U, 679385799U, 479380913U, 681715441U, + 3096225905U, 276813409U, 3854398070U, 2721105350U, 831263315U, + 3276280337U, 2628301522U, 3984868494U, 1466099834U, 2104922114U, + 1412672743U, 820330404U, 3491501010U, 942735832U, 710652807U, + 3972652090U, 679881088U, 40577009U, 3705286397U, 2815423480U, + 3566262429U, 663396513U, 3777887429U, 4016670678U, 404539370U, + 1142712925U, 1140173408U, 2913248352U, 2872321286U, 263751841U, + 3175196073U, 3162557581U, 2878996619U, 75498548U, 3836833140U, + 3284664959U, 1157523805U, 112847376U, 207855609U, 1337979698U, + 1222578451U, 157107174U, 901174378U, 3883717063U, 1618632639U, + 1767889440U, 4264698824U, 1582999313U, 884471997U, 2508825098U, + 3756370771U, 2457213553U, 3565776881U, 3709583214U, 915609601U, + 460833524U, 1091049576U, 85522880U, 2553251U, 132102809U, + 2429882442U, 2562084610U, 1386507633U, 4112471229U, 21965213U, + 1981516006U, 2418435617U, 3054872091U, 4251511224U, 2025783543U, + 1916911512U, 2454491136U, 3938440891U, 3825869115U, 1121698605U, + 3463052265U, 802340101U, 1912886800U, 4031997367U, 3550640406U, + 1596096923U, 610150600U, 431464457U, 2541325046U, 486478003U, + 739704936U, 2862696430U, 3037903166U, 1129749694U, 2611481261U, + 1228993498U, 510075548U, 3424962587U, 2458689681U, 818934833U, + 4233309125U, 1608196251U, 3419476016U, 1858543939U, 2682166524U, + 3317854285U, 631986188U, 3008214764U, 613826412U, 3567358221U, + 3512343882U, 1552467474U, 3316162670U, 1275841024U, 4142173454U, + 565267881U, 768644821U, 198310105U, 2396688616U, 1837659011U, + 203429334U, 854539004U, 4235811518U, 3338304926U, 3730418692U, + 3852254981U, 3032046452U, 2329811860U, 2303590566U, 2696092212U, + 3894665932U, 145835667U, 249563655U, 1932210840U, 2431696407U, + 3312636759U, 214962629U, 2092026914U, 3020145527U, 4073039873U, + 2739105705U, 1308336752U, 855104522U, 2391715321U, 67448785U, + 547989482U, 854411802U, 3608633740U, 431731530U, 537375589U, + 3888005760U, 696099141U, 397343236U, 1864511780U, 44029739U, + 1729526891U, 1993398655U, 2010173426U, 2591546756U, 275223291U, + 1503900299U, 4217765081U, 2185635252U, 1122436015U, 3550155364U, + 681707194U, 3260479338U, 933579397U, 2983029282U, 2505504587U, + 2667410393U, 2962684490U, 4139721708U, 2658172284U, 2452602383U, + 2607631612U, 1344296217U, 3075398709U, 2949785295U, 1049956168U, + 3917185129U, 2155660174U, 3280524475U, 1503827867U, 674380765U, + 1918468193U, 3843983676U, 634358221U, 2538335643U, 1873351298U, + 3368723763U, 2129144130U, 3203528633U, 3087174986U, 2691698871U, + 2516284287U, 24437745U, 1118381474U, 2816314867U, 2448576035U, + 4281989654U, 217287825U, 165872888U, 2628995722U, 3533525116U, + 2721669106U, 872340568U, 3429930655U, 3309047304U, 3916704967U, + 3270160355U, 1348884255U, 1634797670U, 881214967U, 4259633554U, + 174613027U, 1103974314U, 1625224232U, 2678368291U, 1133866707U, + 3853082619U, 4073196549U, 1189620777U, 637238656U, 930241537U, + 4042750792U, 3842136042U, 2417007212U, 2524907510U, 1243036827U, + 1282059441U, 3764588774U, 1394459615U, 2323620015U, 1166152231U, + 3307479609U, 3849322257U, 3507445699U, 4247696636U, 758393720U, + 967665141U, 1095244571U, 1319812152U, 407678762U, 2640605208U, + 2170766134U, 3663594275U, 4039329364U, 2512175520U, 725523154U, + 2249807004U, 3312617979U, 2414634172U, 1278482215U, 349206484U, + 1573063308U, 1196429124U, 3873264116U, 2400067801U, 268795167U, + 226175489U, 2961367263U, 1968719665U, 42656370U, 1010790699U, + 561600615U, 2422453992U, 3082197735U, 1636700484U, 3977715296U, + 3125350482U, 3478021514U, 2227819446U, 1540868045U, 3061908980U, + 1087362407U, 3625200291U, 361937537U, 580441897U, 1520043666U, + 2270875402U, 1009161260U, 2502355842U, 4278769785U, 473902412U, + 1057239083U, 1905829039U, 1483781177U, 2080011417U, 1207494246U, + 1806991954U, 2194674403U, 3455972205U, 807207678U, 3655655687U, + 674112918U, 195425752U, 3917890095U, 1874364234U, 1837892715U, + 3663478166U, 1548892014U, 2570748714U, 2049929836U, 2167029704U, + 697543767U, 3499545023U, 3342496315U, 1725251190U, 3561387469U, + 2905606616U, 1580182447U, 3934525927U, 4103172792U, 1365672522U, + 1534795737U, 3308667416U, 2841911405U, 3943182730U, 4072020313U, + 3494770452U, 3332626671U, 55327267U, 478030603U, 411080625U, + 3419529010U, 1604767823U, 3513468014U, 570668510U, 913790824U, + 2283967995U, 695159462U, 3825542932U, 4150698144U, 1829758699U, + 202895590U, 1609122645U, 1267651008U, 2910315509U, 2511475445U, + 2477423819U, 3932081579U, 900879979U, 2145588390U, 2670007504U, + 580819444U, 1864996828U, 2526325979U, 1019124258U, 815508628U, + 2765933989U, 1277301341U, 3006021786U, 855540956U, 288025710U, + 1919594237U, 2331223864U, 177452412U, 2475870369U, 2689291749U, + 865194284U, 253432152U, 2628531804U, 2861208555U, 2361597573U, + 1653952120U, 1039661024U, 2159959078U, 3709040440U, 3564718533U, + 2596878672U, 2041442161U, 31164696U, 2662962485U, 3665637339U, + 1678115244U, 2699839832U, 3651968520U, 3521595541U, 458433303U, + 2423096824U, 21831741U, 380011703U, 2498168716U, 861806087U, + 1673574843U, 4188794405U, 2520563651U, 2632279153U, 2170465525U, + 4171949898U, 3886039621U, 1661344005U, 3424285243U, 992588372U, + 2500984144U, 2993248497U, 3590193895U, 1535327365U, 515645636U, + 131633450U, 3729760261U, 1613045101U, 3254194278U, 15889678U, + 1493590689U, 244148718U, 2991472662U, 1401629333U, 777349878U, + 2501401703U, 4285518317U, 3794656178U, 955526526U, 3442142820U, + 3970298374U, 736025417U, 2737370764U, 1271509744U, 440570731U, + 136141826U, 1596189518U, 923399175U, 257541519U, 3505774281U, + 2194358432U, 2518162991U, 1379893637U, 2667767062U, 3748146247U, + 1821712620U, 3923161384U, 1947811444U, 2392527197U, 4127419685U, + 1423694998U, 4156576871U, 1382885582U, 3420127279U, 3617499534U, + 2994377493U, 4038063986U, 1918458672U, 2983166794U, 4200449033U, + 353294540U, 1609232588U, 243926648U, 2332803291U, 507996832U, + 2392838793U, 4075145196U, 2060984340U, 4287475136U, 88232602U, + 2491531140U, 4159725633U, 2272075455U, 759298618U, 201384554U, + 838356250U, 1416268324U, 674476934U, 90795364U, 141672229U, + 3660399588U, 4196417251U, 3249270244U, 3774530247U, 59587265U, + 3683164208U, 19392575U, 1463123697U, 1882205379U, 293780489U, + 2553160622U, 2933904694U, 675638239U, 2851336944U, 1435238743U, + 2448730183U, 804436302U, 2119845972U, 322560608U, 4097732704U, + 2987802540U, 641492617U, 2575442710U, 4217822703U, 3271835300U, + 2836418300U, 3739921620U, 2138378768U, 2879771855U, 4294903423U, + 3121097946U, 2603440486U, 2560820391U, 1012930944U, 2313499967U, + 584489368U, 3431165766U, 897384869U, 2062537737U, 2847889234U, + 3742362450U, 2951174585U, 4204621084U, 1109373893U, 3668075775U, + 2750138839U, 3518055702U, 733072558U, 4169325400U, 788493625U +}; +static const uint64_t init_gen_rand_64_expected[] = { + QU(16924766246869039260LLU), QU( 8201438687333352714LLU), + QU( 2265290287015001750LLU), QU(18397264611805473832LLU), + QU( 3375255223302384358LLU), QU( 6345559975416828796LLU), + QU(18229739242790328073LLU), QU( 7596792742098800905LLU), + QU( 255338647169685981LLU), QU( 2052747240048610300LLU), + QU(18328151576097299343LLU), QU(12472905421133796567LLU), + QU(11315245349717600863LLU), QU(16594110197775871209LLU), + QU(15708751964632456450LLU), QU(10452031272054632535LLU), + QU(11097646720811454386LLU), QU( 4556090668445745441LLU), + QU(17116187693090663106LLU), QU(14931526836144510645LLU), + QU( 9190752218020552591LLU), QU( 9625800285771901401LLU), + QU(13995141077659972832LLU), QU( 5194209094927829625LLU), + QU( 4156788379151063303LLU), QU( 8523452593770139494LLU), + QU(14082382103049296727LLU), QU( 2462601863986088483LLU), + QU( 3030583461592840678LLU), QU( 5221622077872827681LLU), + QU( 3084210671228981236LLU), QU(13956758381389953823LLU), + QU(13503889856213423831LLU), QU(15696904024189836170LLU), + QU( 4612584152877036206LLU), QU( 6231135538447867881LLU), + QU(10172457294158869468LLU), QU( 6452258628466708150LLU), + QU(14044432824917330221LLU), QU( 370168364480044279LLU), + QU(10102144686427193359LLU), QU( 667870489994776076LLU), + QU( 2732271956925885858LLU), QU(18027788905977284151LLU), + QU(15009842788582923859LLU), QU( 7136357960180199542LLU), + QU(15901736243475578127LLU), QU(16951293785352615701LLU), + QU(10551492125243691632LLU), QU(17668869969146434804LLU), + QU(13646002971174390445LLU), QU( 9804471050759613248LLU), + QU( 5511670439655935493LLU), QU(18103342091070400926LLU), + QU(17224512747665137533LLU), QU(15534627482992618168LLU), + QU( 1423813266186582647LLU), QU(15821176807932930024LLU), + QU( 30323369733607156LLU), QU(11599382494723479403LLU), + QU( 653856076586810062LLU), QU( 3176437395144899659LLU), + QU(14028076268147963917LLU), QU(16156398271809666195LLU), + QU( 3166955484848201676LLU), QU( 5746805620136919390LLU), + QU(17297845208891256593LLU), QU(11691653183226428483LLU), + QU(17900026146506981577LLU), QU(15387382115755971042LLU), + QU(16923567681040845943LLU), QU( 8039057517199388606LLU), + QU(11748409241468629263LLU), QU( 794358245539076095LLU), + QU(13438501964693401242LLU), QU(14036803236515618962LLU), + QU( 5252311215205424721LLU), QU(17806589612915509081LLU), + QU( 6802767092397596006LLU), QU(14212120431184557140LLU), + QU( 1072951366761385712LLU), QU(13098491780722836296LLU), + QU( 9466676828710797353LLU), QU(12673056849042830081LLU), + QU(12763726623645357580LLU), QU(16468961652999309493LLU), + QU(15305979875636438926LLU), QU(17444713151223449734LLU), + QU( 5692214267627883674LLU), QU(13049589139196151505LLU), + QU( 880115207831670745LLU), QU( 1776529075789695498LLU), + QU(16695225897801466485LLU), QU(10666901778795346845LLU), + QU( 6164389346722833869LLU), QU( 2863817793264300475LLU), + QU( 9464049921886304754LLU), QU( 3993566636740015468LLU), + QU( 9983749692528514136LLU), QU(16375286075057755211LLU), + QU(16042643417005440820LLU), QU(11445419662923489877LLU), + QU( 7999038846885158836LLU), QU( 6721913661721511535LLU), + QU( 5363052654139357320LLU), QU( 1817788761173584205LLU), + QU(13290974386445856444LLU), QU( 4650350818937984680LLU), + QU( 8219183528102484836LLU), QU( 1569862923500819899LLU), + QU( 4189359732136641860LLU), QU(14202822961683148583LLU), + QU( 4457498315309429058LLU), QU(13089067387019074834LLU), + QU(11075517153328927293LLU), QU(10277016248336668389LLU), + QU( 7070509725324401122LLU), QU(17808892017780289380LLU), + QU(13143367339909287349LLU), QU( 1377743745360085151LLU), + QU( 5749341807421286485LLU), QU(14832814616770931325LLU), + QU( 7688820635324359492LLU), QU(10960474011539770045LLU), + QU( 81970066653179790LLU), QU(12619476072607878022LLU), + QU( 4419566616271201744LLU), QU(15147917311750568503LLU), + QU( 5549739182852706345LLU), QU( 7308198397975204770LLU), + QU(13580425496671289278LLU), QU(17070764785210130301LLU), + QU( 8202832846285604405LLU), QU( 6873046287640887249LLU), + QU( 6927424434308206114LLU), QU( 6139014645937224874LLU), + QU(10290373645978487639LLU), QU(15904261291701523804LLU), + QU( 9628743442057826883LLU), QU(18383429096255546714LLU), + QU( 4977413265753686967LLU), QU( 7714317492425012869LLU), + QU( 9025232586309926193LLU), QU(14627338359776709107LLU), + QU(14759849896467790763LLU), QU(10931129435864423252LLU), + QU( 4588456988775014359LLU), QU(10699388531797056724LLU), + QU( 468652268869238792LLU), QU( 5755943035328078086LLU), + QU( 2102437379988580216LLU), QU( 9986312786506674028LLU), + QU( 2654207180040945604LLU), QU( 8726634790559960062LLU), + QU( 100497234871808137LLU), QU( 2800137176951425819LLU), + QU( 6076627612918553487LLU), QU( 5780186919186152796LLU), + QU( 8179183595769929098LLU), QU( 6009426283716221169LLU), + QU( 2796662551397449358LLU), QU( 1756961367041986764LLU), + QU( 6972897917355606205LLU), QU(14524774345368968243LLU), + QU( 2773529684745706940LLU), QU( 4853632376213075959LLU), + QU( 4198177923731358102LLU), QU( 8271224913084139776LLU), + QU( 2741753121611092226LLU), QU(16782366145996731181LLU), + QU(15426125238972640790LLU), QU(13595497100671260342LLU), + QU( 3173531022836259898LLU), QU( 6573264560319511662LLU), + QU(18041111951511157441LLU), QU( 2351433581833135952LLU), + QU( 3113255578908173487LLU), QU( 1739371330877858784LLU), + QU(16046126562789165480LLU), QU( 8072101652214192925LLU), + QU(15267091584090664910LLU), QU( 9309579200403648940LLU), + QU( 5218892439752408722LLU), QU(14492477246004337115LLU), + QU(17431037586679770619LLU), QU( 7385248135963250480LLU), + QU( 9580144956565560660LLU), QU( 4919546228040008720LLU), + QU(15261542469145035584LLU), QU(18233297270822253102LLU), + QU( 5453248417992302857LLU), QU( 9309519155931460285LLU), + QU(10342813012345291756LLU), QU(15676085186784762381LLU), + QU(15912092950691300645LLU), QU( 9371053121499003195LLU), + QU( 9897186478226866746LLU), QU(14061858287188196327LLU), + QU( 122575971620788119LLU), QU(12146750969116317754LLU), + QU( 4438317272813245201LLU), QU( 8332576791009527119LLU), + QU(13907785691786542057LLU), QU(10374194887283287467LLU), + QU( 2098798755649059566LLU), QU( 3416235197748288894LLU), + QU( 8688269957320773484LLU), QU( 7503964602397371571LLU), + QU(16724977015147478236LLU), QU( 9461512855439858184LLU), + QU(13259049744534534727LLU), QU( 3583094952542899294LLU), + QU( 8764245731305528292LLU), QU(13240823595462088985LLU), + QU(13716141617617910448LLU), QU(18114969519935960955LLU), + QU( 2297553615798302206LLU), QU( 4585521442944663362LLU), + QU(17776858680630198686LLU), QU( 4685873229192163363LLU), + QU( 152558080671135627LLU), QU(15424900540842670088LLU), + QU(13229630297130024108LLU), QU(17530268788245718717LLU), + QU(16675633913065714144LLU), QU( 3158912717897568068LLU), + QU(15399132185380087288LLU), QU( 7401418744515677872LLU), + QU(13135412922344398535LLU), QU( 6385314346100509511LLU), + QU(13962867001134161139LLU), QU(10272780155442671999LLU), + QU(12894856086597769142LLU), QU(13340877795287554994LLU), + QU(12913630602094607396LLU), QU(12543167911119793857LLU), + QU(17343570372251873096LLU), QU(10959487764494150545LLU), + QU( 6966737953093821128LLU), QU(13780699135496988601LLU), + QU( 4405070719380142046LLU), QU(14923788365607284982LLU), + QU( 2869487678905148380LLU), QU( 6416272754197188403LLU), + QU(15017380475943612591LLU), QU( 1995636220918429487LLU), + QU( 3402016804620122716LLU), QU(15800188663407057080LLU), + QU(11362369990390932882LLU), QU(15262183501637986147LLU), + QU(10239175385387371494LLU), QU( 9352042420365748334LLU), + QU( 1682457034285119875LLU), QU( 1724710651376289644LLU), + QU( 2038157098893817966LLU), QU( 9897825558324608773LLU), + QU( 1477666236519164736LLU), QU(16835397314511233640LLU), + QU(10370866327005346508LLU), QU(10157504370660621982LLU), + QU(12113904045335882069LLU), QU(13326444439742783008LLU), + QU(11302769043000765804LLU), QU(13594979923955228484LLU), + QU(11779351762613475968LLU), QU( 3786101619539298383LLU), + QU( 8021122969180846063LLU), QU(15745904401162500495LLU), + QU(10762168465993897267LLU), QU(13552058957896319026LLU), + QU(11200228655252462013LLU), QU( 5035370357337441226LLU), + QU( 7593918984545500013LLU), QU( 5418554918361528700LLU), + QU( 4858270799405446371LLU), QU( 9974659566876282544LLU), + QU(18227595922273957859LLU), QU( 2772778443635656220LLU), + QU(14285143053182085385LLU), QU( 9939700992429600469LLU), + QU(12756185904545598068LLU), QU( 2020783375367345262LLU), + QU( 57026775058331227LLU), QU( 950827867930065454LLU), + QU( 6602279670145371217LLU), QU( 2291171535443566929LLU), + QU( 5832380724425010313LLU), QU( 1220343904715982285LLU), + QU(17045542598598037633LLU), QU(15460481779702820971LLU), + QU(13948388779949365130LLU), QU(13975040175430829518LLU), + QU(17477538238425541763LLU), QU(11104663041851745725LLU), + QU(15860992957141157587LLU), QU(14529434633012950138LLU), + QU( 2504838019075394203LLU), QU( 7512113882611121886LLU), + QU( 4859973559980886617LLU), QU( 1258601555703250219LLU), + QU(15594548157514316394LLU), QU( 4516730171963773048LLU), + QU(11380103193905031983LLU), QU( 6809282239982353344LLU), + QU(18045256930420065002LLU), QU( 2453702683108791859LLU), + QU( 977214582986981460LLU), QU( 2006410402232713466LLU), + QU( 6192236267216378358LLU), QU( 3429468402195675253LLU), + QU(18146933153017348921LLU), QU(17369978576367231139LLU), + QU( 1246940717230386603LLU), QU(11335758870083327110LLU), + QU(14166488801730353682LLU), QU( 9008573127269635732LLU), + QU(10776025389820643815LLU), QU(15087605441903942962LLU), + QU( 1359542462712147922LLU), QU(13898874411226454206LLU), + QU(17911176066536804411LLU), QU( 9435590428600085274LLU), + QU( 294488509967864007LLU), QU( 8890111397567922046LLU), + QU( 7987823476034328778LLU), QU(13263827582440967651LLU), + QU( 7503774813106751573LLU), QU(14974747296185646837LLU), + QU( 8504765037032103375LLU), QU(17340303357444536213LLU), + QU( 7704610912964485743LLU), QU( 8107533670327205061LLU), + QU( 9062969835083315985LLU), QU(16968963142126734184LLU), + QU(12958041214190810180LLU), QU( 2720170147759570200LLU), + QU( 2986358963942189566LLU), QU(14884226322219356580LLU), + QU( 286224325144368520LLU), QU(11313800433154279797LLU), + QU(18366849528439673248LLU), QU(17899725929482368789LLU), + QU( 3730004284609106799LLU), QU( 1654474302052767205LLU), + QU( 5006698007047077032LLU), QU( 8196893913601182838LLU), + QU(15214541774425211640LLU), QU(17391346045606626073LLU), + QU( 8369003584076969089LLU), QU( 3939046733368550293LLU), + QU(10178639720308707785LLU), QU( 2180248669304388697LLU), + QU( 62894391300126322LLU), QU( 9205708961736223191LLU), + QU( 6837431058165360438LLU), QU( 3150743890848308214LLU), + QU(17849330658111464583LLU), QU(12214815643135450865LLU), + QU(13410713840519603402LLU), QU( 3200778126692046802LLU), + QU(13354780043041779313LLU), QU( 800850022756886036LLU), + QU(15660052933953067433LLU), QU( 6572823544154375676LLU), + QU(11030281857015819266LLU), QU(12682241941471433835LLU), + QU(11654136407300274693LLU), QU( 4517795492388641109LLU), + QU( 9757017371504524244LLU), QU(17833043400781889277LLU), + QU(12685085201747792227LLU), QU(10408057728835019573LLU), + QU( 98370418513455221LLU), QU( 6732663555696848598LLU), + QU(13248530959948529780LLU), QU( 3530441401230622826LLU), + QU(18188251992895660615LLU), QU( 1847918354186383756LLU), + QU( 1127392190402660921LLU), QU(11293734643143819463LLU), + QU( 3015506344578682982LLU), QU(13852645444071153329LLU), + QU( 2121359659091349142LLU), QU( 1294604376116677694LLU), + QU( 5616576231286352318LLU), QU( 7112502442954235625LLU), + QU(11676228199551561689LLU), QU(12925182803007305359LLU), + QU( 7852375518160493082LLU), QU( 1136513130539296154LLU), + QU( 5636923900916593195LLU), QU( 3221077517612607747LLU), + QU(17784790465798152513LLU), QU( 3554210049056995938LLU), + QU(17476839685878225874LLU), QU( 3206836372585575732LLU), + QU( 2765333945644823430LLU), QU(10080070903718799528LLU), + QU( 5412370818878286353LLU), QU( 9689685887726257728LLU), + QU( 8236117509123533998LLU), QU( 1951139137165040214LLU), + QU( 4492205209227980349LLU), QU(16541291230861602967LLU), + QU( 1424371548301437940LLU), QU( 9117562079669206794LLU), + QU(14374681563251691625LLU), QU(13873164030199921303LLU), + QU( 6680317946770936731LLU), QU(15586334026918276214LLU), + QU(10896213950976109802LLU), QU( 9506261949596413689LLU), + QU( 9903949574308040616LLU), QU( 6038397344557204470LLU), + QU( 174601465422373648LLU), QU(15946141191338238030LLU), + QU(17142225620992044937LLU), QU( 7552030283784477064LLU), + QU( 2947372384532947997LLU), QU( 510797021688197711LLU), + QU( 4962499439249363461LLU), QU( 23770320158385357LLU), + QU( 959774499105138124LLU), QU( 1468396011518788276LLU), + QU( 2015698006852312308LLU), QU( 4149400718489980136LLU), + QU( 5992916099522371188LLU), QU(10819182935265531076LLU), + QU(16189787999192351131LLU), QU( 342833961790261950LLU), + QU(12470830319550495336LLU), QU(18128495041912812501LLU), + QU( 1193600899723524337LLU), QU( 9056793666590079770LLU), + QU( 2154021227041669041LLU), QU( 4963570213951235735LLU), + QU( 4865075960209211409LLU), QU( 2097724599039942963LLU), + QU( 2024080278583179845LLU), QU(11527054549196576736LLU), + QU(10650256084182390252LLU), QU( 4808408648695766755LLU), + QU( 1642839215013788844LLU), QU(10607187948250398390LLU), + QU( 7076868166085913508LLU), QU( 730522571106887032LLU), + QU(12500579240208524895LLU), QU( 4484390097311355324LLU), + QU(15145801330700623870LLU), QU( 8055827661392944028LLU), + QU( 5865092976832712268LLU), QU(15159212508053625143LLU), + QU( 3560964582876483341LLU), QU( 4070052741344438280LLU), + QU( 6032585709886855634LLU), QU(15643262320904604873LLU), + QU( 2565119772293371111LLU), QU( 318314293065348260LLU), + QU(15047458749141511872LLU), QU( 7772788389811528730LLU), + QU( 7081187494343801976LLU), QU( 6465136009467253947LLU), + QU(10425940692543362069LLU), QU( 554608190318339115LLU), + QU(14796699860302125214LLU), QU( 1638153134431111443LLU), + QU(10336967447052276248LLU), QU( 8412308070396592958LLU), + QU( 4004557277152051226LLU), QU( 8143598997278774834LLU), + QU(16413323996508783221LLU), QU(13139418758033994949LLU), + QU( 9772709138335006667LLU), QU( 2818167159287157659LLU), + QU(17091740573832523669LLU), QU(14629199013130751608LLU), + QU(18268322711500338185LLU), QU( 8290963415675493063LLU), + QU( 8830864907452542588LLU), QU( 1614839084637494849LLU), + QU(14855358500870422231LLU), QU( 3472996748392519937LLU), + QU(15317151166268877716LLU), QU( 5825895018698400362LLU), + QU(16730208429367544129LLU), QU(10481156578141202800LLU), + QU( 4746166512382823750LLU), QU(12720876014472464998LLU), + QU( 8825177124486735972LLU), QU(13733447296837467838LLU), + QU( 6412293741681359625LLU), QU( 8313213138756135033LLU), + QU(11421481194803712517LLU), QU( 7997007691544174032LLU), + QU( 6812963847917605930LLU), QU( 9683091901227558641LLU), + QU(14703594165860324713LLU), QU( 1775476144519618309LLU), + QU( 2724283288516469519LLU), QU( 717642555185856868LLU), + QU( 8736402192215092346LLU), QU(11878800336431381021LLU), + QU( 4348816066017061293LLU), QU( 6115112756583631307LLU), + QU( 9176597239667142976LLU), QU(12615622714894259204LLU), + QU(10283406711301385987LLU), QU( 5111762509485379420LLU), + QU( 3118290051198688449LLU), QU( 7345123071632232145LLU), + QU( 9176423451688682359LLU), QU( 4843865456157868971LLU), + QU(12008036363752566088LLU), QU(12058837181919397720LLU), + QU( 2145073958457347366LLU), QU( 1526504881672818067LLU), + QU( 3488830105567134848LLU), QU(13208362960674805143LLU), + QU( 4077549672899572192LLU), QU( 7770995684693818365LLU), + QU( 1398532341546313593LLU), QU(12711859908703927840LLU), + QU( 1417561172594446813LLU), QU(17045191024194170604LLU), + QU( 4101933177604931713LLU), QU(14708428834203480320LLU), + QU(17447509264469407724LLU), QU(14314821973983434255LLU), + QU(17990472271061617265LLU), QU( 5087756685841673942LLU), + QU(12797820586893859939LLU), QU( 1778128952671092879LLU), + QU( 3535918530508665898LLU), QU( 9035729701042481301LLU), + QU(14808661568277079962LLU), QU(14587345077537747914LLU), + QU(11920080002323122708LLU), QU( 6426515805197278753LLU), + QU( 3295612216725984831LLU), QU(11040722532100876120LLU), + QU(12305952936387598754LLU), QU(16097391899742004253LLU), + QU( 4908537335606182208LLU), QU(12446674552196795504LLU), + QU(16010497855816895177LLU), QU( 9194378874788615551LLU), + QU( 3382957529567613384LLU), QU( 5154647600754974077LLU), + QU( 9801822865328396141LLU), QU( 9023662173919288143LLU), + QU(17623115353825147868LLU), QU( 8238115767443015816LLU), + QU(15811444159859002560LLU), QU( 9085612528904059661LLU), + QU( 6888601089398614254LLU), QU( 258252992894160189LLU), + QU( 6704363880792428622LLU), QU( 6114966032147235763LLU), + QU(11075393882690261875LLU), QU( 8797664238933620407LLU), + QU( 5901892006476726920LLU), QU( 5309780159285518958LLU), + QU(14940808387240817367LLU), QU(14642032021449656698LLU), + QU( 9808256672068504139LLU), QU( 3670135111380607658LLU), + QU(11211211097845960152LLU), QU( 1474304506716695808LLU), + QU(15843166204506876239LLU), QU( 7661051252471780561LLU), + QU(10170905502249418476LLU), QU( 7801416045582028589LLU), + QU( 2763981484737053050LLU), QU( 9491377905499253054LLU), + QU(16201395896336915095LLU), QU( 9256513756442782198LLU), + QU( 5411283157972456034LLU), QU( 5059433122288321676LLU), + QU( 4327408006721123357LLU), QU( 9278544078834433377LLU), + QU( 7601527110882281612LLU), QU(11848295896975505251LLU), + QU(12096998801094735560LLU), QU(14773480339823506413LLU), + QU(15586227433895802149LLU), QU(12786541257830242872LLU), + QU( 6904692985140503067LLU), QU( 5309011515263103959LLU), + QU(12105257191179371066LLU), QU(14654380212442225037LLU), + QU( 2556774974190695009LLU), QU( 4461297399927600261LLU), + QU(14888225660915118646LLU), QU(14915459341148291824LLU), + QU( 2738802166252327631LLU), QU( 6047155789239131512LLU), + QU(12920545353217010338LLU), QU(10697617257007840205LLU), + QU( 2751585253158203504LLU), QU(13252729159780047496LLU), + QU(14700326134672815469LLU), QU(14082527904374600529LLU), + QU(16852962273496542070LLU), QU(17446675504235853907LLU), + QU(15019600398527572311LLU), QU(12312781346344081551LLU), + QU(14524667935039810450LLU), QU( 5634005663377195738LLU), + QU(11375574739525000569LLU), QU( 2423665396433260040LLU), + QU( 5222836914796015410LLU), QU( 4397666386492647387LLU), + QU( 4619294441691707638LLU), QU( 665088602354770716LLU), + QU(13246495665281593610LLU), QU( 6564144270549729409LLU), + QU(10223216188145661688LLU), QU( 3961556907299230585LLU), + QU(11543262515492439914LLU), QU(16118031437285993790LLU), + QU( 7143417964520166465LLU), QU(13295053515909486772LLU), + QU( 40434666004899675LLU), QU(17127804194038347164LLU), + QU( 8599165966560586269LLU), QU( 8214016749011284903LLU), + QU(13725130352140465239LLU), QU( 5467254474431726291LLU), + QU( 7748584297438219877LLU), QU(16933551114829772472LLU), + QU( 2169618439506799400LLU), QU( 2169787627665113463LLU), + QU(17314493571267943764LLU), QU(18053575102911354912LLU), + QU(11928303275378476973LLU), QU(11593850925061715550LLU), + QU(17782269923473589362LLU), QU( 3280235307704747039LLU), + QU( 6145343578598685149LLU), QU(17080117031114086090LLU), + QU(18066839902983594755LLU), QU( 6517508430331020706LLU), + QU( 8092908893950411541LLU), QU(12558378233386153732LLU), + QU( 4476532167973132976LLU), QU(16081642430367025016LLU), + QU( 4233154094369139361LLU), QU( 8693630486693161027LLU), + QU(11244959343027742285LLU), QU(12273503967768513508LLU), + QU(14108978636385284876LLU), QU( 7242414665378826984LLU), + QU( 6561316938846562432LLU), QU( 8601038474994665795LLU), + QU(17532942353612365904LLU), QU(17940076637020912186LLU), + QU( 7340260368823171304LLU), QU( 7061807613916067905LLU), + QU(10561734935039519326LLU), QU(17990796503724650862LLU), + QU( 6208732943911827159LLU), QU( 359077562804090617LLU), + QU(14177751537784403113LLU), QU(10659599444915362902LLU), + QU(15081727220615085833LLU), QU(13417573895659757486LLU), + QU(15513842342017811524LLU), QU(11814141516204288231LLU), + QU( 1827312513875101814LLU), QU( 2804611699894603103LLU), + QU(17116500469975602763LLU), QU(12270191815211952087LLU), + QU(12256358467786024988LLU), QU(18435021722453971267LLU), + QU( 671330264390865618LLU), QU( 476504300460286050LLU), + QU(16465470901027093441LLU), QU( 4047724406247136402LLU), + QU( 1322305451411883346LLU), QU( 1388308688834322280LLU), + QU( 7303989085269758176LLU), QU( 9323792664765233642LLU), + QU( 4542762575316368936LLU), QU(17342696132794337618LLU), + QU( 4588025054768498379LLU), QU(13415475057390330804LLU), + QU(17880279491733405570LLU), QU(10610553400618620353LLU), + QU( 3180842072658960139LLU), QU(13002966655454270120LLU), + QU( 1665301181064982826LLU), QU( 7083673946791258979LLU), + QU( 190522247122496820LLU), QU(17388280237250677740LLU), + QU( 8430770379923642945LLU), QU(12987180971921668584LLU), + QU( 2311086108365390642LLU), QU( 2870984383579822345LLU), + QU(14014682609164653318LLU), QU(14467187293062251484LLU), + QU( 192186361147413298LLU), QU(15171951713531796524LLU), + QU( 9900305495015948728LLU), QU(17958004775615466344LLU), + QU(14346380954498606514LLU), QU(18040047357617407096LLU), + QU( 5035237584833424532LLU), QU(15089555460613972287LLU), + QU( 4131411873749729831LLU), QU( 1329013581168250330LLU), + QU(10095353333051193949LLU), QU(10749518561022462716LLU), + QU( 9050611429810755847LLU), QU(15022028840236655649LLU), + QU( 8775554279239748298LLU), QU(13105754025489230502LLU), + QU(15471300118574167585LLU), QU( 89864764002355628LLU), + QU( 8776416323420466637LLU), QU( 5280258630612040891LLU), + QU( 2719174488591862912LLU), QU( 7599309137399661994LLU), + QU(15012887256778039979LLU), QU(14062981725630928925LLU), + QU(12038536286991689603LLU), QU( 7089756544681775245LLU), + QU(10376661532744718039LLU), QU( 1265198725901533130LLU), + QU(13807996727081142408LLU), QU( 2935019626765036403LLU), + QU( 7651672460680700141LLU), QU( 3644093016200370795LLU), + QU( 2840982578090080674LLU), QU(17956262740157449201LLU), + QU(18267979450492880548LLU), QU(11799503659796848070LLU), + QU( 9942537025669672388LLU), QU(11886606816406990297LLU), + QU( 5488594946437447576LLU), QU( 7226714353282744302LLU), + QU( 3784851653123877043LLU), QU( 878018453244803041LLU), + QU(12110022586268616085LLU), QU( 734072179404675123LLU), + QU(11869573627998248542LLU), QU( 469150421297783998LLU), + QU( 260151124912803804LLU), QU(11639179410120968649LLU), + QU( 9318165193840846253LLU), QU(12795671722734758075LLU), + QU(15318410297267253933LLU), QU( 691524703570062620LLU), + QU( 5837129010576994601LLU), QU(15045963859726941052LLU), + QU( 5850056944932238169LLU), QU(12017434144750943807LLU), + QU( 7447139064928956574LLU), QU( 3101711812658245019LLU), + QU(16052940704474982954LLU), QU(18195745945986994042LLU), + QU( 8932252132785575659LLU), QU(13390817488106794834LLU), + QU(11582771836502517453LLU), QU( 4964411326683611686LLU), + QU( 2195093981702694011LLU), QU(14145229538389675669LLU), + QU(16459605532062271798LLU), QU( 866316924816482864LLU), + QU( 4593041209937286377LLU), QU( 8415491391910972138LLU), + QU( 4171236715600528969LLU), QU(16637569303336782889LLU), + QU( 2002011073439212680LLU), QU(17695124661097601411LLU), + QU( 4627687053598611702LLU), QU( 7895831936020190403LLU), + QU( 8455951300917267802LLU), QU( 2923861649108534854LLU), + QU( 8344557563927786255LLU), QU( 6408671940373352556LLU), + QU(12210227354536675772LLU), QU(14294804157294222295LLU), + QU(10103022425071085127LLU), QU(10092959489504123771LLU), + QU( 6554774405376736268LLU), QU(12629917718410641774LLU), + QU( 6260933257596067126LLU), QU( 2460827021439369673LLU), + QU( 2541962996717103668LLU), QU( 597377203127351475LLU), + QU( 5316984203117315309LLU), QU( 4811211393563241961LLU), + QU(13119698597255811641LLU), QU( 8048691512862388981LLU), + QU(10216818971194073842LLU), QU( 4612229970165291764LLU), + QU(10000980798419974770LLU), QU( 6877640812402540687LLU), + QU( 1488727563290436992LLU), QU( 2227774069895697318LLU), + QU(11237754507523316593LLU), QU(13478948605382290972LLU), + QU( 1963583846976858124LLU), QU( 5512309205269276457LLU), + QU( 3972770164717652347LLU), QU( 3841751276198975037LLU), + QU(10283343042181903117LLU), QU( 8564001259792872199LLU), + QU(16472187244722489221LLU), QU( 8953493499268945921LLU), + QU( 3518747340357279580LLU), QU( 4003157546223963073LLU), + QU( 3270305958289814590LLU), QU( 3966704458129482496LLU), + QU( 8122141865926661939LLU), QU(14627734748099506653LLU), + QU(13064426990862560568LLU), QU( 2414079187889870829LLU), + QU( 5378461209354225306LLU), QU(10841985740128255566LLU), + QU( 538582442885401738LLU), QU( 7535089183482905946LLU), + QU(16117559957598879095LLU), QU( 8477890721414539741LLU), + QU( 1459127491209533386LLU), QU(17035126360733620462LLU), + QU( 8517668552872379126LLU), QU(10292151468337355014LLU), + QU(17081267732745344157LLU), QU(13751455337946087178LLU), + QU(14026945459523832966LLU), QU( 6653278775061723516LLU), + QU(10619085543856390441LLU), QU( 2196343631481122885LLU), + QU(10045966074702826136LLU), QU(10082317330452718282LLU), + QU( 5920859259504831242LLU), QU( 9951879073426540617LLU), + QU( 7074696649151414158LLU), QU(15808193543879464318LLU), + QU( 7385247772746953374LLU), QU( 3192003544283864292LLU), + QU(18153684490917593847LLU), QU(12423498260668568905LLU), + QU(10957758099756378169LLU), QU(11488762179911016040LLU), + QU( 2099931186465333782LLU), QU(11180979581250294432LLU), + QU( 8098916250668367933LLU), QU( 3529200436790763465LLU), + QU(12988418908674681745LLU), QU( 6147567275954808580LLU), + QU( 3207503344604030989LLU), QU(10761592604898615360LLU), + QU( 229854861031893504LLU), QU( 8809853962667144291LLU), + QU(13957364469005693860LLU), QU( 7634287665224495886LLU), + QU(12353487366976556874LLU), QU( 1134423796317152034LLU), + QU( 2088992471334107068LLU), QU( 7393372127190799698LLU), + QU( 1845367839871058391LLU), QU( 207922563987322884LLU), + QU(11960870813159944976LLU), QU(12182120053317317363LLU), + QU(17307358132571709283LLU), QU(13871081155552824936LLU), + QU(18304446751741566262LLU), QU( 7178705220184302849LLU), + QU(10929605677758824425LLU), QU(16446976977835806844LLU), + QU(13723874412159769044LLU), QU( 6942854352100915216LLU), + QU( 1726308474365729390LLU), QU( 2150078766445323155LLU), + QU(15345558947919656626LLU), QU(12145453828874527201LLU), + QU( 2054448620739726849LLU), QU( 2740102003352628137LLU), + QU(11294462163577610655LLU), QU( 756164283387413743LLU), + QU(17841144758438810880LLU), QU(10802406021185415861LLU), + QU( 8716455530476737846LLU), QU( 6321788834517649606LLU), + QU(14681322910577468426LLU), QU(17330043563884336387LLU), + QU(12701802180050071614LLU), QU(14695105111079727151LLU), + QU( 5112098511654172830LLU), QU( 4957505496794139973LLU), + QU( 8270979451952045982LLU), QU(12307685939199120969LLU), + QU(12425799408953443032LLU), QU( 8376410143634796588LLU), + QU(16621778679680060464LLU), QU( 3580497854566660073LLU), + QU( 1122515747803382416LLU), QU( 857664980960597599LLU), + QU( 6343640119895925918LLU), QU(12878473260854462891LLU), + QU(10036813920765722626LLU), QU(14451335468363173812LLU), + QU( 5476809692401102807LLU), QU(16442255173514366342LLU), + QU(13060203194757167104LLU), QU(14354124071243177715LLU), + QU(15961249405696125227LLU), QU(13703893649690872584LLU), + QU( 363907326340340064LLU), QU( 6247455540491754842LLU), + QU(12242249332757832361LLU), QU( 156065475679796717LLU), + QU( 9351116235749732355LLU), QU( 4590350628677701405LLU), + QU( 1671195940982350389LLU), QU(13501398458898451905LLU), + QU( 6526341991225002255LLU), QU( 1689782913778157592LLU), + QU( 7439222350869010334LLU), QU(13975150263226478308LLU), + QU(11411961169932682710LLU), QU(17204271834833847277LLU), + QU( 541534742544435367LLU), QU( 6591191931218949684LLU), + QU( 2645454775478232486LLU), QU( 4322857481256485321LLU), + QU( 8477416487553065110LLU), QU(12902505428548435048LLU), + QU( 971445777981341415LLU), QU(14995104682744976712LLU), + QU( 4243341648807158063LLU), QU( 8695061252721927661LLU), + QU( 5028202003270177222LLU), QU( 2289257340915567840LLU), + QU(13870416345121866007LLU), QU(13994481698072092233LLU), + QU( 6912785400753196481LLU), QU( 2278309315841980139LLU), + QU( 4329765449648304839LLU), QU( 5963108095785485298LLU), + QU( 4880024847478722478LLU), QU(16015608779890240947LLU), + QU( 1866679034261393544LLU), QU( 914821179919731519LLU), + QU( 9643404035648760131LLU), QU( 2418114953615593915LLU), + QU( 944756836073702374LLU), QU(15186388048737296834LLU), + QU( 7723355336128442206LLU), QU( 7500747479679599691LLU), + QU(18013961306453293634LLU), QU( 2315274808095756456LLU), + QU(13655308255424029566LLU), QU(17203800273561677098LLU), + QU( 1382158694422087756LLU), QU( 5090390250309588976LLU), + QU( 517170818384213989LLU), QU( 1612709252627729621LLU), + QU( 1330118955572449606LLU), QU( 300922478056709885LLU), + QU(18115693291289091987LLU), QU(13491407109725238321LLU), + QU(15293714633593827320LLU), QU( 5151539373053314504LLU), + QU( 5951523243743139207LLU), QU(14459112015249527975LLU), + QU( 5456113959000700739LLU), QU( 3877918438464873016LLU), + QU(12534071654260163555LLU), QU(15871678376893555041LLU), + QU(11005484805712025549LLU), QU(16353066973143374252LLU), + QU( 4358331472063256685LLU), QU( 8268349332210859288LLU), + QU(12485161590939658075LLU), QU(13955993592854471343LLU), + QU( 5911446886848367039LLU), QU(14925834086813706974LLU), + QU( 6590362597857994805LLU), QU( 1280544923533661875LLU), + QU( 1637756018947988164LLU), QU( 4734090064512686329LLU), + QU(16693705263131485912LLU), QU( 6834882340494360958LLU), + QU( 8120732176159658505LLU), QU( 2244371958905329346LLU), + QU(10447499707729734021LLU), QU( 7318742361446942194LLU), + QU( 8032857516355555296LLU), QU(14023605983059313116LLU), + QU( 1032336061815461376LLU), QU( 9840995337876562612LLU), + QU( 9869256223029203587LLU), QU(12227975697177267636LLU), + QU(12728115115844186033LLU), QU( 7752058479783205470LLU), + QU( 729733219713393087LLU), QU(12954017801239007622LLU) +}; +static const uint64_t init_by_array_64_expected[] = { + QU( 2100341266307895239LLU), QU( 8344256300489757943LLU), + QU(15687933285484243894LLU), QU( 8268620370277076319LLU), + QU(12371852309826545459LLU), QU( 8800491541730110238LLU), + QU(18113268950100835773LLU), QU( 2886823658884438119LLU), + QU( 3293667307248180724LLU), QU( 9307928143300172731LLU), + QU( 7688082017574293629LLU), QU( 900986224735166665LLU), + QU( 9977972710722265039LLU), QU( 6008205004994830552LLU), + QU( 546909104521689292LLU), QU( 7428471521869107594LLU), + QU(14777563419314721179LLU), QU(16116143076567350053LLU), + QU( 5322685342003142329LLU), QU( 4200427048445863473LLU), + QU( 4693092150132559146LLU), QU(13671425863759338582LLU), + QU( 6747117460737639916LLU), QU( 4732666080236551150LLU), + QU( 5912839950611941263LLU), QU( 3903717554504704909LLU), + QU( 2615667650256786818LLU), QU(10844129913887006352LLU), + QU(13786467861810997820LLU), QU(14267853002994021570LLU), + QU(13767807302847237439LLU), QU(16407963253707224617LLU), + QU( 4802498363698583497LLU), QU( 2523802839317209764LLU), + QU( 3822579397797475589LLU), QU( 8950320572212130610LLU), + QU( 3745623504978342534LLU), QU(16092609066068482806LLU), + QU( 9817016950274642398LLU), QU(10591660660323829098LLU), + QU(11751606650792815920LLU), QU( 5122873818577122211LLU), + QU(17209553764913936624LLU), QU( 6249057709284380343LLU), + QU(15088791264695071830LLU), QU(15344673071709851930LLU), + QU( 4345751415293646084LLU), QU( 2542865750703067928LLU), + QU(13520525127852368784LLU), QU(18294188662880997241LLU), + QU( 3871781938044881523LLU), QU( 2873487268122812184LLU), + QU(15099676759482679005LLU), QU(15442599127239350490LLU), + QU( 6311893274367710888LLU), QU( 3286118760484672933LLU), + QU( 4146067961333542189LLU), QU(13303942567897208770LLU), + QU( 8196013722255630418LLU), QU( 4437815439340979989LLU), + QU(15433791533450605135LLU), QU( 4254828956815687049LLU), + QU( 1310903207708286015LLU), QU(10529182764462398549LLU), + QU(14900231311660638810LLU), QU( 9727017277104609793LLU), + QU( 1821308310948199033LLU), QU(11628861435066772084LLU), + QU( 9469019138491546924LLU), QU( 3145812670532604988LLU), + QU( 9938468915045491919LLU), QU( 1562447430672662142LLU), + QU(13963995266697989134LLU), QU( 3356884357625028695LLU), + QU( 4499850304584309747LLU), QU( 8456825817023658122LLU), + QU(10859039922814285279LLU), QU( 8099512337972526555LLU), + QU( 348006375109672149LLU), QU(11919893998241688603LLU), + QU( 1104199577402948826LLU), QU(16689191854356060289LLU), + QU(10992552041730168078LLU), QU( 7243733172705465836LLU), + QU( 5668075606180319560LLU), QU(18182847037333286970LLU), + QU( 4290215357664631322LLU), QU( 4061414220791828613LLU), + QU(13006291061652989604LLU), QU( 7140491178917128798LLU), + QU(12703446217663283481LLU), QU( 5500220597564558267LLU), + QU(10330551509971296358LLU), QU(15958554768648714492LLU), + QU( 5174555954515360045LLU), QU( 1731318837687577735LLU), + QU( 3557700801048354857LLU), QU(13764012341928616198LLU), + QU(13115166194379119043LLU), QU( 7989321021560255519LLU), + QU( 2103584280905877040LLU), QU( 9230788662155228488LLU), + QU(16396629323325547654LLU), QU( 657926409811318051LLU), + QU(15046700264391400727LLU), QU( 5120132858771880830LLU), + QU( 7934160097989028561LLU), QU( 6963121488531976245LLU), + QU(17412329602621742089LLU), QU(15144843053931774092LLU), + QU(17204176651763054532LLU), QU(13166595387554065870LLU), + QU( 8590377810513960213LLU), QU( 5834365135373991938LLU), + QU( 7640913007182226243LLU), QU( 3479394703859418425LLU), + QU(16402784452644521040LLU), QU( 4993979809687083980LLU), + QU(13254522168097688865LLU), QU(15643659095244365219LLU), + QU( 5881437660538424982LLU), QU(11174892200618987379LLU), + QU( 254409966159711077LLU), QU(17158413043140549909LLU), + QU( 3638048789290376272LLU), QU( 1376816930299489190LLU), + QU( 4622462095217761923LLU), QU(15086407973010263515LLU), + QU(13253971772784692238LLU), QU( 5270549043541649236LLU), + QU(11182714186805411604LLU), QU(12283846437495577140LLU), + QU( 5297647149908953219LLU), QU(10047451738316836654LLU), + QU( 4938228100367874746LLU), QU(12328523025304077923LLU), + QU( 3601049438595312361LLU), QU( 9313624118352733770LLU), + QU(13322966086117661798LLU), QU(16660005705644029394LLU), + QU(11337677526988872373LLU), QU(13869299102574417795LLU), + QU(15642043183045645437LLU), QU( 3021755569085880019LLU), + QU( 4979741767761188161LLU), QU(13679979092079279587LLU), + QU( 3344685842861071743LLU), QU(13947960059899588104LLU), + QU( 305806934293368007LLU), QU( 5749173929201650029LLU), + QU(11123724852118844098LLU), QU(15128987688788879802LLU), + QU(15251651211024665009LLU), QU( 7689925933816577776LLU), + QU(16732804392695859449LLU), QU(17087345401014078468LLU), + QU(14315108589159048871LLU), QU( 4820700266619778917LLU), + QU(16709637539357958441LLU), QU( 4936227875177351374LLU), + QU( 2137907697912987247LLU), QU(11628565601408395420LLU), + QU( 2333250549241556786LLU), QU( 5711200379577778637LLU), + QU( 5170680131529031729LLU), QU(12620392043061335164LLU), + QU( 95363390101096078LLU), QU( 5487981914081709462LLU), + QU( 1763109823981838620LLU), QU( 3395861271473224396LLU), + QU( 1300496844282213595LLU), QU( 6894316212820232902LLU), + QU(10673859651135576674LLU), QU( 5911839658857903252LLU), + QU(17407110743387299102LLU), QU( 8257427154623140385LLU), + QU(11389003026741800267LLU), QU( 4070043211095013717LLU), + QU(11663806997145259025LLU), QU(15265598950648798210LLU), + QU( 630585789434030934LLU), QU( 3524446529213587334LLU), + QU( 7186424168495184211LLU), QU(10806585451386379021LLU), + QU(11120017753500499273LLU), QU( 1586837651387701301LLU), + QU(17530454400954415544LLU), QU( 9991670045077880430LLU), + QU( 7550997268990730180LLU), QU( 8640249196597379304LLU), + QU( 3522203892786893823LLU), QU(10401116549878854788LLU), + QU(13690285544733124852LLU), QU( 8295785675455774586LLU), + QU(15535716172155117603LLU), QU( 3112108583723722511LLU), + QU(17633179955339271113LLU), QU(18154208056063759375LLU), + QU( 1866409236285815666LLU), QU(13326075895396412882LLU), + QU( 8756261842948020025LLU), QU( 6281852999868439131LLU), + QU(15087653361275292858LLU), QU(10333923911152949397LLU), + QU( 5265567645757408500LLU), QU(12728041843210352184LLU), + QU( 6347959327507828759LLU), QU( 154112802625564758LLU), + QU(18235228308679780218LLU), QU( 3253805274673352418LLU), + QU( 4849171610689031197LLU), QU(17948529398340432518LLU), + QU(13803510475637409167LLU), QU(13506570190409883095LLU), + QU(15870801273282960805LLU), QU( 8451286481299170773LLU), + QU( 9562190620034457541LLU), QU( 8518905387449138364LLU), + QU(12681306401363385655LLU), QU( 3788073690559762558LLU), + QU( 5256820289573487769LLU), QU( 2752021372314875467LLU), + QU( 6354035166862520716LLU), QU( 4328956378309739069LLU), + QU( 449087441228269600LLU), QU( 5533508742653090868LLU), + QU( 1260389420404746988LLU), QU(18175394473289055097LLU), + QU( 1535467109660399420LLU), QU( 8818894282874061442LLU), + QU(12140873243824811213LLU), QU(15031386653823014946LLU), + QU( 1286028221456149232LLU), QU( 6329608889367858784LLU), + QU( 9419654354945132725LLU), QU( 6094576547061672379LLU), + QU(17706217251847450255LLU), QU( 1733495073065878126LLU), + QU(16918923754607552663LLU), QU( 8881949849954945044LLU), + QU(12938977706896313891LLU), QU(14043628638299793407LLU), + QU(18393874581723718233LLU), QU( 6886318534846892044LLU), + QU(14577870878038334081LLU), QU(13541558383439414119LLU), + QU(13570472158807588273LLU), QU(18300760537910283361LLU), + QU( 818368572800609205LLU), QU( 1417000585112573219LLU), + QU(12337533143867683655LLU), QU(12433180994702314480LLU), + QU( 778190005829189083LLU), QU(13667356216206524711LLU), + QU( 9866149895295225230LLU), QU(11043240490417111999LLU), + QU( 1123933826541378598LLU), QU( 6469631933605123610LLU), + QU(14508554074431980040LLU), QU(13918931242962026714LLU), + QU( 2870785929342348285LLU), QU(14786362626740736974LLU), + QU(13176680060902695786LLU), QU( 9591778613541679456LLU), + QU( 9097662885117436706LLU), QU( 749262234240924947LLU), + QU( 1944844067793307093LLU), QU( 4339214904577487742LLU), + QU( 8009584152961946551LLU), QU(16073159501225501777LLU), + QU( 3335870590499306217LLU), QU(17088312653151202847LLU), + QU( 3108893142681931848LLU), QU(16636841767202792021LLU), + QU(10423316431118400637LLU), QU( 8008357368674443506LLU), + QU(11340015231914677875LLU), QU(17687896501594936090LLU), + QU(15173627921763199958LLU), QU( 542569482243721959LLU), + QU(15071714982769812975LLU), QU( 4466624872151386956LLU), + QU( 1901780715602332461LLU), QU( 9822227742154351098LLU), + QU( 1479332892928648780LLU), QU( 6981611948382474400LLU), + QU( 7620824924456077376LLU), QU(14095973329429406782LLU), + QU( 7902744005696185404LLU), QU(15830577219375036920LLU), + QU(10287076667317764416LLU), QU(12334872764071724025LLU), + QU( 4419302088133544331LLU), QU(14455842851266090520LLU), + QU(12488077416504654222LLU), QU( 7953892017701886766LLU), + QU( 6331484925529519007LLU), QU( 4902145853785030022LLU), + QU(17010159216096443073LLU), QU(11945354668653886087LLU), + QU(15112022728645230829LLU), QU(17363484484522986742LLU), + QU( 4423497825896692887LLU), QU( 8155489510809067471LLU), + QU( 258966605622576285LLU), QU( 5462958075742020534LLU), + QU( 6763710214913276228LLU), QU( 2368935183451109054LLU), + QU(14209506165246453811LLU), QU( 2646257040978514881LLU), + QU( 3776001911922207672LLU), QU( 1419304601390147631LLU), + QU(14987366598022458284LLU), QU( 3977770701065815721LLU), + QU( 730820417451838898LLU), QU( 3982991703612885327LLU), + QU( 2803544519671388477LLU), QU(17067667221114424649LLU), + QU( 2922555119737867166LLU), QU( 1989477584121460932LLU), + QU(15020387605892337354LLU), QU( 9293277796427533547LLU), + QU(10722181424063557247LLU), QU(16704542332047511651LLU), + QU( 5008286236142089514LLU), QU(16174732308747382540LLU), + QU(17597019485798338402LLU), QU(13081745199110622093LLU), + QU( 8850305883842258115LLU), QU(12723629125624589005LLU), + QU( 8140566453402805978LLU), QU(15356684607680935061LLU), + QU(14222190387342648650LLU), QU(11134610460665975178LLU), + QU( 1259799058620984266LLU), QU(13281656268025610041LLU), + QU( 298262561068153992LLU), QU(12277871700239212922LLU), + QU(13911297774719779438LLU), QU(16556727962761474934LLU), + QU(17903010316654728010LLU), QU( 9682617699648434744LLU), + QU(14757681836838592850LLU), QU( 1327242446558524473LLU), + QU(11126645098780572792LLU), QU( 1883602329313221774LLU), + QU( 2543897783922776873LLU), QU(15029168513767772842LLU), + QU(12710270651039129878LLU), QU(16118202956069604504LLU), + QU(15010759372168680524LLU), QU( 2296827082251923948LLU), + QU(10793729742623518101LLU), QU(13829764151845413046LLU), + QU(17769301223184451213LLU), QU( 3118268169210783372LLU), + QU(17626204544105123127LLU), QU( 7416718488974352644LLU), + QU(10450751996212925994LLU), QU( 9352529519128770586LLU), + QU( 259347569641110140LLU), QU( 8048588892269692697LLU), + QU( 1774414152306494058LLU), QU(10669548347214355622LLU), + QU(13061992253816795081LLU), QU(18432677803063861659LLU), + QU( 8879191055593984333LLU), QU(12433753195199268041LLU), + QU(14919392415439730602LLU), QU( 6612848378595332963LLU), + QU( 6320986812036143628LLU), QU(10465592420226092859LLU), + QU( 4196009278962570808LLU), QU( 3747816564473572224LLU), + QU(17941203486133732898LLU), QU( 2350310037040505198LLU), + QU( 5811779859134370113LLU), QU(10492109599506195126LLU), + QU( 7699650690179541274LLU), QU( 1954338494306022961LLU), + QU(14095816969027231152LLU), QU( 5841346919964852061LLU), + QU(14945969510148214735LLU), QU( 3680200305887550992LLU), + QU( 6218047466131695792LLU), QU( 8242165745175775096LLU), + QU(11021371934053307357LLU), QU( 1265099502753169797LLU), + QU( 4644347436111321718LLU), QU( 3609296916782832859LLU), + QU( 8109807992218521571LLU), QU(18387884215648662020LLU), + QU(14656324896296392902LLU), QU(17386819091238216751LLU), + QU(17788300878582317152LLU), QU( 7919446259742399591LLU), + QU( 4466613134576358004LLU), QU(12928181023667938509LLU), + QU(13147446154454932030LLU), QU(16552129038252734620LLU), + QU( 8395299403738822450LLU), QU(11313817655275361164LLU), + QU( 434258809499511718LLU), QU( 2074882104954788676LLU), + QU( 7929892178759395518LLU), QU( 9006461629105745388LLU), + QU( 5176475650000323086LLU), QU(11128357033468341069LLU), + QU(12026158851559118955LLU), QU(14699716249471156500LLU), + QU( 448982497120206757LLU), QU( 4156475356685519900LLU), + QU( 6063816103417215727LLU), QU(10073289387954971479LLU), + QU( 8174466846138590962LLU), QU( 2675777452363449006LLU), + QU( 9090685420572474281LLU), QU( 6659652652765562060LLU), + QU(12923120304018106621LLU), QU(11117480560334526775LLU), + QU( 937910473424587511LLU), QU( 1838692113502346645LLU), + QU(11133914074648726180LLU), QU( 7922600945143884053LLU), + QU(13435287702700959550LLU), QU( 5287964921251123332LLU), + QU(11354875374575318947LLU), QU(17955724760748238133LLU), + QU(13728617396297106512LLU), QU( 4107449660118101255LLU), + QU( 1210269794886589623LLU), QU(11408687205733456282LLU), + QU( 4538354710392677887LLU), QU(13566803319341319267LLU), + QU(17870798107734050771LLU), QU( 3354318982568089135LLU), + QU( 9034450839405133651LLU), QU(13087431795753424314LLU), + QU( 950333102820688239LLU), QU( 1968360654535604116LLU), + QU(16840551645563314995LLU), QU( 8867501803892924995LLU), + QU(11395388644490626845LLU), QU( 1529815836300732204LLU), + QU(13330848522996608842LLU), QU( 1813432878817504265LLU), + QU( 2336867432693429560LLU), QU(15192805445973385902LLU), + QU( 2528593071076407877LLU), QU( 128459777936689248LLU), + QU( 9976345382867214866LLU), QU( 6208885766767996043LLU), + QU(14982349522273141706LLU), QU( 3099654362410737822LLU), + QU(13776700761947297661LLU), QU( 8806185470684925550LLU), + QU( 8151717890410585321LLU), QU( 640860591588072925LLU), + QU(14592096303937307465LLU), QU( 9056472419613564846LLU), + QU(14861544647742266352LLU), QU(12703771500398470216LLU), + QU( 3142372800384138465LLU), QU( 6201105606917248196LLU), + QU(18337516409359270184LLU), QU(15042268695665115339LLU), + QU(15188246541383283846LLU), QU(12800028693090114519LLU), + QU( 5992859621101493472LLU), QU(18278043971816803521LLU), + QU( 9002773075219424560LLU), QU( 7325707116943598353LLU), + QU( 7930571931248040822LLU), QU( 5645275869617023448LLU), + QU( 7266107455295958487LLU), QU( 4363664528273524411LLU), + QU(14313875763787479809LLU), QU(17059695613553486802LLU), + QU( 9247761425889940932LLU), QU(13704726459237593128LLU), + QU( 2701312427328909832LLU), QU(17235532008287243115LLU), + QU(14093147761491729538LLU), QU( 6247352273768386516LLU), + QU( 8268710048153268415LLU), QU( 7985295214477182083LLU), + QU(15624495190888896807LLU), QU( 3772753430045262788LLU), + QU( 9133991620474991698LLU), QU( 5665791943316256028LLU), + QU( 7551996832462193473LLU), QU(13163729206798953877LLU), + QU( 9263532074153846374LLU), QU( 1015460703698618353LLU), + QU(17929874696989519390LLU), QU(18257884721466153847LLU), + QU(16271867543011222991LLU), QU( 3905971519021791941LLU), + QU(16814488397137052085LLU), QU( 1321197685504621613LLU), + QU( 2870359191894002181LLU), QU(14317282970323395450LLU), + QU(13663920845511074366LLU), QU( 2052463995796539594LLU), + QU(14126345686431444337LLU), QU( 1727572121947022534LLU), + QU(17793552254485594241LLU), QU( 6738857418849205750LLU), + QU( 1282987123157442952LLU), QU(16655480021581159251LLU), + QU( 6784587032080183866LLU), QU(14726758805359965162LLU), + QU( 7577995933961987349LLU), QU(12539609320311114036LLU), + QU(10789773033385439494LLU), QU( 8517001497411158227LLU), + QU(10075543932136339710LLU), QU(14838152340938811081LLU), + QU( 9560840631794044194LLU), QU(17445736541454117475LLU), + QU(10633026464336393186LLU), QU(15705729708242246293LLU), + QU( 1117517596891411098LLU), QU( 4305657943415886942LLU), + QU( 4948856840533979263LLU), QU(16071681989041789593LLU), + QU(13723031429272486527LLU), QU( 7639567622306509462LLU), + QU(12670424537483090390LLU), QU( 9715223453097197134LLU), + QU( 5457173389992686394LLU), QU( 289857129276135145LLU), + QU(17048610270521972512LLU), QU( 692768013309835485LLU), + QU(14823232360546632057LLU), QU(18218002361317895936LLU), + QU( 3281724260212650204LLU), QU(16453957266549513795LLU), + QU( 8592711109774511881LLU), QU( 929825123473369579LLU), + QU(15966784769764367791LLU), QU( 9627344291450607588LLU), + QU(10849555504977813287LLU), QU( 9234566913936339275LLU), + QU( 6413807690366911210LLU), QU(10862389016184219267LLU), + QU(13842504799335374048LLU), QU( 1531994113376881174LLU), + QU( 2081314867544364459LLU), QU(16430628791616959932LLU), + QU( 8314714038654394368LLU), QU( 9155473892098431813LLU), + QU(12577843786670475704LLU), QU( 4399161106452401017LLU), + QU( 1668083091682623186LLU), QU( 1741383777203714216LLU), + QU( 2162597285417794374LLU), QU(15841980159165218736LLU), + QU( 1971354603551467079LLU), QU( 1206714764913205968LLU), + QU( 4790860439591272330LLU), QU(14699375615594055799LLU), + QU( 8374423871657449988LLU), QU(10950685736472937738LLU), + QU( 697344331343267176LLU), QU(10084998763118059810LLU), + QU(12897369539795983124LLU), QU(12351260292144383605LLU), + QU( 1268810970176811234LLU), QU( 7406287800414582768LLU), + QU( 516169557043807831LLU), QU( 5077568278710520380LLU), + QU( 3828791738309039304LLU), QU( 7721974069946943610LLU), + QU( 3534670260981096460LLU), QU( 4865792189600584891LLU), + QU(16892578493734337298LLU), QU( 9161499464278042590LLU), + QU(11976149624067055931LLU), QU(13219479887277343990LLU), + QU(14161556738111500680LLU), QU(14670715255011223056LLU), + QU( 4671205678403576558LLU), QU(12633022931454259781LLU), + QU(14821376219869187646LLU), QU( 751181776484317028LLU), + QU( 2192211308839047070LLU), QU(11787306362361245189LLU), + QU(10672375120744095707LLU), QU( 4601972328345244467LLU), + QU(15457217788831125879LLU), QU( 8464345256775460809LLU), + QU(10191938789487159478LLU), QU( 6184348739615197613LLU), + QU(11425436778806882100LLU), QU( 2739227089124319793LLU), + QU( 461464518456000551LLU), QU( 4689850170029177442LLU), + QU( 6120307814374078625LLU), QU(11153579230681708671LLU), + QU( 7891721473905347926LLU), QU(10281646937824872400LLU), + QU( 3026099648191332248LLU), QU( 8666750296953273818LLU), + QU(14978499698844363232LLU), QU(13303395102890132065LLU), + QU( 8182358205292864080LLU), QU(10560547713972971291LLU), + QU(11981635489418959093LLU), QU( 3134621354935288409LLU), + QU(11580681977404383968LLU), QU(14205530317404088650LLU), + QU( 5997789011854923157LLU), QU(13659151593432238041LLU), + QU(11664332114338865086LLU), QU( 7490351383220929386LLU), + QU( 7189290499881530378LLU), QU(15039262734271020220LLU), + QU( 2057217285976980055LLU), QU( 555570804905355739LLU), + QU(11235311968348555110LLU), QU(13824557146269603217LLU), + QU(16906788840653099693LLU), QU( 7222878245455661677LLU), + QU( 5245139444332423756LLU), QU( 4723748462805674292LLU), + QU(12216509815698568612LLU), QU(17402362976648951187LLU), + QU(17389614836810366768LLU), QU( 4880936484146667711LLU), + QU( 9085007839292639880LLU), QU(13837353458498535449LLU), + QU(11914419854360366677LLU), QU(16595890135313864103LLU), + QU( 6313969847197627222LLU), QU(18296909792163910431LLU), + QU(10041780113382084042LLU), QU( 2499478551172884794LLU), + QU(11057894246241189489LLU), QU( 9742243032389068555LLU), + QU(12838934582673196228LLU), QU(13437023235248490367LLU), + QU(13372420669446163240LLU), QU( 6752564244716909224LLU), + QU( 7157333073400313737LLU), QU(12230281516370654308LLU), + QU( 1182884552219419117LLU), QU( 2955125381312499218LLU), + QU(10308827097079443249LLU), QU( 1337648572986534958LLU), + QU(16378788590020343939LLU), QU( 108619126514420935LLU), + QU( 3990981009621629188LLU), QU( 5460953070230946410LLU), + QU( 9703328329366531883LLU), QU(13166631489188077236LLU), + QU( 1104768831213675170LLU), QU( 3447930458553877908LLU), + QU( 8067172487769945676LLU), QU( 5445802098190775347LLU), + QU( 3244840981648973873LLU), QU(17314668322981950060LLU), + QU( 5006812527827763807LLU), QU(18158695070225526260LLU), + QU( 2824536478852417853LLU), QU(13974775809127519886LLU), + QU( 9814362769074067392LLU), QU(17276205156374862128LLU), + QU(11361680725379306967LLU), QU( 3422581970382012542LLU), + QU(11003189603753241266LLU), QU(11194292945277862261LLU), + QU( 6839623313908521348LLU), QU(11935326462707324634LLU), + QU( 1611456788685878444LLU), QU(13112620989475558907LLU), + QU( 517659108904450427LLU), QU(13558114318574407624LLU), + QU(15699089742731633077LLU), QU( 4988979278862685458LLU), + QU( 8111373583056521297LLU), QU( 3891258746615399627LLU), + QU( 8137298251469718086LLU), QU(12748663295624701649LLU), + QU( 4389835683495292062LLU), QU( 5775217872128831729LLU), + QU( 9462091896405534927LLU), QU( 8498124108820263989LLU), + QU( 8059131278842839525LLU), QU(10503167994254090892LLU), + QU(11613153541070396656LLU), QU(18069248738504647790LLU), + QU( 570657419109768508LLU), QU( 3950574167771159665LLU), + QU( 5514655599604313077LLU), QU( 2908460854428484165LLU), + QU(10777722615935663114LLU), QU(12007363304839279486LLU), + QU( 9800646187569484767LLU), QU( 8795423564889864287LLU), + QU(14257396680131028419LLU), QU( 6405465117315096498LLU), + QU( 7939411072208774878LLU), QU(17577572378528990006LLU), + QU(14785873806715994850LLU), QU(16770572680854747390LLU), + QU(18127549474419396481LLU), QU(11637013449455757750LLU), + QU(14371851933996761086LLU), QU( 3601181063650110280LLU), + QU( 4126442845019316144LLU), QU(10198287239244320669LLU), + QU(18000169628555379659LLU), QU(18392482400739978269LLU), + QU( 6219919037686919957LLU), QU( 3610085377719446052LLU), + QU( 2513925039981776336LLU), QU(16679413537926716955LLU), + QU(12903302131714909434LLU), QU( 5581145789762985009LLU), + QU(12325955044293303233LLU), QU(17216111180742141204LLU), + QU( 6321919595276545740LLU), QU( 3507521147216174501LLU), + QU( 9659194593319481840LLU), QU(11473976005975358326LLU), + QU(14742730101435987026LLU), QU( 492845897709954780LLU), + QU(16976371186162599676LLU), QU(17712703422837648655LLU), + QU( 9881254778587061697LLU), QU( 8413223156302299551LLU), + QU( 1563841828254089168LLU), QU( 9996032758786671975LLU), + QU( 138877700583772667LLU), QU(13003043368574995989LLU), + QU( 4390573668650456587LLU), QU( 8610287390568126755LLU), + QU(15126904974266642199LLU), QU( 6703637238986057662LLU), + QU( 2873075592956810157LLU), QU( 6035080933946049418LLU), + QU(13382846581202353014LLU), QU( 7303971031814642463LLU), + QU(18418024405307444267LLU), QU( 5847096731675404647LLU), + QU( 4035880699639842500LLU), QU(11525348625112218478LLU), + QU( 3041162365459574102LLU), QU( 2604734487727986558LLU), + QU(15526341771636983145LLU), QU(14556052310697370254LLU), + QU(12997787077930808155LLU), QU( 9601806501755554499LLU), + QU(11349677952521423389LLU), QU(14956777807644899350LLU), + QU(16559736957742852721LLU), QU(12360828274778140726LLU), + QU( 6685373272009662513LLU), QU(16932258748055324130LLU), + QU(15918051131954158508LLU), QU( 1692312913140790144LLU), + QU( 546653826801637367LLU), QU( 5341587076045986652LLU), + QU(14975057236342585662LLU), QU(12374976357340622412LLU), + QU(10328833995181940552LLU), QU(12831807101710443149LLU), + QU(10548514914382545716LLU), QU( 2217806727199715993LLU), + QU(12627067369242845138LLU), QU( 4598965364035438158LLU), + QU( 150923352751318171LLU), QU(14274109544442257283LLU), + QU( 4696661475093863031LLU), QU( 1505764114384654516LLU), + QU(10699185831891495147LLU), QU( 2392353847713620519LLU), + QU( 3652870166711788383LLU), QU( 8640653276221911108LLU), + QU( 3894077592275889704LLU), QU( 4918592872135964845LLU), + QU(16379121273281400789LLU), QU(12058465483591683656LLU), + QU(11250106829302924945LLU), QU( 1147537556296983005LLU), + QU( 6376342756004613268LLU), QU(14967128191709280506LLU), + QU(18007449949790627628LLU), QU( 9497178279316537841LLU), + QU( 7920174844809394893LLU), QU(10037752595255719907LLU), + QU(15875342784985217697LLU), QU(15311615921712850696LLU), + QU( 9552902652110992950LLU), QU(14054979450099721140LLU), + QU( 5998709773566417349LLU), QU(18027910339276320187LLU), + QU( 8223099053868585554LLU), QU( 7842270354824999767LLU), + QU( 4896315688770080292LLU), QU(12969320296569787895LLU), + QU( 2674321489185759961LLU), QU( 4053615936864718439LLU), + QU(11349775270588617578LLU), QU( 4743019256284553975LLU), + QU( 5602100217469723769LLU), QU(14398995691411527813LLU), + QU( 7412170493796825470LLU), QU( 836262406131744846LLU), + QU( 8231086633845153022LLU), QU( 5161377920438552287LLU), + QU( 8828731196169924949LLU), QU(16211142246465502680LLU), + QU( 3307990879253687818LLU), QU( 5193405406899782022LLU), + QU( 8510842117467566693LLU), QU( 6070955181022405365LLU), + QU(14482950231361409799LLU), QU(12585159371331138077LLU), + QU( 3511537678933588148LLU), QU( 2041849474531116417LLU), + QU(10944936685095345792LLU), QU(18303116923079107729LLU), + QU( 2720566371239725320LLU), QU( 4958672473562397622LLU), + QU( 3032326668253243412LLU), QU(13689418691726908338LLU), + QU( 1895205511728843996LLU), QU( 8146303515271990527LLU), + QU(16507343500056113480LLU), QU( 473996939105902919LLU), + QU( 9897686885246881481LLU), QU(14606433762712790575LLU), + QU( 6732796251605566368LLU), QU( 1399778120855368916LLU), + QU( 935023885182833777LLU), QU(16066282816186753477LLU), + QU( 7291270991820612055LLU), QU(17530230393129853844LLU), + QU(10223493623477451366LLU), QU(15841725630495676683LLU), + QU(17379567246435515824LLU), QU( 8588251429375561971LLU), + QU(18339511210887206423LLU), QU(17349587430725976100LLU), + QU(12244876521394838088LLU), QU( 6382187714147161259LLU), + QU(12335807181848950831LLU), QU(16948885622305460665LLU), + QU(13755097796371520506LLU), QU(14806740373324947801LLU), + QU( 4828699633859287703LLU), QU( 8209879281452301604LLU), + QU(12435716669553736437LLU), QU(13970976859588452131LLU), + QU( 6233960842566773148LLU), QU(12507096267900505759LLU), + QU( 1198713114381279421LLU), QU(14989862731124149015LLU), + QU(15932189508707978949LLU), QU( 2526406641432708722LLU), + QU( 29187427817271982LLU), QU( 1499802773054556353LLU), + QU(10816638187021897173LLU), QU( 5436139270839738132LLU), + QU( 6659882287036010082LLU), QU( 2154048955317173697LLU), + QU(10887317019333757642LLU), QU(16281091802634424955LLU), + QU(10754549879915384901LLU), QU(10760611745769249815LLU), + QU( 2161505946972504002LLU), QU( 5243132808986265107LLU), + QU(10129852179873415416LLU), QU( 710339480008649081LLU), + QU( 7802129453068808528LLU), QU(17967213567178907213LLU), + QU(15730859124668605599LLU), QU(13058356168962376502LLU), + QU( 3701224985413645909LLU), QU(14464065869149109264LLU), + QU( 9959272418844311646LLU), QU(10157426099515958752LLU), + QU(14013736814538268528LLU), QU(17797456992065653951LLU), + QU(17418878140257344806LLU), QU(15457429073540561521LLU), + QU( 2184426881360949378LLU), QU( 2062193041154712416LLU), + QU( 8553463347406931661LLU), QU( 4913057625202871854LLU), + QU( 2668943682126618425LLU), QU(17064444737891172288LLU), + QU( 4997115903913298637LLU), QU(12019402608892327416LLU), + QU(17603584559765897352LLU), QU(11367529582073647975LLU), + QU( 8211476043518436050LLU), QU( 8676849804070323674LLU), + QU(18431829230394475730LLU), QU(10490177861361247904LLU), + QU( 9508720602025651349LLU), QU( 7409627448555722700LLU), + QU( 5804047018862729008LLU), QU(11943858176893142594LLU), + QU(11908095418933847092LLU), QU( 5415449345715887652LLU), + QU( 1554022699166156407LLU), QU( 9073322106406017161LLU), + QU( 7080630967969047082LLU), QU(18049736940860732943LLU), + QU(12748714242594196794LLU), QU( 1226992415735156741LLU), + QU(17900981019609531193LLU), QU(11720739744008710999LLU), + QU( 3006400683394775434LLU), QU(11347974011751996028LLU), + QU( 3316999628257954608LLU), QU( 8384484563557639101LLU), + QU(18117794685961729767LLU), QU( 1900145025596618194LLU), + QU(17459527840632892676LLU), QU( 5634784101865710994LLU), + QU( 7918619300292897158LLU), QU( 3146577625026301350LLU), + QU( 9955212856499068767LLU), QU( 1873995843681746975LLU), + QU( 1561487759967972194LLU), QU( 8322718804375878474LLU), + QU(11300284215327028366LLU), QU( 4667391032508998982LLU), + QU( 9820104494306625580LLU), QU(17922397968599970610LLU), + QU( 1784690461886786712LLU), QU(14940365084341346821LLU), + QU( 5348719575594186181LLU), QU(10720419084507855261LLU), + QU(14210394354145143274LLU), QU( 2426468692164000131LLU), + QU(16271062114607059202LLU), QU(14851904092357070247LLU), + QU( 6524493015693121897LLU), QU( 9825473835127138531LLU), + QU(14222500616268569578LLU), QU(15521484052007487468LLU), + QU(14462579404124614699LLU), QU(11012375590820665520LLU), + QU(11625327350536084927LLU), QU(14452017765243785417LLU), + QU( 9989342263518766305LLU), QU( 3640105471101803790LLU), + QU( 4749866455897513242LLU), QU(13963064946736312044LLU), + QU(10007416591973223791LLU), QU(18314132234717431115LLU), + QU( 3286596588617483450LLU), QU( 7726163455370818765LLU), + QU( 7575454721115379328LLU), QU( 5308331576437663422LLU), + QU(18288821894903530934LLU), QU( 8028405805410554106LLU), + QU(15744019832103296628LLU), QU( 149765559630932100LLU), + QU( 6137705557200071977LLU), QU(14513416315434803615LLU), + QU(11665702820128984473LLU), QU( 218926670505601386LLU), + QU( 6868675028717769519LLU), QU(15282016569441512302LLU), + QU( 5707000497782960236LLU), QU( 6671120586555079567LLU), + QU( 2194098052618985448LLU), QU(16849577895477330978LLU), + QU(12957148471017466283LLU), QU( 1997805535404859393LLU), + QU( 1180721060263860490LLU), QU(13206391310193756958LLU), + QU(12980208674461861797LLU), QU( 3825967775058875366LLU), + QU(17543433670782042631LLU), QU( 1518339070120322730LLU), + QU(16344584340890991669LLU), QU( 2611327165318529819LLU), + QU(11265022723283422529LLU), QU( 4001552800373196817LLU), + QU(14509595890079346161LLU), QU( 3528717165416234562LLU), + QU(18153222571501914072LLU), QU( 9387182977209744425LLU), + QU(10064342315985580021LLU), QU(11373678413215253977LLU), + QU( 2308457853228798099LLU), QU( 9729042942839545302LLU), + QU( 7833785471140127746LLU), QU( 6351049900319844436LLU), + QU(14454610627133496067LLU), QU(12533175683634819111LLU), + QU(15570163926716513029LLU), QU(13356980519185762498LLU) +}; + +TEST_BEGIN(test_gen_rand_32) +{ + uint32_t array32[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + uint32_t array32_2[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + int i; + uint32_t r32; + sfmt_t *ctx; + + assert_d_le(get_min_array_size32(), BLOCK_SIZE, + "Array size too small"); + ctx = init_gen_rand(1234); + fill_array32(ctx, array32, BLOCK_SIZE); + fill_array32(ctx, array32_2, BLOCK_SIZE); + fini_gen_rand(ctx); + + ctx = init_gen_rand(1234); + for (i = 0; i < BLOCK_SIZE; i++) { + if (i < COUNT_1) { + assert_u32_eq(array32[i], init_gen_rand_32_expected[i], + "Output mismatch for i=%d", i); + } + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32[i], + "Mismatch at array32[%d]=%x, gen=%x", i, array32[i], r32); + } + for (i = 0; i < COUNT_2; i++) { + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32_2[i], + "Mismatch at array32_2[%d]=%x, gen=%x", i, array32_2[i], + r32); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_by_array_32) +{ + uint32_t array32[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + uint32_t array32_2[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + int i; + uint32_t ini[4] = {0x1234, 0x5678, 0x9abc, 0xdef0}; + uint32_t r32; + sfmt_t *ctx; + + assert_d_le(get_min_array_size32(), BLOCK_SIZE, + "Array size too small"); + ctx = init_by_array(ini, 4); + fill_array32(ctx, array32, BLOCK_SIZE); + fill_array32(ctx, array32_2, BLOCK_SIZE); + fini_gen_rand(ctx); + + ctx = init_by_array(ini, 4); + for (i = 0; i < BLOCK_SIZE; i++) { + if (i < COUNT_1) { + assert_u32_eq(array32[i], init_by_array_32_expected[i], + "Output mismatch for i=%d", i); + } + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32[i], + "Mismatch at array32[%d]=%x, gen=%x", i, array32[i], r32); + } + for (i = 0; i < COUNT_2; i++) { + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32_2[i], + "Mismatch at array32_2[%d]=%x, gen=%x", i, array32_2[i], + r32); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_gen_rand_64) +{ + uint64_t array64[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + uint64_t array64_2[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + int i; + uint64_t r; + sfmt_t *ctx; + + assert_d_le(get_min_array_size64(), BLOCK_SIZE64, + "Array size too small"); + ctx = init_gen_rand(4321); + fill_array64(ctx, array64, BLOCK_SIZE64); + fill_array64(ctx, array64_2, BLOCK_SIZE64); + fini_gen_rand(ctx); + + ctx = init_gen_rand(4321); + for (i = 0; i < BLOCK_SIZE64; i++) { + if (i < COUNT_1) { + assert_u64_eq(array64[i], init_gen_rand_64_expected[i], + "Output mismatch for i=%d", i); + } + r = gen_rand64(ctx); + assert_u64_eq(r, array64[i], + "Mismatch at array64[%d]=%"PRIx64", gen=%"PRIx64, i, + array64[i], r); + } + for (i = 0; i < COUNT_2; i++) { + r = gen_rand64(ctx); + assert_u64_eq(r, array64_2[i], + "Mismatch at array64_2[%d]=%"PRIx64" gen=%"PRIx64"", i, + array64_2[i], r); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_by_array_64) +{ + uint64_t array64[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + uint64_t array64_2[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + int i; + uint64_t r; + uint32_t ini[] = {5, 4, 3, 2, 1}; + sfmt_t *ctx; + + assert_d_le(get_min_array_size64(), BLOCK_SIZE64, + "Array size too small"); + ctx = init_by_array(ini, 5); + fill_array64(ctx, array64, BLOCK_SIZE64); + fill_array64(ctx, array64_2, BLOCK_SIZE64); + fini_gen_rand(ctx); + + ctx = init_by_array(ini, 5); + for (i = 0; i < BLOCK_SIZE64; i++) { + if (i < COUNT_1) { + assert_u64_eq(array64[i], init_by_array_64_expected[i], + "Output mismatch for i=%d", i); + } + r = gen_rand64(ctx); + assert_u64_eq(r, array64[i], + "Mismatch at array64[%d]=%"PRIx64" gen=%"PRIx64, i, + array64[i], r); + } + for (i = 0; i < COUNT_2; i++) { + r = gen_rand64(ctx); + assert_u64_eq(r, array64_2[i], + "Mismatch at array64_2[%d]=%"PRIx64" gen=%"PRIx64, i, + array64_2[i], r); + } + fini_gen_rand(ctx); +} +TEST_END + +int +main(void) +{ + + return (test( + test_gen_rand_32, + test_by_array_32, + test_gen_rand_64, + test_by_array_64)); +} diff --git a/deps/jemalloc/test/unit/bitmap.c b/deps/jemalloc/test/unit/bitmap.c new file mode 100644 index 00000000000..8086b8885fc --- /dev/null +++ b/deps/jemalloc/test/unit/bitmap.c @@ -0,0 +1,165 @@ +#include "test/jemalloc_test.h" + +#if (LG_BITMAP_MAXBITS > 12) +# define MAXBITS 4500 +#else +# define MAXBITS (1U << LG_BITMAP_MAXBITS) +#endif + +TEST_BEGIN(test_bitmap_size) +{ + size_t i, prev_size; + + prev_size = 0; + for (i = 1; i <= MAXBITS; i++) { + size_t size = bitmap_size(i); + assert_true(size >= prev_size, + "Bitmap size is smaller than expected"); + prev_size = size; + } +} +TEST_END + +TEST_BEGIN(test_bitmap_init) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) { + assert_false(bitmap_get(bitmap, &binfo, j), + "Bit should be unset"); + } + free(bitmap); + } + } +} +TEST_END + +TEST_BEGIN(test_bitmap_set) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); + free(bitmap); + } + } +} +TEST_END + +TEST_BEGIN(test_bitmap_unset) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); + for (j = 0; j < i; j++) + bitmap_unset(bitmap, &binfo, j); + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); + free(bitmap); + } + } +} +TEST_END + +TEST_BEGIN(test_bitmap_sfu) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + ssize_t j; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); + bitmap_init(bitmap, &binfo); + + /* Iteratively set bits starting at the beginning. */ + for (j = 0; j < i; j++) { + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should be just after " + "previous first unset bit"); + } + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); + + /* + * Iteratively unset bits starting at the end, and + * verify that bitmap_sfu() reaches the unset bits. + */ + for (j = i - 1; j >= 0; j--) { + bitmap_unset(bitmap, &binfo, j); + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should the bit previously " + "unset"); + bitmap_unset(bitmap, &binfo, j); + } + assert_false(bitmap_get(bitmap, &binfo, 0), + "Bit should be unset"); + + /* + * Iteratively set bits starting at the beginning, and + * verify that bitmap_sfu() looks past them. + */ + for (j = 1; j < i; j++) { + bitmap_set(bitmap, &binfo, j - 1); + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should be just after the " + "bit previously set"); + bitmap_unset(bitmap, &binfo, j); + } + assert_zd_eq(bitmap_sfu(bitmap, &binfo), i - 1, + "First unset bit should be the last bit"); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); + free(bitmap); + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_bitmap_size, + test_bitmap_init, + test_bitmap_set, + test_bitmap_unset, + test_bitmap_sfu)); +} diff --git a/deps/jemalloc/test/unit/ckh.c b/deps/jemalloc/test/unit/ckh.c new file mode 100644 index 00000000000..b214c279a5e --- /dev/null +++ b/deps/jemalloc/test/unit/ckh.c @@ -0,0 +1,206 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_new_delete) +{ + ckh_t ckh; + + assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), + "Unexpected ckh_new() error"); + ckh_delete(&ckh); + + assert_false(ckh_new(&ckh, 3, ckh_pointer_hash, ckh_pointer_keycomp), + "Unexpected ckh_new() error"); + ckh_delete(&ckh); +} +TEST_END + +TEST_BEGIN(test_count_insert_search_remove) +{ + ckh_t ckh; + const char *strs[] = { + "a string", + "A string", + "a string.", + "A string." + }; + const char *missing = "A string not in the hash table."; + size_t i; + + assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), + "Unexpected ckh_new() error"); + assert_zu_eq(ckh_count(&ckh), 0, + "ckh_count() should return %zu, but it returned %zu", ZU(0), + ckh_count(&ckh)); + + /* Insert. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + ckh_insert(&ckh, strs[i], strs[i]); + assert_zu_eq(ckh_count(&ckh), i+1, + "ckh_count() should return %zu, but it returned %zu", i+1, + ckh_count(&ckh)); + } + + /* Search. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + union { + void *p; + const char *s; + } k, v; + void **kp, **vp; + const char *ks, *vs; + + kp = (i & 1) ? &k.p : NULL; + vp = (i & 2) ? &v.p : NULL; + k.p = NULL; + v.p = NULL; + assert_false(ckh_search(&ckh, strs[i], kp, vp), + "Unexpected ckh_search() error"); + + ks = (i & 1) ? strs[i] : (const char *)NULL; + vs = (i & 2) ? strs[i] : (const char *)NULL; + assert_ptr_eq((void *)ks, (void *)k.s, + "Key mismatch, i=%zu", i); + assert_ptr_eq((void *)vs, (void *)v.s, + "Value mismatch, i=%zu", i); + } + assert_true(ckh_search(&ckh, missing, NULL, NULL), + "Unexpected ckh_search() success"); + + /* Remove. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + union { + void *p; + const char *s; + } k, v; + void **kp, **vp; + const char *ks, *vs; + + kp = (i & 1) ? &k.p : NULL; + vp = (i & 2) ? &v.p : NULL; + k.p = NULL; + v.p = NULL; + assert_false(ckh_remove(&ckh, strs[i], kp, vp), + "Unexpected ckh_remove() error"); + + ks = (i & 1) ? strs[i] : (const char *)NULL; + vs = (i & 2) ? strs[i] : (const char *)NULL; + assert_ptr_eq((void *)ks, (void *)k.s, + "Key mismatch, i=%zu", i); + assert_ptr_eq((void *)vs, (void *)v.s, + "Value mismatch, i=%zu", i); + assert_zu_eq(ckh_count(&ckh), + sizeof(strs)/sizeof(const char *) - i - 1, + "ckh_count() should return %zu, but it returned %zu", + sizeof(strs)/sizeof(const char *) - i - 1, + ckh_count(&ckh)); + } + + ckh_delete(&ckh); +} +TEST_END + +TEST_BEGIN(test_insert_iter_remove) +{ +#define NITEMS ZU(1000) + ckh_t ckh; + void **p[NITEMS]; + void *q, *r; + size_t i; + + assert_false(ckh_new(&ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp), + "Unexpected ckh_new() error"); + + for (i = 0; i < NITEMS; i++) { + p[i] = mallocx(i+1, 0); + assert_ptr_not_null(p[i], "Unexpected mallocx() failure"); + } + + for (i = 0; i < NITEMS; i++) { + size_t j; + + for (j = i; j < NITEMS; j++) { + assert_false(ckh_insert(&ckh, p[j], p[j]), + "Unexpected ckh_insert() failure"); + assert_false(ckh_search(&ckh, p[j], &q, &r), + "Unexpected ckh_search() failure"); + assert_ptr_eq(p[j], q, "Key pointer mismatch"); + assert_ptr_eq(p[j], r, "Value pointer mismatch"); + } + + assert_zu_eq(ckh_count(&ckh), NITEMS, + "ckh_count() should return %zu, but it returned %zu", + NITEMS, ckh_count(&ckh)); + + for (j = i + 1; j < NITEMS; j++) { + assert_false(ckh_search(&ckh, p[j], NULL, NULL), + "Unexpected ckh_search() failure"); + assert_false(ckh_remove(&ckh, p[j], &q, &r), + "Unexpected ckh_remove() failure"); + assert_ptr_eq(p[j], q, "Key pointer mismatch"); + assert_ptr_eq(p[j], r, "Value pointer mismatch"); + assert_true(ckh_search(&ckh, p[j], NULL, NULL), + "Unexpected ckh_search() success"); + assert_true(ckh_remove(&ckh, p[j], &q, &r), + "Unexpected ckh_remove() success"); + } + + { + bool seen[NITEMS]; + size_t tabind; + + memset(seen, 0, sizeof(seen)); + + for (tabind = 0; ckh_iter(&ckh, &tabind, &q, &r) == + false;) { + size_t k; + + assert_ptr_eq(q, r, "Key and val not equal"); + + for (k = 0; k < NITEMS; k++) { + if (p[k] == q) { + assert_false(seen[k], + "Item %zu already seen", k); + seen[k] = true; + break; + } + } + } + + for (j = 0; j < i + 1; j++) + assert_true(seen[j], "Item %zu not seen", j); + for (; j < NITEMS; j++) + assert_false(seen[j], "Item %zu seen", j); + } + } + + for (i = 0; i < NITEMS; i++) { + assert_false(ckh_search(&ckh, p[i], NULL, NULL), + "Unexpected ckh_search() failure"); + assert_false(ckh_remove(&ckh, p[i], &q, &r), + "Unexpected ckh_remove() failure"); + assert_ptr_eq(p[i], q, "Key pointer mismatch"); + assert_ptr_eq(p[i], r, "Value pointer mismatch"); + assert_true(ckh_search(&ckh, p[i], NULL, NULL), + "Unexpected ckh_search() success"); + assert_true(ckh_remove(&ckh, p[i], &q, &r), + "Unexpected ckh_remove() success"); + dallocx(p[i], 0); + } + + assert_zu_eq(ckh_count(&ckh), 0, + "ckh_count() should return %zu, but it returned %zu", ZU(0), + ckh_count(&ckh)); + ckh_delete(&ckh); +#undef NITEMS +} +TEST_END + +int +main(void) +{ + + return (test( + test_new_delete, + test_count_insert_search_remove, + test_insert_iter_remove)); +} diff --git a/deps/jemalloc/test/unit/hash.c b/deps/jemalloc/test/unit/hash.c new file mode 100644 index 00000000000..abb394ac077 --- /dev/null +++ b/deps/jemalloc/test/unit/hash.c @@ -0,0 +1,171 @@ +/* + * This file is based on code that is part of SMHasher + * (https://code.google.com/p/smhasher/), and is subject to the MIT license + * (http://www.opensource.org/licenses/mit-license.php). Both email addresses + * associated with the source code's revision history belong to Austin Appleby, + * and the revision history ranges from 2010 to 2012. Therefore the copyright + * and license are here taken to be: + * + * Copyright (c) 2010-2012 Austin Appleby + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "test/jemalloc_test.h" + +typedef enum { + hash_variant_x86_32, + hash_variant_x86_128, + hash_variant_x64_128 +} hash_variant_t; + +static size_t +hash_variant_bits(hash_variant_t variant) +{ + + switch (variant) { + case hash_variant_x86_32: return (32); + case hash_variant_x86_128: return (128); + case hash_variant_x64_128: return (128); + default: not_reached(); + } +} + +static const char * +hash_variant_string(hash_variant_t variant) +{ + + switch (variant) { + case hash_variant_x86_32: return ("hash_x86_32"); + case hash_variant_x86_128: return ("hash_x86_128"); + case hash_variant_x64_128: return ("hash_x64_128"); + default: not_reached(); + } +} + +static void +hash_variant_verify(hash_variant_t variant) +{ + const size_t hashbytes = hash_variant_bits(variant) / 8; + uint8_t key[256]; + uint8_t hashes[hashbytes * 256]; + uint8_t final[hashbytes]; + unsigned i; + uint32_t computed, expected; + + memset(key, 0, sizeof(key)); + memset(hashes, 0, sizeof(hashes)); + memset(final, 0, sizeof(final)); + + /* + * Hash keys of the form {0}, {0,1}, {0,1,2}, ..., {0,1,...,255} as the + * seed. + */ + for (i = 0; i < 256; i++) { + key[i] = (uint8_t)i; + switch (variant) { + case hash_variant_x86_32: { + uint32_t out; + out = hash_x86_32(key, i, 256-i); + memcpy(&hashes[i*hashbytes], &out, hashbytes); + break; + } case hash_variant_x86_128: { + uint64_t out[2]; + hash_x86_128(key, i, 256-i, out); + memcpy(&hashes[i*hashbytes], out, hashbytes); + break; + } case hash_variant_x64_128: { + uint64_t out[2]; + hash_x64_128(key, i, 256-i, out); + memcpy(&hashes[i*hashbytes], out, hashbytes); + break; + } default: not_reached(); + } + } + + /* Hash the result array. */ + switch (variant) { + case hash_variant_x86_32: { + uint32_t out = hash_x86_32(hashes, hashbytes*256, 0); + memcpy(final, &out, sizeof(out)); + break; + } case hash_variant_x86_128: { + uint64_t out[2]; + hash_x86_128(hashes, hashbytes*256, 0, out); + memcpy(final, out, sizeof(out)); + break; + } case hash_variant_x64_128: { + uint64_t out[2]; + hash_x64_128(hashes, hashbytes*256, 0, out); + memcpy(final, out, sizeof(out)); + break; + } default: not_reached(); + } + + computed = (final[0] << 0) | (final[1] << 8) | (final[2] << 16) | + (final[3] << 24); + + switch (variant) { +#ifdef JEMALLOC_BIG_ENDIAN + case hash_variant_x86_32: expected = 0x6213303eU; break; + case hash_variant_x86_128: expected = 0x266820caU; break; + case hash_variant_x64_128: expected = 0xcc622b6fU; break; +#else + case hash_variant_x86_32: expected = 0xb0f57ee3U; break; + case hash_variant_x86_128: expected = 0xb3ece62aU; break; + case hash_variant_x64_128: expected = 0x6384ba69U; break; +#endif + default: not_reached(); + } + + assert_u32_eq(computed, expected, + "Hash mismatch for %s(): expected %#x but got %#x", + hash_variant_string(variant), expected, computed); +} + +TEST_BEGIN(test_hash_x86_32) +{ + + hash_variant_verify(hash_variant_x86_32); +} +TEST_END + +TEST_BEGIN(test_hash_x86_128) +{ + + hash_variant_verify(hash_variant_x86_128); +} +TEST_END + +TEST_BEGIN(test_hash_x64_128) +{ + + hash_variant_verify(hash_variant_x64_128); +} +TEST_END + +int +main(void) +{ + + return (test( + test_hash_x86_32, + test_hash_x86_128, + test_hash_x64_128)); +} diff --git a/deps/jemalloc/test/unit/junk.c b/deps/jemalloc/test/unit/junk.c new file mode 100644 index 00000000000..85bbf9e2bd3 --- /dev/null +++ b/deps/jemalloc/test/unit/junk.c @@ -0,0 +1,222 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_FILL +const char *malloc_conf = + "abort:false,junk:true,zero:false,redzone:true,quarantine:0"; +#endif + +static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; +static arena_dalloc_junk_large_t *arena_dalloc_junk_large_orig; +static huge_dalloc_junk_t *huge_dalloc_junk_orig; +static void *most_recently_junked; + +static void +arena_dalloc_junk_small_intercept(void *ptr, arena_bin_info_t *bin_info) +{ + size_t i; + + arena_dalloc_junk_small_orig(ptr, bin_info); + for (i = 0; i < bin_info->reg_size; i++) { + assert_c_eq(((char *)ptr)[i], 0x5a, + "Missing junk fill for byte %zu/%zu of deallocated region", + i, bin_info->reg_size); + } + most_recently_junked = ptr; +} + +static void +arena_dalloc_junk_large_intercept(void *ptr, size_t usize) +{ + size_t i; + + arena_dalloc_junk_large_orig(ptr, usize); + for (i = 0; i < usize; i++) { + assert_c_eq(((char *)ptr)[i], 0x5a, + "Missing junk fill for byte %zu/%zu of deallocated region", + i, usize); + } + most_recently_junked = ptr; +} + +static void +huge_dalloc_junk_intercept(void *ptr, size_t usize) +{ + + huge_dalloc_junk_orig(ptr, usize); + /* + * The conditions under which junk filling actually occurs are nuanced + * enough that it doesn't make sense to duplicate the decision logic in + * test code, so don't actually check that the region is junk-filled. + */ + most_recently_junked = ptr; +} + +static void +test_junk(size_t sz_min, size_t sz_max) +{ + char *s; + size_t sz_prev, sz, i; + + arena_dalloc_junk_small_orig = arena_dalloc_junk_small; + arena_dalloc_junk_small = arena_dalloc_junk_small_intercept; + arena_dalloc_junk_large_orig = arena_dalloc_junk_large; + arena_dalloc_junk_large = arena_dalloc_junk_large_intercept; + huge_dalloc_junk_orig = huge_dalloc_junk; + huge_dalloc_junk = huge_dalloc_junk_intercept; + + sz_prev = 0; + s = (char *)mallocx(sz_min, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + + for (sz = sallocx(s, 0); sz <= sz_max; + sz_prev = sz, sz = sallocx(s, 0)) { + if (sz_prev > 0) { + assert_c_eq(s[0], 'a', + "Previously allocated byte %zu/%zu is corrupted", + ZU(0), sz_prev); + assert_c_eq(s[sz_prev-1], 'a', + "Previously allocated byte %zu/%zu is corrupted", + sz_prev-1, sz_prev); + } + + for (i = sz_prev; i < sz; i++) { + assert_c_eq(s[i], 0xa5, + "Newly allocated byte %zu/%zu isn't junk-filled", + i, sz); + s[i] = 'a'; + } + + if (xallocx(s, sz+1, 0, 0) == sz) { + void *junked = (void *)s; + + s = (char *)rallocx(s, sz+1, 0); + assert_ptr_not_null((void *)s, + "Unexpected rallocx() failure"); + if (!config_mremap || sz+1 <= arena_maxclass) { + assert_ptr_eq(most_recently_junked, junked, + "Expected region of size %zu to be " + "junk-filled", + sz); + } + } + } + + dallocx(s, 0); + assert_ptr_eq(most_recently_junked, (void *)s, + "Expected region of size %zu to be junk-filled", sz); + + arena_dalloc_junk_small = arena_dalloc_junk_small_orig; + arena_dalloc_junk_large = arena_dalloc_junk_large_orig; + huge_dalloc_junk = huge_dalloc_junk_orig; +} + +TEST_BEGIN(test_junk_small) +{ + + test_skip_if(!config_fill); + test_junk(1, SMALL_MAXCLASS-1); +} +TEST_END + +TEST_BEGIN(test_junk_large) +{ + + test_skip_if(!config_fill); + test_junk(SMALL_MAXCLASS+1, arena_maxclass); +} +TEST_END + +TEST_BEGIN(test_junk_huge) +{ + + test_skip_if(!config_fill); + test_junk(arena_maxclass+1, chunksize*2); +} +TEST_END + +arena_ralloc_junk_large_t *arena_ralloc_junk_large_orig; +static void *most_recently_trimmed; + +static void +arena_ralloc_junk_large_intercept(void *ptr, size_t old_usize, size_t usize) +{ + + arena_ralloc_junk_large_orig(ptr, old_usize, usize); + assert_zu_eq(old_usize, arena_maxclass, "Unexpected old_usize"); + assert_zu_eq(usize, arena_maxclass-PAGE, "Unexpected usize"); + most_recently_trimmed = ptr; +} + +TEST_BEGIN(test_junk_large_ralloc_shrink) +{ + void *p1, *p2; + + p1 = mallocx(arena_maxclass, 0); + assert_ptr_not_null(p1, "Unexpected mallocx() failure"); + + arena_ralloc_junk_large_orig = arena_ralloc_junk_large; + arena_ralloc_junk_large = arena_ralloc_junk_large_intercept; + + p2 = rallocx(p1, arena_maxclass-PAGE, 0); + assert_ptr_eq(p1, p2, "Unexpected move during shrink"); + + arena_ralloc_junk_large = arena_ralloc_junk_large_orig; + + assert_ptr_eq(most_recently_trimmed, p1, + "Expected trimmed portion of region to be junk-filled"); +} +TEST_END + +static bool detected_redzone_corruption; + +static void +arena_redzone_corruption_replacement(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + detected_redzone_corruption = true; +} + +TEST_BEGIN(test_junk_redzone) +{ + char *s; + arena_redzone_corruption_t *arena_redzone_corruption_orig; + + test_skip_if(!config_fill); + + arena_redzone_corruption_orig = arena_redzone_corruption; + arena_redzone_corruption = arena_redzone_corruption_replacement; + + /* Test underflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[-1] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + /* Test overflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[sallocx(s, 0)] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + arena_redzone_corruption = arena_redzone_corruption_orig; +} +TEST_END + +int +main(void) +{ + + return (test( + test_junk_small, + test_junk_large, + test_junk_huge, + test_junk_large_ralloc_shrink, + test_junk_redzone)); +} diff --git a/deps/jemalloc/test/unit/mallctl.c b/deps/jemalloc/test/unit/mallctl.c new file mode 100644 index 00000000000..31fb8105764 --- /dev/null +++ b/deps/jemalloc/test/unit/mallctl.c @@ -0,0 +1,415 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_mallctl_errors) +{ + uint64_t epoch; + size_t sz; + + assert_d_eq(mallctl("no_such_name", NULL, NULL, NULL, 0), ENOENT, + "mallctl() should return ENOENT for non-existent names"); + + assert_d_eq(mallctl("version", NULL, NULL, "0.0.0", strlen("0.0.0")), + EPERM, "mallctl() should return EPERM on attempt to write " + "read-only value"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)-1), + EINVAL, "mallctl() should return EINVAL for input size mismatch"); + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)+1), + EINVAL, "mallctl() should return EINVAL for input size mismatch"); + + sz = sizeof(epoch)-1; + assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL, + "mallctl() should return EINVAL for output size mismatch"); + sz = sizeof(epoch)+1; + assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL, + "mallctl() should return EINVAL for output size mismatch"); +} +TEST_END + +TEST_BEGIN(test_mallctlnametomib_errors) +{ + size_t mib[1]; + size_t miblen; + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("no_such_name", mib, &miblen), ENOENT, + "mallctlnametomib() should return ENOENT for non-existent names"); +} +TEST_END + +TEST_BEGIN(test_mallctlbymib_errors) +{ + uint64_t epoch; + size_t sz; + size_t mib[1]; + size_t miblen; + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("version", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, "0.0.0", + strlen("0.0.0")), EPERM, "mallctl() should return EPERM on " + "attempt to write read-only value"); + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("epoch", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch, + sizeof(epoch)-1), EINVAL, + "mallctlbymib() should return EINVAL for input size mismatch"); + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch, + sizeof(epoch)+1), EINVAL, + "mallctlbymib() should return EINVAL for input size mismatch"); + + sz = sizeof(epoch)-1; + assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL, + "mallctlbymib() should return EINVAL for output size mismatch"); + sz = sizeof(epoch)+1; + assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL, + "mallctlbymib() should return EINVAL for output size mismatch"); +} +TEST_END + +TEST_BEGIN(test_mallctl_read_write) +{ + uint64_t old_epoch, new_epoch; + size_t sz = sizeof(old_epoch); + + /* Blind. */ + assert_d_eq(mallctl("epoch", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Read. */ + assert_d_eq(mallctl("epoch", &old_epoch, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Write. */ + assert_d_eq(mallctl("epoch", NULL, NULL, &new_epoch, sizeof(new_epoch)), + 0, "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Read+write. */ + assert_d_eq(mallctl("epoch", &old_epoch, &sz, &new_epoch, + sizeof(new_epoch)), 0, "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); +} +TEST_END + +TEST_BEGIN(test_mallctlnametomib_short_mib) +{ + size_t mib[4]; + size_t miblen; + + miblen = 3; + mib[3] = 42; + assert_d_eq(mallctlnametomib("arenas.bin.0.nregs", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + assert_zu_eq(miblen, 3, "Unexpected mib output length"); + assert_zu_eq(mib[3], 42, + "mallctlnametomib() wrote past the end of the input mib"); +} +TEST_END + +TEST_BEGIN(test_mallctl_config) +{ + +#define TEST_MALLCTL_CONFIG(config) do { \ + bool oldval; \ + size_t sz = sizeof(oldval); \ + assert_d_eq(mallctl("config."#config, &oldval, &sz, NULL, 0), \ + 0, "Unexpected mallctl() failure"); \ + assert_b_eq(oldval, config_##config, "Incorrect config value"); \ + assert_zu_eq(sz, sizeof(oldval), "Unexpected output size"); \ +} while (0) + + TEST_MALLCTL_CONFIG(debug); + TEST_MALLCTL_CONFIG(dss); + TEST_MALLCTL_CONFIG(fill); + TEST_MALLCTL_CONFIG(lazy_lock); + TEST_MALLCTL_CONFIG(mremap); + TEST_MALLCTL_CONFIG(munmap); + TEST_MALLCTL_CONFIG(prof); + TEST_MALLCTL_CONFIG(prof_libgcc); + TEST_MALLCTL_CONFIG(prof_libunwind); + TEST_MALLCTL_CONFIG(stats); + TEST_MALLCTL_CONFIG(tcache); + TEST_MALLCTL_CONFIG(tls); + TEST_MALLCTL_CONFIG(utrace); + TEST_MALLCTL_CONFIG(valgrind); + TEST_MALLCTL_CONFIG(xmalloc); + +#undef TEST_MALLCTL_CONFIG +} +TEST_END + +TEST_BEGIN(test_mallctl_opt) +{ + bool config_always = true; + +#define TEST_MALLCTL_OPT(t, opt, config) do { \ + t oldval; \ + size_t sz = sizeof(oldval); \ + int expected = config_##config ? 0 : ENOENT; \ + int result = mallctl("opt."#opt, &oldval, &sz, NULL, 0); \ + assert_d_eq(result, expected, \ + "Unexpected mallctl() result for opt."#opt); \ + assert_zu_eq(sz, sizeof(oldval), "Unexpected output size"); \ +} while (0) + + TEST_MALLCTL_OPT(bool, abort, always); + TEST_MALLCTL_OPT(size_t, lg_chunk, always); + TEST_MALLCTL_OPT(const char *, dss, always); + TEST_MALLCTL_OPT(size_t, narenas, always); + TEST_MALLCTL_OPT(ssize_t, lg_dirty_mult, always); + TEST_MALLCTL_OPT(bool, stats_print, always); + TEST_MALLCTL_OPT(bool, junk, fill); + TEST_MALLCTL_OPT(size_t, quarantine, fill); + TEST_MALLCTL_OPT(bool, redzone, fill); + TEST_MALLCTL_OPT(bool, zero, fill); + TEST_MALLCTL_OPT(bool, utrace, utrace); + TEST_MALLCTL_OPT(bool, valgrind, valgrind); + TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); + TEST_MALLCTL_OPT(bool, tcache, tcache); + TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache); + TEST_MALLCTL_OPT(bool, prof, prof); + TEST_MALLCTL_OPT(const char *, prof_prefix, prof); + TEST_MALLCTL_OPT(bool, prof_active, prof); + TEST_MALLCTL_OPT(ssize_t, lg_prof_sample, prof); + TEST_MALLCTL_OPT(bool, prof_accum, prof); + TEST_MALLCTL_OPT(ssize_t, lg_prof_interval, prof); + TEST_MALLCTL_OPT(bool, prof_gdump, prof); + TEST_MALLCTL_OPT(bool, prof_final, prof); + TEST_MALLCTL_OPT(bool, prof_leak, prof); + +#undef TEST_MALLCTL_OPT +} +TEST_END + +TEST_BEGIN(test_manpage_example) +{ + unsigned nbins, i; + size_t mib[4]; + size_t len, miblen; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0, + "Unexpected mallctl() failure"); + + miblen = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + for (i = 0; i < nbins; i++) { + size_t bin_size; + + mib[2] = i; + len = sizeof(bin_size); + assert_d_eq(mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0), + 0, "Unexpected mallctlbymib() failure"); + /* Do something with bin_size... */ + } +} +TEST_END + +TEST_BEGIN(test_thread_arena) +{ + unsigned arena_old, arena_new, narenas; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect"); + arena_new = narenas - 1; + assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new, + sizeof(unsigned)), 0, "Unexpected mallctl() failure"); + arena_new = 0; + assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new, + sizeof(unsigned)), 0, "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arena_i_purge) +{ + unsigned narenas; + size_t sz = sizeof(unsigned); + size_t mib[3]; + size_t miblen = 3; + + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[1] = narenas; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); +} +TEST_END + +TEST_BEGIN(test_arena_i_dss) +{ + const char *dss_prec_old, *dss_prec_new; + size_t sz = sizeof(dss_prec_old); + + dss_prec_new = "primary"; + assert_d_eq(mallctl("arena.0.dss", &dss_prec_old, &sz, &dss_prec_new, + sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); + assert_str_ne(dss_prec_old, "primary", + "Unexpected default for dss precedence"); + + assert_d_eq(mallctl("arena.0.dss", &dss_prec_new, &sz, &dss_prec_old, + sizeof(dss_prec_old)), 0, "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arenas_purge) +{ + unsigned arena = 0; + + assert_d_eq(mallctl("arenas.purge", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("arenas.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arenas_initialized) +{ + unsigned narenas; + size_t sz = sizeof(narenas); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + { + bool initialized[narenas]; + + sz = narenas * sizeof(bool); + assert_d_eq(mallctl("arenas.initialized", initialized, &sz, + NULL, 0), 0, "Unexpected mallctl() failure"); + } +} +TEST_END + +TEST_BEGIN(test_arenas_constants) +{ + +#define TEST_ARENAS_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas."#name, &name, &sz, NULL, 0), 0, \ + "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); + TEST_ARENAS_CONSTANT(size_t, page, PAGE); + TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); + TEST_ARENAS_CONSTANT(size_t, nlruns, nlclasses); + +#undef TEST_ARENAS_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_bin_constants) +{ + +#define TEST_ARENAS_BIN_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas.bin.0."#name, &name, &sz, NULL, 0), \ + 0, "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_BIN_CONSTANT(size_t, size, arena_bin_info[0].reg_size); + TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, arena_bin_info[0].nregs); + TEST_ARENAS_BIN_CONSTANT(size_t, run_size, arena_bin_info[0].run_size); + +#undef TEST_ARENAS_BIN_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_lrun_constants) +{ + +#define TEST_ARENAS_LRUN_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas.lrun.0."#name, &name, &sz, NULL, \ + 0), 0, "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_LRUN_CONSTANT(size_t, size, (1 << LG_PAGE)); + +#undef TEST_ARENAS_LRUN_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_extend) +{ + unsigned narenas_before, arena, narenas_after; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.narenas", &narenas_before, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctl("arenas.extend", &arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctl("arenas.narenas", &narenas_after, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_u_eq(narenas_before+1, narenas_after, + "Unexpected number of arenas before versus after extension"); + assert_u_eq(arena, narenas_after-1, "Unexpected arena index"); +} +TEST_END + +TEST_BEGIN(test_stats_arenas) +{ + +#define TEST_STATS_ARENAS(t, name) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("stats.arenas.0."#name, &name, &sz, NULL, \ + 0), 0, "Unexpected mallctl() failure"); \ +} while (0) + + TEST_STATS_ARENAS(const char *, dss); + TEST_STATS_ARENAS(unsigned, nthreads); + TEST_STATS_ARENAS(size_t, pactive); + TEST_STATS_ARENAS(size_t, pdirty); + +#undef TEST_STATS_ARENAS +} +TEST_END + +int +main(void) +{ + + return (test( + test_mallctl_errors, + test_mallctlnametomib_errors, + test_mallctlbymib_errors, + test_mallctl_read_write, + test_mallctlnametomib_short_mib, + test_mallctl_config, + test_mallctl_opt, + test_manpage_example, + test_thread_arena, + test_arena_i_purge, + test_arena_i_dss, + test_arenas_purge, + test_arenas_initialized, + test_arenas_constants, + test_arenas_bin_constants, + test_arenas_lrun_constants, + test_arenas_extend, + test_stats_arenas)); +} diff --git a/deps/jemalloc/test/unit/math.c b/deps/jemalloc/test/unit/math.c new file mode 100644 index 00000000000..a1b288ea19c --- /dev/null +++ b/deps/jemalloc/test/unit/math.c @@ -0,0 +1,388 @@ +#include "test/jemalloc_test.h" + +#define MAX_REL_ERR 1.0e-9 +#define MAX_ABS_ERR 1.0e-9 + +static bool +double_eq_rel(double a, double b, double max_rel_err, double max_abs_err) +{ + double rel_err; + + if (fabs(a - b) < max_abs_err) + return (true); + rel_err = (fabs(b) > fabs(a)) ? fabs((a-b)/b) : fabs((a-b)/a); + return (rel_err < max_rel_err); +} + +static uint64_t +factorial(unsigned x) +{ + uint64_t ret = 1; + unsigned i; + + for (i = 2; i <= x; i++) + ret *= (uint64_t)i; + + return (ret); +} + +TEST_BEGIN(test_ln_gamma_factorial) +{ + unsigned x; + + /* exp(ln_gamma(x)) == (x-1)! for integer x. */ + for (x = 1; x <= 21; x++) { + assert_true(double_eq_rel(exp(ln_gamma(x)), + (double)factorial(x-1), MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect factorial result for x=%u", x); + } +} +TEST_END + +/* Expected ln_gamma([0.0..100.0] increment=0.25). */ +static const double ln_gamma_misc_expected[] = { + INFINITY, + 1.28802252469807743, 0.57236494292470008, 0.20328095143129538, + 0.00000000000000000, -0.09827183642181320, -0.12078223763524518, + -0.08440112102048555, 0.00000000000000000, 0.12487171489239651, + 0.28468287047291918, 0.47521466691493719, 0.69314718055994529, + 0.93580193110872523, 1.20097360234707429, 1.48681557859341718, + 1.79175946922805496, 2.11445692745037128, 2.45373657084244234, + 2.80857141857573644, 3.17805383034794575, 3.56137591038669710, + 3.95781396761871651, 4.36671603662228680, 4.78749174278204581, + 5.21960398699022932, 5.66256205985714178, 6.11591589143154568, + 6.57925121201010121, 7.05218545073853953, 7.53436423675873268, + 8.02545839631598312, 8.52516136106541467, 9.03318691960512332, + 9.54926725730099690, 10.07315123968123949, 10.60460290274525086, + 11.14340011995171231, 11.68933342079726856, 12.24220494005076176, + 12.80182748008146909, 13.36802367147604720, 13.94062521940376342, + 14.51947222506051816, 15.10441257307551943, 15.69530137706046524, + 16.29200047656724237, 16.89437797963419285, 17.50230784587389010, + 18.11566950571089407, 18.73434751193644843, 19.35823122022435427, + 19.98721449566188468, 20.62119544270163018, 21.26007615624470048, + 21.90376249182879320, 22.55216385312342098, 23.20519299513386002, + 23.86276584168908954, 24.52480131594137802, 25.19122118273868338, + 25.86194990184851861, 26.53691449111561340, 27.21604439872720604, + 27.89927138384089389, 28.58652940490193828, 29.27775451504081516, + 29.97288476399884871, 30.67186010608067548, 31.37462231367769050, + 32.08111489594735843, 32.79128302226991565, 33.50507345013689076, + 34.22243445715505317, 34.94331577687681545, 35.66766853819134298, + 36.39544520803305261, 37.12659953718355865, 37.86108650896109395, + 38.59886229060776230, 39.33988418719949465, 40.08411059791735198, + 40.83150097453079752, 41.58201578195490100, 42.33561646075348506, + 43.09226539146988699, 43.85192586067515208, 44.61456202863158893, + 45.38013889847690052, 46.14862228684032885, 46.91997879580877395, + 47.69417578616628361, 48.47118135183522014, 49.25096429545256882, + 50.03349410501914463, 50.81874093156324790, 51.60667556776436982, + 52.39726942748592364, 53.19049452616926743, 53.98632346204390586, + 54.78472939811231157, 55.58568604486942633, 56.38916764371992940, + 57.19514895105859864, 58.00360522298051080, 58.81451220059079787, + 59.62784609588432261, 60.44358357816834371, 61.26170176100199427, + 62.08217818962842927, 62.90499082887649962, 63.73011805151035958, + 64.55753862700632340, 65.38723171073768015, 66.21917683354901385, + 67.05335389170279825, 67.88974313718154008, 68.72832516833013017, + 69.56908092082363737, 70.41199165894616385, 71.25703896716800045, + 72.10420474200799390, 72.95347118416940191, 73.80482079093779646, + 74.65823634883015814, 75.51370092648485866, 76.37119786778275454, + 77.23071078519033961, 78.09222355331530707, 78.95572030266725960, + 79.82118541361435859, 80.68860351052903468, 81.55795945611502873, + 82.42923834590904164, 83.30242550295004378, 84.17750647261028973, + 85.05446701758152983, 85.93329311301090456, 86.81397094178107920, + 87.69648688992882057, 88.58082754219766741, 89.46697967771913795, + 90.35493026581838194, 91.24466646193963015, 92.13617560368709292, + 93.02944520697742803, 93.92446296229978486, 94.82121673107967297, + 95.71969454214321615, 96.61988458827809723, 97.52177522288820910, + 98.42535495673848800, 99.33061245478741341, 100.23753653310367895, + 101.14611615586458981, 102.05634043243354370, 102.96819861451382394, + 103.88168009337621811, 104.79677439715833032, 105.71347118823287303, + 106.63176026064346047, 107.55163153760463501, 108.47307506906540198, + 109.39608102933323153, 110.32063971475740516, 111.24674154146920557, + 112.17437704317786995, 113.10353686902013237, 114.03421178146170689, + 114.96639265424990128, 115.90007047041454769, 116.83523632031698014, + 117.77188139974506953, 118.70999700805310795, 119.64957454634490830, + 120.59060551569974962, 121.53308151543865279, 122.47699424143097247, + 123.42233548443955726, 124.36909712850338394, 125.31727114935689826, + 126.26684961288492559, 127.21782467361175861, 128.17018857322420899, + 129.12393363912724453, 130.07905228303084755, 131.03553699956862033, + 131.99338036494577864, 132.95257503561629164, 133.91311374698926784, + 134.87498931216194364, 135.83819462068046846, 136.80272263732638294, + 137.76856640092901785, 138.73571902320256299, 139.70417368760718091, + 140.67392364823425055, 141.64496222871400732, 142.61728282114600574, + 143.59087888505104047, 144.56574394634486680, 145.54187159633210058, + 146.51925549072063859, 147.49788934865566148, 148.47776695177302031, + 149.45888214327129617, 150.44122882700193600, 151.42480096657754984, + 152.40959258449737490, 153.39559776128982094, 154.38281063467164245, + 155.37122539872302696, 156.36083630307879844, 157.35163765213474107, + 158.34362380426921391, 159.33678917107920370, 160.33112821663092973, + 161.32663545672428995, 162.32330545817117695, 163.32113283808695314, + 164.32011226319519892, 165.32023844914485267, 166.32150615984036790, + 167.32391020678358018, 168.32744544842768164, 169.33210678954270634, + 170.33788918059275375, 171.34478761712384198, 172.35279713916281707, + 173.36191283062726143, 174.37212981874515094, 175.38344327348534080, + 176.39584840699734514, 177.40934047306160437, 178.42391476654847793, + 179.43956662288721304, 180.45629141754378111, 181.47408456550741107, + 182.49294152078630304, 183.51285777591152737, 184.53382886144947861, + 185.55585034552262869, 186.57891783333786861, 187.60302696672312095, + 188.62817342367162610, 189.65435291789341932, 190.68156119837468054, + 191.70979404894376330, 192.73904728784492590, 193.76931676731820176, + 194.80059837318714244, 195.83288802445184729, 196.86618167288995096, + 197.90047530266301123, 198.93576492992946214, 199.97204660246373464, + 201.00931639928148797, 202.04757043027063901, 203.08680483582807597, + 204.12701578650228385, 205.16819948264117102, 206.21035215404597807, + 207.25347005962987623, 208.29754948708190909, 209.34258675253678916, + 210.38857820024875878, 211.43552020227099320, 212.48340915813977858, + 213.53224149456323744, 214.58201366511514152, 215.63272214993284592, + 216.68436345542014010, 217.73693411395422004, 218.79043068359703739, + 219.84484974781133815, 220.90018791517996988, 221.95644181913033322, + 223.01360811766215875, 224.07168349307951871, 225.13066465172661879, + 226.19054832372759734, 227.25133126272962159, 228.31301024565024704, + 229.37558207242807384, 230.43904356577689896, 231.50339157094342113, + 232.56862295546847008, 233.63473460895144740, 234.70172344281823484, + 235.76958639009222907, 236.83832040516844586, 237.90792246359117712, + 238.97838956183431947, 240.04971871708477238, 241.12190696702904802, + 242.19495136964280846, 243.26884900298270509, 244.34359696498191283, + 245.41919237324782443, 246.49563236486270057, 247.57291409618682110, + 248.65103474266476269, 249.72999149863338175, 250.80978157713354904, + 251.89040220972316320, 252.97185064629374551, 254.05412415488834199, + 255.13722002152300661, 256.22113555000953511, 257.30586806178126835, + 258.39141489572085675, 259.47777340799029844, 260.56494097186322279, + 261.65291497755913497, 262.74169283208021852, 263.83127195904967266, + 264.92164979855277807, 266.01282380697938379, 267.10479145686849733, + 268.19755023675537586, 269.29109765101975427, 270.38543121973674488, + 271.48054847852881721, 272.57644697842033565, 273.67312428569374561, + 274.77057798174683967, 275.86880566295326389, 276.96780494052313770, + 278.06757344036617496, 279.16810880295668085, 280.26940868320008349, + 281.37147075030043197, 282.47429268763045229, 283.57787219260217171, + 284.68220697654078322, 285.78729476455760050, 286.89313329542699194, + 287.99972032146268930, 289.10705360839756395, 290.21513093526289140, + 291.32395009427028754, 292.43350889069523646, 293.54380514276073200, + 294.65483668152336350, 295.76660135076059532, 296.87909700685889902, + 297.99232151870342022, 299.10627276756946458, 300.22094864701409733, + 301.33634706277030091, 302.45246593264130297, 303.56930318639643929, + 304.68685676566872189, 305.80512462385280514, 306.92410472600477078, + 308.04379504874236773, 309.16419358014690033, 310.28529831966631036, + 311.40710727801865687, 312.52961847709792664, 313.65282994987899201, + 314.77673974032603610, 315.90134590329950015, 317.02664650446632777, + 318.15263962020929966, 319.27932333753892635, 320.40669575400545455, + 321.53475497761127144, 322.66349912672620803, 323.79292633000159185, + 324.92303472628691452, 326.05382246454587403, 327.18528770377525916, + 328.31742861292224234, 329.45024337080525356, 330.58373016603343331, + 331.71788719692847280, 332.85271267144611329, 333.98820480709991898, + 335.12436183088397001, 336.26118197919845443, 337.39866349777429377, + 338.53680464159958774, 339.67560367484657036, 340.81505887079896411, + 341.95516851178109619, 343.09593088908627578, 344.23734430290727460, + 345.37940706226686416, 346.52211748494903532, 347.66547389743118401, + 348.80947463481720661, 349.95411804077025408, 351.09940246744753267, + 352.24532627543504759, 353.39188783368263103, 354.53908551944078908, + 355.68691771819692349, 356.83538282361303118, 357.98447923746385868, + 359.13420536957539753 +}; + +TEST_BEGIN(test_ln_gamma_misc) +{ + unsigned i; + + for (i = 1; i < sizeof(ln_gamma_misc_expected)/sizeof(double); i++) { + double x = (double)i * 0.25; + assert_true(double_eq_rel(ln_gamma(x), + ln_gamma_misc_expected[i], MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect ln_gamma result for i=%u", i); + } +} +TEST_END + +/* Expected pt_norm([0.01..0.99] increment=0.01). */ +static const double pt_norm_expected[] = { + -INFINITY, + -2.32634787404084076, -2.05374891063182252, -1.88079360815125085, + -1.75068607125216946, -1.64485362695147264, -1.55477359459685305, + -1.47579102817917063, -1.40507156030963221, -1.34075503369021654, + -1.28155156554460081, -1.22652812003661049, -1.17498679206608991, + -1.12639112903880045, -1.08031934081495606, -1.03643338949378938, + -0.99445788320975281, -0.95416525314619416, -0.91536508784281390, + -0.87789629505122846, -0.84162123357291418, -0.80642124701824025, + -0.77219321418868492, -0.73884684918521371, -0.70630256284008752, + -0.67448975019608171, -0.64334540539291685, -0.61281299101662701, + -0.58284150727121620, -0.55338471955567281, -0.52440051270804067, + -0.49585034734745320, -0.46769879911450812, -0.43991316567323380, + -0.41246312944140462, -0.38532046640756751, -0.35845879325119373, + -0.33185334643681652, -0.30548078809939738, -0.27931903444745404, + -0.25334710313579978, -0.22754497664114931, -0.20189347914185077, + -0.17637416478086135, -0.15096921549677725, -0.12566134685507399, + -0.10043372051146975, -0.07526986209982976, -0.05015358346473352, + -0.02506890825871106, 0.00000000000000000, 0.02506890825871106, + 0.05015358346473366, 0.07526986209982990, 0.10043372051146990, + 0.12566134685507413, 0.15096921549677739, 0.17637416478086146, + 0.20189347914185105, 0.22754497664114931, 0.25334710313579978, + 0.27931903444745404, 0.30548078809939738, 0.33185334643681652, + 0.35845879325119373, 0.38532046640756762, 0.41246312944140484, + 0.43991316567323391, 0.46769879911450835, 0.49585034734745348, + 0.52440051270804111, 0.55338471955567303, 0.58284150727121620, + 0.61281299101662701, 0.64334540539291685, 0.67448975019608171, + 0.70630256284008752, 0.73884684918521371, 0.77219321418868492, + 0.80642124701824036, 0.84162123357291441, 0.87789629505122879, + 0.91536508784281423, 0.95416525314619460, 0.99445788320975348, + 1.03643338949378938, 1.08031934081495606, 1.12639112903880045, + 1.17498679206608991, 1.22652812003661049, 1.28155156554460081, + 1.34075503369021654, 1.40507156030963265, 1.47579102817917085, + 1.55477359459685394, 1.64485362695147308, 1.75068607125217102, + 1.88079360815125041, 2.05374891063182208, 2.32634787404084076 +}; + +TEST_BEGIN(test_pt_norm) +{ + unsigned i; + + for (i = 1; i < sizeof(pt_norm_expected)/sizeof(double); i++) { + double p = (double)i * 0.01; + assert_true(double_eq_rel(pt_norm(p), pt_norm_expected[i], + MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect pt_norm result for i=%u", i); + } +} +TEST_END + +/* + * Expected pt_chi2(p=[0.01..0.99] increment=0.07, + * df={0.1, 1.1, 10.1, 100.1, 1000.1}). + */ +static const double pt_chi2_df[] = {0.1, 1.1, 10.1, 100.1, 1000.1}; +static const double pt_chi2_expected[] = { + 1.168926411457320e-40, 1.347680397072034e-22, 3.886980416666260e-17, + 8.245951724356564e-14, 2.068936347497604e-11, 1.562561743309233e-09, + 5.459543043426564e-08, 1.114775688149252e-06, 1.532101202364371e-05, + 1.553884683726585e-04, 1.239396954915939e-03, 8.153872320255721e-03, + 4.631183739647523e-02, 2.473187311701327e-01, 2.175254800183617e+00, + + 0.0003729887888876379, 0.0164409238228929513, 0.0521523015190650113, + 0.1064701372271216612, 0.1800913735793082115, 0.2748704281195626931, + 0.3939246282787986497, 0.5420727552260817816, 0.7267265822221973259, + 0.9596554296000253670, 1.2607440376386165326, 1.6671185084541604304, + 2.2604828984738705167, 3.2868613342148607082, 6.9298574921692139839, + + 2.606673548632508, 4.602913725294877, 5.646152813924212, + 6.488971315540869, 7.249823275816285, 7.977314231410841, + 8.700354939944047, 9.441728024225892, 10.224338321374127, + 11.076435368801061, 12.039320937038386, 13.183878752697167, + 14.657791935084575, 16.885728216339373, 23.361991680031817, + + 70.14844087392152, 80.92379498849355, 85.53325420085891, + 88.94433120715347, 91.83732712857017, 94.46719943606301, + 96.96896479994635, 99.43412843510363, 101.94074719829733, + 104.57228644307247, 107.43900093448734, 110.71844673417287, + 114.76616819871325, 120.57422505959563, 135.92318818757556, + + 899.0072447849649, 937.9271278858220, 953.8117189560207, + 965.3079371501154, 974.8974061207954, 983.4936235182347, + 991.5691170518946, 999.4334123954690, 1007.3391826856553, + 1015.5445154999951, 1024.3777075619569, 1034.3538789836223, + 1046.4872561869577, 1063.5717461999654, 1107.0741966053859 +}; + +TEST_BEGIN(test_pt_chi2) +{ + unsigned i, j; + unsigned e = 0; + + for (i = 0; i < sizeof(pt_chi2_df)/sizeof(double); i++) { + double df = pt_chi2_df[i]; + double ln_gamma_df = ln_gamma(df * 0.5); + for (j = 1; j < 100; j += 7) { + double p = (double)j * 0.01; + assert_true(double_eq_rel(pt_chi2(p, df, ln_gamma_df), + pt_chi2_expected[e], MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect pt_chi2 result for i=%u, j=%u", i, j); + e++; + } + } +} +TEST_END + +/* + * Expected pt_gamma(p=[0.1..0.99] increment=0.07, + * shape=[0.5..3.0] increment=0.5). + */ +static const double pt_gamma_shape[] = {0.5, 1.0, 1.5, 2.0, 2.5, 3.0}; +static const double pt_gamma_expected[] = { + 7.854392895485103e-05, 5.043466107888016e-03, 1.788288957794883e-02, + 3.900956150232906e-02, 6.913847560638034e-02, 1.093710833465766e-01, + 1.613412523825817e-01, 2.274682115597864e-01, 3.114117323127083e-01, + 4.189466220207417e-01, 5.598106789059246e-01, 7.521856146202706e-01, + 1.036125427911119e+00, 1.532450860038180e+00, 3.317448300510606e+00, + + 0.01005033585350144, 0.08338160893905107, 0.16251892949777497, + 0.24846135929849966, 0.34249030894677596, 0.44628710262841947, + 0.56211891815354142, 0.69314718055994529, 0.84397007029452920, + 1.02165124753198167, 1.23787435600161766, 1.51412773262977574, + 1.89711998488588196, 2.52572864430825783, 4.60517018598809091, + + 0.05741590094955853, 0.24747378084860744, 0.39888572212236084, + 0.54394139997444901, 0.69048812513915159, 0.84311389861296104, + 1.00580622221479898, 1.18298694218766931, 1.38038096305861213, + 1.60627736383027453, 1.87396970522337947, 2.20749220408081070, + 2.65852391865854942, 3.37934630984842244, 5.67243336507218476, + + 0.1485547402532659, 0.4657458011640391, 0.6832386130709406, + 0.8794297834672100, 1.0700752852474524, 1.2629614217350744, + 1.4638400448580779, 1.6783469900166610, 1.9132338090606940, + 2.1778589228618777, 2.4868823970010991, 2.8664695666264195, + 3.3724415436062114, 4.1682658512758071, 6.6383520679938108, + + 0.2771490383641385, 0.7195001279643727, 0.9969081732265243, + 1.2383497880608061, 1.4675206597269927, 1.6953064251816552, + 1.9291243435606809, 2.1757300955477641, 2.4428032131216391, + 2.7406534569230616, 3.0851445039665513, 3.5043101122033367, + 4.0575997065264637, 4.9182956424675286, 7.5431362346944937, + + 0.4360451650782932, 0.9983600902486267, 1.3306365880734528, + 1.6129750834753802, 1.8767241606994294, 2.1357032436097660, + 2.3988853336865565, 2.6740603137235603, 2.9697561737517959, + 3.2971457713883265, 3.6731795898504660, 4.1275751617770631, + 4.7230515633946677, 5.6417477865306020, 8.4059469148854635 +}; + +TEST_BEGIN(test_pt_gamma_shape) +{ + unsigned i, j; + unsigned e = 0; + + for (i = 0; i < sizeof(pt_gamma_shape)/sizeof(double); i++) { + double shape = pt_gamma_shape[i]; + double ln_gamma_shape = ln_gamma(shape); + for (j = 1; j < 100; j += 7) { + double p = (double)j * 0.01; + assert_true(double_eq_rel(pt_gamma(p, shape, 1.0, + ln_gamma_shape), pt_gamma_expected[e], MAX_REL_ERR, + MAX_ABS_ERR), + "Incorrect pt_gamma result for i=%u, j=%u", i, j); + e++; + } + } +} +TEST_END + +TEST_BEGIN(test_pt_gamma_scale) +{ + double shape = 1.0; + double ln_gamma_shape = ln_gamma(shape); + + assert_true(double_eq_rel( + pt_gamma(0.5, shape, 1.0, ln_gamma_shape) * 10.0, + pt_gamma(0.5, shape, 10.0, ln_gamma_shape), MAX_REL_ERR, + MAX_ABS_ERR), + "Scale should be trivially equivalent to external multiplication"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ln_gamma_factorial, + test_ln_gamma_misc, + test_pt_norm, + test_pt_chi2, + test_pt_gamma_shape, + test_pt_gamma_scale)); +} diff --git a/deps/jemalloc/test/unit/mq.c b/deps/jemalloc/test/unit/mq.c new file mode 100644 index 00000000000..f57e96af1cb --- /dev/null +++ b/deps/jemalloc/test/unit/mq.c @@ -0,0 +1,92 @@ +#include "test/jemalloc_test.h" + +#define NSENDERS 3 +#define NMSGS 100000 + +typedef struct mq_msg_s mq_msg_t; +struct mq_msg_s { + mq_msg(mq_msg_t) link; +}; +mq_gen(static, mq_, mq_t, mq_msg_t, link) + +TEST_BEGIN(test_mq_basic) +{ + mq_t mq; + mq_msg_t msg; + + assert_false(mq_init(&mq), "Unexpected mq_init() failure"); + assert_u_eq(mq_count(&mq), 0, "mq should be empty"); + assert_ptr_null(mq_tryget(&mq), + "mq_tryget() should fail when the queue is empty"); + + mq_put(&mq, &msg); + assert_u_eq(mq_count(&mq), 1, "mq should contain one message"); + assert_ptr_eq(mq_tryget(&mq), &msg, "mq_tryget() should return msg"); + + mq_put(&mq, &msg); + assert_ptr_eq(mq_get(&mq), &msg, "mq_get() should return msg"); + + mq_fini(&mq); +} +TEST_END + +static void * +thd_receiver_start(void *arg) +{ + mq_t *mq = (mq_t *)arg; + unsigned i; + + for (i = 0; i < (NSENDERS * NMSGS); i++) { + mq_msg_t *msg = mq_get(mq); + assert_ptr_not_null(msg, "mq_get() should never return NULL"); + dallocx(msg, 0); + } + return (NULL); +} + +static void * +thd_sender_start(void *arg) +{ + mq_t *mq = (mq_t *)arg; + unsigned i; + + for (i = 0; i < NMSGS; i++) { + mq_msg_t *msg; + void *p; + p = mallocx(sizeof(mq_msg_t), 0); + assert_ptr_not_null(p, "Unexpected allocm() failure"); + msg = (mq_msg_t *)p; + mq_put(mq, msg); + } + return (NULL); +} + +TEST_BEGIN(test_mq_threaded) +{ + mq_t mq; + thd_t receiver; + thd_t senders[NSENDERS]; + unsigned i; + + assert_false(mq_init(&mq), "Unexpected mq_init() failure"); + + thd_create(&receiver, thd_receiver_start, (void *)&mq); + for (i = 0; i < NSENDERS; i++) + thd_create(&senders[i], thd_sender_start, (void *)&mq); + + thd_join(receiver, NULL); + for (i = 0; i < NSENDERS; i++) + thd_join(senders[i], NULL); + + mq_fini(&mq); +} +TEST_END + +int +main(void) +{ + return (test( + test_mq_basic, + test_mq_threaded)); +} + diff --git a/deps/jemalloc/test/unit/mtx.c b/deps/jemalloc/test/unit/mtx.c new file mode 100644 index 00000000000..96ff69486ea --- /dev/null +++ b/deps/jemalloc/test/unit/mtx.c @@ -0,0 +1,60 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 2 +#define NINCRS 2000000 + +TEST_BEGIN(test_mtx_basic) +{ + mtx_t mtx; + + assert_false(mtx_init(&mtx), "Unexpected mtx_init() failure"); + mtx_lock(&mtx); + mtx_unlock(&mtx); + mtx_fini(&mtx); +} +TEST_END + +typedef struct { + mtx_t mtx; + unsigned x; +} thd_start_arg_t; + +static void * +thd_start(void *varg) +{ + thd_start_arg_t *arg = (thd_start_arg_t *)varg; + unsigned i; + + for (i = 0; i < NINCRS; i++) { + mtx_lock(&arg->mtx); + arg->x++; + mtx_unlock(&arg->mtx); + } + return (NULL); +} + +TEST_BEGIN(test_mtx_race) +{ + thd_start_arg_t arg; + thd_t thds[NTHREADS]; + unsigned i; + + assert_false(mtx_init(&arg.mtx), "Unexpected mtx_init() failure"); + arg.x = 0; + for (i = 0; i < NTHREADS; i++) + thd_create(&thds[i], thd_start, (void *)&arg); + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); + assert_u_eq(arg.x, NTHREADS * NINCRS, + "Race-related counter corruption"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_mtx_basic, + test_mtx_race)); +} diff --git a/deps/jemalloc/test/unit/prof_accum.c b/deps/jemalloc/test/unit/prof_accum.c new file mode 100644 index 00000000000..050a8a7ee5d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum.c @@ -0,0 +1,86 @@ +#include "prof_accum.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0"; +#endif + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +static void * +alloc_from_permuted_backtrace(unsigned thd_ind, unsigned iteration) +{ + + return (alloc_0(thd_ind*NALLOCS_PER_THREAD + iteration)); +} + +static void * +thd_start(void *varg) +{ + unsigned thd_ind = *(unsigned *)varg; + size_t bt_count_prev, bt_count; + unsigned i_prev, i; + + i_prev = 0; + bt_count_prev = 0; + for (i = 0; i < NALLOCS_PER_THREAD; i++) { + void *p = alloc_from_permuted_backtrace(thd_ind, i); + dallocx(p, 0); + if (i % DUMP_INTERVAL == 0) { + assert_d_eq(mallctl("prof.dump", NULL, NULL, NULL, 0), + 0, "Unexpected error while dumping heap profile"); + } + + if (i % BT_COUNT_CHECK_INTERVAL == 0 || + i+1 == NALLOCS_PER_THREAD) { + bt_count = prof_bt_count(); + assert_zu_le(bt_count_prev+(i-i_prev), bt_count, + "Expected larger backtrace count increase"); + i_prev = i; + bt_count_prev = bt_count; + } + } + + return (NULL); +} + +TEST_BEGIN(test_idump) +{ + bool active; + thd_t thds[NTHREADS]; + unsigned thd_args[NTHREADS]; + unsigned i; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + for (i = 0; i < NTHREADS; i++) { + thd_args[i] = i; + thd_create(&thds[i], thd_start, (void *)&thd_args[i]); + } + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); +} +TEST_END + +int +main(void) +{ + + return (test( + test_idump)); +} diff --git a/deps/jemalloc/test/unit/prof_accum.h b/deps/jemalloc/test/unit/prof_accum.h new file mode 100644 index 00000000000..109d86b598d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum.h @@ -0,0 +1,35 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 4 +#define NALLOCS_PER_THREAD 50 +#define DUMP_INTERVAL 1 +#define BT_COUNT_CHECK_INTERVAL 5 + +#define alloc_n_proto(n) \ +void *alloc_##n(unsigned bits); +alloc_n_proto(0) +alloc_n_proto(1) + +#define alloc_n_gen(n) \ +void * \ +alloc_##n(unsigned bits) \ +{ \ + void *p; \ + \ + if (bits == 0) \ + p = mallocx(1, 0); \ + else { \ + switch (bits & 0x1U) { \ + case 0: \ + p = (alloc_0(bits >> 1)); \ + break; \ + case 1: \ + p = (alloc_1(bits >> 1)); \ + break; \ + default: not_reached(); \ + } \ + } \ + /* Intentionally sabotage tail call optimization. */ \ + assert_ptr_not_null(p, "Unexpected mallocx() failure"); \ + return (p); \ +} diff --git a/deps/jemalloc/test/unit/prof_accum_a.c b/deps/jemalloc/test/unit/prof_accum_a.c new file mode 100644 index 00000000000..42ad521d8df --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum_a.c @@ -0,0 +1,3 @@ +#include "prof_accum.h" + +alloc_n_gen(0) diff --git a/deps/jemalloc/test/unit/prof_accum_b.c b/deps/jemalloc/test/unit/prof_accum_b.c new file mode 100644 index 00000000000..60d9dab6a8e --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum_b.c @@ -0,0 +1,3 @@ +#include "prof_accum.h" + +alloc_n_gen(1) diff --git a/deps/jemalloc/test/unit/prof_gdump.c b/deps/jemalloc/test/unit/prof_gdump.c new file mode 100644 index 00000000000..a00b1054f1d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_gdump.c @@ -0,0 +1,56 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = "prof:true,prof_active:false,prof_gdump:true"; +#endif + +static bool did_prof_dump_open; + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + did_prof_dump_open = true; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +TEST_BEGIN(test_gdump) +{ + bool active; + void *p, *q; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + did_prof_dump_open = false; + p = mallocx(chunksize, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + assert_true(did_prof_dump_open, "Expected a profile dump"); + + did_prof_dump_open = false; + q = mallocx(chunksize, 0); + assert_ptr_not_null(q, "Unexpected mallocx() failure"); + assert_true(did_prof_dump_open, "Expected a profile dump"); + + dallocx(p, 0); + dallocx(q, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_gdump)); +} diff --git a/deps/jemalloc/test/unit/prof_idump.c b/deps/jemalloc/test/unit/prof_idump.c new file mode 100644 index 00000000000..bdea53ecdd9 --- /dev/null +++ b/deps/jemalloc/test/unit/prof_idump.c @@ -0,0 +1,51 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0," + "lg_prof_interval:0"; +#endif + +static bool did_prof_dump_open; + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + did_prof_dump_open = true; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +TEST_BEGIN(test_idump) +{ + bool active; + void *p; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + did_prof_dump_open = false; + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + dallocx(p, 0); + assert_true(did_prof_dump_open, "Expected a profile dump"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_idump)); +} diff --git a/deps/jemalloc/test/unit/ql.c b/deps/jemalloc/test/unit/ql.c new file mode 100644 index 00000000000..05fad450fc0 --- /dev/null +++ b/deps/jemalloc/test/unit/ql.c @@ -0,0 +1,209 @@ +#include "test/jemalloc_test.h" + +/* Number of ring entries, in [2..26]. */ +#define NENTRIES 9 + +typedef struct list_s list_t; +typedef ql_head(list_t) list_head_t; + +struct list_s { + ql_elm(list_t) link; + char id; +}; + +static void +test_empty_list(list_head_t *head) +{ + list_t *t; + unsigned i; + + assert_ptr_null(ql_first(head), "Unexpected element for empty list"); + assert_ptr_null(ql_last(head, link), + "Unexpected element for empty list"); + + i = 0; + ql_foreach(t, head, link) { + i++; + } + assert_u_eq(i, 0, "Unexpected element for empty list"); + + i = 0; + ql_reverse_foreach(t, head, link) { + i++; + } + assert_u_eq(i, 0, "Unexpected element for empty list"); +} + +TEST_BEGIN(test_ql_empty) +{ + list_head_t head; + + ql_new(&head); + test_empty_list(&head); +} +TEST_END + +static void +init_entries(list_t *entries, unsigned nentries) +{ + unsigned i; + + for (i = 0; i < nentries; i++) { + entries[i].id = 'a' + i; + ql_elm_new(&entries[i], link); + } +} + +static void +test_entries_list(list_head_t *head, list_t *entries, unsigned nentries) +{ + list_t *t; + unsigned i; + + assert_c_eq(ql_first(head)->id, entries[0].id, "Element id mismatch"); + assert_c_eq(ql_last(head, link)->id, entries[nentries-1].id, + "Element id mismatch"); + + i = 0; + ql_foreach(t, head, link) { + assert_c_eq(t->id, entries[i].id, "Element id mismatch"); + i++; + } + + i = 0; + ql_reverse_foreach(t, head, link) { + assert_c_eq(t->id, entries[nentries-i-1].id, + "Element id mismatch"); + i++; + } + + for (i = 0; i < nentries-1; i++) { + t = ql_next(head, &entries[i], link); + assert_c_eq(t->id, entries[i+1].id, "Element id mismatch"); + } + assert_ptr_null(ql_next(head, &entries[nentries-1], link), + "Unexpected element"); + + assert_ptr_null(ql_prev(head, &entries[0], link), "Unexpected element"); + for (i = 1; i < nentries; i++) { + t = ql_prev(head, &entries[i], link); + assert_c_eq(t->id, entries[i-1].id, "Element id mismatch"); + } +} + +TEST_BEGIN(test_ql_tail_insert) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_tail_insert(&head, &entries[i], link); + + test_entries_list(&head, entries, NENTRIES); +} +TEST_END + +TEST_BEGIN(test_ql_tail_remove) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_tail_insert(&head, &entries[i], link); + + for (i = 0; i < NENTRIES; i++) { + test_entries_list(&head, entries, NENTRIES-i); + ql_tail_remove(&head, list_t, link); + } + test_empty_list(&head); +} +TEST_END + +TEST_BEGIN(test_ql_head_insert) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_head_insert(&head, &entries[NENTRIES-i-1], link); + + test_entries_list(&head, entries, NENTRIES); +} +TEST_END + +TEST_BEGIN(test_ql_head_remove) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_head_insert(&head, &entries[NENTRIES-i-1], link); + + for (i = 0; i < NENTRIES; i++) { + test_entries_list(&head, &entries[i], NENTRIES-i); + ql_head_remove(&head, list_t, link); + } + test_empty_list(&head); +} +TEST_END + +TEST_BEGIN(test_ql_insert) +{ + list_head_t head; + list_t entries[8]; + list_t *a, *b, *c, *d, *e, *f, *g, *h; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + a = &entries[0]; + b = &entries[1]; + c = &entries[2]; + d = &entries[3]; + e = &entries[4]; + f = &entries[5]; + g = &entries[6]; + h = &entries[7]; + + /* + * ql_remove(), ql_before_insert(), and ql_after_insert() are used + * internally by other macros that are already tested, so there's no + * need to test them completely. However, insertion/deletion from the + * middle of lists is not otherwise tested; do so here. + */ + ql_tail_insert(&head, f, link); + ql_before_insert(&head, f, b, link); + ql_before_insert(&head, f, c, link); + ql_after_insert(f, h, link); + ql_after_insert(f, g, link); + ql_before_insert(&head, b, a, link); + ql_after_insert(c, d, link); + ql_before_insert(&head, f, e, link); + + test_entries_list(&head, entries, sizeof(entries)/sizeof(list_t)); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ql_empty, + test_ql_tail_insert, + test_ql_tail_remove, + test_ql_head_insert, + test_ql_head_remove, + test_ql_insert)); +} diff --git a/deps/jemalloc/test/unit/qr.c b/deps/jemalloc/test/unit/qr.c new file mode 100644 index 00000000000..a2a2d902b58 --- /dev/null +++ b/deps/jemalloc/test/unit/qr.c @@ -0,0 +1,248 @@ +#include "test/jemalloc_test.h" + +/* Number of ring entries, in [2..26]. */ +#define NENTRIES 9 +/* Split index, in [1..NENTRIES). */ +#define SPLIT_INDEX 5 + +typedef struct ring_s ring_t; + +struct ring_s { + qr(ring_t) link; + char id; +}; + +static void +init_entries(ring_t *entries) +{ + unsigned i; + + for (i = 0; i < NENTRIES; i++) { + qr_new(&entries[i], link); + entries[i].id = 'a' + i; + } +} + +static void +test_independent_entries(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + j++; + } + assert_u_eq(j, 1, + "Iteration over single-element ring should visit precisely " + "one element"); + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + j++; + } + assert_u_eq(j, 1, + "Iteration over single-element ring should visit precisely " + "one element"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_ptr_eq(t, &entries[i], + "Next element in single-element ring should be same as " + "current element"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_ptr_eq(t, &entries[i], + "Previous element in single-element ring should be same as " + "current element"); + } +} + +TEST_BEGIN(test_qr_one) +{ + ring_t entries[NENTRIES]; + + init_entries(entries); + test_independent_entries(entries); +} +TEST_END + +static void +test_entries_ring(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(i+j) % NENTRIES].id, + "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(NENTRIES+i-j-1) % + NENTRIES].id, "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_c_eq(t->id, entries[(i+1) % NENTRIES].id, + "Element id mismatch"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_c_eq(t->id, entries[(NENTRIES+i-1) % NENTRIES].id, + "Element id mismatch"); + } +} + +TEST_BEGIN(test_qr_after_insert) +{ + ring_t entries[NENTRIES]; + unsigned i; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + test_entries_ring(entries); +} +TEST_END + +TEST_BEGIN(test_qr_remove) +{ + ring_t entries[NENTRIES]; + ring_t *t; + unsigned i, j; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[i+j].id, + "Element id mismatch"); + j++; + } + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[NENTRIES - 1 - j].id, + "Element id mismatch"); + j++; + } + qr_remove(&entries[i], link); + } + test_independent_entries(entries); +} +TEST_END + +TEST_BEGIN(test_qr_before_insert) +{ + ring_t entries[NENTRIES]; + ring_t *t; + unsigned i, j; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_before_insert(&entries[i - 1], &entries[i], link); + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(NENTRIES+i-j) % + NENTRIES].id, "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(i+j+1) % NENTRIES].id, + "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_c_eq(t->id, entries[(NENTRIES+i-1) % NENTRIES].id, + "Element id mismatch"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_c_eq(t->id, entries[(i+1) % NENTRIES].id, + "Element id mismatch"); + } +} +TEST_END + +static void +test_split_entries(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + if (i < SPLIT_INDEX) { + assert_c_eq(t->id, + entries[(i+j) % SPLIT_INDEX].id, + "Element id mismatch"); + } else { + assert_c_eq(t->id, entries[(i+j-SPLIT_INDEX) % + (NENTRIES-SPLIT_INDEX) + SPLIT_INDEX].id, + "Element id mismatch"); + } + j++; + } + } +} + +TEST_BEGIN(test_qr_meld_split) +{ + ring_t entries[NENTRIES]; + unsigned i; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + + qr_split(&entries[0], &entries[SPLIT_INDEX], link); + test_split_entries(entries); + + qr_meld(&entries[0], &entries[SPLIT_INDEX], link); + test_entries_ring(entries); + + qr_meld(&entries[0], &entries[SPLIT_INDEX], link); + test_split_entries(entries); + + qr_split(&entries[0], &entries[SPLIT_INDEX], link); + test_entries_ring(entries); + + qr_split(&entries[0], &entries[0], link); + test_entries_ring(entries); + + qr_meld(&entries[0], &entries[0], link); + test_entries_ring(entries); +} +TEST_END + +int +main(void) +{ + + return (test( + test_qr_one, + test_qr_after_insert, + test_qr_remove, + test_qr_before_insert, + test_qr_meld_split)); +} diff --git a/deps/jemalloc/test/unit/quarantine.c b/deps/jemalloc/test/unit/quarantine.c new file mode 100644 index 00000000000..bbd48a51ddb --- /dev/null +++ b/deps/jemalloc/test/unit/quarantine.c @@ -0,0 +1,108 @@ +#include "test/jemalloc_test.h" + +#define QUARANTINE_SIZE 8192 +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + +#ifdef JEMALLOC_FILL +const char *malloc_conf = "abort:false,junk:true,redzone:true,quarantine:" + STRINGIFY(QUARANTINE_SIZE); +#endif + +void +quarantine_clear(void) +{ + void *p; + + p = mallocx(QUARANTINE_SIZE*2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + dallocx(p, 0); +} + +TEST_BEGIN(test_quarantine) +{ +#define SZ ZU(256) +#define NQUARANTINED (QUARANTINE_SIZE/SZ) + void *quarantined[NQUARANTINED+1]; + size_t i, j; + + test_skip_if(!config_fill); + + assert_zu_eq(nallocx(SZ, 0), SZ, + "SZ=%zu does not precisely equal a size class", SZ); + + quarantine_clear(); + + /* + * Allocate enough regions to completely fill the quarantine, plus one + * more. The last iteration occurs with a completely full quarantine, + * but no regions should be drained from the quarantine until the last + * deallocation occurs. Therefore no region recycling should occur + * until after this loop completes. + */ + for (i = 0; i < NQUARANTINED+1; i++) { + void *p = mallocx(SZ, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + quarantined[i] = p; + dallocx(p, 0); + for (j = 0; j < i; j++) { + assert_ptr_ne(p, quarantined[j], + "Quarantined region recycled too early; " + "i=%zu, j=%zu", i, j); + } + } +#undef NQUARANTINED +#undef SZ +} +TEST_END + +static bool detected_redzone_corruption; + +static void +arena_redzone_corruption_replacement(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + detected_redzone_corruption = true; +} + +TEST_BEGIN(test_quarantine_redzone) +{ + char *s; + arena_redzone_corruption_t *arena_redzone_corruption_orig; + + test_skip_if(!config_fill); + + arena_redzone_corruption_orig = arena_redzone_corruption; + arena_redzone_corruption = arena_redzone_corruption_replacement; + + /* Test underflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[-1] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + /* Test overflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[sallocx(s, 0)] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + arena_redzone_corruption = arena_redzone_corruption_orig; +} +TEST_END + +int +main(void) +{ + + return (test( + test_quarantine, + test_quarantine_redzone)); +} diff --git a/deps/jemalloc/test/unit/rb.c b/deps/jemalloc/test/unit/rb.c new file mode 100644 index 00000000000..b737485a7c6 --- /dev/null +++ b/deps/jemalloc/test/unit/rb.c @@ -0,0 +1,333 @@ +#include "test/jemalloc_test.h" + +#define rbtn_black_height(a_type, a_field, a_rbt, r_height) do { \ + a_type *rbp_bh_t; \ + for (rbp_bh_t = (a_rbt)->rbt_root, (r_height) = 0; \ + rbp_bh_t != &(a_rbt)->rbt_nil; \ + rbp_bh_t = rbtn_left_get(a_type, a_field, rbp_bh_t)) { \ + if (rbtn_red_get(a_type, a_field, rbp_bh_t) == false) { \ + (r_height)++; \ + } \ + } \ +} while (0) + +typedef struct node_s node_t; + +struct node_s { +#define NODE_MAGIC 0x9823af7e + uint32_t magic; + rb_node(node_t) link; + uint64_t key; +}; + +static int +node_cmp(node_t *a, node_t *b) { + int ret; + + assert_u32_eq(a->magic, NODE_MAGIC, "Bad magic"); + assert_u32_eq(b->magic, NODE_MAGIC, "Bad magic"); + + ret = (a->key > b->key) - (a->key < b->key); + if (ret == 0) { + /* + * Duplicates are not allowed in the tree, so force an + * arbitrary ordering for non-identical items with equal keys. + */ + ret = (((uintptr_t)a) > ((uintptr_t)b)) + - (((uintptr_t)a) < ((uintptr_t)b)); + } + return (ret); +} + +typedef rb_tree(node_t) tree_t; +rb_gen(static, tree_, tree_t, node_t, link, node_cmp); + +TEST_BEGIN(test_rb_empty) +{ + tree_t tree; + node_t key; + + tree_new(&tree); + + assert_ptr_null(tree_first(&tree), "Unexpected node"); + assert_ptr_null(tree_last(&tree), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_search(&tree, &key), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_nsearch(&tree, &key), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_psearch(&tree, &key), "Unexpected node"); +} +TEST_END + +static unsigned +tree_recurse(node_t *node, unsigned black_height, unsigned black_depth, + node_t *nil) +{ + unsigned ret = 0; + node_t *left_node = rbtn_left_get(node_t, link, node); + node_t *right_node = rbtn_right_get(node_t, link, node); + + if (rbtn_red_get(node_t, link, node) == false) + black_depth++; + + /* Red nodes must be interleaved with black nodes. */ + if (rbtn_red_get(node_t, link, node)) { + assert_false(rbtn_red_get(node_t, link, left_node), + "Node should be black"); + assert_false(rbtn_red_get(node_t, link, right_node), + "Node should be black"); + } + + if (node == nil) + return (ret); + /* Self. */ + assert_u32_eq(node->magic, NODE_MAGIC, "Bad magic"); + + /* Left subtree. */ + if (left_node != nil) + ret += tree_recurse(left_node, black_height, black_depth, nil); + else + ret += (black_depth != black_height); + + /* Right subtree. */ + if (right_node != nil) + ret += tree_recurse(right_node, black_height, black_depth, nil); + else + ret += (black_depth != black_height); + + return (ret); +} + +static node_t * +tree_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *i = (unsigned *)data; + node_t *search_node; + + assert_u32_eq(node->magic, NODE_MAGIC, "Bad magic"); + + /* Test rb_search(). */ + search_node = tree_search(tree, node); + assert_ptr_eq(search_node, node, + "tree_search() returned unexpected node"); + + /* Test rb_nsearch(). */ + search_node = tree_nsearch(tree, node); + assert_ptr_eq(search_node, node, + "tree_nsearch() returned unexpected node"); + + /* Test rb_psearch(). */ + search_node = tree_psearch(tree, node); + assert_ptr_eq(search_node, node, + "tree_psearch() returned unexpected node"); + + (*i)++; + + return (NULL); +} + +static unsigned +tree_iterate(tree_t *tree) +{ + unsigned i; + + i = 0; + tree_iter(tree, NULL, tree_iterate_cb, (void *)&i); + + return (i); +} + +static unsigned +tree_iterate_reverse(tree_t *tree) +{ + unsigned i; + + i = 0; + tree_reverse_iter(tree, NULL, tree_iterate_cb, (void *)&i); + + return (i); +} + +static void +node_remove(tree_t *tree, node_t *node, unsigned nnodes) +{ + node_t *search_node; + unsigned black_height, imbalances; + + tree_remove(tree, node); + + /* Test rb_nsearch(). */ + search_node = tree_nsearch(tree, node); + if (search_node != NULL) { + assert_u64_ge(search_node->key, node->key, + "Key ordering error"); + } + + /* Test rb_psearch(). */ + search_node = tree_psearch(tree, node); + if (search_node != NULL) { + assert_u64_le(search_node->key, node->key, + "Key ordering error"); + } + + node->magic = 0; + + rbtn_black_height(node_t, link, tree, black_height); + imbalances = tree_recurse(tree->rbt_root, black_height, 0, + &(tree->rbt_nil)); + assert_u_eq(imbalances, 0, "Tree is unbalanced"); + assert_u_eq(tree_iterate(tree), nnodes-1, + "Unexpected node iteration count"); + assert_u_eq(tree_iterate_reverse(tree), nnodes-1, + "Unexpected node iteration count"); +} + +static node_t * +remove_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *nnodes = (unsigned *)data; + node_t *ret = tree_next(tree, node); + + node_remove(tree, node, *nnodes); + + return (ret); +} + +static node_t * +remove_reverse_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *nnodes = (unsigned *)data; + node_t *ret = tree_prev(tree, node); + + node_remove(tree, node, *nnodes); + + return (ret); +} + +TEST_BEGIN(test_rb_random) +{ +#define NNODES 25 +#define NBAGS 250 +#define SEED 42 + sfmt_t *sfmt; + uint64_t bag[NNODES]; + tree_t tree; + node_t nodes[NNODES]; + unsigned i, j, k, black_height, imbalances; + + sfmt = init_gen_rand(SEED); + for (i = 0; i < NBAGS; i++) { + switch (i) { + case 0: + /* Insert in order. */ + for (j = 0; j < NNODES; j++) + bag[j] = j; + break; + case 1: + /* Insert in reverse order. */ + for (j = 0; j < NNODES; j++) + bag[j] = NNODES - j - 1; + break; + default: + for (j = 0; j < NNODES; j++) + bag[j] = gen_rand64_range(sfmt, NNODES); + } + + for (j = 1; j <= NNODES; j++) { + /* Initialize tree and nodes. */ + tree_new(&tree); + tree.rbt_nil.magic = 0; + for (k = 0; k < j; k++) { + nodes[k].magic = NODE_MAGIC; + nodes[k].key = bag[k]; + } + + /* Insert nodes. */ + for (k = 0; k < j; k++) { + tree_insert(&tree, &nodes[k]); + + rbtn_black_height(node_t, link, &tree, + black_height); + imbalances = tree_recurse(tree.rbt_root, + black_height, 0, &(tree.rbt_nil)); + assert_u_eq(imbalances, 0, + "Tree is unbalanced"); + + assert_u_eq(tree_iterate(&tree), k+1, + "Unexpected node iteration count"); + assert_u_eq(tree_iterate_reverse(&tree), k+1, + "Unexpected node iteration count"); + + assert_ptr_not_null(tree_first(&tree), + "Tree should not be empty"); + assert_ptr_not_null(tree_last(&tree), + "Tree should not be empty"); + + tree_next(&tree, &nodes[k]); + tree_prev(&tree, &nodes[k]); + } + + /* Remove nodes. */ + switch (i % 4) { + case 0: + for (k = 0; k < j; k++) + node_remove(&tree, &nodes[k], j - k); + break; + case 1: + for (k = j; k > 0; k--) + node_remove(&tree, &nodes[k-1], k); + break; + case 2: { + node_t *start; + unsigned nnodes = j; + + start = NULL; + do { + start = tree_iter(&tree, start, + remove_iterate_cb, (void *)&nnodes); + nnodes--; + } while (start != NULL); + assert_u_eq(nnodes, 0, + "Removal terminated early"); + break; + } case 3: { + node_t *start; + unsigned nnodes = j; + + start = NULL; + do { + start = tree_reverse_iter(&tree, start, + remove_reverse_iterate_cb, + (void *)&nnodes); + nnodes--; + } while (start != NULL); + assert_u_eq(nnodes, 0, + "Removal terminated early"); + break; + } default: + not_reached(); + } + } + } + fini_gen_rand(sfmt); +#undef NNODES +#undef NBAGS +#undef SEED +} +TEST_END + +int +main(void) +{ + + return (test( + test_rb_empty, + test_rb_random)); +} diff --git a/deps/jemalloc/test/unit/rtree.c b/deps/jemalloc/test/unit/rtree.c new file mode 100644 index 00000000000..5463055fe92 --- /dev/null +++ b/deps/jemalloc/test/unit/rtree.c @@ -0,0 +1,118 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_rtree_get_empty) +{ + unsigned i; + + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + assert_u_eq(rtree_get(rtree, 0), 0, + "rtree_get() should return NULL for empty tree"); + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_extrema) +{ + unsigned i; + + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + + rtree_set(rtree, 0, 1); + assert_u_eq(rtree_get(rtree, 0), 1, + "rtree_get() should return previously set value"); + + rtree_set(rtree, ~((uintptr_t)0), 1); + assert_u_eq(rtree_get(rtree, ~((uintptr_t)0)), 1, + "rtree_get() should return previously set value"); + + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_bits) +{ + unsigned i, j, k; + + for (i = 1; i < (sizeof(uintptr_t) << 3); i++) { + uintptr_t keys[] = {0, 1, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1}; + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + + for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { + rtree_set(rtree, keys[j], 1); + for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) { + assert_u_eq(rtree_get(rtree, keys[k]), 1, + "rtree_get() should return previously set " + "value and ignore insignificant key bits; " + "i=%u, j=%u, k=%u, set key=%#"PRIxPTR", " + "get key=%#"PRIxPTR, i, j, k, keys[j], + keys[k]); + } + assert_u_eq(rtree_get(rtree, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), 0, + "Only leftmost rtree leaf should be set; " + "i=%u, j=%u", i, j); + rtree_set(rtree, keys[j], 0); + } + + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_random) +{ + unsigned i; + sfmt_t *sfmt; +#define NSET 100 +#define SEED 42 + + sfmt = init_gen_rand(SEED); + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + uintptr_t keys[NSET]; + unsigned j; + + for (j = 0; j < NSET; j++) { + keys[j] = (uintptr_t)gen_rand64(sfmt); + rtree_set(rtree, keys[j], 1); + assert_u_eq(rtree_get(rtree, keys[j]), 1, + "rtree_get() should return previously set value"); + } + for (j = 0; j < NSET; j++) { + assert_u_eq(rtree_get(rtree, keys[j]), 1, + "rtree_get() should return previously set value"); + } + + for (j = 0; j < NSET; j++) { + rtree_set(rtree, keys[j], 0); + assert_u_eq(rtree_get(rtree, keys[j]), 0, + "rtree_get() should return previously set value"); + } + for (j = 0; j < NSET; j++) { + assert_u_eq(rtree_get(rtree, keys[j]), 0, + "rtree_get() should return previously set value"); + } + + rtree_delete(rtree); + } + fini_gen_rand(sfmt); +#undef NSET +#undef SEED +} +TEST_END + +int +main(void) +{ + + return (test( + test_rtree_get_empty, + test_rtree_extrema, + test_rtree_bits, + test_rtree_random)); +} diff --git a/deps/jemalloc/test/unit/stats.c b/deps/jemalloc/test/unit/stats.c new file mode 100644 index 00000000000..03a55c7fdce --- /dev/null +++ b/deps/jemalloc/test/unit/stats.c @@ -0,0 +1,380 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_stats_summary) +{ + size_t *cactive; + size_t sz, allocated, active, mapped; + int expected = config_stats ? 0 : ENOENT; + + sz = sizeof(cactive); + assert_d_eq(mallctl("stats.cactive", &cactive, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.allocated", &allocated, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.active", &active, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.mapped", &mapped, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_le(active, *cactive, + "active should be no larger than cactive"); + assert_zu_le(allocated, active, + "allocated should be no larger than active"); + assert_zu_le(active, mapped, + "active should be no larger than mapped"); + } +} +TEST_END + +TEST_BEGIN(test_stats_chunks) +{ + size_t current, high; + uint64_t total; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.chunks.current", ¤t, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.chunks.total", &total, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.chunks.high", &high, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_le(current, high, + "current should be no larger than high"); + assert_u64_le((uint64_t)high, total, + "high should be no larger than total"); + } +} +TEST_END + +TEST_BEGIN(test_stats_huge) +{ + void *p; + uint64_t epoch; + size_t allocated; + uint64_t nmalloc, ndalloc; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + + p = mallocx(arena_maxclass+1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.huge.allocated", &allocated, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.huge.nmalloc", &nmalloc, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.huge.ndalloc", &ndalloc, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_summary) +{ + unsigned arena; + void *small, *large; + uint64_t epoch; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + size_t mapped; + uint64_t npurge, nmadvise, purged; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + small = mallocx(SMALL_MAXCLASS, 0); + assert_ptr_not_null(small, "Unexpected mallocx() failure"); + large = mallocx(arena_maxclass, 0); + assert_ptr_not_null(large, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.mapped", &mapped, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.nmadvise", &nmadvise, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.purged", &purged, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + + if (config_stats) { + assert_u64_gt(npurge, 0, + "At least one purge should have occurred"); + assert_u64_le(nmadvise, purged, + "nmadvise should be no greater than purged"); + } + + dallocx(small, 0); + dallocx(large, 0); +} +TEST_END + +void * +thd_start(void *arg) +{ + + return (NULL); +} + +static void +no_lazy_lock(void) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} + +TEST_BEGIN(test_stats_arenas_small) +{ + unsigned arena; + void *p; + size_t sz, allocated; + uint64_t epoch, nmalloc, ndalloc, nrequests; + int expected = config_stats ? 0 : ENOENT; + + no_lazy_lock(); /* Lazy locking would dodge tcache testing. */ + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(SMALL_MAXCLASS, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), + config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.small.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.small.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.small.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.small.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_gt(nmalloc, 0, + "nmalloc should be no greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_large) +{ + unsigned arena; + void *p; + size_t sz, allocated; + uint64_t epoch, nmalloc, ndalloc, nrequests; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(arena_maxclass, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.large.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_zu_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_zu_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_zu_gt(nrequests, 0, + "nrequests should be greater than zero"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_bins) +{ + unsigned arena; + void *p; + size_t sz, allocated, curruns; + uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; + uint64_t nruns, nreruns; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(arena_bin_info[0].reg_size, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), + config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", &nfills, &sz, + NULL, 0), config_tcache ? expected : ENOENT, + "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nflushes", &nflushes, &sz, + NULL, 0), config_tcache ? expected : ENOENT, + "Unexpected mallctl() result"); + + assert_d_eq(mallctl("stats.arenas.0.bins.0.nruns", &nruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nreruns", &nreruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.curruns", &curruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + if (config_tcache) { + assert_u64_gt(nfills, 0, + "At least one fill should have occurred"); + assert_u64_gt(nflushes, 0, + "At least one flush should have occurred"); + } + assert_u64_gt(nruns, 0, + "At least one run should have been allocated"); + assert_zu_gt(curruns, 0, + "At least one run should be currently allocated"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_lruns) +{ + unsigned arena; + void *p; + uint64_t epoch, nmalloc, ndalloc, nrequests; + size_t curruns, sz; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(SMALL_MAXCLASS+1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.curruns", &curruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_u64_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + assert_u64_gt(curruns, 0, + "At least one run should be currently allocated"); + } + + dallocx(p, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_stats_summary, + test_stats_chunks, + test_stats_huge, + test_stats_arenas_summary, + test_stats_arenas_small, + test_stats_arenas_large, + test_stats_arenas_bins, + test_stats_arenas_lruns)); +} diff --git a/deps/jemalloc/test/unit/tsd.c b/deps/jemalloc/test/unit/tsd.c new file mode 100644 index 00000000000..f421c1a3cff --- /dev/null +++ b/deps/jemalloc/test/unit/tsd.c @@ -0,0 +1,71 @@ +#include "test/jemalloc_test.h" + +#define THREAD_DATA 0x72b65c10 + +typedef unsigned int data_t; + +static bool data_cleanup_executed; + +void +data_cleanup(void *arg) +{ + data_t *data = (data_t *)arg; + + assert_x_eq(*data, THREAD_DATA, + "Argument passed into cleanup function should match tsd value"); + data_cleanup_executed = true; +} + +malloc_tsd_protos(, data, data_t) +malloc_tsd_externs(data, data_t) +#define DATA_INIT 0x12345678 +malloc_tsd_data(, data, data_t, DATA_INIT) +malloc_tsd_funcs(, data, data_t, DATA_INIT, data_cleanup) + +static void * +thd_start(void *arg) +{ + data_t d = (data_t)(uintptr_t)arg; + assert_x_eq(*data_tsd_get(), DATA_INIT, + "Initial tsd get should return initialization value"); + + data_tsd_set(&d); + assert_x_eq(*data_tsd_get(), d, + "After tsd set, tsd get should return value that was set"); + + d = 0; + assert_x_eq(*data_tsd_get(), (data_t)(uintptr_t)arg, + "Resetting local data should have no effect on tsd"); + + return (NULL); +} + +TEST_BEGIN(test_tsd_main_thread) +{ + + thd_start((void *) 0xa5f3e329); +} +TEST_END + +TEST_BEGIN(test_tsd_sub_thread) +{ + thd_t thd; + + data_cleanup_executed = false; + thd_create(&thd, thd_start, (void *)THREAD_DATA); + thd_join(thd, NULL); + assert_true(data_cleanup_executed, + "Cleanup function should have executed"); +} +TEST_END + +int +main(void) +{ + + data_tsd_boot(); + + return (test( + test_tsd_main_thread, + test_tsd_sub_thread)); +} diff --git a/deps/jemalloc/test/unit/util.c b/deps/jemalloc/test/unit/util.c new file mode 100644 index 00000000000..dc3cfe8a9db --- /dev/null +++ b/deps/jemalloc/test/unit/util.c @@ -0,0 +1,294 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_pow2_ceil) +{ + unsigned i, pow2; + size_t x; + + assert_zu_eq(pow2_ceil(0), 0, "Unexpected result"); + + for (i = 0; i < sizeof(size_t) * 8; i++) { + assert_zu_eq(pow2_ceil(ZU(1) << i), ZU(1) << i, + "Unexpected result"); + } + + for (i = 2; i < sizeof(size_t) * 8; i++) { + assert_zu_eq(pow2_ceil((ZU(1) << i) - 1), ZU(1) << i, + "Unexpected result"); + } + + for (i = 0; i < sizeof(size_t) * 8 - 1; i++) { + assert_zu_eq(pow2_ceil((ZU(1) << i) + 1), ZU(1) << (i+1), + "Unexpected result"); + } + + for (pow2 = 1; pow2 < 25; pow2++) { + for (x = (ZU(1) << (pow2-1)) + 1; x <= ZU(1) << pow2; x++) { + assert_zu_eq(pow2_ceil(x), ZU(1) << pow2, + "Unexpected result, x=%zu", x); + } + } +} +TEST_END + +TEST_BEGIN(test_malloc_strtoumax_no_endptr) +{ + int err; + + set_errno(0); + assert_ju_eq(malloc_strtoumax("0", NULL, 0), 0, "Unexpected result"); + err = get_errno(); + assert_d_eq(err, 0, "Unexpected failure"); +} +TEST_END + +TEST_BEGIN(test_malloc_strtoumax) +{ + struct test_s { + const char *input; + const char *expected_remainder; + int base; + int expected_errno; + const char *expected_errno_name; + uintmax_t expected_x; + }; +#define ERR(e) e, #e +#define UMAX(x) ((uintmax_t)x##ULL) + struct test_s tests[] = { + {"0", "0", -1, ERR(EINVAL), UINTMAX_MAX}, + {"0", "0", 1, ERR(EINVAL), UINTMAX_MAX}, + {"0", "0", 37, ERR(EINVAL), UINTMAX_MAX}, + + {"", "", 0, ERR(EINVAL), UINTMAX_MAX}, + {"+", "+", 0, ERR(EINVAL), UINTMAX_MAX}, + {"++3", "++3", 0, ERR(EINVAL), UINTMAX_MAX}, + {"-", "-", 0, ERR(EINVAL), UINTMAX_MAX}, + + {"42", "", 0, ERR(0), UMAX(42)}, + {"+42", "", 0, ERR(0), UMAX(42)}, + {"-42", "", 0, ERR(0), UMAX(-42)}, + {"042", "", 0, ERR(0), UMAX(042)}, + {"+042", "", 0, ERR(0), UMAX(042)}, + {"-042", "", 0, ERR(0), UMAX(-042)}, + {"0x42", "", 0, ERR(0), UMAX(0x42)}, + {"+0x42", "", 0, ERR(0), UMAX(0x42)}, + {"-0x42", "", 0, ERR(0), UMAX(-0x42)}, + + {"0", "", 0, ERR(0), UMAX(0)}, + {"1", "", 0, ERR(0), UMAX(1)}, + + {"42", "", 0, ERR(0), UMAX(42)}, + {" 42", "", 0, ERR(0), UMAX(42)}, + {"42 ", " ", 0, ERR(0), UMAX(42)}, + {"0x", "x", 0, ERR(0), UMAX(0)}, + {"42x", "x", 0, ERR(0), UMAX(42)}, + + {"07", "", 0, ERR(0), UMAX(7)}, + {"010", "", 0, ERR(0), UMAX(8)}, + {"08", "8", 0, ERR(0), UMAX(0)}, + {"0_", "_", 0, ERR(0), UMAX(0)}, + + {"0x", "x", 0, ERR(0), UMAX(0)}, + {"0X", "X", 0, ERR(0), UMAX(0)}, + {"0xg", "xg", 0, ERR(0), UMAX(0)}, + {"0XA", "", 0, ERR(0), UMAX(10)}, + + {"010", "", 10, ERR(0), UMAX(10)}, + {"0x3", "x3", 10, ERR(0), UMAX(0)}, + + {"12", "2", 2, ERR(0), UMAX(1)}, + {"78", "8", 8, ERR(0), UMAX(7)}, + {"9a", "a", 10, ERR(0), UMAX(9)}, + {"9A", "A", 10, ERR(0), UMAX(9)}, + {"fg", "g", 16, ERR(0), UMAX(15)}, + {"FG", "G", 16, ERR(0), UMAX(15)}, + {"0xfg", "g", 16, ERR(0), UMAX(15)}, + {"0XFG", "G", 16, ERR(0), UMAX(15)}, + {"z_", "_", 36, ERR(0), UMAX(35)}, + {"Z_", "_", 36, ERR(0), UMAX(35)} + }; +#undef ERR +#undef UMAX + unsigned i; + + for (i = 0; i < sizeof(tests)/sizeof(struct test_s); i++) { + struct test_s *test = &tests[i]; + int err; + uintmax_t result; + char *remainder; + + set_errno(0); + result = malloc_strtoumax(test->input, &remainder, test->base); + err = get_errno(); + assert_d_eq(err, test->expected_errno, + "Expected errno %s for \"%s\", base %d", + test->expected_errno_name, test->input, test->base); + assert_str_eq(remainder, test->expected_remainder, + "Unexpected remainder for \"%s\", base %d", + test->input, test->base); + if (err == 0) { + assert_ju_eq(result, test->expected_x, + "Unexpected result for \"%s\", base %d", + test->input, test->base); + } + } +} +TEST_END + +TEST_BEGIN(test_malloc_snprintf_truncated) +{ +#define BUFLEN 15 + char buf[BUFLEN]; + int result; + size_t len; +#define TEST(expected_str_untruncated, fmt...) do { \ + result = malloc_snprintf(buf, len, fmt); \ + assert_d_eq(strncmp(buf, expected_str_untruncated, len-1), 0, \ + "Unexpected string inequality (\"%s\" vs \"%s\")", \ + buf, expected_str_untruncated); \ + assert_d_eq(result, strlen(expected_str_untruncated), \ + "Unexpected result"); \ +} while (0) + + for (len = 1; len < BUFLEN; len++) { + TEST("012346789", "012346789"); + TEST("a0123b", "a%sb", "0123"); + TEST("a01234567", "a%s%s", "0123", "4567"); + TEST("a0123 ", "a%-6s", "0123"); + TEST("a 0123", "a%6s", "0123"); + TEST("a 012", "a%6.3s", "0123"); + TEST("a 012", "a%*.*s", 6, 3, "0123"); + TEST("a 123b", "a% db", 123); + TEST("a123b", "a%-db", 123); + TEST("a-123b", "a%-db", -123); + TEST("a+123b", "a%+db", 123); + } +#undef BUFLEN +#undef TEST +} +TEST_END + +TEST_BEGIN(test_malloc_snprintf) +{ +#define BUFLEN 128 + char buf[BUFLEN]; + int result; +#define TEST(expected_str, fmt...) do { \ + result = malloc_snprintf(buf, sizeof(buf), fmt); \ + assert_str_eq(buf, expected_str, "Unexpected output"); \ + assert_d_eq(result, strlen(expected_str), "Unexpected result"); \ +} while (0) + + TEST("hello", "hello"); + + TEST("50%, 100%", "50%%, %d%%", 100); + + TEST("a0123b", "a%sb", "0123"); + + TEST("a 0123b", "a%5sb", "0123"); + TEST("a 0123b", "a%*sb", 5, "0123"); + + TEST("a0123 b", "a%-5sb", "0123"); + TEST("a0123b", "a%*sb", -1, "0123"); + TEST("a0123 b", "a%*sb", -5, "0123"); + TEST("a0123 b", "a%-*sb", -5, "0123"); + + TEST("a012b", "a%.3sb", "0123"); + TEST("a012b", "a%.*sb", 3, "0123"); + TEST("a0123b", "a%.*sb", -3, "0123"); + + TEST("a 012b", "a%5.3sb", "0123"); + TEST("a 012b", "a%5.*sb", 3, "0123"); + TEST("a 012b", "a%*.3sb", 5, "0123"); + TEST("a 012b", "a%*.*sb", 5, 3, "0123"); + TEST("a 0123b", "a%*.*sb", 5, -3, "0123"); + + TEST("_abcd_", "_%x_", 0xabcd); + TEST("_0xabcd_", "_%#x_", 0xabcd); + TEST("_1234_", "_%o_", 01234); + TEST("_01234_", "_%#o_", 01234); + TEST("_1234_", "_%u_", 1234); + + TEST("_1234_", "_%d_", 1234); + TEST("_ 1234_", "_% d_", 1234); + TEST("_+1234_", "_%+d_", 1234); + TEST("_-1234_", "_%d_", -1234); + TEST("_-1234_", "_% d_", -1234); + TEST("_-1234_", "_%+d_", -1234); + + TEST("_-1234_", "_%d_", -1234); + TEST("_1234_", "_%d_", 1234); + TEST("_-1234_", "_%i_", -1234); + TEST("_1234_", "_%i_", 1234); + TEST("_01234_", "_%#o_", 01234); + TEST("_1234_", "_%u_", 1234); + TEST("_0x1234abc_", "_%#x_", 0x1234abc); + TEST("_0X1234ABC_", "_%#X_", 0x1234abc); + TEST("_c_", "_%c_", 'c'); + TEST("_string_", "_%s_", "string"); + TEST("_0x42_", "_%p_", ((void *)0x42)); + + TEST("_-1234_", "_%ld_", ((long)-1234)); + TEST("_1234_", "_%ld_", ((long)1234)); + TEST("_-1234_", "_%li_", ((long)-1234)); + TEST("_1234_", "_%li_", ((long)1234)); + TEST("_01234_", "_%#lo_", ((long)01234)); + TEST("_1234_", "_%lu_", ((long)1234)); + TEST("_0x1234abc_", "_%#lx_", ((long)0x1234abc)); + TEST("_0X1234ABC_", "_%#lX_", ((long)0x1234ABC)); + + TEST("_-1234_", "_%lld_", ((long long)-1234)); + TEST("_1234_", "_%lld_", ((long long)1234)); + TEST("_-1234_", "_%lli_", ((long long)-1234)); + TEST("_1234_", "_%lli_", ((long long)1234)); + TEST("_01234_", "_%#llo_", ((long long)01234)); + TEST("_1234_", "_%llu_", ((long long)1234)); + TEST("_0x1234abc_", "_%#llx_", ((long long)0x1234abc)); + TEST("_0X1234ABC_", "_%#llX_", ((long long)0x1234ABC)); + + TEST("_-1234_", "_%qd_", ((long long)-1234)); + TEST("_1234_", "_%qd_", ((long long)1234)); + TEST("_-1234_", "_%qi_", ((long long)-1234)); + TEST("_1234_", "_%qi_", ((long long)1234)); + TEST("_01234_", "_%#qo_", ((long long)01234)); + TEST("_1234_", "_%qu_", ((long long)1234)); + TEST("_0x1234abc_", "_%#qx_", ((long long)0x1234abc)); + TEST("_0X1234ABC_", "_%#qX_", ((long long)0x1234ABC)); + + TEST("_-1234_", "_%jd_", ((intmax_t)-1234)); + TEST("_1234_", "_%jd_", ((intmax_t)1234)); + TEST("_-1234_", "_%ji_", ((intmax_t)-1234)); + TEST("_1234_", "_%ji_", ((intmax_t)1234)); + TEST("_01234_", "_%#jo_", ((intmax_t)01234)); + TEST("_1234_", "_%ju_", ((intmax_t)1234)); + TEST("_0x1234abc_", "_%#jx_", ((intmax_t)0x1234abc)); + TEST("_0X1234ABC_", "_%#jX_", ((intmax_t)0x1234ABC)); + + TEST("_1234_", "_%td_", ((ptrdiff_t)1234)); + TEST("_-1234_", "_%td_", ((ptrdiff_t)-1234)); + TEST("_1234_", "_%ti_", ((ptrdiff_t)1234)); + TEST("_-1234_", "_%ti_", ((ptrdiff_t)-1234)); + + TEST("_-1234_", "_%zd_", ((ssize_t)-1234)); + TEST("_1234_", "_%zd_", ((ssize_t)1234)); + TEST("_-1234_", "_%zi_", ((ssize_t)-1234)); + TEST("_1234_", "_%zi_", ((ssize_t)1234)); + TEST("_01234_", "_%#zo_", ((ssize_t)01234)); + TEST("_1234_", "_%zu_", ((ssize_t)1234)); + TEST("_0x1234abc_", "_%#zx_", ((ssize_t)0x1234abc)); + TEST("_0X1234ABC_", "_%#zX_", ((ssize_t)0x1234ABC)); +#undef BUFLEN +} +TEST_END + +int +main(void) +{ + + return (test( + test_pow2_ceil, + test_malloc_strtoumax_no_endptr, + test_malloc_strtoumax, + test_malloc_snprintf_truncated, + test_malloc_snprintf)); +} diff --git a/deps/jemalloc/test/unit/zero.c b/deps/jemalloc/test/unit/zero.c new file mode 100644 index 00000000000..65a8f0c9c32 --- /dev/null +++ b/deps/jemalloc/test/unit/zero.c @@ -0,0 +1,78 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_FILL +const char *malloc_conf = + "abort:false,junk:false,zero:true,redzone:false,quarantine:0"; +#endif + +static void +test_zero(size_t sz_min, size_t sz_max) +{ + char *s; + size_t sz_prev, sz, i; + + sz_prev = 0; + s = (char *)mallocx(sz_min, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + + for (sz = sallocx(s, 0); sz <= sz_max; + sz_prev = sz, sz = sallocx(s, 0)) { + if (sz_prev > 0) { + assert_c_eq(s[0], 'a', + "Previously allocated byte %zu/%zu is corrupted", + ZU(0), sz_prev); + assert_c_eq(s[sz_prev-1], 'a', + "Previously allocated byte %zu/%zu is corrupted", + sz_prev-1, sz_prev); + } + + for (i = sz_prev; i < sz; i++) { + assert_c_eq(s[i], 0x0, + "Newly allocated byte %zu/%zu isn't zero-filled", + i, sz); + s[i] = 'a'; + } + + if (xallocx(s, sz+1, 0, 0) == sz) { + s = (char *)rallocx(s, sz+1, 0); + assert_ptr_not_null((void *)s, + "Unexpected rallocx() failure"); + } + } + + dallocx(s, 0); +} + +TEST_BEGIN(test_zero_small) +{ + + test_skip_if(!config_fill); + test_zero(1, SMALL_MAXCLASS-1); +} +TEST_END + +TEST_BEGIN(test_zero_large) +{ + + test_skip_if(!config_fill); + test_zero(SMALL_MAXCLASS+1, arena_maxclass); +} +TEST_END + +TEST_BEGIN(test_zero_huge) +{ + + test_skip_if(!config_fill); + test_zero(arena_maxclass+1, chunksize*2); +} +TEST_END + +int +main(void) +{ + + return (test( + test_zero_small, + test_zero_large, + test_zero_huge)); +} diff --git a/deps/linenoise/.gitignore b/deps/linenoise/.gitignore index 28f258a304c..7ab7825f58a 100644 --- a/deps/linenoise/.gitignore +++ b/deps/linenoise/.gitignore @@ -1 +1,3 @@ -linenoise_example* +linenoise_example +*.dSYM +history.txt diff --git a/deps/linenoise/Makefile b/deps/linenoise/Makefile index 841f39072ff..1dd894b49e9 100644 --- a/deps/linenoise/Makefile +++ b/deps/linenoise/Makefile @@ -1,10 +1,21 @@ -linenoise_example: linenoise.h linenoise.c +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +linenoise.o: linenoise.h linenoise.c linenoise_example: linenoise.o example.o - $(CC) $(ARCH) -Wall -W -Os -g -o linenoise_example linenoise.o example.o + $(R_LD) -o $@ $^ .c.o: - $(CC) $(ARCH) -c -Wall -W -Os -g $< + $(R_CC) -c $< clean: rm -f linenoise_example *.o diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown index 9612da47f7c..c845673cd41 100644 --- a/deps/linenoise/README.markdown +++ b/deps/linenoise/README.markdown @@ -1,8 +1,13 @@ # Linenoise -A minimal, zero-config, BSD licensed, readline replacement. +A minimal, zero-config, BSD licensed, readline replacement used in Redis, +MongoDB, and Android. -News: linenoise is now part of [Android](http://android.git.kernel.org/?p=platform/system/core.git;a=tree;f=liblinenoise;h=56450eaed7f783760e5e6a5993ef75cde2e29dea;hb=HEAD Android)! +* Single and multi line editing mode with the usual key bindings implemented. +* History handling. +* Completion. +* About 1,100 lines of BSD license source code. +* Only uses a subset of VT100 escapes (ANSI.SYS compatible). ## Can a line editing library be 20k lines of code? @@ -10,7 +15,7 @@ Line editing with some support for history is a really important feature for com So what usually happens is either: - * Large programs with configure scripts disabling line editing if readline is not present in the system, or not supporting it at all since readline is GPL licensed and libedit (the BSD clone) is not as known and available as readline is (Readl world example of this problem: Tclsh). + * Large programs with configure scripts disabling line editing if readline is not present in the system, or not supporting it at all since readline is GPL licensed and libedit (the BSD clone) is not as known and available as readline is (Real world example of this problem: Tclsh). * Smaller programs not using a configure script not supporting line editing at all (A problem we had with Redis-cli for instance). The result is a pollution of binaries without line editing support. @@ -19,27 +24,29 @@ So I spent more or less two hours doing a reality check resulting in this little ## Terminals, in 2010. -Apparently almost every terminal you can happen to use today has some kind of support for VT100 alike escape sequences. So I tried to write a lib using just very basic VT100 features. The resulting library appears to work everywhere I tried to use it. +Apparently almost every terminal you can happen to use today has some kind of support for basic VT100 escape sequences. So I tried to write a lib using just very basic VT100 features. The resulting library appears to work everywhere I tried to use it, and now can work even on ANSI.SYS compatible terminals, since no +VT220 specific sequences are used anymore. -Since it's so young I guess there are a few bugs, or the lib may not compile or work with some operating system, but it's a matter of a few weeks and eventually we'll get it right, and there will be no excuses for not shipping command line tools without built-in line editing support. - -The library is currently less than 400 lines of code. In order to use it in your project just look at the *example.c* file in the source distribution, it is trivial. Linenoise is BSD code, so you can use both in free software and commercial software. +The library is currently about 1100 lines of code. In order to use it in your project just look at the *example.c* file in the source distribution, it is trivial. Linenoise is BSD code, so you can use both in free software and commercial software. ## Tested with... * Linux text only console ($TERM = linux) * Linux KDE terminal application ($TERM = xterm) * Linux xterm ($TERM = xterm) + * Linux Buildroot ($TERM = vt100) * Mac OS X iTerm ($TERM = xterm) * Mac OS X default Terminal.app ($TERM = xterm) * OpenBSD 4.5 through an OSX Terminal.app ($TERM = screen) * IBM AIX 6.1 * FreeBSD xterm ($TERM = xterm) + * ANSI.SYS Please test it everywhere you can and report back! ## Let's push this forward! -Please fork it and add something interesting and send me a pull request. What's especially interesting are fixes, new key bindings, completion. +Patches should be provided in the respect of linenoise sensibility for small +easy to understand code. Send feedbacks to antirez at gmail diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c index ea0b515c1fc..a2f0936ede4 100644 --- a/deps/linenoise/example.c +++ b/deps/linenoise/example.c @@ -1,5 +1,6 @@ #include #include +#include #include "linenoise.h" @@ -10,16 +11,52 @@ void completion(const char *buf, linenoiseCompletions *lc) { } } -int main(void) { +int main(int argc, char **argv) { char *line; + char *prgname = argv[0]; + /* Parse options, with --multiline we enable multi line editing. */ + while(argc > 1) { + argc--; + argv++; + if (!strcmp(*argv,"--multiline")) { + linenoiseSetMultiLine(1); + printf("Multi-line mode enabled.\n"); + } else if (!strcmp(*argv,"--keycodes")) { + linenoisePrintKeyCodes(); + exit(0); + } else { + fprintf(stderr, "Usage: %s [--multiline] [--keycodes]\n", prgname); + exit(1); + } + } + + /* Set the completion callback. This will be called every time the + * user uses the key. */ linenoiseSetCompletionCallback(completion); + + /* Load history from file. The history file is just a plain text file + * where entries are separated by newlines. */ linenoiseHistoryLoad("history.txt"); /* Load the history at startup */ + + /* Now this is the main loop of the typical linenoise-based application. + * The call to linenoise() will block as long as the user types something + * and presses enter. + * + * The typed string is returned as a malloc() allocated string by + * linenoise, so the user needs to free() it. */ while((line = linenoise("hello> ")) != NULL) { - if (line[0] != '\0') { + /* Do something with the string. */ + if (line[0] != '\0' && line[0] != '/') { printf("echo: '%s'\n", line); - linenoiseHistoryAdd(line); - linenoiseHistorySave("history.txt"); /* Save every new entry */ + linenoiseHistoryAdd(line); /* Add to the history. */ + linenoiseHistorySave("history.txt"); /* Save the history on disk. */ + } else if (!strncmp(line,"/historylen",11)) { + /* The "/historylen" command will change the history len. */ + int len = atoi(line+11); + linenoiseHistorySetMaxLen(len); + } else if (line[0] == '/') { + printf("Unreconized command: %s\n", line); } free(line); } diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index 18a15cc45cc..36c0c5f6d26 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -2,52 +2,53 @@ * line editing lib needs to be 20,000 lines of C code. * * You can find the latest source code at: - * + * * http://github.com/antirez/linenoise * * Does a number of crazy assumptions that happen to be true in 99.9999% of * the 2010 UNIX computers around. * - * Copyright (c) 2010, Salvatore Sanfilippo - * Copyright (c) 2010, Pieter Noordhuis + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2013, Salvatore Sanfilippo + * Copyright (c) 2010-2013, Pieter Noordhuis * * All rights reserved. * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions are + * met: * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ------------------------------------------------------------------------ * * References: * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html * * Todo list: - * - Switch to gets() if $TERM is something we can't support. * - Filter bogus Ctrl+ combinations. * - Win32 support * * Bloat: - * - Completion? * - History search like Ctrl+r in readline? * * List of escape sequences used by this program, we do everything just @@ -55,10 +56,6 @@ * flickering effect with some slow terminal, but the lesser sequences * the more compatible. * - * CHA (Cursor Horizontal Absolute) - * Sequence: ESC [ n G - * Effect: moves cursor to column n - * * EL (Erase Line) * Sequence: ESC [ n K * Effect: if n is 0 or missing, clear from cursor to end of line @@ -67,19 +64,43 @@ * * CUF (CUrsor Forward) * Sequence: ESC [ n C - * Effect: moves cursor forward of n chars + * Effect: moves cursor forward n chars + * + * CUB (CUrsor Backward) + * Sequence: ESC [ n D + * Effect: moves cursor backward n chars + * + * The following is used to get the terminal width if getting + * the width with the TIOCGWINSZ ioctl fails * - * The following are used to clear the screen: ESC [ H ESC [ 2 J - * This is actually composed of two sequences: + * DSR (Device Status Report) + * Sequence: ESC [ 6 n + * Effect: reports the current cusor position as ESC [ n ; m R + * where n is the row and m is the column * - * cursorhome + * When multi line mode is enabled, we also use an additional escape + * sequence. However multi line editing is disabled by default. + * + * CUU (Cursor Up) + * Sequence: ESC [ n A + * Effect: moves cursor up of n chars. + * + * CUD (Cursor Down) + * Sequence: ESC [ n B + * Effect: moves cursor down of n chars. + * + * When linenoiseClearScreen() is called, two additional escape sequences + * are used in order to clear the screen and position the cursor at home + * position. + * + * CUP (Cursor position) * Sequence: ESC [ H * Effect: moves the cursor to upper left corner * - * ED2 (Clear entire screen) + * ED (Erase display) * Sequence: ESC [ 2 J * Effect: clear the whole screen - * + * */ #include @@ -89,6 +110,7 @@ #include #include #include +#include #include #include #include @@ -96,19 +118,89 @@ #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 #define LINENOISE_MAX_LINE 4096 -static char *unsupported_term[] = {"dumb","cons25",NULL}; +static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; static linenoiseCompletionCallback *completionCallback = NULL; -static struct termios orig_termios; /* in order to restore at exit */ -static int rawmode = 0; /* for atexit() function to check if restore is needed*/ -static int atexit_registered = 0; /* register atexit just 1 time */ +static struct termios orig_termios; /* In order to restore at exit.*/ +static int rawmode = 0; /* For atexit() function to check if restore is needed*/ +static int mlmode = 0; /* Multi line mode. Default is single line. */ +static int atexit_registered = 0; /* Register atexit just 1 time. */ static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN; static int history_len = 0; -char **history = NULL; +static char **history = NULL; + +/* The linenoiseState structure represents the state during line editing. + * We pass this state to functions implementing specific editing + * functionalities. */ +struct linenoiseState { + int ifd; /* Terminal stdin file descriptor. */ + int ofd; /* Terminal stdout file descriptor. */ + char *buf; /* Edited line buffer. */ + size_t buflen; /* Edited line buffer size. */ + const char *prompt; /* Prompt to display. */ + size_t plen; /* Prompt length. */ + size_t pos; /* Current cursor position. */ + size_t oldpos; /* Previous refresh cursor position. */ + size_t len; /* Current edited line length. */ + size_t cols; /* Number of columns in terminal. */ + size_t maxrows; /* Maximum num of rows used so far (multiline mode) */ + int history_index; /* The history index we are currently editing. */ +}; + +enum KEY_ACTION{ + KEY_NULL = 0, /* NULL */ + CTRL_A = 1, /* Ctrl+a */ + CTRL_B = 2, /* Ctrl-b */ + CTRL_C = 3, /* Ctrl-c */ + CTRL_D = 4, /* Ctrl-d */ + CTRL_E = 5, /* Ctrl-e */ + CTRL_F = 6, /* Ctrl-f */ + CTRL_H = 8, /* Ctrl-h */ + TAB = 9, /* Tab */ + CTRL_K = 11, /* Ctrl+k */ + CTRL_L = 12, /* Ctrl+l */ + ENTER = 13, /* Enter */ + CTRL_N = 14, /* Ctrl-n */ + CTRL_P = 16, /* Ctrl-p */ + CTRL_T = 20, /* Ctrl-t */ + CTRL_U = 21, /* Ctrl+u */ + CTRL_W = 23, /* Ctrl+w */ + ESC = 27, /* Escape */ + BACKSPACE = 127 /* Backspace */ +}; static void linenoiseAtExit(void); int linenoiseHistoryAdd(const char *line); +static void refreshLine(struct linenoiseState *l); +/* Debugging macro. */ +#if 0 +FILE *lndebug_fp = NULL; +#define lndebug(...) \ + do { \ + if (lndebug_fp == NULL) { \ + lndebug_fp = fopen("/tmp/lndebug.txt","a"); \ + fprintf(lndebug_fp, \ + "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ + (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ + (int)l->maxrows,old_rows); \ + } \ + fprintf(lndebug_fp, ", " __VA_ARGS__); \ + fflush(lndebug_fp); \ + } while (0) +#else +#define lndebug(fmt, ...) +#endif + +/* ======================= Low level terminal handling ====================== */ + +/* Set if to use or not the multi line mode. */ +void linenoiseSetMultiLine(int ml) { + mlmode = ml; +} + +/* Return true if the terminal name is in the list of terminals we know are + * not able to understand basic escape sequences. */ static int isUnsupportedTerm(void) { char *term = getenv("TERM"); int j; @@ -119,16 +211,7 @@ static int isUnsupportedTerm(void) { return 0; } -static void freeHistory(void) { - if (history) { - int j; - - for (j = 0; j < history_len; j++) - free(history[j]); - free(history); - } -} - +/* Raw mode: 1960 magic shit. */ static int enableRawMode(int fd) { struct termios raw; @@ -170,51 +253,83 @@ static void disableRawMode(int fd) { rawmode = 0; } -/* At exit we'll try to fix the terminal to the initial conditions. */ -static void linenoiseAtExit(void) { - disableRawMode(STDIN_FILENO); - freeHistory(); +/* Use the ESC [6n escape sequence to query the horizontal cursor position + * and return it. On error -1 is returned, on success the position of the + * cursor. */ +static int getCursorPosition(int ifd, int ofd) { + char buf[32]; + int cols, rows; + unsigned int i = 0; + + /* Report cursor location */ + if (write(ofd, "\x1b[6n", 4) != 4) return -1; + + /* Read the response: ESC [ rows ; cols R */ + while (i < sizeof(buf)-1) { + if (read(ifd,buf+i,1) != 1) break; + if (buf[i] == 'R') break; + i++; + } + buf[i] = '\0'; + + /* Parse it. */ + if (buf[0] != ESC || buf[1] != '[') return -1; + if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1; + return cols; } -static int getColumns(void) { +/* Try to get the number of columns in the current terminal, or assume 80 + * if it fails. */ +static int getColumns(int ifd, int ofd) { struct winsize ws; - if (ioctl(1, TIOCGWINSZ, &ws) == -1) return 80; - return ws.ws_col; -} + if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) { + /* ioctl() failed. Try to query the terminal itself. */ + int start, cols; -static void refreshLine(int fd, const char *prompt, char *buf, size_t len, size_t pos, size_t cols) { - char seq[64]; - size_t plen = strlen(prompt); - - while((plen+pos) >= cols) { - buf++; - len--; - pos--; - } - while (plen+len > cols) { - len--; + /* Get the initial position so we can restore it later. */ + start = getCursorPosition(ifd,ofd); + if (start == -1) goto failed; + + /* Go to right margin and get position. */ + if (write(ofd,"\x1b[999C",6) != 6) goto failed; + cols = getCursorPosition(ifd,ofd); + if (cols == -1) goto failed; + + /* Restore position. */ + if (cols > start) { + char seq[32]; + snprintf(seq,32,"\x1b[%dD",cols-start); + if (write(ofd,seq,strlen(seq)) == -1) { + /* Can't recover... */ + } + } + return cols; + } else { + return ws.ws_col; } - /* Cursor to left edge */ - snprintf(seq,64,"\x1b[0G"); - if (write(fd,seq,strlen(seq)) == -1) return; - /* Write the prompt and the current buffer content */ - if (write(fd,prompt,strlen(prompt)) == -1) return; - if (write(fd,buf,len) == -1) return; - /* Erase to right */ - snprintf(seq,64,"\x1b[0K"); - if (write(fd,seq,strlen(seq)) == -1) return; - /* Move cursor to original position. */ - snprintf(seq,64,"\x1b[0G\x1b[%dC", (int)(pos+plen)); - if (write(fd,seq,strlen(seq)) == -1) return; +failed: + return 80; } -static void beep() { +/* Clear the screen. Used to handle ctrl+l */ +void linenoiseClearScreen(void) { + if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) { + /* nothing to do, just to avoid warning. */ + } +} + +/* Beep, used for completion when there is nothing to complete or when all + * the choices were already shown. */ +static void linenoiseBeep(void) { fprintf(stderr, "\x7"); fflush(stderr); } +/* ============================== Completion ================================ */ + +/* Free a list of completion option populated by linenoiseAddCompletion(). */ static void freeCompletions(linenoiseCompletions *lc) { size_t i; for (i = 0; i < lc->len; i++) @@ -223,28 +338,39 @@ static void freeCompletions(linenoiseCompletions *lc) { free(lc->cvec); } -static int completeLine(int fd, const char *prompt, char *buf, size_t buflen, size_t *len, size_t *pos, size_t cols) { +/* This is an helper function for linenoiseEdit() and is called when the + * user types the key in order to complete the string currently in the + * input. + * + * The state of the editing is encapsulated into the pointed linenoiseState + * structure as described in the structure definition. */ +static int completeLine(struct linenoiseState *ls) { linenoiseCompletions lc = { 0, NULL }; int nread, nwritten; char c = 0; - completionCallback(buf,&lc); + completionCallback(ls->buf,&lc); if (lc.len == 0) { - beep(); + linenoiseBeep(); } else { size_t stop = 0, i = 0; - size_t clen; while(!stop) { /* Show completion or original buffer */ if (i < lc.len) { - clen = strlen(lc.cvec[i]); - refreshLine(fd,prompt,lc.cvec[i],clen,clen,cols); + struct linenoiseState saved = *ls; + + ls->len = ls->pos = strlen(lc.cvec[i]); + ls->buf = lc.cvec[i]; + refreshLine(ls); + ls->len = saved.len; + ls->pos = saved.pos; + ls->buf = saved.buf; } else { - refreshLine(fd,prompt,buf,*len,*pos,cols); + refreshLine(ls); } - nread = read(fd,&c,1); + nread = read(ls->ifd,&c,1); if (nread <= 0) { freeCompletions(&lc); return -1; @@ -253,20 +379,18 @@ static int completeLine(int fd, const char *prompt, char *buf, size_t buflen, si switch(c) { case 9: /* tab */ i = (i+1) % (lc.len+1); - if (i == lc.len) beep(); + if (i == lc.len) linenoiseBeep(); break; case 27: /* escape */ /* Re-show original buffer */ - if (i < lc.len) { - refreshLine(fd,prompt,buf,*len,*pos,cols); - } + if (i < lc.len) refreshLine(ls); stop = 1; break; default: /* Update buffer and return */ if (i < lc.len) { - nwritten = snprintf(buf,buflen,"%s",lc.cvec[i]); - *len = *pos = nwritten; + nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]); + ls->len = ls->pos = nwritten; } stop = 1; break; @@ -278,200 +402,532 @@ static int completeLine(int fd, const char *prompt, char *buf, size_t buflen, si return c; /* Return last read character */ } -void linenoiseClearScreen(void) { - if (write(STDIN_FILENO,"\x1b[H\x1b[2J",7) <= 0) { - /* nothing to do, just to avoid warning. */ +/* Register a callback function to be called for tab-completion. */ +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { + completionCallback = fn; +} + +/* This function is used by the callback function registered by the user + * in order to add completion options given the input string when the + * user typed . See the example.c source code for a very easy to + * understand example. */ +void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { + size_t len = strlen(str); + char *copy, **cvec; + + copy = malloc(len+1); + if (copy == NULL) return; + memcpy(copy,str,len+1); + cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + if (cvec == NULL) { + free(copy); + return; + } + lc->cvec = cvec; + lc->cvec[lc->len++] = copy; +} + +/* =========================== Line editing ================================= */ + +/* We define a very simple "append buffer" structure, that is an heap + * allocated string where we can append to. This is useful in order to + * write all the escape sequences in a buffer and flush them to the standard + * output in a single call, to avoid flickering effects. */ +struct abuf { + char *b; + int len; +}; + +static void abInit(struct abuf *ab) { + ab->b = NULL; + ab->len = 0; +} + +static void abAppend(struct abuf *ab, const char *s, int len) { + char *new = realloc(ab->b,ab->len+len); + + if (new == NULL) return; + memcpy(new+ab->len,s,len); + ab->b = new; + ab->len += len; +} + +static void abFree(struct abuf *ab) { + free(ab->b); +} + +/* Single line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshSingleLine(struct linenoiseState *l) { + char seq[64]; + size_t plen = strlen(l->prompt); + int fd = l->ofd; + char *buf = l->buf; + size_t len = l->len; + size_t pos = l->pos; + struct abuf ab; + + while((plen+pos) >= l->cols) { + buf++; + len--; + pos--; + } + while (plen+len > l->cols) { + len--; } + + abInit(&ab); + /* Cursor to left edge */ + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,buf,len); + /* Erase to right */ + snprintf(seq,64,"\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + /* Move cursor to original position. */ + snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen)); + abAppend(&ab,seq,strlen(seq)); + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); } -static int linenoisePrompt(int fd, char *buf, size_t buflen, const char *prompt) { - size_t plen = strlen(prompt); - size_t pos = 0; - size_t len = 0; - size_t cols = getColumns(); - int history_index = 0; +/* Multi line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshMultiLine(struct linenoiseState *l) { + char seq[64]; + int plen = strlen(l->prompt); + int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */ + int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */ + int rpos2; /* rpos after refresh. */ + int col; /* colum position, zero-based. */ + int old_rows = l->maxrows; + int fd = l->ofd, j; + struct abuf ab; + + /* Update maxrows if needed. */ + if (rows > (int)l->maxrows) l->maxrows = rows; + + /* First step: clear all the lines used before. To do so start by + * going to the last row. */ + abInit(&ab); + if (old_rows-rpos > 0) { + lndebug("go down %d", old_rows-rpos); + snprintf(seq,64,"\x1b[%dB", old_rows-rpos); + abAppend(&ab,seq,strlen(seq)); + } + + /* Now for every row clear it, go up. */ + for (j = 0; j < old_rows-1; j++) { + lndebug("clear+up"); + snprintf(seq,64,"\r\x1b[0K\x1b[1A"); + abAppend(&ab,seq,strlen(seq)); + } + + /* Clean the top line. */ + lndebug("clear"); + snprintf(seq,64,"\r\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,l->buf,l->len); + + /* If we are at the very end of the screen with our prompt, we need to + * emit a newline and move the prompt to the first column. */ + if (l->pos && + l->pos == l->len && + (l->pos+plen) % l->cols == 0) + { + lndebug(""); + abAppend(&ab,"\n",1); + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + rows++; + if (rows > (int)l->maxrows) l->maxrows = rows; + } - buf[0] = '\0'; - buflen--; /* Make sure there is always space for the nulterm */ + /* Move cursor to right position. */ + rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ + lndebug("rpos2 %d", rpos2); + + /* Go up till we reach the expected positon. */ + if (rows-rpos2 > 0) { + lndebug("go-up %d", rows-rpos2); + snprintf(seq,64,"\x1b[%dA", rows-rpos2); + abAppend(&ab,seq,strlen(seq)); + } + + /* Set column. */ + col = (plen+(int)l->pos) % (int)l->cols; + lndebug("set col %d", 1+col); + if (col) + snprintf(seq,64,"\r\x1b[%dC", col); + else + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + + lndebug("\n"); + l->oldpos = l->pos; + + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Calls the two low level functions refreshSingleLine() or + * refreshMultiLine() according to the selected mode. */ +static void refreshLine(struct linenoiseState *l) { + if (mlmode) + refreshMultiLine(l); + else + refreshSingleLine(l); +} + +/* Insert the character 'c' at cursor current position. + * + * On error writing to the terminal -1 is returned, otherwise 0. */ +int linenoiseEditInsert(struct linenoiseState *l, char c) { + if (l->len < l->buflen) { + if (l->len == l->pos) { + l->buf[l->pos] = c; + l->pos++; + l->len++; + l->buf[l->len] = '\0'; + if ((!mlmode && l->plen+l->len < l->cols) /* || mlmode */) { + /* Avoid a full update of the line in the + * trivial case. */ + if (write(l->ofd,&c,1) == -1) return -1; + } else { + refreshLine(l); + } + } else { + memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos); + l->buf[l->pos] = c; + l->len++; + l->pos++; + l->buf[l->len] = '\0'; + refreshLine(l); + } + } + return 0; +} + +/* Move cursor on the left. */ +void linenoiseEditMoveLeft(struct linenoiseState *l) { + if (l->pos > 0) { + l->pos--; + refreshLine(l); + } +} + +/* Move cursor on the right. */ +void linenoiseEditMoveRight(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos++; + refreshLine(l); + } +} + +/* Move cursor to the start of the line. */ +void linenoiseEditMoveHome(struct linenoiseState *l) { + if (l->pos != 0) { + l->pos = 0; + refreshLine(l); + } +} + +/* Move cursor to the end of the line. */ +void linenoiseEditMoveEnd(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos = l->len; + refreshLine(l); + } +} + +/* Substitute the currently edited line with the next or previous history + * entry as specified by 'dir'. */ +#define LINENOISE_HISTORY_NEXT 0 +#define LINENOISE_HISTORY_PREV 1 +void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) { + if (history_len > 1) { + /* Update the current history entry before to + * overwrite it with the next one. */ + free(history[history_len - 1 - l->history_index]); + history[history_len - 1 - l->history_index] = strdup(l->buf); + /* Show the new entry */ + l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1; + if (l->history_index < 0) { + l->history_index = 0; + return; + } else if (l->history_index >= history_len) { + l->history_index = history_len-1; + return; + } + strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen); + l->buf[l->buflen-1] = '\0'; + l->len = l->pos = strlen(l->buf); + refreshLine(l); + } +} + +/* Delete the character at the right of the cursor without altering the cursor + * position. Basically this is what happens with the "Delete" keyboard key. */ +void linenoiseEditDelete(struct linenoiseState *l) { + if (l->len > 0 && l->pos < l->len) { + memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1); + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Backspace implementation. */ +void linenoiseEditBackspace(struct linenoiseState *l) { + if (l->pos > 0 && l->len > 0) { + memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos); + l->pos--; + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Delete the previosu word, maintaining the cursor at the start of the + * current word. */ +void linenoiseEditDeletePrevWord(struct linenoiseState *l) { + size_t old_pos = l->pos; + size_t diff; + + while (l->pos > 0 && l->buf[l->pos-1] == ' ') + l->pos--; + while (l->pos > 0 && l->buf[l->pos-1] != ' ') + l->pos--; + diff = old_pos - l->pos; + memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1); + l->len -= diff; + refreshLine(l); +} + +/* This function is the core of the line editing capability of linenoise. + * It expects 'fd' to be already in "raw mode" so that every key pressed + * will be returned ASAP to read(). + * + * The resulting string is put into 'buf' when the user type enter, or + * when ctrl+d is typed. + * + * The function returns the length of the current buffer. */ +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +{ + struct linenoiseState l; + + /* Populate the linenoise state that we pass to functions implementing + * specific editing functionalities. */ + l.ifd = stdin_fd; + l.ofd = stdout_fd; + l.buf = buf; + l.buflen = buflen; + l.prompt = prompt; + l.plen = strlen(prompt); + l.oldpos = l.pos = 0; + l.len = 0; + l.cols = getColumns(stdin_fd, stdout_fd); + l.maxrows = 0; + l.history_index = 0; + + /* Buffer starts empty. */ + l.buf[0] = '\0'; + l.buflen--; /* Make sure there is always space for the nulterm */ /* The latest history entry is always our current buffer, that * initially is just an empty string. */ linenoiseHistoryAdd(""); - - if (write(fd,prompt,plen) == -1) return -1; + + if (write(l.ofd,prompt,l.plen) == -1) return -1; while(1) { char c; int nread; - char seq[2], seq2[2]; + char seq[3]; - nread = read(fd,&c,1); - if (nread <= 0) return len; + nread = read(l.ifd,&c,1); + if (nread <= 0) return l.len; /* Only autocomplete when the callback is set. It returns < 0 when * there was an error reading from fd. Otherwise it will return the * character that should be handled next. */ if (c == 9 && completionCallback != NULL) { - c = completeLine(fd,prompt,buf,buflen,&len,&pos,cols); + c = completeLine(&l); /* Return on errors */ - if (c < 0) return len; + if (c < 0) return l.len; /* Read next character when 0 */ if (c == 0) continue; } switch(c) { - case 13: /* enter */ + case ENTER: /* enter */ history_len--; free(history[history_len]); - return (int)len; - case 3: /* ctrl-c */ + if (mlmode) linenoiseEditMoveEnd(&l); + return (int)l.len; + case CTRL_C: /* ctrl-c */ errno = EAGAIN; return -1; - case 127: /* backspace */ + case BACKSPACE: /* backspace */ case 8: /* ctrl-h */ - if (pos > 0 && len > 0) { - memmove(buf+pos-1,buf+pos,len-pos); - pos--; - len--; - buf[len] = '\0'; - refreshLine(fd,prompt,buf,len,pos,cols); - } + linenoiseEditBackspace(&l); break; - case 4: /* ctrl-d, remove char at right of cursor */ - if (len > 1 && pos < (len-1)) { - memmove(buf+pos,buf+pos+1,len-pos); - len--; - buf[len] = '\0'; - refreshLine(fd,prompt,buf,len,pos,cols); - } else if (len == 0) { + case CTRL_D: /* ctrl-d, remove char at right of cursor, or if the + line is empty, act as end-of-file. */ + if (l.len > 0) { + linenoiseEditDelete(&l); + } else { history_len--; free(history[history_len]); return -1; } break; - case 20: /* ctrl-t */ - if (pos > 0 && pos < len) { - int aux = buf[pos-1]; - buf[pos-1] = buf[pos]; - buf[pos] = aux; - if (pos != len-1) pos++; - refreshLine(fd,prompt,buf,len,pos,cols); + case CTRL_T: /* ctrl-t, swaps current character with previous. */ + if (l.pos > 0 && l.pos < l.len) { + int aux = buf[l.pos-1]; + buf[l.pos-1] = buf[l.pos]; + buf[l.pos] = aux; + if (l.pos != l.len-1) l.pos++; + refreshLine(&l); } break; - case 2: /* ctrl-b */ - goto left_arrow; - case 6: /* ctrl-f */ - goto right_arrow; - case 16: /* ctrl-p */ - seq[1] = 65; - goto up_down_arrow; - case 14: /* ctrl-n */ - seq[1] = 66; - goto up_down_arrow; + case CTRL_B: /* ctrl-b */ + linenoiseEditMoveLeft(&l); break; - case 27: /* escape sequence */ - if (read(fd,seq,2) == -1) break; - if (seq[0] == 91 && seq[1] == 68) { -left_arrow: - /* left arrow */ - if (pos > 0) { - pos--; - refreshLine(fd,prompt,buf,len,pos,cols); - } - } else if (seq[0] == 91 && seq[1] == 67) { -right_arrow: - /* right arrow */ - if (pos != len) { - pos++; - refreshLine(fd,prompt,buf,len,pos,cols); - } - } else if (seq[0] == 91 && (seq[1] == 65 || seq[1] == 66)) { -up_down_arrow: - /* up and down arrow: history */ - if (history_len > 1) { - /* Update the current history entry before to - * overwrite it with tne next one. */ - free(history[history_len-1-history_index]); - history[history_len-1-history_index] = strdup(buf); - /* Show the new entry */ - history_index += (seq[1] == 65) ? 1 : -1; - if (history_index < 0) { - history_index = 0; + case CTRL_F: /* ctrl-f */ + linenoiseEditMoveRight(&l); + break; + case CTRL_P: /* ctrl-p */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case CTRL_N: /* ctrl-n */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case ESC: /* escape sequence */ + /* Read the next two bytes representing the escape sequence. + * Use two calls to handle slow terminals returning the two + * chars at different times. */ + if (read(l.ifd,seq,1) == -1) break; + if (read(l.ifd,seq+1,1) == -1) break; + + /* ESC [ sequences. */ + if (seq[0] == '[') { + if (seq[1] >= '0' && seq[1] <= '9') { + /* Extended escape, read additional byte. */ + if (read(l.ifd,seq+2,1) == -1) break; + if (seq[2] == '~') { + switch(seq[1]) { + case '3': /* Delete key. */ + linenoiseEditDelete(&l); + break; + } + } + } else { + switch(seq[1]) { + case 'A': /* Up */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case 'B': /* Down */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case 'C': /* Right */ + linenoiseEditMoveRight(&l); + break; + case 'D': /* Left */ + linenoiseEditMoveLeft(&l); break; - } else if (history_index >= history_len) { - history_index = history_len-1; + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); break; } - strncpy(buf,history[history_len-1-history_index],buflen); - buf[buflen] = '\0'; - len = pos = strlen(buf); - refreshLine(fd,prompt,buf,len,pos,cols); } - } else if (seq[0] == 91 && seq[1] > 48 && seq[1] < 55) { - /* extended escape */ - if (read(fd,seq2,2) == -1) break; - if (seq[1] == 51 && seq2[0] == 126) { - /* delete */ - if (len > 0 && pos < len) { - memmove(buf+pos,buf+pos+1,len-pos-1); - len--; - buf[len] = '\0'; - refreshLine(fd,prompt,buf,len,pos,cols); - } + } + + /* ESC O sequences. */ + else if (seq[0] == 'O') { + switch(seq[1]) { + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; } } break; default: - if (len < buflen) { - if (len == pos) { - buf[pos] = c; - pos++; - len++; - buf[len] = '\0'; - if (plen+len < cols) { - /* Avoid a full update of the line in the - * trivial case. */ - if (write(fd,&c,1) == -1) return -1; - } else { - refreshLine(fd,prompt,buf,len,pos,cols); - } - } else { - memmove(buf+pos+1,buf+pos,len-pos); - buf[pos] = c; - len++; - pos++; - buf[len] = '\0'; - refreshLine(fd,prompt,buf,len,pos,cols); - } - } + if (linenoiseEditInsert(&l,c)) return -1; break; - case 21: /* Ctrl+u, delete the whole line. */ + case CTRL_U: /* Ctrl+u, delete the whole line. */ buf[0] = '\0'; - pos = len = 0; - refreshLine(fd,prompt,buf,len,pos,cols); + l.pos = l.len = 0; + refreshLine(&l); break; - case 11: /* Ctrl+k, delete from current to end of line. */ - buf[pos] = '\0'; - len = pos; - refreshLine(fd,prompt,buf,len,pos,cols); + case CTRL_K: /* Ctrl+k, delete from current to end of line. */ + buf[l.pos] = '\0'; + l.len = l.pos; + refreshLine(&l); break; - case 1: /* Ctrl+a, go to the start of the line */ - pos = 0; - refreshLine(fd,prompt,buf,len,pos,cols); + case CTRL_A: /* Ctrl+a, go to the start of the line */ + linenoiseEditMoveHome(&l); break; - case 5: /* ctrl+e, go to the end of the line */ - pos = len; - refreshLine(fd,prompt,buf,len,pos,cols); + case CTRL_E: /* ctrl+e, go to the end of the line */ + linenoiseEditMoveEnd(&l); break; - case 12: /* ctrl+l, clear screen */ + case CTRL_L: /* ctrl+l, clear screen */ linenoiseClearScreen(); - refreshLine(fd,prompt,buf,len,pos,cols); + refreshLine(&l); + break; + case CTRL_W: /* ctrl+w, delete previous word */ + linenoiseEditDeletePrevWord(&l); + break; } } - return len; + return l.len; +} + +/* This special mode is used by linenoise in order to print scan codes + * on screen for debugging / development purposes. It is implemented + * by the linenoise_example program using the --keycodes option. */ +void linenoisePrintKeyCodes(void) { + char quit[4]; + + printf("Linenoise key codes debugging mode.\n" + "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); + if (enableRawMode(STDIN_FILENO) == -1) return; + memset(quit,' ',4); + while(1) { + char c; + int nread; + + nread = read(STDIN_FILENO,&c,1); + if (nread <= 0) continue; + memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */ + quit[sizeof(quit)-1] = c; /* Insert current char on the right. */ + if (memcmp(quit,"quit",sizeof(quit)) == 0) break; + + printf("'%c' %02x (%d) (type quit to exit)\n", + isprint(c) ? c : '?', (int)c, (int)c); + printf("\r"); /* Go left edge manually, we are in raw mode. */ + fflush(stdout); + } + disableRawMode(STDIN_FILENO); } +/* This function calls the line editing function linenoiseEdit() using + * the STDIN file descriptor set in raw mode. */ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { - int fd = STDIN_FILENO; int count; if (buflen == 0) { @@ -479,6 +935,7 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { return -1; } if (!isatty(STDIN_FILENO)) { + /* Not a tty: read from file / pipe. */ if (fgets(buf, buflen, stdin) == NULL) return -1; count = strlen(buf); if (count && buf[count-1] == '\n') { @@ -486,14 +943,20 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { buf[count] = '\0'; } } else { - if (enableRawMode(fd) == -1) return -1; - count = linenoisePrompt(fd, buf, buflen, prompt); - disableRawMode(fd); + /* Interactive editing. */ + if (enableRawMode(STDIN_FILENO) == -1) return -1; + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); + disableRawMode(STDIN_FILENO); printf("\n"); } return count; } +/* The high level function that is the main API of the linenoise library. + * This function checks if the terminal has basic capabilities, just checking + * for a blacklist of stupid terminals, and later either calls the line + * editing function or uses dummy fgets() so that you will be able to type + * something even in the most desperate of the conditions. */ char *linenoise(const char *prompt) { char buf[LINENOISE_MAX_LINE]; int count; @@ -517,29 +980,50 @@ char *linenoise(const char *prompt) { } } -/* Register a callback function to be called for tab-completion. */ -void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { - completionCallback = fn; +/* ================================ History ================================= */ + +/* Free the history, but does not reset it. Only used when we have to + * exit() to avoid memory leaks are reported by valgrind & co. */ +static void freeHistory(void) { + if (history) { + int j; + + for (j = 0; j < history_len; j++) + free(history[j]); + free(history); + } } -void linenoiseAddCompletion(linenoiseCompletions *lc, char *str) { - size_t len = strlen(str); - char *copy = malloc(len+1); - memcpy(copy,str,len+1); - lc->cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); - lc->cvec[lc->len++] = copy; +/* At exit we'll try to fix the terminal to the initial conditions. */ +static void linenoiseAtExit(void) { + disableRawMode(STDIN_FILENO); + freeHistory(); } -/* Using a circular buffer is smarter, but a bit more complex to handle. */ +/* This is the API call to add a new entry in the linenoise history. + * It uses a fixed array of char pointers that are shifted (memmoved) + * when the history max length is reached in order to remove the older + * entry and make room for the new one, so it is not exactly suitable for huge + * histories, but will work well for a few hundred of entries. + * + * Using a circular buffer is smarter, but a bit more complex to handle. */ int linenoiseHistoryAdd(const char *line) { char *linecopy; if (history_max_len == 0) return 0; + + /* Initialization on first call. */ if (history == NULL) { history = malloc(sizeof(char*)*history_max_len); if (history == NULL) return 0; memset(history,0,(sizeof(char*)*history_max_len)); } + + /* Don't add duplicated lines. */ + if (history_len && !strcmp(history[history_len-1], line)) return 0; + + /* Add an heap allocated copy of the line in the history. + * If we reached the max length, remove the older line. */ linecopy = strdup(line); if (!linecopy) return 0; if (history_len == history_max_len) { @@ -552,6 +1036,10 @@ int linenoiseHistoryAdd(const char *line) { return 1; } +/* Set the maximum length for the history. This function can be called even + * if there is already some history, the function will make sure to retain + * just the latest 'len' elements if the new history length value is smaller + * than the amount of items already inside the history. */ int linenoiseHistorySetMaxLen(int len) { char **new; @@ -561,8 +1049,16 @@ int linenoiseHistorySetMaxLen(int len) { new = malloc(sizeof(char*)*len); if (new == NULL) return 0; - if (len < tocopy) tocopy = len; - memcpy(new,history+(history_max_len-tocopy), sizeof(char*)*tocopy); + + /* If we can't copy everything, free the elements we'll not use. */ + if (len < tocopy) { + int j; + + for (j = 0; j < tocopy-len; j++) free(history[j]); + tocopy = len; + } + memset(new,0,sizeof(char*)*len); + memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); free(history); history = new; } @@ -574,10 +1070,10 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ -int linenoiseHistorySave(char *filename) { +int linenoiseHistorySave(const char *filename) { FILE *fp = fopen(filename,"w"); int j; - + if (fp == NULL) return -1; for (j = 0; j < history_len; j++) fprintf(fp,"%s\n",history[j]); @@ -590,15 +1086,15 @@ int linenoiseHistorySave(char *filename) { * * If the file exists and the operation succeeded 0 is returned, otherwise * on error -1 is returned. */ -int linenoiseHistoryLoad(char *filename) { +int linenoiseHistoryLoad(const char *filename) { FILE *fp = fopen(filename,"r"); char buf[LINENOISE_MAX_LINE]; - + if (fp == NULL) return -1; while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) { char *p; - + p = strchr(buf,'\r'); if (!p) p = strchr(buf,'\n'); if (p) *p = '\0'; diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h index 76a703c2848..36394eb99ce 100644 --- a/deps/linenoise/linenoise.h +++ b/deps/linenoise/linenoise.h @@ -3,39 +3,44 @@ * * See linenoise.c for more information. * + * ------------------------------------------------------------------------ + * * Copyright (c) 2010, Salvatore Sanfilippo * Copyright (c) 2010, Pieter Noordhuis * * All rights reserved. * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright + * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __LINENOISE_H #define __LINENOISE_H +#ifdef __cplusplus +extern "C" { +#endif + typedef struct linenoiseCompletions { size_t len; char **cvec; @@ -43,13 +48,19 @@ typedef struct linenoiseCompletions { typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); -void linenoiseAddCompletion(linenoiseCompletions *, char *); +void linenoiseAddCompletion(linenoiseCompletions *, const char *); char *linenoise(const char *prompt); int linenoiseHistoryAdd(const char *line); int linenoiseHistorySetMaxLen(int len); -int linenoiseHistorySave(char *filename); -int linenoiseHistoryLoad(char *filename); +int linenoiseHistorySave(const char *filename); +int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); +void linenoiseSetMultiLine(int ml); +void linenoisePrintKeyCodes(void); + +#ifdef __cplusplus +} +#endif #endif /* __LINENOISE_H */ diff --git a/deps/lua/COPYRIGHT b/deps/lua/COPYRIGHT index 3a53e741e03..a86026803a0 100644 --- a/deps/lua/COPYRIGHT +++ b/deps/lua/COPYRIGHT @@ -9,7 +9,7 @@ For details and rationale, see http://www.lua.org/license.html . =============================================================================== -Copyright (C) 1994-2008 Lua.org, PUC-Rio. +Copyright (C) 1994-2012 Lua.org, PUC-Rio. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/deps/lua/Makefile b/deps/lua/Makefile index 6e78f66fa5b..209a1324418 100644 --- a/deps/lua/Makefile +++ b/deps/lua/Makefile @@ -48,7 +48,7 @@ TO_MAN= lua.1 luac.1 # Lua version and release. V= 5.1 -R= 5.1.4 +R= 5.1.5 all: $(PLAT) diff --git a/deps/lua/doc/amazon.gif b/deps/lua/doc/amazon.gif deleted file mode 100644 index f2586d57653..00000000000 Binary files a/deps/lua/doc/amazon.gif and /dev/null differ diff --git a/deps/lua/doc/contents.html b/deps/lua/doc/contents.html index 8e58e18ca9b..3d83da98a6d 100644 --- a/deps/lua/doc/contents.html +++ b/deps/lua/doc/contents.html @@ -3,11 +3,10 @@ Lua 5.1 Reference Manual - contents - + @@ -20,7 +19,13 @@

Lua 5.1 Reference Manual

-This is an online version of +

+The reference manual is the official definition of the Lua language. +For a complete introduction to Lua programming, see the book +Programming in Lua. + +

+This manual is also available as a book:

@@ -29,128 +34,119 @@


by R. Ierusalimschy, L. H. de Figueiredo, W. Celes
Lua.org, August 2006
ISBN 85-903798-3-3 -
 -[Buy from Amazon]

-

-Buy a copy of this book and +

+Buy a copy +of this book and help to support the Lua project. -

-The reference manual is the official definition of the Lua language. -For a complete introduction to Lua programming, see the book -Programming in Lua.

- start · contents · index · -português -· -español +other versions

-Copyright © 2006-2008 Lua.org, PUC-Rio. +Copyright © 2006–2012 Lua.org, PUC-Rio. Freely available under the terms of the -Lua license. +Lua license. -

Contents

Index

@@ -160,6 +156,8 @@

Index

Lua functions

_G
_VERSION
+

+ assert
collectgarbage
dofile
@@ -487,12 +485,12 @@

auxiliary library

- + Last update: -Sat Jan 19 13:24:29 BRST 2008 +Mon Feb 13 18:53:32 BRST 2012 diff --git a/deps/lua/doc/lua.css b/deps/lua/doc/lua.css index 039cf11698c..7fafbb1bb63 100644 --- a/deps/lua/doc/lua.css +++ b/deps/lua/doc/lua.css @@ -1,17 +1,37 @@ body { color: #000000 ; background-color: #FFFFFF ; - font-family: sans-serif ; + font-family: Helvetica, Arial, sans-serif ; text-align: justify ; - margin-right: 20px ; - margin-left: 20px ; + margin-right: 30px ; + margin-left: 30px ; } h1, h2, h3, h4 { + font-family: Verdana, Geneva, sans-serif ; font-weight: normal ; font-style: italic ; } +h2 { + padding-top: 0.4em ; + padding-bottom: 0.4em ; + padding-left: 30px ; + padding-right: 30px ; + margin-left: -30px ; + background-color: #E0E0FF ; +} + +h3 { + padding-left: 0.5em ; + border-left: solid #E0E0FF 1em ; +} + +table h3 { + padding-left: 0px ; + border-left: none ; +} + a:link { color: #000080 ; background-color: inherit ; @@ -39,3 +59,25 @@ hr { background-color: #a0a0a0 ; } +:target { + background-color: #F8F8F8 ; + padding: 8px ; + border: solid #a0a0a0 2px ; +} + +.footer { + color: gray ; + font-size: small ; +} + +input[type=text] { + border: solid #a0a0a0 2px ; + border-radius: 2em ; + -moz-border-radius: 2em ; + background-image: url('images/search.png') ; + background-repeat: no-repeat; + background-position: 4px center ; + padding-left: 20px ; + height: 2em ; +} + diff --git a/deps/lua/doc/manual.css b/deps/lua/doc/manual.css index eed5afd9eef..b49b362937a 100644 --- a/deps/lua/doc/manual.css +++ b/deps/lua/doc/manual.css @@ -1,13 +1,24 @@ h3 code { font-family: inherit ; + font-size: inherit ; } -pre { - font-size: 105% ; +pre, code { + font-size: 12pt ; } span.apii { float: right ; font-family: inherit ; + font-style: normal ; + font-size: small ; + color: gray ; } +p+h1, ul+h1 { + padding-top: 0.4em ; + padding-bottom: 0.4em ; + padding-left: 30px ; + margin-left: -30px ; + background-color: #E0E0FF ; +} diff --git a/deps/lua/doc/manual.html b/deps/lua/doc/manual.html index f46f17c8ec1..4e41683d028 100644 --- a/deps/lua/doc/manual.html +++ b/deps/lua/doc/manual.html @@ -19,9 +19,9 @@

by Roberto Ierusalimschy, Luiz Henrique de Figueiredo, Waldemar Celes

-Copyright © 2006-2008 Lua.org, PUC-Rio. +Copyright © 2006–2012 Lua.org, PUC-Rio. Freely available under the terms of the -Lua license. +Lua license.

@@ -29,11 +29,13 @@

contents · index +· +other versions

- + @@ -5645,7 +5647,7 @@

5 - Standard Libraries

    -
  • basic library,
    • which includes the coroutine sub-library; +
  • basic library, which includes the coroutine sub-library;
  • package library;
    • @@ -5709,7 +5711,7 @@

    5.1 - Basic Functions

    -

    collectgarbage (opt [, arg])

    +

    collectgarbage ([opt [, arg]])

    @@ -5718,6 +5720,11 @@

    5.1 - Basic Functions

      +
    • "collect": +performs a full garbage-collection cycle. +This is the default option. +
      • +
    • "stop": stops the garbage collector.
      • @@ -5726,10 +5733,6 @@

      5.1 - Basic Functions

      restarts the garbage collector. -
    • "collect": -performs a full garbage-collection cycle. -
      • -
    • "count": returns the total memory in use by Lua (in Kbytes).
      • @@ -5760,7 +5763,7 @@

      5.1 - Basic Functions

      -

      dofile (filename)

      +

      dofile ([filename])

      Opens the named file and executes its contents as a Lua chunk. When called without arguments, dofile executes the contents of the standard input (stdin). @@ -8421,7 +8424,7 @@

      5.9 - The Debug Library

      -

      debug.traceback ([thread,] [message] [, level])

      +

      debug.traceback ([thread,] [message [, level]])

      @@ -8789,12 +8792,12 @@

      8 - The Complete Syntax of Lua

      - + Last update: -Mon Aug 18 13:25:46 BRT 2008 +Mon Feb 13 18:54:19 BRST 2012 diff --git a/deps/lua/doc/readme.html b/deps/lua/doc/readme.html index 38be6dbbfc7..3ed6a81895b 100644 --- a/deps/lua/doc/readme.html +++ b/deps/lua/doc/readme.html @@ -12,7 +12,7 @@

      Documentation

      -This is the documentation included in the source distribution of Lua 5.1.4. +This is the documentation included in the source distribution of Lua 5.1.5.
      • Reference manual @@ -33,7 +33,7 @@

        Last update: -Tue Aug 12 14:46:07 BRT 2008 +Fri Feb 3 09:44:42 BRST 2012 diff --git a/deps/lua/etc/lua.pc b/deps/lua/etc/lua.pc index f52f55b0123..07e2852b0a7 100644 --- a/deps/lua/etc/lua.pc +++ b/deps/lua/etc/lua.pc @@ -5,7 +5,7 @@ # grep '^V=' ../Makefile V= 5.1 # grep '^R=' ../Makefile -R= 5.1.4 +R= 5.1.5 # grep '^INSTALL_.*=' ../Makefile | sed 's/INSTALL_TOP/prefix/' prefix= /usr/local diff --git a/deps/lua/src/Makefile b/deps/lua/src/Makefile index 77d6a48b4b1..f3bba2f8123 100644 --- a/deps/lua/src/Makefile +++ b/deps/lua/src/Makefile @@ -7,7 +7,7 @@ # Your platform. See PLATS for possible values. PLAT= none -CC= gcc +CC?= gcc CFLAGS= -O2 -Wall $(MYCFLAGS) AR= ar rcu RANLIB= ranlib @@ -25,9 +25,10 @@ PLATS= aix ansi bsd freebsd generic linux macosx mingw posix solaris LUA_A= liblua.a CORE_O= lapi.o lcode.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o lmem.o \ lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o ltm.o \ - lundump.o lvm.o lzio.o strbuf.o + lundump.o lvm.o lzio.o strbuf.o fpconv.o LIB_O= lauxlib.o lbaselib.o ldblib.o liolib.o lmathlib.o loslib.o ltablib.o \ - lstrlib.o loadlib.o linit.o lua_cjson.o lua_struct.o lua_cmsgpack.o + lstrlib.o loadlib.o linit.o lua_cjson.o lua_struct.o lua_cmsgpack.o \ + lua_bit.o LUA_T= lua LUA_O= lua.o @@ -48,7 +49,7 @@ o: $(ALL_O) a: $(ALL_A) $(LUA_A): $(CORE_O) $(LIB_O) - $(AR) $@ $? + $(AR) $@ $(CORE_O) $(LIB_O) # DLL needs all object files $(RANLIB) $@ $(LUA_T): $(LUA_O) $(LUA_A) diff --git a/deps/lua/src/fpconv.c b/deps/lua/src/fpconv.c new file mode 100644 index 00000000000..79908317a45 --- /dev/null +++ b/deps/lua/src/fpconv.c @@ -0,0 +1,205 @@ +/* fpconv - Floating point conversion routines + * + * Copyright (c) 2011-2012 Mark Pulford + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* JSON uses a '.' decimal separator. strtod() / sprintf() under C libraries + * with locale support will break when the decimal separator is a comma. + * + * fpconv_* will around these issues with a translation buffer if required. + */ + +#include +#include +#include +#include + +#include "fpconv.h" + +/* Lua CJSON assumes the locale is the same for all threads within a + * process and doesn't change after initialisation. + * + * This avoids the need for per thread storage or expensive checks + * for call. */ +static char locale_decimal_point = '.'; + +/* In theory multibyte decimal_points are possible, but + * Lua CJSON only supports UTF-8 and known locales only have + * single byte decimal points ([.,]). + * + * localconv() may not be thread safe (=>crash), and nl_langinfo() is + * not supported on some platforms. Use sprintf() instead - if the + * locale does change, at least Lua CJSON won't crash. */ +static void fpconv_update_locale() +{ + char buf[8]; + + snprintf(buf, sizeof(buf), "%g", 0.5); + + /* Failing this test might imply the platform has a buggy dtoa + * implementation or wide characters */ + if (buf[0] != '0' || buf[2] != '5' || buf[3] != 0) { + fprintf(stderr, "Error: wide characters found or printf() bug."); + abort(); + } + + locale_decimal_point = buf[1]; +} + +/* Check for a valid number character: [-+0-9a-yA-Y.] + * Eg: -0.6e+5, infinity, 0xF0.F0pF0 + * + * Used to find the probable end of a number. It doesn't matter if + * invalid characters are counted - strtod() will find the valid + * number if it exists. The risk is that slightly more memory might + * be allocated before a parse error occurs. */ +static inline int valid_number_character(char ch) +{ + char lower_ch; + + if ('0' <= ch && ch <= '9') + return 1; + if (ch == '-' || ch == '+' || ch == '.') + return 1; + + /* Hex digits, exponent (e), base (p), "infinity",.. */ + lower_ch = ch | 0x20; + if ('a' <= lower_ch && lower_ch <= 'y') + return 1; + + return 0; +} + +/* Calculate the size of the buffer required for a strtod locale + * conversion. */ +static int strtod_buffer_size(const char *s) +{ + const char *p = s; + + while (valid_number_character(*p)) + p++; + + return p - s; +} + +/* Similar to strtod(), but must be passed the current locale's decimal point + * character. Guaranteed to be called at the start of any valid number in a string */ +double fpconv_strtod(const char *nptr, char **endptr) +{ + char localbuf[FPCONV_G_FMT_BUFSIZE]; + char *buf, *endbuf, *dp; + int buflen; + double value; + + /* System strtod() is fine when decimal point is '.' */ + if (locale_decimal_point == '.') + return strtod(nptr, endptr); + + buflen = strtod_buffer_size(nptr); + if (!buflen) { + /* No valid characters found, standard strtod() return */ + *endptr = (char *)nptr; + return 0; + } + + /* Duplicate number into buffer */ + if (buflen >= FPCONV_G_FMT_BUFSIZE) { + /* Handle unusually large numbers */ + buf = malloc(buflen + 1); + if (!buf) { + fprintf(stderr, "Out of memory"); + abort(); + } + } else { + /* This is the common case.. */ + buf = localbuf; + } + memcpy(buf, nptr, buflen); + buf[buflen] = 0; + + /* Update decimal point character if found */ + dp = strchr(buf, '.'); + if (dp) + *dp = locale_decimal_point; + + value = strtod(buf, &endbuf); + *endptr = (char *)&nptr[endbuf - buf]; + if (buflen >= FPCONV_G_FMT_BUFSIZE) + free(buf); + + return value; +} + +/* "fmt" must point to a buffer of at least 6 characters */ +static void set_number_format(char *fmt, int precision) +{ + int d1, d2, i; + + assert(1 <= precision && precision <= 14); + + /* Create printf format (%.14g) from precision */ + d1 = precision / 10; + d2 = precision % 10; + fmt[0] = '%'; + fmt[1] = '.'; + i = 2; + if (d1) { + fmt[i++] = '0' + d1; + } + fmt[i++] = '0' + d2; + fmt[i++] = 'g'; + fmt[i] = 0; +} + +/* Assumes there is always at least 32 characters available in the target buffer */ +int fpconv_g_fmt(char *str, double num, int precision) +{ + char buf[FPCONV_G_FMT_BUFSIZE]; + char fmt[6]; + int len; + char *b; + + set_number_format(fmt, precision); + + /* Pass through when decimal point character is dot. */ + if (locale_decimal_point == '.') + return snprintf(str, FPCONV_G_FMT_BUFSIZE, fmt, num); + + /* snprintf() to a buffer then translate for other decimal point characters */ + len = snprintf(buf, FPCONV_G_FMT_BUFSIZE, fmt, num); + + /* Copy into target location. Translate decimal point if required */ + b = buf; + do { + *str++ = (*b == locale_decimal_point ? '.' : *b); + } while(*b++); + + return len; +} + +void fpconv_init() +{ + fpconv_update_locale(); +} + +/* vi:ai et sw=4 ts=4: + */ diff --git a/deps/lua/src/fpconv.h b/deps/lua/src/fpconv.h new file mode 100644 index 00000000000..7b0d0ee313d --- /dev/null +++ b/deps/lua/src/fpconv.h @@ -0,0 +1,22 @@ +/* Lua CJSON floating point conversion routines */ + +/* Buffer required to store the largest string representation of a double. + * + * Longest double printed with %.14g is 21 characters long: + * -1.7976931348623e+308 */ +# define FPCONV_G_FMT_BUFSIZE 32 + +#ifdef USE_INTERNAL_FPCONV +static inline void fpconv_init() +{ + /* Do nothing - not required */ +} +#else +extern void fpconv_init(); +#endif + +extern int fpconv_g_fmt(char*, double, int); +extern double fpconv_strtod(const char*, char**); + +/* vi:ai et sw=4 ts=4: + */ diff --git a/deps/lua/src/lbaselib.c b/deps/lua/src/lbaselib.c index 2a4c079d3b0..2ab550bd48d 100644 --- a/deps/lua/src/lbaselib.c +++ b/deps/lua/src/lbaselib.c @@ -631,7 +631,7 @@ static void base_open (lua_State *L) { luaL_register(L, "_G", base_funcs); lua_pushliteral(L, LUA_VERSION); lua_setglobal(L, "_VERSION"); /* set global _VERSION */ - /* `ipairs' and `pairs' need auxliliary functions as upvalues */ + /* `ipairs' and `pairs' need auxiliary functions as upvalues */ auxopen(L, "ipairs", luaB_ipairs, ipairsaux); auxopen(L, "pairs", luaB_pairs, luaB_next); /* `newproxy' needs a weaktable as upvalue */ diff --git a/deps/lua/src/lcode.c b/deps/lua/src/lcode.c index cff626b7fa6..679cb9cfd98 100644 --- a/deps/lua/src/lcode.c +++ b/deps/lua/src/lcode.c @@ -1,5 +1,5 @@ /* -** $Id: lcode.c,v 2.25.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lcode.c,v 2.25.1.5 2011/01/31 14:53:16 roberto Exp $ ** Code generator for Lua ** See Copyright Notice in lua.h */ @@ -544,10 +544,6 @@ void luaK_goiftrue (FuncState *fs, expdesc *e) { pc = NO_JUMP; /* always true; do nothing */ break; } - case VFALSE: { - pc = luaK_jump(fs); /* always jump */ - break; - } case VJMP: { invertjump(fs, e); pc = e->u.s.info; @@ -572,10 +568,6 @@ static void luaK_goiffalse (FuncState *fs, expdesc *e) { pc = NO_JUMP; /* always false; do nothing */ break; } - case VTRUE: { - pc = luaK_jump(fs); /* always jump */ - break; - } case VJMP: { pc = e->u.s.info; break; diff --git a/deps/lua/src/ldblib.c b/deps/lua/src/ldblib.c index 67de1222a94..2027eda5983 100644 --- a/deps/lua/src/ldblib.c +++ b/deps/lua/src/ldblib.c @@ -1,5 +1,5 @@ /* -** $Id: ldblib.c,v 1.104.1.3 2008/01/21 13:11:21 roberto Exp $ +** $Id: ldblib.c,v 1.104.1.4 2009/08/04 18:50:18 roberto Exp $ ** Interface from Lua to its debug API ** See Copyright Notice in lua.h */ @@ -45,6 +45,7 @@ static int db_setmetatable (lua_State *L) { static int db_getfenv (lua_State *L) { + luaL_checkany(L, 1); lua_getfenv(L, 1); return 1; } diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c index 8de05f728e7..514f7a2a31f 100644 --- a/deps/lua/src/ldo.c +++ b/deps/lua/src/ldo.c @@ -1,5 +1,5 @@ /* -** $Id: ldo.c,v 2.38.1.3 2008/01/18 22:31:22 roberto Exp $ +** $Id: ldo.c,v 2.38.1.4 2012/01/18 02:27:10 roberto Exp $ ** Stack and Call structure of Lua ** See Copyright Notice in lua.h */ @@ -217,6 +217,7 @@ static StkId adjust_varargs (lua_State *L, Proto *p, int actual) { int nvar = actual - nfixargs; /* number of extra arguments */ lua_assert(p->is_vararg & VARARG_HASARG); luaC_checkGC(L); + luaD_checkstack(L, p->maxstacksize); htab = luaH_new(L, nvar, 1); /* create `arg' table */ for (i=0; itop - nvar + i); @@ -494,7 +495,7 @@ static void f_parser (lua_State *L, void *ud) { struct SParser *p = cast(struct SParser *, ud); int c = luaZ_lookahead(p->z); luaC_checkGC(L); - tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z, + tf = (luaY_parser)(L, p->z, &p->buff, p->name); cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L))); cl->l.p = tf; diff --git a/deps/lua/src/lgc.c b/deps/lua/src/lgc.c index d9e0b78294e..e909c79a969 100644 --- a/deps/lua/src/lgc.c +++ b/deps/lua/src/lgc.c @@ -1,5 +1,5 @@ /* -** $Id: lgc.c,v 2.38.1.1 2007/12/27 13:02:25 roberto Exp $ +** $Id: lgc.c,v 2.38.1.2 2011/03/18 18:05:38 roberto Exp $ ** Garbage Collector ** See Copyright Notice in lua.h */ @@ -627,7 +627,6 @@ void luaC_step (lua_State *L) { } } else { - lua_assert(g->totalbytes >= g->estimate); setthreshold(g); } } diff --git a/deps/lua/src/liolib.c b/deps/lua/src/liolib.c index e79ed1cb2e2..649f9a59515 100644 --- a/deps/lua/src/liolib.c +++ b/deps/lua/src/liolib.c @@ -1,5 +1,5 @@ /* -** $Id: liolib.c,v 2.73.1.3 2008/01/18 17:47:43 roberto Exp $ +** $Id: liolib.c,v 2.73.1.4 2010/05/14 15:33:51 roberto Exp $ ** Standard I/O (and system) library ** See Copyright Notice in lua.h */ @@ -276,7 +276,10 @@ static int read_number (lua_State *L, FILE *f) { lua_pushnumber(L, d); return 1; } - else return 0; /* read fails */ + else { + lua_pushnil(L); /* "result" to be removed */ + return 0; /* read fails */ + } } diff --git a/deps/lua/src/llex.c b/deps/lua/src/llex.c index 6dc319358c0..88c6790c076 100644 --- a/deps/lua/src/llex.c +++ b/deps/lua/src/llex.c @@ -1,5 +1,5 @@ /* -** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $ +** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $ ** Lexical Analyzer ** See Copyright Notice in lua.h */ @@ -118,8 +118,10 @@ TString *luaX_newstring (LexState *ls, const char *str, size_t l) { lua_State *L = ls->L; TString *ts = luaS_newlstr(L, str, l); TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ - if (ttisnil(o)) + if (ttisnil(o)) { setbvalue(o, 1); /* make sure `str' will not be collected */ + luaC_checkGC(L); + } return ts; } diff --git a/deps/lua/src/loadlib.c b/deps/lua/src/loadlib.c index 0d401eba1cf..6158c5353df 100644 --- a/deps/lua/src/loadlib.c +++ b/deps/lua/src/loadlib.c @@ -1,5 +1,5 @@ /* -** $Id: loadlib.c,v 1.52.1.3 2008/08/06 13:29:28 roberto Exp $ +** $Id: loadlib.c,v 1.52.1.4 2009/09/09 13:17:16 roberto Exp $ ** Dynamic library loader for Lua ** See Copyright Notice in lua.h ** @@ -639,7 +639,7 @@ LUALIB_API int luaopen_package (lua_State *L) { lua_pushvalue(L, -1); lua_replace(L, LUA_ENVIRONINDEX); /* create `loaders' table */ - lua_createtable(L, 0, sizeof(loaders)/sizeof(loaders[0]) - 1); + lua_createtable(L, sizeof(loaders)/sizeof(loaders[0]) - 1, 0); /* fill it with pre-defined loaders */ for (i=0; loaders[i] != NULL; i++) { lua_pushcfunction(L, loaders[i]); diff --git a/deps/lua/src/lparser.c b/deps/lua/src/lparser.c index 1e2a9a88b79..dda7488dcad 100644 --- a/deps/lua/src/lparser.c +++ b/deps/lua/src/lparser.c @@ -1,5 +1,5 @@ /* -** $Id: lparser.c,v 2.42.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lparser.c,v 2.42.1.4 2011/10/21 19:31:42 roberto Exp $ ** Lua Parser ** See Copyright Notice in lua.h */ @@ -374,9 +374,9 @@ static void close_func (LexState *ls) { lua_assert(luaG_checkcode(f)); lua_assert(fs->bl == NULL); ls->fs = fs->prev; - L->top -= 2; /* remove table and prototype from the stack */ /* last token read was anchored in defunct function; must reanchor it */ if (fs) anchor_token(ls); + L->top -= 2; /* remove table and prototype from the stack */ } diff --git a/deps/lua/src/lstrlib.c b/deps/lua/src/lstrlib.c index 1b4763d4ee1..7a03489bebd 100644 --- a/deps/lua/src/lstrlib.c +++ b/deps/lua/src/lstrlib.c @@ -1,5 +1,5 @@ /* -** $Id: lstrlib.c,v 1.132.1.4 2008/07/11 17:27:21 roberto Exp $ +** $Id: lstrlib.c,v 1.132.1.5 2010/05/14 15:34:19 roberto Exp $ ** Standard library for string operations and pattern-matching ** See Copyright Notice in lua.h */ @@ -754,6 +754,7 @@ static void addintlen (char *form) { static int str_format (lua_State *L) { + int top = lua_gettop(L); int arg = 1; size_t sfl; const char *strfrmt = luaL_checklstring(L, arg, &sfl); @@ -768,7 +769,8 @@ static int str_format (lua_State *L) { else { /* format item */ char form[MAX_FORMAT]; /* to store the format (`%...') */ char buff[MAX_ITEM]; /* to store the formatted item */ - arg++; + if (++arg > top) + luaL_argerror(L, arg, "no value"); strfrmt = scanformat(L, strfrmt, form); switch (*strfrmt++) { case 'c': { diff --git a/deps/lua/src/lua.h b/deps/lua/src/lua.h index e4bdfd3b94f..a4b73e743ed 100644 --- a/deps/lua/src/lua.h +++ b/deps/lua/src/lua.h @@ -1,5 +1,5 @@ /* -** $Id: lua.h,v 1.218.1.5 2008/08/06 13:30:12 roberto Exp $ +** $Id: lua.h,v 1.218.1.7 2012/01/13 20:36:20 roberto Exp $ ** Lua - An Extensible Extension Language ** Lua.org, PUC-Rio, Brazil (http://www.lua.org) ** See Copyright Notice at the end of this file @@ -17,9 +17,9 @@ #define LUA_VERSION "Lua 5.1" -#define LUA_RELEASE "Lua 5.1.4" +#define LUA_RELEASE "Lua 5.1.5" #define LUA_VERSION_NUM 501 -#define LUA_COPYRIGHT "Copyright (C) 1994-2008 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT "Copyright (C) 1994-2012 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" @@ -362,7 +362,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2008 Lua.org, PUC-Rio. All rights reserved. +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/deps/lua/src/lua_bit.c b/deps/lua/src/lua_bit.c new file mode 100644 index 00000000000..690df7d3ce6 --- /dev/null +++ b/deps/lua/src/lua_bit.c @@ -0,0 +1,189 @@ +/* +** Lua BitOp -- a bit operations library for Lua 5.1/5.2. +** http://bitop.luajit.org/ +** +** Copyright (C) 2008-2012 Mike Pall. All rights reserved. +** +** Permission is hereby granted, free of charge, to any person obtaining +** a copy of this software and associated documentation files (the +** "Software"), to deal in the Software without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Software, and to +** permit persons to whom the Software is furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be +** included in all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +** +** [ MIT license: http://www.opensource.org/licenses/mit-license.php ] +*/ + +#define LUA_BITOP_VERSION "1.0.2" + +#define LUA_LIB +#include "lua.h" +#include "lauxlib.h" + +#ifdef _MSC_VER +/* MSVC is stuck in the last century and doesn't have C99's stdint.h. */ +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#else +#include +#endif + +typedef int32_t SBits; +typedef uint32_t UBits; + +typedef union { + lua_Number n; +#ifdef LUA_NUMBER_DOUBLE + uint64_t b; +#else + UBits b; +#endif +} BitNum; + +/* Convert argument to bit type. */ +static UBits barg(lua_State *L, int idx) +{ + BitNum bn; + UBits b; +#if LUA_VERSION_NUM < 502 + bn.n = lua_tonumber(L, idx); +#else + bn.n = luaL_checknumber(L, idx); +#endif +#if defined(LUA_NUMBER_DOUBLE) + bn.n += 6755399441055744.0; /* 2^52+2^51 */ +#ifdef SWAPPED_DOUBLE + b = (UBits)(bn.b >> 32); +#else + b = (UBits)bn.b; +#endif +#elif defined(LUA_NUMBER_INT) || defined(LUA_NUMBER_LONG) || \ + defined(LUA_NUMBER_LONGLONG) || defined(LUA_NUMBER_LONG_LONG) || \ + defined(LUA_NUMBER_LLONG) + if (sizeof(UBits) == sizeof(lua_Number)) + b = bn.b; + else + b = (UBits)(SBits)bn.n; +#elif defined(LUA_NUMBER_FLOAT) +#error "A 'float' lua_Number type is incompatible with this library" +#else +#error "Unknown number type, check LUA_NUMBER_* in luaconf.h" +#endif +#if LUA_VERSION_NUM < 502 + if (b == 0 && !lua_isnumber(L, idx)) { + luaL_typerror(L, idx, "number"); + } +#endif + return b; +} + +/* Return bit type. */ +#define BRET(b) lua_pushnumber(L, (lua_Number)(SBits)(b)); return 1; + +static int bit_tobit(lua_State *L) { BRET(barg(L, 1)) } +static int bit_bnot(lua_State *L) { BRET(~barg(L, 1)) } + +#define BIT_OP(func, opr) \ + static int func(lua_State *L) { int i; UBits b = barg(L, 1); \ + for (i = lua_gettop(L); i > 1; i--) b opr barg(L, i); BRET(b) } +BIT_OP(bit_band, &=) +BIT_OP(bit_bor, |=) +BIT_OP(bit_bxor, ^=) + +#define bshl(b, n) (b << n) +#define bshr(b, n) (b >> n) +#define bsar(b, n) ((SBits)b >> n) +#define brol(b, n) ((b << n) | (b >> (32-n))) +#define bror(b, n) ((b << (32-n)) | (b >> n)) +#define BIT_SH(func, fn) \ + static int func(lua_State *L) { \ + UBits b = barg(L, 1); UBits n = barg(L, 2) & 31; BRET(fn(b, n)) } +BIT_SH(bit_lshift, bshl) +BIT_SH(bit_rshift, bshr) +BIT_SH(bit_arshift, bsar) +BIT_SH(bit_rol, brol) +BIT_SH(bit_ror, bror) + +static int bit_bswap(lua_State *L) +{ + UBits b = barg(L, 1); + b = (b >> 24) | ((b >> 8) & 0xff00) | ((b & 0xff00) << 8) | (b << 24); + BRET(b) +} + +static int bit_tohex(lua_State *L) +{ + UBits b = barg(L, 1); + SBits n = lua_isnone(L, 2) ? 8 : (SBits)barg(L, 2); + const char *hexdigits = "0123456789abcdef"; + char buf[8]; + int i; + if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; } + if (n > 8) n = 8; + for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; } + lua_pushlstring(L, buf, (size_t)n); + return 1; +} + +static const struct luaL_Reg bit_funcs[] = { + { "tobit", bit_tobit }, + { "bnot", bit_bnot }, + { "band", bit_band }, + { "bor", bit_bor }, + { "bxor", bit_bxor }, + { "lshift", bit_lshift }, + { "rshift", bit_rshift }, + { "arshift", bit_arshift }, + { "rol", bit_rol }, + { "ror", bit_ror }, + { "bswap", bit_bswap }, + { "tohex", bit_tohex }, + { NULL, NULL } +}; + +/* Signed right-shifts are implementation-defined per C89/C99. +** But the de facto standard are arithmetic right-shifts on two's +** complement CPUs. This behaviour is required here, so test for it. +*/ +#define BAD_SAR (bsar(-8, 2) != (SBits)-2) + +LUALIB_API int luaopen_bit(lua_State *L) +{ + UBits b; + lua_pushnumber(L, (lua_Number)1437217655L); + b = barg(L, -1); + if (b != (UBits)1437217655L || BAD_SAR) { /* Perform a simple self-test. */ + const char *msg = "compiled with incompatible luaconf.h"; +#ifdef LUA_NUMBER_DOUBLE +#ifdef _WIN32 + if (b == (UBits)1610612736L) + msg = "use D3DCREATE_FPU_PRESERVE with DirectX"; +#endif + if (b == (UBits)1127743488L) + msg = "not compiled with SWAPPED_DOUBLE"; +#endif + if (BAD_SAR) + msg = "arithmetic right-shift broken"; + luaL_error(L, "bit library self-test failed (%s)", msg); + } +#if LUA_VERSION_NUM < 502 + luaL_register(L, "bit", bit_funcs); +#else + luaL_newlib(L, bit_funcs); +#endif + return 1; +} + diff --git a/deps/lua/src/lua_cjson.c b/deps/lua/src/lua_cjson.c index 2e272b00700..c26c0d7b8ea 100644 --- a/deps/lua/src/lua_cjson.c +++ b/deps/lua/src/lua_cjson.c @@ -1,8 +1,6 @@ -#define VERSION "1.0.3" - -/* CJSON - JSON support for Lua +/* Lua CJSON - JSON support for Lua * - * Copyright (c) 2010-2011 Mark Pulford + * Copyright (c) 2010-2012 Mark Pulford * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -41,22 +39,42 @@ #include #include #include +#include #include "lua.h" #include "lauxlib.h" #include "strbuf.h" +#include "fpconv.h" + +#include "../../../src/solarisfixes.h" + +#ifndef CJSON_MODNAME +#define CJSON_MODNAME "cjson" +#endif + +#ifndef CJSON_VERSION +#define CJSON_VERSION "2.1.0" +#endif -#ifdef MISSING_ISINF +/* Workaround for Solaris platforms missing isinf() */ +#if !defined(isinf) && (defined(USE_INTERNAL_ISINF) || defined(MISSING_ISINF)) #define isinf(x) (!isnan(x) && isnan((x) - (x))) #endif #define DEFAULT_SPARSE_CONVERT 0 #define DEFAULT_SPARSE_RATIO 2 #define DEFAULT_SPARSE_SAFE 10 -#define DEFAULT_MAX_DEPTH 20 -#define DEFAULT_ENCODE_REFUSE_BADNUM 1 -#define DEFAULT_DECODE_REFUSE_BADNUM 0 +#define DEFAULT_ENCODE_MAX_DEPTH 1000 +#define DEFAULT_DECODE_MAX_DEPTH 1000 +#define DEFAULT_ENCODE_INVALID_NUMBERS 0 +#define DEFAULT_DECODE_INVALID_NUMBERS 1 #define DEFAULT_ENCODE_KEEP_BUFFER 1 +#define DEFAULT_ENCODE_NUMBER_PRECISION 14 + +#ifdef DISABLE_INVALID_NUMBERS +#undef DEFAULT_DECODE_INVALID_NUMBERS +#define DEFAULT_DECODE_INVALID_NUMBERS 0 +#endif typedef enum { T_OBJ_BEGIN, @@ -96,29 +114,29 @@ static const char *json_token_type_name[] = { typedef struct { json_token_type_t ch2token[256]; char escape2char[256]; /* Decoding */ -#if 0 - char escapes[35][8]; /* Pre-generated escape string buffer */ - char *char2escape[256]; /* Encoding */ -#endif + + /* encode_buf is only allocated and used when + * encode_keep_buffer is set */ strbuf_t encode_buf; - char number_fmt[8]; /* "%.XXg\0" */ - int current_depth; int encode_sparse_convert; int encode_sparse_ratio; int encode_sparse_safe; int encode_max_depth; - int encode_refuse_badnum; - int decode_refuse_badnum; - int encode_keep_buffer; + int encode_invalid_numbers; /* 2 => Encode as "null" */ int encode_number_precision; + int encode_keep_buffer; + + int decode_invalid_numbers; + int decode_max_depth; } json_config_t; typedef struct { const char *data; - int index; + const char *ptr; strbuf_t *tmp; /* Temporary storage for strings */ json_config_t *cfg; + int current_depth; } json_parse_t; typedef struct { @@ -171,29 +189,76 @@ static const char *char2escape[256] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, }; -static int json_config_key; - /* ===== CONFIGURATION ===== */ static json_config_t *json_fetch_config(lua_State *l) { json_config_t *cfg; - lua_pushlightuserdata(l, &json_config_key); - lua_gettable(l, LUA_REGISTRYINDEX); - cfg = lua_touserdata(l, -1); + cfg = lua_touserdata(l, lua_upvalueindex(1)); if (!cfg) luaL_error(l, "BUG: Unable to fetch CJSON configuration"); - lua_pop(l, 1); - return cfg; } -static void json_verify_arg_count(lua_State *l, int args) +/* Ensure the correct number of arguments have been provided. + * Pad with nil to allow other functions to simply check arg[i] + * to find whether an argument was provided */ +static json_config_t *json_arg_init(lua_State *l, int args) { luaL_argcheck(l, lua_gettop(l) <= args, args + 1, "found too many arguments"); + + while (lua_gettop(l) < args) + lua_pushnil(l); + + return json_fetch_config(l); +} + +/* Process integer options for configuration functions */ +static int json_integer_option(lua_State *l, int optindex, int *setting, + int min, int max) +{ + char errmsg[64]; + int value; + + if (!lua_isnil(l, optindex)) { + value = luaL_checkinteger(l, optindex); + snprintf(errmsg, sizeof(errmsg), "expected integer between %d and %d", min, max); + luaL_argcheck(l, min <= value && value <= max, 1, errmsg); + *setting = value; + } + + lua_pushinteger(l, *setting); + + return 1; +} + +/* Process enumerated arguments for a configuration function */ +static int json_enum_option(lua_State *l, int optindex, int *setting, + const char **options, int bool_true) +{ + static const char *bool_options[] = { "off", "on", NULL }; + + if (!options) { + options = bool_options; + bool_true = 1; + } + + if (!lua_isnil(l, optindex)) { + if (bool_true && lua_isboolean(l, optindex)) + *setting = lua_toboolean(l, optindex) * bool_true; + else + *setting = luaL_checkoption(l, optindex, NULL, options); + } + + if (bool_true && (*setting == 0 || *setting == bool_true)) + lua_pushboolean(l, *setting); + else + lua_pushstring(l, options[*setting]); + + return 1; } /* Configures handling of extremely sparse arrays: @@ -202,29 +267,11 @@ static void json_verify_arg_count(lua_State *l, int args) * safe: Always use an array when the max index <= safe */ static int json_cfg_encode_sparse_array(lua_State *l) { - json_config_t *cfg; - int val; - - json_verify_arg_count(l, 3); - cfg = json_fetch_config(l); - - switch (lua_gettop(l)) { - case 3: - val = luaL_checkinteger(l, 3); - luaL_argcheck(l, val >= 0, 3, "expected integer >= 0"); - cfg->encode_sparse_safe = val; - case 2: - val = luaL_checkinteger(l, 2); - luaL_argcheck(l, val >= 0, 2, "expected integer >= 0"); - cfg->encode_sparse_ratio = val; - case 1: - luaL_argcheck(l, lua_isboolean(l, 1), 1, "expected boolean"); - cfg->encode_sparse_convert = lua_toboolean(l, 1); - } + json_config_t *cfg = json_arg_init(l, 3); - lua_pushboolean(l, cfg->encode_sparse_convert); - lua_pushinteger(l, cfg->encode_sparse_ratio); - lua_pushinteger(l, cfg->encode_sparse_safe); + json_enum_option(l, 1, &cfg->encode_sparse_convert, NULL, 1); + json_integer_option(l, 2, &cfg->encode_sparse_ratio, 0, INT_MAX); + json_integer_option(l, 3, &cfg->encode_sparse_safe, 0, INT_MAX); return 3; } @@ -233,108 +280,80 @@ static int json_cfg_encode_sparse_array(lua_State *l) * encoding */ static int json_cfg_encode_max_depth(lua_State *l) { - json_config_t *cfg; - int depth; + json_config_t *cfg = json_arg_init(l, 1); - json_verify_arg_count(l, 1); - cfg = json_fetch_config(l); - - if (lua_gettop(l)) { - depth = luaL_checkinteger(l, 1); - luaL_argcheck(l, depth > 0, 1, "expected positive integer"); - cfg->encode_max_depth = depth; - } - - lua_pushinteger(l, cfg->encode_max_depth); - - return 1; + return json_integer_option(l, 1, &cfg->encode_max_depth, 1, INT_MAX); } -static void json_set_number_precision(json_config_t *cfg, int prec) +/* Configures the maximum number of nested arrays/objects allowed when + * encoding */ +static int json_cfg_decode_max_depth(lua_State *l) { - cfg->encode_number_precision = prec; - sprintf(cfg->number_fmt, "%%.%dg", prec); + json_config_t *cfg = json_arg_init(l, 1); + + return json_integer_option(l, 1, &cfg->decode_max_depth, 1, INT_MAX); } /* Configures number precision when converting doubles to text */ static int json_cfg_encode_number_precision(lua_State *l) { - json_config_t *cfg; - int precision; - - json_verify_arg_count(l, 1); - cfg = json_fetch_config(l); - - if (lua_gettop(l)) { - precision = luaL_checkinteger(l, 1); - luaL_argcheck(l, 1 <= precision && precision <= 14, 1, - "expected integer between 1 and 14"); - json_set_number_precision(cfg, precision); - } + json_config_t *cfg = json_arg_init(l, 1); - lua_pushinteger(l, cfg->encode_number_precision); - - return 1; + return json_integer_option(l, 1, &cfg->encode_number_precision, 1, 14); } /* Configures JSON encoding buffer persistence */ static int json_cfg_encode_keep_buffer(lua_State *l) { - json_config_t *cfg; + json_config_t *cfg = json_arg_init(l, 1); + int old_value; - json_verify_arg_count(l, 1); - cfg = json_fetch_config(l); + old_value = cfg->encode_keep_buffer; - if (lua_gettop(l)) { - luaL_checktype(l, 1, LUA_TBOOLEAN); - cfg->encode_keep_buffer = lua_toboolean(l, 1); - } + json_enum_option(l, 1, &cfg->encode_keep_buffer, NULL, 1); - lua_pushboolean(l, cfg->encode_keep_buffer); + /* Init / free the buffer if the setting has changed */ + if (old_value ^ cfg->encode_keep_buffer) { + if (cfg->encode_keep_buffer) + strbuf_init(&cfg->encode_buf, 0); + else + strbuf_free(&cfg->encode_buf); + } return 1; } -/* On argument: decode enum and set config variables - * **options must point to a NULL terminated array of 4 enums - * Returns: current enum value */ -static void json_enum_option(lua_State *l, const char **options, - int *opt1, int *opt2) +#if defined(DISABLE_INVALID_NUMBERS) && !defined(USE_INTERNAL_FPCONV) +void json_verify_invalid_number_setting(lua_State *l, int *setting) { - int setting; + if (*setting == 1) { + *setting = 0; + luaL_error(l, "Infinity, NaN, and/or hexadecimal numbers are not supported."); + } +} +#else +#define json_verify_invalid_number_setting(l, s) do { } while(0) +#endif - if (lua_gettop(l)) { - if (lua_isboolean(l, 1)) - setting = lua_toboolean(l, 1) * 3; - else - setting = luaL_checkoption(l, 1, NULL, options); +static int json_cfg_encode_invalid_numbers(lua_State *l) +{ + static const char *options[] = { "off", "on", "null", NULL }; + json_config_t *cfg = json_arg_init(l, 1); - *opt1 = setting & 1 ? 1 : 0; - *opt2 = setting & 2 ? 1 : 0; - } else { - setting = *opt1 | (*opt2 << 1); - } + json_enum_option(l, 1, &cfg->encode_invalid_numbers, options, 1); - if (setting) - lua_pushstring(l, options[setting]); - else - lua_pushboolean(l, 0); -} + json_verify_invalid_number_setting(l, &cfg->encode_invalid_numbers); + return 1; +} -/* When enabled, rejects: NaN, Infinity, hexidecimal numbers */ -static int json_cfg_refuse_invalid_numbers(lua_State *l) +static int json_cfg_decode_invalid_numbers(lua_State *l) { - static const char *options_enc_dec[] = { "none", "encode", "decode", - "both", NULL }; - json_config_t *cfg; + json_config_t *cfg = json_arg_init(l, 1); - json_verify_arg_count(l, 1); - cfg = json_fetch_config(l); + json_enum_option(l, 1, &cfg->decode_invalid_numbers, NULL, 1); - json_enum_option(l, options_enc_dec, - &cfg->encode_refuse_badnum, - &cfg->decode_refuse_badnum); + json_verify_invalid_number_setting(l, &cfg->encode_invalid_numbers); return 1; } @@ -364,16 +383,19 @@ static void json_create_config(lua_State *l) lua_setfield(l, -2, "__gc"); lua_setmetatable(l, -2); - strbuf_init(&cfg->encode_buf, 0); - cfg->encode_sparse_convert = DEFAULT_SPARSE_CONVERT; cfg->encode_sparse_ratio = DEFAULT_SPARSE_RATIO; cfg->encode_sparse_safe = DEFAULT_SPARSE_SAFE; - cfg->encode_max_depth = DEFAULT_MAX_DEPTH; - cfg->encode_refuse_badnum = DEFAULT_ENCODE_REFUSE_BADNUM; - cfg->decode_refuse_badnum = DEFAULT_DECODE_REFUSE_BADNUM; + cfg->encode_max_depth = DEFAULT_ENCODE_MAX_DEPTH; + cfg->decode_max_depth = DEFAULT_DECODE_MAX_DEPTH; + cfg->encode_invalid_numbers = DEFAULT_ENCODE_INVALID_NUMBERS; + cfg->decode_invalid_numbers = DEFAULT_DECODE_INVALID_NUMBERS; cfg->encode_keep_buffer = DEFAULT_ENCODE_KEEP_BUFFER; - json_set_number_precision(cfg, 14); + cfg->encode_number_precision = DEFAULT_ENCODE_NUMBER_PRECISION; + +#if DEFAULT_ENCODE_KEEP_BUFFER > 0 + strbuf_init(&cfg->encode_buf, 0); +#endif /* Decoding init */ @@ -419,41 +441,15 @@ static void json_create_config(lua_State *l) cfg->escape2char['f'] = '\f'; cfg->escape2char['r'] = '\r'; cfg->escape2char['u'] = 'u'; /* Unicode parsing required */ - - -#if 0 - /* Initialise separate storage for pre-generated escape codes. - * Escapes 0-31 map directly, 34, 92, 127 follow afterwards to - * save memory. */ - for (i = 0 ; i < 32; i++) - sprintf(cfg->escapes[i], "\\u%04x", i); - strcpy(cfg->escapes[8], "\b"); /* Override simpler escapes */ - strcpy(cfg->escapes[9], "\t"); - strcpy(cfg->escapes[10], "\n"); - strcpy(cfg->escapes[12], "\f"); - strcpy(cfg->escapes[13], "\r"); - strcpy(cfg->escapes[32], "\\\""); /* chr(34) */ - strcpy(cfg->escapes[33], "\\\\"); /* chr(92) */ - sprintf(cfg->escapes[34], "\\u%04x", 127); /* char(127) */ - - /* Initialise encoding escape lookup table */ - for (i = 0; i < 32; i++) - cfg->char2escape[i] = cfg->escapes[i]; - for (i = 32; i < 256; i++) - cfg->char2escape[i] = NULL; - cfg->char2escape[34] = cfg->escapes[32]; - cfg->char2escape[92] = cfg->escapes[33]; - cfg->char2escape[127] = cfg->escapes[34]; -#endif } /* ===== ENCODING ===== */ -static void json_encode_exception(lua_State *l, json_config_t *cfg, int lindex, +static void json_encode_exception(lua_State *l, json_config_t *cfg, strbuf_t *json, int lindex, const char *reason) { if (!cfg->encode_keep_buffer) - strbuf_free(&cfg->encode_buf); + strbuf_free(json); luaL_error(l, "Cannot serialise %s: %s", lua_typename(l, lua_type(l, lindex)), reason); } @@ -494,7 +490,7 @@ static void json_append_string(lua_State *l, strbuf_t *json, int lindex) * -1 object (not a pure array) * >=0 elements in array */ -static int lua_array_length(lua_State *l, json_config_t *cfg) +static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json) { double k; int max; @@ -529,7 +525,7 @@ static int lua_array_length(lua_State *l, json_config_t *cfg) max > items * cfg->encode_sparse_ratio && max > cfg->encode_sparse_safe) { if (!cfg->encode_sparse_convert) - json_encode_exception(l, cfg, -1, "excessively sparse array"); + json_encode_exception(l, cfg, json, -1, "excessively sparse array"); return -1; } @@ -537,31 +533,41 @@ static int lua_array_length(lua_State *l, json_config_t *cfg) return max; } -static void json_encode_descend(lua_State *l, json_config_t *cfg) +static void json_check_encode_depth(lua_State *l, json_config_t *cfg, + int current_depth, strbuf_t *json) { - cfg->current_depth++; + /* Ensure there are enough slots free to traverse a table (key, + * value) and push a string for a potential error message. + * + * Unlike "decode", the key and value are still on the stack when + * lua_checkstack() is called. Hence an extra slot for luaL_error() + * below is required just in case the next check to lua_checkstack() + * fails. + * + * While this won't cause a crash due to the EXTRA_STACK reserve + * slots, it would still be an improper use of the API. */ + if (current_depth <= cfg->encode_max_depth && lua_checkstack(l, 3)) + return; - if (cfg->current_depth > cfg->encode_max_depth) { - if (!cfg->encode_keep_buffer) - strbuf_free(&cfg->encode_buf); - luaL_error(l, "Cannot serialise, excessive nesting (%d)", - cfg->current_depth); - } + if (!cfg->encode_keep_buffer) + strbuf_free(json); + + luaL_error(l, "Cannot serialise, excessive nesting (%d)", + current_depth); } -static void json_append_data(lua_State *l, json_config_t *cfg, strbuf_t *json); +static void json_append_data(lua_State *l, json_config_t *cfg, + int current_depth, strbuf_t *json); /* json_append_array args: * - lua_State * - JSON strbuf * - Size of passwd Lua array (top of stack) */ -static void json_append_array(lua_State *l, json_config_t *cfg, strbuf_t *json, - int array_length) +static void json_append_array(lua_State *l, json_config_t *cfg, int current_depth, + strbuf_t *json, int array_length) { int comma, i; - json_encode_descend(l, cfg); - strbuf_append_char(json, '['); comma = 0; @@ -572,38 +578,48 @@ static void json_append_array(lua_State *l, json_config_t *cfg, strbuf_t *json, comma = 1; lua_rawgeti(l, -1, i); - json_append_data(l, cfg, json); + json_append_data(l, cfg, current_depth, json); lua_pop(l, 1); } strbuf_append_char(json, ']'); - - cfg->current_depth--; } -static void json_append_number(lua_State *l, strbuf_t *json, int index, - json_config_t *cfg) +static void json_append_number(lua_State *l, json_config_t *cfg, + strbuf_t *json, int lindex) { - double num = lua_tonumber(l, index); + double num = lua_tonumber(l, lindex); + int len; - if (cfg->encode_refuse_badnum && (isinf(num) || isnan(num))) - json_encode_exception(l, cfg, index, "must not be NaN or Inf"); + if (cfg->encode_invalid_numbers == 0) { + /* Prevent encoding invalid numbers */ + if (isinf(num) || isnan(num)) + json_encode_exception(l, cfg, json, lindex, "must not be NaN or Inf"); + } else if (cfg->encode_invalid_numbers == 1) { + /* Encode invalid numbers, but handle "nan" separately + * since some platforms may encode as "-nan". */ + if (isnan(num)) { + strbuf_append_mem(json, "nan", 3); + return; + } + } else { + /* Encode invalid numbers as "null" */ + if (isinf(num) || isnan(num)) { + strbuf_append_mem(json, "null", 4); + return; + } + } - /* Lowest double printed with %.14g is 21 characters long: - * -1.7976931348623e+308 - * - * Use 32 to include the \0, and a few extra just in case.. - */ - strbuf_append_fmt(json, 32, cfg->number_fmt, num); + strbuf_ensure_empty_length(json, FPCONV_G_FMT_BUFSIZE); + len = fpconv_g_fmt(strbuf_empty_ptr(json), num, cfg->encode_number_precision); + strbuf_extend_length(json, len); } static void json_append_object(lua_State *l, json_config_t *cfg, - strbuf_t *json) + int current_depth, strbuf_t *json) { int comma, keytype; - json_encode_descend(l, cfg); - /* Object */ strbuf_append_char(json, '{'); @@ -620,30 +636,29 @@ static void json_append_object(lua_State *l, json_config_t *cfg, keytype = lua_type(l, -2); if (keytype == LUA_TNUMBER) { strbuf_append_char(json, '"'); - json_append_number(l, json, -2, cfg); + json_append_number(l, cfg, json, -2); strbuf_append_mem(json, "\":", 2); } else if (keytype == LUA_TSTRING) { json_append_string(l, json, -2); strbuf_append_char(json, ':'); } else { - json_encode_exception(l, cfg, -2, + json_encode_exception(l, cfg, json, -2, "table key must be a number or string"); /* never returns */ } /* table, key, value */ - json_append_data(l, cfg, json); + json_append_data(l, cfg, current_depth, json); lua_pop(l, 1); /* table, key */ } strbuf_append_char(json, '}'); - - cfg->current_depth--; } /* Serialise Lua data into JSON string. */ -static void json_append_data(lua_State *l, json_config_t *cfg, strbuf_t *json) +static void json_append_data(lua_State *l, json_config_t *cfg, + int current_depth, strbuf_t *json) { int len; @@ -652,7 +667,7 @@ static void json_append_data(lua_State *l, json_config_t *cfg, strbuf_t *json) json_append_string(l, json, -1); break; case LUA_TNUMBER: - json_append_number(l, json, -1, cfg); + json_append_number(l, cfg, json, -1); break; case LUA_TBOOLEAN: if (lua_toboolean(l, -1)) @@ -661,11 +676,13 @@ static void json_append_data(lua_State *l, json_config_t *cfg, strbuf_t *json) strbuf_append_mem(json, "false", 5); break; case LUA_TTABLE: - len = lua_array_length(l, cfg); + current_depth++; + json_check_encode_depth(l, cfg, current_depth, json); + len = lua_array_length(l, cfg, json); if (len > 0) - json_append_array(l, cfg, json, len); + json_append_array(l, cfg, current_depth, json, len); else - json_append_object(l, cfg, json); + json_append_object(l, cfg, current_depth, json); break; case LUA_TNIL: strbuf_append_mem(json, "null", 4); @@ -678,38 +695,38 @@ static void json_append_data(lua_State *l, json_config_t *cfg, strbuf_t *json) default: /* Remaining types (LUA_TFUNCTION, LUA_TUSERDATA, LUA_TTHREAD, * and LUA_TLIGHTUSERDATA) cannot be serialised */ - json_encode_exception(l, cfg, -1, "type not supported"); + json_encode_exception(l, cfg, json, -1, "type not supported"); /* never returns */ } } static int json_encode(lua_State *l) { - json_config_t *cfg; + json_config_t *cfg = json_fetch_config(l); + strbuf_t local_encode_buf; + strbuf_t *encode_buf; char *json; int len; - /* Can't use json_verify_arg_count() since we need to ensure - * there is only 1 argument */ luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); - cfg = json_fetch_config(l); - cfg->current_depth = 0; - - /* Reset the persistent buffer if it exists. - * Otherwise allocate a new buffer. */ - if (strbuf_allocated(&cfg->encode_buf)) - strbuf_reset(&cfg->encode_buf); - else - strbuf_init(&cfg->encode_buf, 0); + if (!cfg->encode_keep_buffer) { + /* Use private buffer */ + encode_buf = &local_encode_buf; + strbuf_init(encode_buf, 0); + } else { + /* Reuse existing buffer */ + encode_buf = &cfg->encode_buf; + strbuf_reset(encode_buf); + } - json_append_data(l, cfg, &cfg->encode_buf); - json = strbuf_string(&cfg->encode_buf, &len); + json_append_data(l, cfg, 0, encode_buf); + json = strbuf_string(encode_buf, &len); lua_pushlstring(l, json, len); if (!cfg->encode_keep_buffer) - strbuf_free(&cfg->encode_buf); + strbuf_free(encode_buf); return 1; } @@ -808,7 +825,7 @@ static int json_append_unicode_escape(json_parse_t *json) int escape_len = 6; /* Fetch UTF-16 code unit */ - codepoint = decode_hex4(&json->data[json->index + 2]); + codepoint = decode_hex4(json->ptr + 2); if (codepoint < 0) return -1; @@ -824,13 +841,13 @@ static int json_append_unicode_escape(json_parse_t *json) return -1; /* Ensure the next code is a unicode escape */ - if (json->data[json->index + escape_len] != '\\' || - json->data[json->index + escape_len + 1] != 'u') { + if (*(json->ptr + escape_len) != '\\' || + *(json->ptr + escape_len + 1) != 'u') { return -1; } /* Fetch the next codepoint */ - surrogate_low = decode_hex4(&json->data[json->index + 2 + escape_len]); + surrogate_low = decode_hex4(json->ptr + 2 + escape_len); if (surrogate_low < 0) return -1; @@ -852,7 +869,7 @@ static int json_append_unicode_escape(json_parse_t *json) /* Append bytes and advance parse index */ strbuf_append_mem_unsafe(json->tmp, utf8, len); - json->index += escape_len; + json->ptr += escape_len; return 0; } @@ -861,7 +878,7 @@ static void json_set_token_error(json_token_t *token, json_parse_t *json, const char *errtype) { token->type = T_ERROR; - token->index = json->index; + token->index = json->ptr - json->data; token->value.string = errtype; } @@ -871,15 +888,18 @@ static void json_next_string_token(json_parse_t *json, json_token_t *token) char ch; /* Caller must ensure a string is next */ - assert(json->data[json->index] == '"'); + assert(*json->ptr == '"'); /* Skip " */ - json->index++; + json->ptr++; /* json->tmp is the temporary strbuf used to accumulate the - * decoded string value. */ + * decoded string value. + * json->tmp is sized to handle JSON containing only a string value. + */ strbuf_reset(json->tmp); - while ((ch = json->data[json->index]) != '"') { + + while ((ch = *json->ptr) != '"') { if (!ch) { /* Premature end of the string */ json_set_token_error(token, json, "unexpected end of string"); @@ -889,7 +909,7 @@ static void json_next_string_token(json_parse_t *json, json_token_t *token) /* Handle escapes */ if (ch == '\\') { /* Fetch escape character */ - ch = json->data[json->index + 1]; + ch = *(json->ptr + 1); /* Translate escape code and append to tmp string */ ch = escape2char[(unsigned char)ch]; @@ -907,14 +927,14 @@ static void json_next_string_token(json_parse_t *json, json_token_t *token) } /* Skip '\' */ - json->index++; + json->ptr++; } /* Append normal character or translated single character * Unicode escapes are handled above */ strbuf_append_char_unsafe(json->tmp, ch); - json->index++; + json->ptr++; } - json->index++; /* Eat final quote (") */ + json->ptr++; /* Eat final quote (") */ strbuf_ensure_null(json->tmp); @@ -928,7 +948,7 @@ static void json_next_string_token(json_parse_t *json, json_token_t *token) * json_next_number_token() uses strtod() which allows other forms: * - numbers starting with '+' * - NaN, -NaN, infinity, -infinity - * - hexidecimal numbers + * - hexadecimal numbers * - numbers with leading zeros * * json_is_invalid_number() detects "numbers" which may pass strtod()'s @@ -939,34 +959,33 @@ static void json_next_string_token(json_parse_t *json, json_token_t *token) */ static int json_is_invalid_number(json_parse_t *json) { - int i = json->index; + const char *p = json->ptr; /* Reject numbers starting with + */ - if (json->data[i] == '+') + if (*p == '+') return 1; /* Skip minus sign if it exists */ - if (json->data[i] == '-') - i++; + if (*p == '-') + p++; /* Reject numbers starting with 0x, or leading zeros */ - if (json->data[i] == '0') { - int ch2 = json->data[i + 1]; + if (*p == '0') { + int ch2 = *(p + 1); if ((ch2 | 0x20) == 'x' || /* Hex */ ('0' <= ch2 && ch2 <= '9')) /* Leading zero */ return 1; return 0; - } else if (json->data[i] <= '9') { + } else if (*p <= '9') { return 0; /* Ordinary number */ } - /* Reject inf/nan */ - if (!strncasecmp(&json->data[i], "inf", 3)) + if (!strncasecmp(p, "inf", 3)) return 1; - if (!strncasecmp(&json->data[i], "nan", 3)) + if (!strncasecmp(p, "nan", 3)) return 1; /* Pass all other numbers which may still be invalid, but @@ -976,35 +995,39 @@ static int json_is_invalid_number(json_parse_t *json) static void json_next_number_token(json_parse_t *json, json_token_t *token) { - const char *startptr; char *endptr; token->type = T_NUMBER; - startptr = &json->data[json->index]; - token->value.number = strtod(&json->data[json->index], &endptr); - if (startptr == endptr) + token->value.number = fpconv_strtod(json->ptr, &endptr); + if (json->ptr == endptr) json_set_token_error(token, json, "invalid number"); else - json->index += endptr - startptr; /* Skip the processed number */ + json->ptr = endptr; /* Skip the processed number */ return; } /* Fills in the token struct. * T_STRING will return a pointer to the json_parse_t temporary string - * T_ERROR will leave the json->index pointer at the error. + * T_ERROR will leave the json->ptr pointer at the error. */ static void json_next_token(json_parse_t *json, json_token_t *token) { - json_token_type_t *ch2token = json->cfg->ch2token; + const json_token_type_t *ch2token = json->cfg->ch2token; int ch; - /* Eat whitespace. FIXME: UGLY */ - token->type = ch2token[(unsigned char)json->data[json->index]]; - while (token->type == T_WHITESPACE) - token->type = ch2token[(unsigned char)json->data[++json->index]]; + /* Eat whitespace. */ + while (1) { + ch = (unsigned char)*(json->ptr); + token->type = ch2token[ch]; + if (token->type != T_WHITESPACE) + break; + json->ptr++; + } - token->index = json->index; + /* Store location of new token. Required when throwing errors + * for unexpected tokens (syntax errors). */ + token->index = json->ptr - json->data; /* Don't advance the pointer for an error or the end */ if (token->type == T_ERROR) { @@ -1018,14 +1041,13 @@ static void json_next_token(json_parse_t *json, json_token_t *token) /* Found a known single character token, advance index and return */ if (token->type != T_UNKNOWN) { - json->index++; + json->ptr++; return; } - /* Process characters which triggered T_UNKNOWN */ - ch = json->data[json->index]; - - /* Must use strncmp() to match the front of the JSON string. + /* Process characters which triggered T_UNKNOWN + * + * Must use strncmp() to match the front of the JSON string. * JSON identifier must be lowercase. * When strict_numbers if disabled, either case is allowed for * Infinity/NaN (since we are no longer following the spec..) */ @@ -1033,29 +1055,29 @@ static void json_next_token(json_parse_t *json, json_token_t *token) json_next_string_token(json, token); return; } else if (ch == '-' || ('0' <= ch && ch <= '9')) { - if (json->cfg->decode_refuse_badnum && json_is_invalid_number(json)) { + if (!json->cfg->decode_invalid_numbers && json_is_invalid_number(json)) { json_set_token_error(token, json, "invalid number"); return; } json_next_number_token(json, token); return; - } else if (!strncmp(&json->data[json->index], "true", 4)) { + } else if (!strncmp(json->ptr, "true", 4)) { token->type = T_BOOLEAN; token->value.boolean = 1; - json->index += 4; + json->ptr += 4; return; - } else if (!strncmp(&json->data[json->index], "false", 5)) { + } else if (!strncmp(json->ptr, "false", 5)) { token->type = T_BOOLEAN; token->value.boolean = 0; - json->index += 5; + json->ptr += 5; return; - } else if (!strncmp(&json->data[json->index], "null", 4)) { + } else if (!strncmp(json->ptr, "null", 4)) { token->type = T_NULL; - json->index += 4; + json->ptr += 4; return; - } else if (!json->cfg->decode_refuse_badnum && + } else if (json->cfg->decode_invalid_numbers && json_is_invalid_number(json)) { - /* When refuse_badnum is disabled, only attempt to process + /* When decode_invalid_numbers is enabled, only attempt to process * numbers we know are invalid JSON (Inf, NaN, hex) * This is required to generate an appropriate token error, * otherwise all bad tokens will register as "invalid number" @@ -1091,13 +1113,23 @@ static void json_throw_parse_error(lua_State *l, json_parse_t *json, exp, found, token->index + 1); } -static void json_decode_checkstack(lua_State *l, json_parse_t *json, int n) +static inline void json_decode_ascend(json_parse_t *json) { - if (lua_checkstack(l, n)) + json->current_depth--; +} + +static void json_decode_descend(lua_State *l, json_parse_t *json, int slots) +{ + json->current_depth++; + + if (json->current_depth <= json->cfg->decode_max_depth && + lua_checkstack(l, slots)) { return; + } strbuf_free(json->tmp); - luaL_error(l, "Too many nested data structures"); + luaL_error(l, "Found too many nested data structures (%d) at character %d", + json->current_depth, json->ptr - json->data); } static void json_parse_object_context(lua_State *l, json_parse_t *json) @@ -1106,7 +1138,7 @@ static void json_parse_object_context(lua_State *l, json_parse_t *json) /* 3 slots required: * .., table, key, value */ - json_decode_checkstack(l, json, 3); + json_decode_descend(l, json, 3); lua_newtable(l); @@ -1114,6 +1146,7 @@ static void json_parse_object_context(lua_State *l, json_parse_t *json) /* Handle empty objects */ if (token.type == T_OBJ_END) { + json_decode_ascend(json); return; } @@ -1137,8 +1170,10 @@ static void json_parse_object_context(lua_State *l, json_parse_t *json) json_next_token(json, &token); - if (token.type == T_OBJ_END) + if (token.type == T_OBJ_END) { + json_decode_ascend(json); return; + } if (token.type != T_COMMA) json_throw_parse_error(l, json, "comma or object end", &token); @@ -1155,15 +1190,17 @@ static void json_parse_array_context(lua_State *l, json_parse_t *json) /* 2 slots required: * .., table, value */ - json_decode_checkstack(l, json, 2); + json_decode_descend(l, json, 2); lua_newtable(l); json_next_token(json, &token); /* Handle empty arrays */ - if (token.type == T_ARR_END) + if (token.type == T_ARR_END) { + json_decode_ascend(json); return; + } for (i = 1; ; i++) { json_process_value(l, json, &token); @@ -1171,8 +1208,10 @@ static void json_parse_array_context(lua_State *l, json_parse_t *json) json_next_token(json, &token); - if (token.type == T_ARR_END) + if (token.type == T_ARR_END) { + json_decode_ascend(json); return; + } if (token.type != T_COMMA) json_throw_parse_error(l, json, "comma or array end", &token); @@ -1211,15 +1250,26 @@ static void json_process_value(lua_State *l, json_parse_t *json, } } -/* json_text must be null terminated string */ -static void lua_json_decode(lua_State *l, const char *json_text, int json_len) +static int json_decode(lua_State *l) { json_parse_t json; json_token_t token; + size_t json_len; + + luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); json.cfg = json_fetch_config(l); - json.data = json_text; - json.index = 0; + json.data = luaL_checklstring(l, 1, &json_len); + json.current_depth = 0; + json.ptr = json.data; + + /* Detect Unicode other than UTF-8 (see RFC 4627, Sec 3) + * + * CJSON can support any simple data type, hence only the first + * character is guaranteed to be ASCII (at worst: '"'). This is + * still enough to detect whether the wrong encoding is in use. */ + if (json_len >= 2 && (!json.data[0] || !json.data[1])) + luaL_error(l, "JSON parser does not support UTF-16 or UTF-32"); /* Ensure the temporary buffer can hold the entire string. * This means we no longer need to do length checks since the decoded @@ -1236,64 +1286,142 @@ static void lua_json_decode(lua_State *l, const char *json_text, int json_len) json_throw_parse_error(l, &json, "the end", &token); strbuf_free(json.tmp); + + return 1; } -static int json_decode(lua_State *l) +/* ===== INITIALISATION ===== */ + +#if !defined(LUA_VERSION_NUM) || LUA_VERSION_NUM < 502 +/* Compatibility for Lua 5.1. + * + * luaL_setfuncs() is used to create a module table where the functions have + * json_config_t as their first upvalue. Code borrowed from Lua 5.2 source. */ +static void luaL_setfuncs (lua_State *l, const luaL_Reg *reg, int nup) { - const char *json; - size_t len; + int i; - json_verify_arg_count(l, 1); + luaL_checkstack(l, nup, "too many upvalues"); + for (; reg->name != NULL; reg++) { /* fill the table with given functions */ + for (i = 0; i < nup; i++) /* copy upvalues to the top */ + lua_pushvalue(l, -nup); + lua_pushcclosure(l, reg->func, nup); /* closure with those upvalues */ + lua_setfield(l, -(nup + 2), reg->name); + } + lua_pop(l, nup); /* remove upvalues */ +} +#endif - json = luaL_checklstring(l, 1, &len); +/* Call target function in protected mode with all supplied args. + * Assumes target function only returns a single non-nil value. + * Convert and return thrown errors as: nil, "error message" */ +static int json_protect_conversion(lua_State *l) +{ + int err; - /* Detect Unicode other than UTF-8 (see RFC 4627, Sec 3) - * - * CJSON can support any simple data type, hence only the first - * character is guaranteed to be ASCII (at worst: '"'). This is - * still enough to detect whether the wrong encoding is in use. */ - if (len >= 2 && (!json[0] || !json[1])) - luaL_error(l, "JSON parser does not support UTF-16 or UTF-32"); + /* Deliberately throw an error for invalid arguments */ + luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); - lua_json_decode(l, json, len); + /* pcall() the function stored as upvalue(1) */ + lua_pushvalue(l, lua_upvalueindex(1)); + lua_insert(l, 1); + err = lua_pcall(l, 1, 1, 0); + if (!err) + return 1; - return 1; -} + if (err == LUA_ERRRUN) { + lua_pushnil(l); + lua_insert(l, -2); + return 2; + } -/* ===== INITIALISATION ===== */ + /* Since we are not using a custom error handler, the only remaining + * errors are memory related */ + return luaL_error(l, "Memory allocation error in CJSON protected call"); +} -int luaopen_cjson(lua_State *l) +/* Return cjson module table */ +static int lua_cjson_new(lua_State *l) { luaL_Reg reg[] = { { "encode", json_encode }, { "decode", json_decode }, { "encode_sparse_array", json_cfg_encode_sparse_array }, { "encode_max_depth", json_cfg_encode_max_depth }, + { "decode_max_depth", json_cfg_decode_max_depth }, { "encode_number_precision", json_cfg_encode_number_precision }, { "encode_keep_buffer", json_cfg_encode_keep_buffer }, - { "refuse_invalid_numbers", json_cfg_refuse_invalid_numbers }, + { "encode_invalid_numbers", json_cfg_encode_invalid_numbers }, + { "decode_invalid_numbers", json_cfg_decode_invalid_numbers }, + { "new", lua_cjson_new }, { NULL, NULL } }; - /* Use json_fetch_config as a pointer. - * It's faster than using a config string, and more unique */ - lua_pushlightuserdata(l, &json_config_key); - json_create_config(l); - lua_settable(l, LUA_REGISTRYINDEX); + /* Initialise number conversions */ + fpconv_init(); - luaL_register(l, "cjson", reg); + /* cjson module table */ + lua_newtable(l); + + /* Register functions with config data as upvalue */ + json_create_config(l); + luaL_setfuncs(l, reg, 1); /* Set cjson.null */ lua_pushlightuserdata(l, NULL); lua_setfield(l, -2, "null"); - /* Set cjson.version */ - lua_pushliteral(l, VERSION); - lua_setfield(l, -2, "version"); + /* Set module name / version fields */ + lua_pushliteral(l, CJSON_MODNAME); + lua_setfield(l, -2, "_NAME"); + lua_pushliteral(l, CJSON_VERSION); + lua_setfield(l, -2, "_VERSION"); + + return 1; +} + +/* Return cjson.safe module table */ +static int lua_cjson_safe_new(lua_State *l) +{ + const char *func[] = { "decode", "encode", NULL }; + int i; + + lua_cjson_new(l); + + /* Fix new() method */ + lua_pushcfunction(l, lua_cjson_safe_new); + lua_setfield(l, -2, "new"); + + for (i = 0; func[i]; i++) { + lua_getfield(l, -1, func[i]); + lua_pushcclosure(l, json_protect_conversion, 1); + lua_setfield(l, -2, func[i]); + } + + return 1; +} + +int luaopen_cjson(lua_State *l) +{ + lua_cjson_new(l); + +#ifdef ENABLE_CJSON_GLOBAL + /* Register a global "cjson" table. */ + lua_pushvalue(l, -1); + lua_setglobal(l, CJSON_MODNAME); +#endif /* Return cjson table */ return 1; } +int luaopen_cjson_safe(lua_State *l) +{ + lua_cjson_safe_new(l); + + /* Return cjson.safe table */ + return 1; +} + /* vi:ai et sw=4 ts=4: */ diff --git a/deps/lua/src/lua_cmsgpack.c b/deps/lua/src/lua_cmsgpack.c index 90612136169..90a388f3f3f 100644 --- a/deps/lua/src/lua_cmsgpack.c +++ b/deps/lua/src/lua_cmsgpack.c @@ -7,14 +7,38 @@ #include "lua.h" #include "lauxlib.h" -#define LUACMSGPACK_VERSION "lua-cmsgpack 0.3.0" +#define LUACMSGPACK_NAME "cmsgpack" +#define LUACMSGPACK_SAFE_NAME "cmsgpack_safe" +#define LUACMSGPACK_VERSION "lua-cmsgpack 0.4.0" #define LUACMSGPACK_COPYRIGHT "Copyright (C) 2012, Salvatore Sanfilippo" #define LUACMSGPACK_DESCRIPTION "MessagePack C implementation for Lua" -#define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ +/* Allows a preprocessor directive to override MAX_NESTING */ +#ifndef LUACMSGPACK_MAX_NESTING + #define LUACMSGPACK_MAX_NESTING 16 /* Max tables nesting. */ +#endif -/* ============================================================================== - * MessagePack implementation and bindings for Lua 5.1. +/* Check if float or double can be an integer without loss of precision */ +#define IS_INT_TYPE_EQUIVALENT(x, T) (!isinf(x) && (T)(x) == (x)) + +#define IS_INT64_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int64_t) +#define IS_INT_EQUIVALENT(x) IS_INT_TYPE_EQUIVALENT(x, int) + +/* If size of pointer is equal to a 4 byte integer, we're on 32 bits. */ +#if UINTPTR_MAX == UINT_MAX + #define BITS_32 1 +#else + #define BITS_32 0 +#endif + +#if BITS_32 + #define lua_pushunsigned(L, n) lua_pushnumber(L, n) +#else + #define lua_pushunsigned(L, n) lua_pushinteger(L, n) +#endif + +/* ============================================================================= + * MessagePack implementation and bindings for Lua 5.1/5.2. * Copyright(C) 2012 Salvatore Sanfilippo * * http://github.com/antirez/lua-cmsgpack @@ -29,23 +53,27 @@ * 20-Feb-2012 (ver 0.2.0): Tables encoding improved. * 20-Feb-2012 (ver 0.2.1): Minor bug fixing. * 20-Feb-2012 (ver 0.3.0): Module renamed lua-cmsgpack (was lua-msgpack). - * ============================================================================ */ + * 04-Apr-2014 (ver 0.3.1): Lua 5.2 support and minor bug fix. + * 07-Apr-2014 (ver 0.4.0): Multiple pack/unpack, lua allocator, efficiency. + * ========================================================================== */ -/* --------------------------- Endian conversion -------------------------------- - * We use it only for floats and doubles, all the other conversions are performed +/* -------------------------- Endian conversion -------------------------------- + * We use it only for floats and doubles, all the other conversions performed * in an endian independent fashion. So the only thing we need is a function - * that swaps a binary string if the arch is little endian (and left it untouched + * that swaps a binary string if arch is little endian (and left it untouched * otherwise). */ /* Reverse memory bytes if arch is little endian. Given the conceptual - * simplicity of the Lua build system we prefer to check for endianess at runtime. + * simplicity of the Lua build system we prefer check for endianess at runtime. * The performance difference should be acceptable. */ -static void memrevifle(void *ptr, size_t len) { - unsigned char *p = ptr, *e = p+len-1, aux; +void memrevifle(void *ptr, size_t len) { + unsigned char *p = (unsigned char *)ptr, + *e = (unsigned char *)p+len-1, + aux; int test = 1; unsigned char *testp = (unsigned char*) &test; - if (testp[0] == 0) return; /* Big endian, nothign to do. */ + if (testp[0] == 0) return; /* Big endian, nothing to do. */ len /= 2; while(len--) { aux = *p; @@ -56,8 +84,8 @@ static void memrevifle(void *ptr, size_t len) { } } -/* ----------------------------- String buffer ---------------------------------- - * This is a simple implementation of string buffers. The only opereation +/* ---------------------------- String buffer ---------------------------------- + * This is a simple implementation of string buffers. The only operation * supported is creating empty buffers and appending bytes to it. * The string buffer uses 2x preallocation on every realloc for O(N) append * behavior. */ @@ -67,32 +95,44 @@ typedef struct mp_buf { size_t len, free; } mp_buf; -static mp_buf *mp_buf_new(void) { - mp_buf *buf = malloc(sizeof(*buf)); - +void *mp_realloc(lua_State *L, void *target, size_t osize,size_t nsize) { + void *(*local_realloc) (void *, void *, size_t osize, size_t nsize) = NULL; + void *ud; + + local_realloc = lua_getallocf(L, &ud); + + return local_realloc(ud, target, osize, nsize); +} + +mp_buf *mp_buf_new(lua_State *L) { + mp_buf *buf = NULL; + + /* Old size = 0; new size = sizeof(*buf) */ + buf = (mp_buf*)mp_realloc(L, NULL, 0, sizeof(*buf)); + buf->b = NULL; buf->len = buf->free = 0; return buf; } -void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { +void mp_buf_append(lua_State *L, mp_buf *buf, const unsigned char *s, size_t len) { if (buf->free < len) { - size_t newlen = buf->len+len; + size_t newsize = (buf->len+len)*2; - buf->b = realloc(buf->b,newlen*2); - buf->free = newlen; + buf->b = (unsigned char*)mp_realloc(L, buf->b, buf->len + buf->free, newsize); + buf->free = newsize - buf->len; } memcpy(buf->b+buf->len,s,len); buf->len += len; buf->free -= len; } -void mp_buf_free(mp_buf *buf) { - free(buf->b); - free(buf); +void mp_buf_free(lua_State *L, mp_buf *buf) { + mp_realloc(L, buf->b, buf->len + buf->free, 0); /* realloc to 0 = free */ + mp_realloc(L, buf, sizeof(*buf), 0); } -/* ------------------------------ String cursor ---------------------------------- +/* ---------------------------- String cursor ---------------------------------- * This simple data structure is used for parsing. Basically you create a cursor * using a string pointer and a length, then it is possible to access the * current string position with cursor->p, check the remaining length @@ -102,7 +142,7 @@ void mp_buf_free(mp_buf *buf) { * be used to report errors. */ #define MP_CUR_ERROR_NONE 0 -#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete the opereation. */ +#define MP_CUR_ERROR_EOF 1 /* Not enough data to complete operation. */ #define MP_CUR_ERROR_BADFMT 2 /* Bad data format */ typedef struct mp_cur { @@ -111,22 +151,15 @@ typedef struct mp_cur { int err; } mp_cur; -static mp_cur *mp_cur_new(const unsigned char *s, size_t len) { - mp_cur *cursor = malloc(sizeof(*cursor)); - +void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { cursor->p = s; cursor->left = len; cursor->err = MP_CUR_ERROR_NONE; - return cursor; -} - -static void mp_cur_free(mp_cur *cursor) { - free(cursor); } #define mp_cur_consume(_c,_len) do { _c->p += _len; _c->left -= _len; } while(0) -/* When there is not enough room we set an error in the cursor and return, this +/* When there is not enough room we set an error in the cursor and return. This * is very common across the code so we have a macro to make the code look * a bit simpler. */ #define mp_cur_need(_c,_len) do { \ @@ -136,15 +169,19 @@ static void mp_cur_free(mp_cur *cursor) { } \ } while(0) -/* --------------------------- Low level MP encoding -------------------------- */ +/* ------------------------- Low level MP encoding -------------------------- */ -static void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { +void mp_encode_bytes(lua_State *L, mp_buf *buf, const unsigned char *s, size_t len) { unsigned char hdr[5]; int hdrlen; if (len < 32) { hdr[0] = 0xa0 | (len&0xff); /* fix raw */ hdrlen = 1; + } else if (len <= 0xff) { + hdr[0] = 0xd9; + hdr[1] = len; + hdrlen = 2; } else if (len <= 0xffff) { hdr[0] = 0xda; hdr[1] = (len&0xff00)>>8; @@ -158,12 +195,12 @@ static void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { hdr[4] = len&0xff; hdrlen = 5; } - mp_buf_append(buf,hdr,hdrlen); - mp_buf_append(buf,s,len); + mp_buf_append(L,buf,hdr,hdrlen); + mp_buf_append(L,buf,s,len); } /* we assume IEEE 754 internal format for single and double precision floats. */ -static void mp_encode_double(mp_buf *buf, double d) { +void mp_encode_double(lua_State *L, mp_buf *buf, double d) { unsigned char b[9]; float f = d; @@ -172,16 +209,16 @@ static void mp_encode_double(mp_buf *buf, double d) { b[0] = 0xca; /* float IEEE 754 */ memcpy(b+1,&f,4); memrevifle(b+1,4); - mp_buf_append(buf,b,5); + mp_buf_append(L,buf,b,5); } else if (sizeof(d) == 8) { b[0] = 0xcb; /* double IEEE 754 */ memcpy(b+1,&d,8); memrevifle(b+1,8); - mp_buf_append(buf,b,9); + mp_buf_append(L,buf,b,9); } } -static void mp_encode_int(mp_buf *buf, int64_t n) { +void mp_encode_int(lua_State *L, mp_buf *buf, int64_t n) { unsigned char b[9]; int enclen; @@ -219,7 +256,7 @@ static void mp_encode_int(mp_buf *buf, int64_t n) { } } else { if (n >= -32) { - b[0] = ((char)n); /* negative fixnum */ + b[0] = ((signed char)n); /* negative fixnum */ enclen = 1; } else if (n >= -128) { b[0] = 0xd0; /* int 8 */ @@ -250,10 +287,10 @@ static void mp_encode_int(mp_buf *buf, int64_t n) { enclen = 9; } } - mp_buf_append(buf,b,enclen); + mp_buf_append(L,buf,b,enclen); } -static void mp_encode_array(mp_buf *buf, int64_t n) { +void mp_encode_array(lua_State *L, mp_buf *buf, int64_t n) { unsigned char b[5]; int enclen; @@ -273,10 +310,10 @@ static void mp_encode_array(mp_buf *buf, int64_t n) { b[4] = n & 0xff; enclen = 5; } - mp_buf_append(buf,b,enclen); + mp_buf_append(L,buf,b,enclen); } -static void mp_encode_map(mp_buf *buf, int64_t n) { +void mp_encode_map(lua_State *L, mp_buf *buf, int64_t n) { unsigned char b[5]; int enclen; @@ -296,41 +333,58 @@ static void mp_encode_map(mp_buf *buf, int64_t n) { b[4] = n & 0xff; enclen = 5; } - mp_buf_append(buf,b,enclen); + mp_buf_append(L,buf,b,enclen); } -/* ----------------------------- Lua types encoding --------------------------- */ +/* --------------------------- Lua types encoding --------------------------- */ -static void mp_encode_lua_string(lua_State *L, mp_buf *buf) { +void mp_encode_lua_string(lua_State *L, mp_buf *buf) { size_t len; const char *s; s = lua_tolstring(L,-1,&len); - mp_encode_bytes(buf,(const unsigned char*)s,len); + mp_encode_bytes(L,buf,(const unsigned char*)s,len); } -static void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { +void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { unsigned char b = lua_toboolean(L,-1) ? 0xc3 : 0xc2; - mp_buf_append(buf,&b,1); + mp_buf_append(L,buf,&b,1); +} + +/* Lua 5.3 has a built in 64-bit integer type */ +void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { +#if (LUA_VERSION_NUM < 503) && BITS_32 + lua_Number i = lua_tonumber(L,-1); +#else + lua_Integer i = lua_tointeger(L,-1); +#endif + mp_encode_int(L, buf, (int64_t)i); } -static void mp_encode_lua_number(lua_State *L, mp_buf *buf) { +/* Lua 5.2 and lower only has 64-bit doubles, so we need to + * detect if the double may be representable as an int + * for Lua < 5.3 */ +void mp_encode_lua_number(lua_State *L, mp_buf *buf) { lua_Number n = lua_tonumber(L,-1); - if (floor(n) != n) { - mp_encode_double(buf,(double)n); + if (IS_INT64_EQUIVALENT(n)) { + mp_encode_lua_integer(L, buf); } else { - mp_encode_int(buf,(int64_t)n); + mp_encode_double(L,buf,(double)n); } } -static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level); +void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level); /* Convert a lua table into a message pack list. */ -static void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { +#if LUA_VERSION_NUM < 502 size_t len = lua_objlen(L,-1), j; +#else + size_t len = lua_rawlen(L,-1), j; +#endif - mp_encode_array(buf,len); + mp_encode_array(L,buf,len); for (j = 1; j <= len; j++) { lua_pushnumber(L,j); lua_gettable(L,-2); @@ -339,13 +393,13 @@ static void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { } /* Convert a lua table into a message pack key-value map. */ -static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { size_t len = 0; /* First step: count keys into table. No other way to do it with the * Lua API, we need to iterate a first time. Note that an alternative * would be to do a single run, and then hack the buffer to insert the - * map opcodes for message pack. Too hachish for this lib. */ + * map opcodes for message pack. Too hackish for this lib. */ lua_pushnil(L); while(lua_next(L,-2)) { lua_pop(L,1); /* remove value, keep key for next iteration. */ @@ -353,7 +407,7 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { } /* Step two: actually encoding of the map. */ - mp_encode_map(buf,len); + mp_encode_map(L,buf,len); lua_pushnil(L); while(lua_next(L,-2)) { /* Stack: ... key value */ @@ -366,80 +420,130 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { /* Returns true if the Lua table on top of the stack is exclusively composed * of keys from numerical keys from 1 up to N, with N being the total number * of elements, without any hole in the middle. */ -static int table_is_an_array(lua_State *L) { - long count = 0, max = 0, idx = 0; +int table_is_an_array(lua_State *L) { + int count = 0, max = 0; +#if LUA_VERSION_NUM < 503 lua_Number n; +#else + lua_Integer n; +#endif + + /* Stack top on function entry */ + int stacktop; + + stacktop = lua_gettop(L); lua_pushnil(L); while(lua_next(L,-2)) { /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ - if (!lua_isnumber(L,-1)) goto not_array; - n = lua_tonumber(L,-1); - idx = n; - if (idx != n || idx < 1) goto not_array; + /* The <= 0 check is valid here because we're comparing indexes. */ +#if LUA_VERSION_NUM < 503 + if ((LUA_TNUMBER != lua_type(L,-1)) || (n = lua_tonumber(L, -1)) <= 0 || + !IS_INT_EQUIVALENT(n)) +#else + if (!lua_isinteger(L,-1) || (n = lua_tointeger(L, -1)) <= 0) +#endif + { + lua_settop(L, stacktop); + return 0; + } + max = (n > max ? n : max); count++; - max = idx; } /* We have the total number of elements in "count". Also we have - * the max index encountered in "idx". We can't reach this code + * the max index encountered in "max". We can't reach this code * if there are indexes <= 0. If you also note that there can not be - * repeated keys into a table, you have that if idx==count you are sure + * repeated keys into a table, you have that if max==count you are sure * that there are all the keys form 1 to count (both included). */ - return idx == count; - -not_array: - lua_pop(L,1); - return 0; + lua_settop(L, stacktop); + return max == count; } /* If the length operator returns non-zero, that is, there is at least * an object at key '1', we serialize to message pack list. Otherwise * we use a map. */ -static void mp_encode_lua_table(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table(lua_State *L, mp_buf *buf, int level) { if (table_is_an_array(L)) mp_encode_lua_table_as_array(L,buf,level); else mp_encode_lua_table_as_map(L,buf,level); } -static void mp_encode_lua_null(lua_State *L, mp_buf *buf) { +void mp_encode_lua_null(lua_State *L, mp_buf *buf) { unsigned char b[1]; b[0] = 0xc0; - mp_buf_append(buf,b,1); + mp_buf_append(L,buf,b,1); } -static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { int t = lua_type(L,-1); - /* Limit the encoding of nested tables to a specfiied maximum depth, so that + /* Limit the encoding of nested tables to a specified maximum depth, so that * we survive when called against circular references in tables. */ if (t == LUA_TTABLE && level == LUACMSGPACK_MAX_NESTING) t = LUA_TNIL; switch(t) { case LUA_TSTRING: mp_encode_lua_string(L,buf); break; case LUA_TBOOLEAN: mp_encode_lua_bool(L,buf); break; - case LUA_TNUMBER: mp_encode_lua_number(L,buf); break; + case LUA_TNUMBER: + #if LUA_VERSION_NUM < 503 + mp_encode_lua_number(L,buf); break; + #else + if (lua_isinteger(L, -1)) { + mp_encode_lua_integer(L, buf); + } else { + mp_encode_lua_number(L, buf); + } + break; + #endif case LUA_TTABLE: mp_encode_lua_table(L,buf,level); break; default: mp_encode_lua_null(L,buf); break; } lua_pop(L,1); } -static int mp_pack(lua_State *L) { - mp_buf *buf = mp_buf_new(); +/* + * Packs all arguments as a stream for multiple upacking later. + * Returns error if no arguments provided. + */ +int mp_pack(lua_State *L) { + int nargs = lua_gettop(L); + int i; + mp_buf *buf; + + if (nargs == 0) + return luaL_argerror(L, 0, "MessagePack pack needs input."); + + buf = mp_buf_new(L); + for(i = 1; i <= nargs; i++) { + /* Copy argument i to top of stack for _encode processing; + * the encode function pops it from the stack when complete. */ + lua_pushvalue(L, i); + + mp_encode_lua_type(L,buf,0); + + lua_pushlstring(L,(char*)buf->b,buf->len); + + /* Reuse the buffer for the next operation by + * setting its free count to the total buffer size + * and the current position to zero. */ + buf->free += buf->len; + buf->len = 0; + } + mp_buf_free(L, buf); - mp_encode_lua_type(L,buf,0); - lua_pushlstring(L,(char*)buf->b,buf->len); - mp_buf_free(buf); + /* Concatenate all nargs buffers together */ + lua_concat(L, nargs); return 1; } -/* --------------------------------- Decoding --------------------------------- */ +/* ------------------------------- Decoding --------------------------------- */ void mp_decode_to_lua_type(lua_State *L, mp_cur *c); void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { + assert(len <= UINT_MAX); int index = 1; lua_newtable(L); @@ -452,6 +556,7 @@ void mp_decode_to_lua_array(lua_State *L, mp_cur *c, size_t len) { } void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { + assert(len <= UINT_MAX); lua_newtable(L); while(len--) { mp_decode_to_lua_type(L,c); /* key */ @@ -466,34 +571,44 @@ void mp_decode_to_lua_hash(lua_State *L, mp_cur *c, size_t len) { * a Lua type, that is left as the only result on the stack. */ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { mp_cur_need(c,1); + + /* If we return more than 18 elements, we must resize the stack to + * fit all our return values. But, there is no way to + * determine how many objects a msgpack will unpack to up front, so + * we request a +1 larger stack on each iteration (noop if stack is + * big enough, and when stack does require resize it doubles in size) */ + luaL_checkstack(L, 1, + "too many return values at once; " + "use unpack_one or unpack_limit instead."); + switch(c->p[0]) { case 0xcc: /* uint 8 */ mp_cur_need(c,2); - lua_pushnumber(L,c->p[1]); + lua_pushunsigned(L,c->p[1]); mp_cur_consume(c,2); break; case 0xd0: /* int 8 */ mp_cur_need(c,2); - lua_pushnumber(L,(char)c->p[1]); + lua_pushinteger(L,(signed char)c->p[1]); mp_cur_consume(c,2); break; case 0xcd: /* uint 16 */ mp_cur_need(c,3); - lua_pushnumber(L, + lua_pushunsigned(L, (c->p[1] << 8) | c->p[2]); mp_cur_consume(c,3); break; case 0xd1: /* int 16 */ mp_cur_need(c,3); - lua_pushnumber(L,(int16_t) + lua_pushinteger(L,(int16_t) (c->p[1] << 8) | c->p[2]); mp_cur_consume(c,3); break; case 0xce: /* uint 32 */ mp_cur_need(c,5); - lua_pushnumber(L, + lua_pushunsigned(L, ((uint32_t)c->p[1] << 24) | ((uint32_t)c->p[2] << 16) | ((uint32_t)c->p[3] << 8) | @@ -502,7 +617,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xd2: /* int 32 */ mp_cur_need(c,5); - lua_pushnumber(L, + lua_pushinteger(L, ((int32_t)c->p[1] << 24) | ((int32_t)c->p[2] << 16) | ((int32_t)c->p[3] << 8) | @@ -511,7 +626,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xcf: /* uint 64 */ mp_cur_need(c,9); - lua_pushnumber(L, + lua_pushunsigned(L, ((uint64_t)c->p[1] << 56) | ((uint64_t)c->p[2] << 48) | ((uint64_t)c->p[3] << 40) | @@ -524,7 +639,11 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { break; case 0xd3: /* int 64 */ mp_cur_need(c,9); +#if LUA_VERSION_NUM < 503 lua_pushnumber(L, +#else + lua_pushinteger(L, +#endif ((int64_t)c->p[1] << 56) | ((int64_t)c->p[2] << 48) | ((int64_t)c->p[3] << 40) | @@ -569,6 +688,15 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { mp_cur_consume(c,9); } break; + case 0xd9: /* raw 8 */ + mp_cur_need(c,2); + { + size_t l = c->p[1]; + mp_cur_need(c,2+l); + lua_pushlstring(L,(char*)c->p+2,l); + mp_cur_consume(c,2+l); + } + break; case 0xda: /* raw 16 */ mp_cur_need(c,3); { @@ -581,13 +709,14 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { case 0xdb: /* raw 32 */ mp_cur_need(c,5); { - size_t l = (c->p[1] << 24) | - (c->p[2] << 16) | - (c->p[3] << 8) | - c->p[4]; - mp_cur_need(c,5+l); - lua_pushlstring(L,(char*)c->p+5,l); - mp_cur_consume(c,5+l); + size_t l = ((size_t)c->p[1] << 24) | + ((size_t)c->p[2] << 16) | + ((size_t)c->p[3] << 8) | + (size_t)c->p[4]; + mp_cur_consume(c,5); + mp_cur_need(c,l); + lua_pushlstring(L,(char*)c->p,l); + mp_cur_consume(c,l); } break; case 0xdc: /* array 16 */ @@ -601,10 +730,10 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { case 0xdd: /* array 32 */ mp_cur_need(c,5); { - size_t l = (c->p[1] << 24) | - (c->p[2] << 16) | - (c->p[3] << 8) | - c->p[4]; + size_t l = ((size_t)c->p[1] << 24) | + ((size_t)c->p[2] << 16) | + ((size_t)c->p[3] << 8) | + (size_t)c->p[4]; mp_cur_consume(c,5); mp_decode_to_lua_array(L,c,l); } @@ -620,20 +749,20 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { case 0xdf: /* map 32 */ mp_cur_need(c,5); { - size_t l = (c->p[1] << 24) | - (c->p[2] << 16) | - (c->p[3] << 8) | - c->p[4]; + size_t l = ((size_t)c->p[1] << 24) | + ((size_t)c->p[2] << 16) | + ((size_t)c->p[3] << 8) | + (size_t)c->p[4]; mp_cur_consume(c,5); mp_decode_to_lua_hash(L,c,l); } break; default: /* types that can't be idenitified by first byte value. */ if ((c->p[0] & 0x80) == 0) { /* positive fixnum */ - lua_pushnumber(L,c->p[0]); + lua_pushunsigned(L,c->p[0]); mp_cur_consume(c,1); } else if ((c->p[0] & 0xe0) == 0xe0) { /* negative fixnum */ - lua_pushnumber(L,(signed char)c->p[0]); + lua_pushinteger(L,(signed char)c->p[0]); mp_cur_consume(c,1); } else if ((c->p[0] & 0xe0) == 0xa0) { /* fix raw */ size_t l = c->p[0] & 0x1f; @@ -654,54 +783,163 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { } } -static int mp_unpack(lua_State *L) { +int mp_unpack_full(lua_State *L, int limit, int offset) { size_t len; - const unsigned char *s; - mp_cur *c; + const char *s; + mp_cur c; + int cnt; /* Number of objects unpacked */ + int decode_all = (!limit && !offset); + + s = luaL_checklstring(L,1,&len); /* if no match, exits */ - if (!lua_isstring(L,-1)) { - lua_pushstring(L,"MessagePack decoding needs a string as input."); - lua_error(L); + if (offset < 0 || limit < 0) /* requesting negative off or lim is invalid */ + return luaL_error(L, + "Invalid request to unpack with offset of %d and limit of %d.", + offset, len); + else if (offset > len) + return luaL_error(L, + "Start offset %d greater than input length %d.", offset, len); + + if (decode_all) limit = INT_MAX; + + mp_cur_init(&c,(const unsigned char *)s+offset,len-offset); + + /* We loop over the decode because this could be a stream + * of multiple top-level values serialized together */ + for(cnt = 0; c.left > 0 && cnt < limit; cnt++) { + mp_decode_to_lua_type(L,&c); + + if (c.err == MP_CUR_ERROR_EOF) { + return luaL_error(L,"Missing bytes in input."); + } else if (c.err == MP_CUR_ERROR_BADFMT) { + return luaL_error(L,"Bad data format in input."); + } } - s = (const unsigned char*) lua_tolstring(L,-1,&len); - c = mp_cur_new(s,len); - mp_decode_to_lua_type(L,c); - - if (c->err == MP_CUR_ERROR_EOF) { - mp_cur_free(c); - lua_pushstring(L,"Missing bytes in input."); - lua_error(L); - } else if (c->err == MP_CUR_ERROR_BADFMT) { - mp_cur_free(c); - lua_pushstring(L,"Bad data format in input."); - lua_error(L); - } else if (c->left != 0) { - mp_cur_free(c); - lua_pushstring(L,"Extra bytes in input."); - lua_error(L); + if (!decode_all) { + /* c->left is the remaining size of the input buffer. + * subtract the entire buffer size from the unprocessed size + * to get our next start offset */ + int offset = len - c.left; + /* Return offset -1 when we have have processed the entire buffer. */ + lua_pushinteger(L, c.left == 0 ? -1 : offset); + /* Results are returned with the arg elements still + * in place. Lua takes care of only returning + * elements above the args for us. + * In this case, we have one arg on the stack + * for this function, so we insert our first return + * value at position 2. */ + lua_insert(L, 2); + cnt += 1; /* increase return count by one to make room for offset */ } - mp_cur_free(c); - return 1; + + return cnt; } -/* ---------------------------------------------------------------------------- */ +int mp_unpack(lua_State *L) { + return mp_unpack_full(L, 0, 0); +} + +int mp_unpack_one(lua_State *L) { + int offset = luaL_optinteger(L, 2, 0); + /* Variable pop because offset may not exist */ + lua_pop(L, lua_gettop(L)-1); + return mp_unpack_full(L, 1, offset); +} + +int mp_unpack_limit(lua_State *L) { + int limit = luaL_checkinteger(L, 2); + int offset = luaL_optinteger(L, 3, 0); + /* Variable pop because offset may not exist */ + lua_pop(L, lua_gettop(L)-1); + + return mp_unpack_full(L, limit, offset); +} + +int mp_safe(lua_State *L) { + int argc, err, total_results; + + argc = lua_gettop(L); + + /* This adds our function to the bottom of the stack + * (the "call this function" position) */ + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + + err = lua_pcall(L, argc, LUA_MULTRET, 0); + total_results = lua_gettop(L); + + if (!err) { + return total_results; + } else { + lua_pushnil(L); + lua_insert(L,-2); + return 2; + } +} -static const struct luaL_reg thislib[] = { +/* -------------------------------------------------------------------------- */ +const struct luaL_Reg cmds[] = { {"pack", mp_pack}, {"unpack", mp_unpack}, - {NULL, NULL} + {"unpack_one", mp_unpack_one}, + {"unpack_limit", mp_unpack_limit}, + {0} }; -LUALIB_API int luaopen_cmsgpack (lua_State *L) { - luaL_register(L, "cmsgpack", thislib); +int luaopen_create(lua_State *L) { + int i; + /* Manually construct our module table instead of + * relying on _register or _newlib */ + lua_newtable(L); + + for (i = 0; i < (sizeof(cmds)/sizeof(*cmds) - 1); i++) { + lua_pushcfunction(L, cmds[i].func); + lua_setfield(L, -2, cmds[i].name); + } + /* Add metadata */ + lua_pushliteral(L, LUACMSGPACK_NAME); + lua_setfield(L, -2, "_NAME"); lua_pushliteral(L, LUACMSGPACK_VERSION); lua_setfield(L, -2, "_VERSION"); lua_pushliteral(L, LUACMSGPACK_COPYRIGHT); lua_setfield(L, -2, "_COPYRIGHT"); lua_pushliteral(L, LUACMSGPACK_DESCRIPTION); - lua_setfield(L, -2, "_DESCRIPTION"); + lua_setfield(L, -2, "_DESCRIPTION"); + return 1; +} + +LUALIB_API int luaopen_cmsgpack(lua_State *L) { + luaopen_create(L); + +#if LUA_VERSION_NUM < 502 + /* Register name globally for 5.1 */ + lua_pushvalue(L, -1); + lua_setglobal(L, LUACMSGPACK_NAME); +#endif + + return 1; +} + +LUALIB_API int luaopen_cmsgpack_safe(lua_State *L) { + int i; + + luaopen_cmsgpack(L); + + /* Wrap all functions in the safe handler */ + for (i = 0; i < (sizeof(cmds)/sizeof(*cmds) - 1); i++) { + lua_getfield(L, -1, cmds[i].name); + lua_pushcclosure(L, mp_safe, 1); + lua_setfield(L, -2, cmds[i].name); + } + +#if LUA_VERSION_NUM < 502 + /* Register name globally for 5.1 */ + lua_pushvalue(L, -1); + lua_setglobal(L, LUACMSGPACK_SAFE_NAME); +#endif + return 1; } diff --git a/deps/lua/src/lua_struct.c b/deps/lua/src/lua_struct.c index 6807c838748..a602bb4309e 100644 --- a/deps/lua/src/lua_struct.c +++ b/deps/lua/src/lua_struct.c @@ -1,18 +1,7 @@ - -#include -#include -#include -#include - - -#include "lua.h" -#include "lauxlib.h" - - /* ** {====================================================== ** Library for packing/unpacking structures. -** $Id: struct.c,v 1.2 2008/04/18 20:06:01 roberto Exp $ +** $Id: struct.c,v 1.4 2012/07/04 18:54:29 roberto Exp $ ** See Copyright Notice at the end of this file ** ======================================================= */ @@ -25,6 +14,7 @@ ** b/B - signed/unsigned byte ** h/H - signed/unsigned short ** l/L - signed/unsigned long +** T - size_t ** i/In - signed/unsigned integer with size `n' (default is size of int) ** cn - sequence of `n' chars (from/to a string); when packing, n==0 means the whole string; when unpacking, n==0 means use the previous @@ -36,6 +26,38 @@ */ +#include +#include +#include +#include +#include + + +#include "lua.h" +#include "lauxlib.h" + + +#if (LUA_VERSION_NUM >= 502) + +#define luaL_register(L,n,f) luaL_newlib(L,f) + +#endif + + +/* basic integer type */ +#if !defined(STRUCT_INT) +#define STRUCT_INT long +#endif + +typedef STRUCT_INT Inttype; + +/* corresponding unsigned version */ +typedef unsigned STRUCT_INT Uinttype; + + +/* maximum size (in bytes) for integral types */ +#define MAXINTSIZE 32 + /* is 'x' a power of 2? */ #define isp2(x) ((x) > 0 && ((x) & ((x) - 1)) == 0) @@ -67,12 +89,14 @@ typedef struct Header { } Header; -static size_t getnum (const char **fmt, size_t df) { +static int getnum (lua_State *L, const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { - size_t a = 0; + int a = 0; do { + if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0'))) + luaL_error(L, "integral size overflow"); a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); return a; @@ -89,45 +113,55 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'B': case 'b': return sizeof(char); case 'H': case 'h': return sizeof(short); case 'L': case 'l': return sizeof(long); + case 'T': return sizeof(size_t); case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; - case 'c': return getnum(fmt, 1); - case 's': case ' ': case '<': case '>': case '!': return 0; + case 'c': return getnum(L, fmt, 1); case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); - if (!isp2(sz)) - luaL_error(L, "integral size %d is not a power of 2", sz); + int sz = getnum(L, fmt, sizeof(int)); + if (sz > MAXINTSIZE) + luaL_error(L, "integral size %d is larger than limit of %d", + sz, MAXINTSIZE); return sz; } - default: { - const char *msg = lua_pushfstring(L, "invalid format option [%c]", opt); - return luaL_argerror(L, 1, msg); - } + default: return 0; /* other cases do not need alignment */ } } +/* +** return number of bytes needed to align an element of size 'size' +** at current position 'len' +*/ static int gettoalign (size_t len, Header *h, int opt, size_t size) { if (size == 0 || opt == 'c') return 0; - if (size > (size_t)h->align) size = h->align; /* respect max. alignment */ - return (size - (len & (size - 1))) & (size - 1); + if (size > (size_t)h->align) + size = h->align; /* respect max. alignment */ + return (size - (len & (size - 1))) & (size - 1); } -static void commoncases (lua_State *L, int opt, const char **fmt, Header *h) { +/* +** options to control endianess and alignment +*/ +static void controloptions (lua_State *L, int opt, const char **fmt, + Header *h) { switch (opt) { case ' ': return; /* ignore white spaces */ case '>': h->endian = BIG; return; case '<': h->endian = LITTLE; return; case '!': { - int a = getnum(fmt, MAXALIGN); + int a = getnum(L, fmt, MAXALIGN); if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); h->align = a; return; } - default: assert(0); + default: { + const char *msg = lua_pushfstring(L, "invalid format option '%c'", opt); + luaL_argerror(L, 1, msg); + } } } @@ -135,21 +169,27 @@ static void commoncases (lua_State *L, int opt, const char **fmt, Header *h) { static void putinteger (lua_State *L, luaL_Buffer *b, int arg, int endian, int size) { lua_Number n = luaL_checknumber(L, arg); - unsigned long value; - if (n < (lua_Number)LONG_MAX) - value = (long)n; + Uinttype value; + char buff[MAXINTSIZE]; + if (n < 0) + value = (Uinttype)(Inttype)n; else - value = (unsigned long)n; + value = (Uinttype)n; if (endian == LITTLE) { int i; - for (i = 0; i < size; i++) - luaL_addchar(b, (value >> 8*i) & 0xff); + for (i = 0; i < size; i++) { + buff[i] = (value & 0xff); + value >>= 8; + } } else { int i; - for (i = size - 1; i >= 0; i--) - luaL_addchar(b, (value >> 8*i) & 0xff); + for (i = size - 1; i >= 0; i--) { + buff[i] = (value & 0xff); + value >>= 8; + } } + luaL_addlstring(b, buff, size); } @@ -179,15 +219,15 @@ static int b_pack (lua_State *L) { size_t size = optsize(L, opt, &fmt); int toalign = gettoalign(totalsize, &h, opt, size); totalsize += toalign; - while (toalign-- > 0) luaL_putchar(&b, '\0'); + while (toalign-- > 0) luaL_addchar(&b, '\0'); switch (opt) { case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'i': case 'I': { /* integer types */ + case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ putinteger(L, &b, arg++, h.endian, size); break; } case 'x': { - luaL_putchar(&b, '\0'); + luaL_addchar(&b, '\0'); break; } case 'f': { @@ -209,12 +249,12 @@ static int b_pack (lua_State *L) { luaL_argcheck(L, l >= (size_t)size, arg, "string too short"); luaL_addlstring(&b, s, size); if (opt == 's') { - luaL_putchar(&b, '\0'); /* add zero at the end */ + luaL_addchar(&b, '\0'); /* add zero at the end */ size++; } break; } - default: commoncases(L, opt, &fmt, &h); + default: controloptions(L, opt, &fmt, &h); } totalsize += size; } @@ -225,24 +265,27 @@ static int b_pack (lua_State *L) { static lua_Number getinteger (const char *buff, int endian, int issigned, int size) { - unsigned long l = 0; + Uinttype l = 0; + int i; if (endian == BIG) { - int i; - for (i = 0; i < size; i++) - l |= (unsigned long)(unsigned char)buff[size - i - 1] << (i*8); + for (i = 0; i < size; i++) { + l <<= 8; + l |= (Uinttype)(unsigned char)buff[i]; + } } else { - int i; - for (i = 0; i < size; i++) - l |= (unsigned long)(unsigned char)buff[i] << (i*8); + for (i = size - 1; i >= 0; i--) { + l <<= 8; + l |= (Uinttype)(unsigned char)buff[i]; + } } if (!issigned) return (lua_Number)l; else { /* signed format */ - unsigned long mask = ~(0UL) << (size*8 - 1); + Uinttype mask = (Uinttype)(~((Uinttype)0)) << (size*8 - 1); if (l & mask) /* negative value? */ l |= mask; /* signal extension */ - return (lua_Number)(long)l; + return (lua_Number)(Inttype)l; } } @@ -260,9 +303,10 @@ static int b_unpack (lua_State *L) { size_t size = optsize(L, opt, &fmt); pos += gettoalign(pos, &h, opt, size); luaL_argcheck(L, pos+size <= ld, 2, "data string too short"); + luaL_checkstack(L, 1, "too many results"); switch (opt) { case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'i': case 'I': { /* integer types */ + case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ int issigned = islower(opt); lua_Number res = getinteger(data+pos, h.endian, issigned, size); lua_pushnumber(L, res); @@ -304,7 +348,7 @@ static int b_unpack (lua_State *L) { lua_pushlstring(L, data+pos, size - 1); break; } - default: commoncases(L, opt, &fmt, &h); + default: controloptions(L, opt, &fmt, &h); } pos += size; } @@ -312,26 +356,50 @@ static int b_unpack (lua_State *L) { return lua_gettop(L) - 2; } + +static int b_size (lua_State *L) { + Header h; + const char *fmt = luaL_checkstring(L, 1); + size_t pos = 0; + defaultoptions(&h); + while (*fmt) { + int opt = *fmt++; + size_t size = optsize(L, opt, &fmt); + pos += gettoalign(pos, &h, opt, size); + if (opt == 's') + luaL_argerror(L, 1, "option 's' has no fixed size"); + else if (opt == 'c' && size == 0) + luaL_argerror(L, 1, "option 'c0' has no fixed size"); + if (!isalnum(opt)) + controloptions(L, opt, &fmt, &h); + pos += size; + } + lua_pushinteger(L, pos); + return 1; +} + /* }====================================================== */ -static const struct luaL_reg thislib[] = { +static const struct luaL_Reg thislib[] = { {"pack", b_pack}, {"unpack", b_unpack}, + {"size", b_size}, {NULL, NULL} }; +LUALIB_API int luaopen_struct (lua_State *L); + LUALIB_API int luaopen_struct (lua_State *L) { luaL_register(L, "struct", thislib); return 1; } - /****************************************************************************** -* Copyright (C) 2010 Lua.org, PUC-Rio. All rights reserved. +* Copyright (C) 2010-2012 Lua.org, PUC-Rio. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -352,3 +420,4 @@ LUALIB_API int luaopen_struct (lua_State *L) { * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ + diff --git a/deps/lua/src/lvm.c b/deps/lua/src/lvm.c index ee3256ab94d..e0a0cd85219 100644 --- a/deps/lua/src/lvm.c +++ b/deps/lua/src/lvm.c @@ -1,5 +1,5 @@ /* -** $Id: lvm.c,v 2.63.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lvm.c,v 2.63.1.5 2011/08/17 20:43:11 roberto Exp $ ** Lua virtual machine ** See Copyright Notice in lua.h */ @@ -133,6 +133,7 @@ void luaV_gettable (lua_State *L, const TValue *t, TValue *key, StkId val) { void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { int loop; + TValue temp; for (loop = 0; loop < MAXTAGLOOP; loop++) { const TValue *tm; if (ttistable(t)) { /* `t' is a table? */ @@ -141,6 +142,7 @@ void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { if (!ttisnil(oldval) || /* result is no nil? */ (tm = fasttm(L, h->metatable, TM_NEWINDEX)) == NULL) { /* or no TM? */ setobj2t(L, oldval, val); + h->flags = 0; luaC_barriert(L, h, val); return; } @@ -152,7 +154,9 @@ void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { callTM(L, tm, t, key, val); return; } - t = tm; /* else repeat with `tm' */ + /* else repeat with `tm' */ + setobj(L, &temp, tm); /* avoid pointing inside table (may rehash) */ + t = &temp; } luaG_runerror(L, "loop in settable"); } diff --git a/deps/lua/src/strbuf.c b/deps/lua/src/strbuf.c index 976925a883b..f0f7f4b9a36 100644 --- a/deps/lua/src/strbuf.c +++ b/deps/lua/src/strbuf.c @@ -1,6 +1,6 @@ -/* strbuf - string buffer routines +/* strbuf - String buffer routines * - * Copyright (c) 2010-2011 Mark Pulford + * Copyright (c) 2010-2012 Mark Pulford * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -29,7 +29,7 @@ #include "strbuf.h" -void die(const char *fmt, ...) +static void die(const char *fmt, ...) { va_list arg; diff --git a/deps/lua/src/strbuf.h b/deps/lua/src/strbuf.h index f856543ad5e..d861108c14c 100644 --- a/deps/lua/src/strbuf.h +++ b/deps/lua/src/strbuf.h @@ -1,6 +1,6 @@ /* strbuf - String buffer routines * - * Copyright (c) 2010-2011 Mark Pulford + * Copyright (c) 2010-2012 Mark Pulford * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -62,7 +62,9 @@ extern void strbuf_resize(strbuf_t *s, int len); static int strbuf_empty_length(strbuf_t *s); static int strbuf_length(strbuf_t *s); static char *strbuf_string(strbuf_t *s, int *len); -static void strbuf_ensure_empty_length(strbuf_t *s, int len); +static void strbuf_ensure_empty_length(strbuf_t *s, int len); +static char *strbuf_empty_ptr(strbuf_t *s); +static void strbuf_extend_length(strbuf_t *s, int len); /* Update */ extern void strbuf_append_fmt(strbuf_t *s, int len, const char *fmt, ...); @@ -96,6 +98,16 @@ static inline void strbuf_ensure_empty_length(strbuf_t *s, int len) strbuf_resize(s, s->length + len); } +static inline char *strbuf_empty_ptr(strbuf_t *s) +{ + return s->buf + s->length; +} + +static inline void strbuf_extend_length(strbuf_t *s, int len) +{ + s->length += len; +} + static inline int strbuf_length(strbuf_t *s) { return s->length; diff --git a/deps/update-jemalloc.sh b/deps/update-jemalloc.sh new file mode 100755 index 00000000000..8ffbe1d70b9 --- /dev/null +++ b/deps/update-jemalloc.sh @@ -0,0 +1,9 @@ +#!/bin/bash +VER=$1 +URL="http://www.canonware.com/download/jemalloc/jemalloc-${VER}.tar.bz2" +echo "Downloading $URL" +curl $URL > /tmp/jemalloc.tar.bz2 +tar xvjf /tmp/jemalloc.tar.bz2 +rm -rf jemalloc +mv jemalloc-${VER} jemalloc +echo "Use git status, add all files and commit changes." diff --git a/redis.conf b/redis.conf index 80e14ad953d..870959f3a44 100644 --- a/redis.conf +++ b/redis.conf @@ -1,4 +1,9 @@ -# Redis configuration file example +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -12,6 +17,26 @@ # # units are case insensitive so 1GB 1Gb 1gB are all the same. +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. daemonize no @@ -24,33 +49,63 @@ pidfile /var/run/redis.pid # If port 0 is specified Redis will not listen on a TCP socket. port 6379 -# If you want you can bind a single interface, if the bind option is not -# specified all the interfaces will listen for incoming connections. +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# By default Redis listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. # +# Examples: +# +# bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 -# Specify the path for the unix socket that will be used to listen for +# Specify the path for the Unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # # unixsocket /tmp/redis.sock -# unixsocketperm 755 +# unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 -# Set server verbosity to 'debug' -# it can be one of: +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 0 + +# Specify the server verbosity level. +# This can be one of: # debug (a lot of information, useful for development/testing) # verbose (many rarely useful info, but not a mess like the debug level) # notice (moderately verbose, what you want in production probably) # warning (only very important / critical messages are logged) loglevel notice -# Specify the log file name. Also 'stdout' can be used to force +# Specify the log file name. Also the empty string can be used to force # Redis to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null -logfile stdout +logfile "" # To enable logging to the system logger, just set 'syslog-enabled' to yes, # and optionally update the other syslog parameters to suit your needs. @@ -59,7 +114,7 @@ logfile stdout # Specify the syslog identity. # syslog-ident redis -# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 # Set the number of databases. The default database is DB 0, you can select @@ -67,7 +122,7 @@ logfile stdout # dbid is a number between 0 and 'databases'-1 databases 16 -################################ SNAPSHOTTING ################################# +################################ SNAPSHOTTING ################################ # # Save the DB on disk: # @@ -81,7 +136,7 @@ databases 16 # after 300 sec (5 min) if at least 10 keys changed # after 60 sec if at least 10000 keys changed # -# Note: you can disable saving at all commenting all the "save" lines. +# Note: you can disable saving completely by commenting out all "save" lines. # # It is also possible to remove all the previously configured save # points by adding a save directive with a single empty string argument @@ -95,16 +150,16 @@ save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. -# This will make the user aware (in an hard way) that data is not persisting +# This will make the user aware (in a hard way) that data is not persisting # on disk properly, otherwise chances are that no one will notice and some -# distater will happen. +# disaster will happen. # # If the background saving process will start working again Redis will # automatically allow writes again. # # However if you have setup your proper monitoring of the Redis server # and persistence, you may want to disable this feature so that Redis will -# continue to work as usually even if there are problems with disk, +# continue to work as usual even if there are problems with disk, # permissions, and so forth. stop-writes-on-bgsave-error yes @@ -114,6 +169,15 @@ stop-writes-on-bgsave-error yes # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + # The filename where to dump the DB dbfilename dump.rdb @@ -121,18 +185,27 @@ dbfilename dump.rdb # # The DB will be written inside this directory, with the filename specified # above using the 'dbfilename' configuration directive. -# -# Also the Append Only File will be created inside this directory. -# +# +# The Append Only File will also be created inside this directory. +# # Note that you must specify a directory here, not a file name. dir ./ ################################# REPLICATION ################################# # Master-Slave replication. Use slaveof to make a Redis instance a copy of -# another Redis server. Note that the configuration is local to the slave -# so for example it is possible to configure the slave to save the DB with a -# different interval, or to listen to another port, and so on. +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. # # slaveof @@ -143,27 +216,89 @@ dir ./ # # masterauth -# When a slave lost the connection with the master, or when the replication +# When a slave loses its connection with the master, or when the replication # is still in progress, the slave can act in two different ways: # # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale data is set to 'no' the slave will reply with +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with # an error "SYNC with master in progress" to all the kind of commands # but to INFO and SLAVEOF. # slave-serve-stale-data yes +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that trnasfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + # Slaves send PINGs to server in a predefined interval. It's possible to change # this interval with the repl_ping_slave_period option. The default value is 10 # seconds. # # repl-ping-slave-period 10 -# The following option sets a timeout for both Bulk transfer I/O timeout and -# master data or ping response timeout. The default value is 60 seconds. +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value # specified for repl-ping-slave-period otherwise a timeout will be detected @@ -171,6 +306,80 @@ slave-serve-stale-data yes # # repl-timeout 60 +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + ################################## SECURITY ################################### # Require clients to issue AUTH before processing any other @@ -179,7 +388,7 @@ slave-serve-stale-data yes # # This should stay commented out for backward compatibility and because most # people do not need auth (e.g. they run their own servers). -# +# # Warning: since Redis is pretty fast an outside user can try up to # 150k passwords per second against a good box. This means that you should # use a very strong password otherwise it will be very easy to break. @@ -190,23 +399,26 @@ slave-serve-stale-data yes # # It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something -# of hard to guess so that it will be still available for internal-use -# tools but not available for general clients. +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. # # Example: # # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 # -# It is also possible to completely kill a command renaming it into +# It is also possible to completely kill a command by renaming it into # an empty string: # # rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. ################################### LIMITS #################################### # Set the max number of connected clients at the same time. By default # this limit is set to 10000 clients, however if the Redis server is not -# able ot configure the process file limit to allow for the specified limit +# able to configure the process file limit to allow for the specified limit # the max number of allowed clients is set to the current file limit # minus 32 (as Redis reserves a few file descriptors for internal uses). # @@ -217,7 +429,7 @@ slave-serve-stale-data yes # Don't use more memory than the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys -# accordingly to the eviction policy selected (see maxmemmory-policy). +# according to the eviction policy selected (see maxmemory-policy). # # If Redis can't remove keys according to the policy, or if the policy is # set to 'noeviction', Redis will start to reply with errors to commands @@ -225,7 +437,7 @@ slave-serve-stale-data yes # to reply to read-only commands like GET. # # This option is usually useful when using Redis as an LRU cache, or to set -# an hard memory limit for an instance (using the 'noeviction' policy). +# a hard memory limit for an instance (using the 'noeviction' policy). # # WARNING: If you have slaves attached to an instance with maxmemory on, # the size of the output buffers needed to feed the slaves are subtracted @@ -241,19 +453,19 @@ slave-serve-stale-data yes # maxmemory # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached? You can select among five behavior: -# +# is reached. You can select among five behaviors: +# # volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key accordingly to the LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm # volatile-random -> remove a random key with an expire set -# allkeys->random -> remove a random key, any key +# allkeys-random -> remove a random key, any key # volatile-ttl -> remove the key with the nearest expire time (minor TTL) # noeviction -> don't expire at all, just return an error on write operations -# -# Note: with all the kind of policies, Redis will return an error on write -# operations, when there are not suitable keys for eviction. # -# At the date of writing this commands are: set setnx setex append +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby @@ -273,38 +485,41 @@ slave-serve-stale-data yes ############################## APPEND ONLY MODE ############################### -# By default Redis asynchronously dumps the dataset on disk. If you can live -# with the idea that the latest records will be lost if something like a crash -# happens this is the preferred way to run Redis. If instead you care a lot -# about your data and don't want to that a single record can get lost you should -# enable the append only mode: when this mode is enabled Redis will append -# every write operation received in the file appendonly.aof. This file will -# be read on startup in order to rebuild the full dataset in memory. +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. # -# Note that you can have both the async dumps and the append only file if you -# like (you have to comment the "save" statements above to disable the dumps). -# Still if append only mode is enabled Redis will load the data from the -# log file at startup ignoring the dump.rdb file. +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. # -# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append -# log file in background when it gets too big. +# Please check http://redis.io/topics/persistence for more information. appendonly no # The name of the append only file (default: "appendonly.aof") -# appendfilename appendonly.aof + +appendfilename "appendonly.aof" # The fsync() call tells the Operating System to actually write data on disk -# instead to wait for more data in the output buffer. Some OS will really flush +# instead of waiting for more data in the output buffer. Some OS will really flush # data on disk, some other OS will just try to do it ASAP. # # Redis supports three different modes: # # no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log . Slow, Safest. -# everysec: fsync only if one second passed since the last fsync. Compromise. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. # -# The default is "everysec" that's usually the right compromise between +# The default is "everysec", as that's usually the right compromise between # speed and data safety. It's up to you to understand if you can relax this to # "no" that will let the operating system flush the output buffer when # it wants, for better performances (but if you can live with the idea of @@ -312,6 +527,9 @@ appendonly no # or on the contrary, use "always" that's very slow but a bit safer than # everysec. # +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# # If unsure, use "everysec". # appendfsync always @@ -329,21 +547,22 @@ appendfsync everysec # that will prevent fsync() from being called in the main process while a # BGSAVE or BGREWRITEAOF is in progress. # -# This means that while another child is saving the durability of Redis is -# the same as "appendfsync none", that in practical terms means that it is -# possible to lost up to 30 seconds of log in the worst scenario (with the +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the # default Linux settings). -# +# # If you have latency problems turn this to "yes". Otherwise leave it as # "no" that is the safest pick from the point of view of durability. + no-appendfsync-on-rewrite no # Automatic rewrite of the append only file. # Redis is able to automatically rewrite the log file implicitly calling -# BGREWRITEAOF when the AOF log size will growth by the specified percentage. -# +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# # This is how it works: Redis remembers the size of the AOF file after the -# latest rewrite (or if no rewrite happened since the restart, the size of +# latest rewrite (if no rewrite has happened since the restart, the size of # the AOF at startup is used). # # This base size is compared to the current size. If the current size is @@ -358,6 +577,30 @@ no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + ################################ LUA SCRIPTING ############################### # Max execution time of a Lua script in milliseconds. @@ -366,35 +609,16 @@ auto-aof-rewrite-min-size 64mb # still in execution after the maximum allowed time and will start to # reply to queries with an error. # -# When a long running script exceed the maximum execution time only the +# When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be # used to stop a script that did not yet called write commands. The second -# is the only way to shut down the server in the case a write commands was -# already issue by the script but the user don't want to wait for the natural +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural # termination of the script. # # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 -################################ REDIS CLUSTER ############################### -# -# Normal Redis instances can't be part of a Redis Cluster, only nodes that are -# started as cluster nodes can. In order to start a Redis instance as a -# cluster node enable the cluster support uncommenting the following: -# -# cluster-enabled yes - -# Every cluster node has a cluster configuration file. This file is not -# intended to be edited by hand. It is created and updated by Redis nodes. -# Every Redis Cluster node requires a different cluster configuration file. -# Make sure that instances running in the same system does not have -# overlapping cluster configuration file names. -# -# cluster-config-file nodes-6379.conf - -# In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. - ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified @@ -403,7 +627,7 @@ lua-time-limit 5000 # but just the time needed to actually execute the command (this is the only # stage of command execution where the thread is blocked and can not serve # other requests in the meantime). -# +# # You can configure the slow log with two parameters: one tells Redis # what is the execution time, in microseconds, to exceed in order for the # command to get logged, and the other parameter is the length of the @@ -417,7 +641,74 @@ slowlog-log-slower-than 10000 # There is no limit to this length. Just be aware that it will consume memory. # You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 1024 +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enalbed at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" ############################### ADVANCED CONFIG ############################### @@ -434,7 +725,7 @@ list-max-ziplist-entries 512 list-max-ziplist-value 64 # Sets have a special encoding in just one case: when a set is composed -# of just strings that happens to be integers in radix 10 in the range +# of just strings that happen to be integers in radix 10 in the range # of 64 bit signed integers. # The following configuration setting sets the limit in the size of the # set in order to use this special memory saving encoding. @@ -446,20 +737,34 @@ set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation Redis uses (see dict.c) -# performs a lazy rehashing: the more operation you run into an hash table +# performs a lazy rehashing: the more operation you run into a hash table # that is rehashing, the more rehashing "steps" are performed, so if the # server is idle the rehashing is never complete and some more memory is used # by the hash table. -# +# # The default is to use this millisecond 10 times every second in order to -# active rehashing the main dictionaries, freeing memory when possible. +# actively rehash the main dictionaries, freeing memory when possible. # # If unsure: # use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply form time to time +# not a good thing in your environment that Redis can reply from time to time # to queries with 2 milliseconds delay. # # use "activerehashing yes" if you don't have such hard requirements but @@ -473,9 +778,9 @@ activerehashing yes # # The limit can be set differently for the three different classes of clients: # -# normal -> normal clients -# slave -> slave clients and MONITOR clients -# pubsub -> clients subcribed to at least one pubsub channel or pattern +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: # @@ -498,17 +803,30 @@ activerehashing yes # Instead there is a default limit for pubsub and slave clients, since # subscribers and slaves receive data in a push fashion. # -# Both the hard or the soft limit can be disabled just setting it to zero. +# Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 -################################## INCLUDES ################################### - -# Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis server but also need -# to customize a few per-server settings. Include files can include -# other files, so use this wisely. +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. # -# include /path/to/local.conf -# include /path/to/other.conf +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/runtest b/runtest index 0eb384c24dc..d8451df57eb 100755 --- a/runtest +++ b/runtest @@ -1,9 +1,14 @@ #!/bin/sh -TCL=tclsh8.5 -which $TCL -if [ "$?" != "0" ] +TCL_VERSIONS="8.5 8.6" +TCLSH="" + +for VERSION in $TCL_VERSIONS; do + TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL +done + +if [ -z $TCLSH ] then - echo "You need '$TCL' in order to run the Redis test" + echo "You need tcl 8.5 or newer in order to run the Redis test" exit 1 fi -$TCL tests/test_helper.tcl $* +$TCLSH tests/test_helper.tcl $* diff --git a/runtest-sentinel b/runtest-sentinel new file mode 100755 index 00000000000..3fb1ef61561 --- /dev/null +++ b/runtest-sentinel @@ -0,0 +1,14 @@ +#!/bin/sh +TCL_VERSIONS="8.5 8.6" +TCLSH="" + +for VERSION in $TCL_VERSIONS; do + TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL +done + +if [ -z $TCLSH ] +then + echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test" + exit 1 +fi +$TCLSH tests/sentinel/run.tcl $* diff --git a/sentinel.conf b/sentinel.conf new file mode 100644 index 00000000000..39d1044e2a8 --- /dev/null +++ b/sentinel.conf @@ -0,0 +1,181 @@ +# Example sentinel.conf + +# port +# The port that this sentinel instance will run on +port 26379 + +# sentinel announce-ip +# sentinel announce-port +# +# The above two configuration directives are useful in environments where, +# because of NAT, Sentinel is reachable from outside via a non-local address. +# +# When announce-ip is provided, the Sentinel will claim the specified IP address +# in HELLO messages used to gossip its presence, instead of auto-detecting the +# local address as it usually does. +# +# Similarly when announce-port is provided and is valid and non-zero, Sentinel +# will announce the specified TCP port. +# +# The two options don't need to be used together, if only announce-ip is +# provided, the Sentinel will announce the specified IP and the server port +# as specified by the "port" option. If only announce-port is provided, the +# Sentinel will announce the auto-detected local IP and the specified port. +# +# Example: +# +# sentinel announce-ip 1.2.3.4 + +# dir +# Every long running process should have a well-defined working directory. +# For Redis Sentinel to chdir to /tmp at startup is the simplest thing +# for the process to don't interfere with administrative tasks such as +# unmounting filesystems. +dir /tmp + +# sentinel monitor +# +# Tells Sentinel to monitor this master, and to consider it in O_DOWN +# (Objectively Down) state only if at least sentinels agree. +# +# Note that whatever is the ODOWN quorum, a Sentinel will require to +# be elected by the majority of the known Sentinels in order to +# start a failover, so no failover can be performed in minority. +# +# Slaves are auto-discovered, so you don't need to specify slaves in +# any way. Sentinel itself will rewrite this configuration file adding +# the slaves using additional configuration options. +# Also note that the configuration file is rewritten when a +# slave is promoted to master. +# +# Note: master name should not include special characters or spaces. +# The valid charset is A-z 0-9 and the three characters ".-_". +sentinel monitor mymaster 127.0.0.1 6379 2 + +# sentinel auth-pass +# +# Set the password to use to authenticate with the master and slaves. +# Useful if there is a password set in the Redis instances to monitor. +# +# Note that the master password is also used for slaves, so it is not +# possible to set a different password in masters and slaves instances +# if you want to be able to monitor these instances with Sentinel. +# +# However you can have Redis instances without the authentication enabled +# mixed with Redis instances requiring the authentication (as long as the +# password set is the same for all the instances requiring the password) as +# the AUTH command will have no effect in Redis instances with authentication +# switched off. +# +# Example: +# +# sentinel auth-pass mymaster MySUPER--secret-0123passw0rd + +# sentinel down-after-milliseconds +# +# Number of milliseconds the master (or any attached slave or sentinel) should +# be unreachable (as in, not acceptable reply to PING, continuously, for the +# specified period) in order to consider it in S_DOWN state (Subjectively +# Down). +# +# Default is 30 seconds. +sentinel down-after-milliseconds mymaster 30000 + +# sentinel parallel-syncs +# +# How many slaves we can reconfigure to point to the new slave simultaneously +# during the failover. Use a low number if you use the slaves to serve query +# to avoid that all the slaves will be unreachable at about the same +# time while performing the synchronization with the master. +sentinel parallel-syncs mymaster 1 + +# sentinel failover-timeout +# +# Specifies the failover timeout in milliseconds. It is used in many ways: +# +# - The time needed to re-start a failover after a previous failover was +# already tried against the same master by a given Sentinel, is two +# times the failover timeout. +# +# - The time needed for a slave replicating to a wrong master according +# to a Sentinel current configuration, to be forced to replicate +# with the right master, is exactly the failover timeout (counting since +# the moment a Sentinel detected the misconfiguration). +# +# - The time needed to cancel a failover that is already in progress but +# did not produced any configuration change (SLAVEOF NO ONE yet not +# acknowledged by the promoted slave). +# +# - The maximum time a failover in progress waits for all the slaves to be +# reconfigured as slaves of the new master. However even after this time +# the slaves will be reconfigured by the Sentinels anyway, but not with +# the exact parallel-syncs progression as specified. +# +# Default is 3 minutes. +sentinel failover-timeout mymaster 180000 + +# SCRIPTS EXECUTION +# +# sentinel notification-script and sentinel reconfig-script are used in order +# to configure scripts that are called to notify the system administrator +# or to reconfigure clients after a failover. The scripts are executed +# with the following rules for error handling: +# +# If script exits with "1" the execution is retried later (up to a maximum +# number of times currently set to 10). +# +# If script exits with "2" (or an higher value) the script execution is +# not retried. +# +# If script terminates because it receives a signal the behavior is the same +# as exit code 1. +# +# A script has a maximum running time of 60 seconds. After this limit is +# reached the script is terminated with a SIGKILL and the execution retried. + +# NOTIFICATION SCRIPT +# +# sentinel notification-script +# +# Call the specified notification script for any sentinel event that is +# generated in the WARNING level (for instance -sdown, -odown, and so forth). +# This script should notify the system administrator via email, SMS, or any +# other messaging system, that there is something wrong with the monitored +# Redis systems. +# +# The script is called with just two arguments: the first is the event type +# and the second the event description. +# +# The script must exist and be executable in order for sentinel to start if +# this option is provided. +# +# Example: +# +# sentinel notification-script mymaster /var/redis/notify.sh + +# CLIENTS RECONFIGURATION SCRIPT +# +# sentinel client-reconfig-script +# +# When the master changed because of a failover a script can be called in +# order to perform application-specific tasks to notify the clients that the +# configuration has changed and the master is at a different address. +# +# The following arguments are passed to the script: +# +# +# +# is currently always "failover" +# is either "leader" or "observer" +# +# The arguments from-ip, from-port, to-ip, to-port are used to communicate +# the old address of the master and the new address of the elected slave +# (now a master). +# +# This script should be resistant to multiple invocations. +# +# Example: +# +# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh + + diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 00000000000..aee7aacf09f --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,5 @@ +*.gcda +*.gcno +*.gcov +redis.info +lcov-html diff --git a/src/Makefile b/src/Makefile index cca785cbc2f..a65e7675227 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,65 +1,102 @@ # Redis Makefile # Copyright (C) 2009 Salvatore Sanfilippo # This file is released under the BSD license, see the COPYING file +# +# The Makefile composes the final FINAL_CFLAGS and FINAL_LDFLAGS using +# what is needed for Redis plus the standard CFLAGS and LDFLAGS passed. +# However when building the dependencies (Jemalloc, Lua, Hiredis, ...) +# CFLAGS and LDFLAGS are propagated to the dependencies, so to pass +# flags only to be used when compiling / linking Redis itself REDIS_CFLAGS +# and REDIS_LDFLAGS are used instead (this is the case of 'make gcov'). +# +# Dependencies are stored in the Makefile.dep file. To rebuild this file +# Just use 'make dep', but this is only needed by developers. release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua -ifeq ($(uname_S),SunOS) - CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 - CCLINK?=-ldl -lnsl -lsocket -lm -lpthread - DEBUG?=-g -ggdb -else - CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF) - CCLINK?=-lm -pthread - DEBUG?=-g -rdynamic -ggdb -endif +# Default settings +STD=-std=c99 -pedantic +WARN=-Wall -W +OPT=$(OPTIMIZATION) + +PREFIX?=/usr/local +INSTALL_BIN=$(PREFIX)/bin +INSTALL=install # Default allocator ifeq ($(uname_S),Linux) - MALLOC?=jemalloc + MALLOC=jemalloc else - MALLOC?=libc + MALLOC=libc endif # Backwards compatibility for selecting an allocator ifeq ($(USE_TCMALLOC),yes) - MALLOC=tcmalloc + MALLOC=tcmalloc endif ifeq ($(USE_TCMALLOC_MINIMAL),yes) - MALLOC=tcmalloc_minimal + MALLOC=tcmalloc_minimal endif ifeq ($(USE_JEMALLOC),yes) - MALLOC=jemalloc + MALLOC=jemalloc +endif + +# Override default settings if possible +-include .make-settings + +FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) +FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) +FINAL_LIBS=-lm +DEBUG=-g -ggdb + +ifeq ($(uname_S),SunOS) + # SunOS + INSTALL=cp -pf + FINAL_CFLAGS+= -D__EXTENSIONS__ -D_XPG6 + FINAL_LIBS+= -ldl -lnsl -lsocket -lresolv -lpthread -lrt +else +ifeq ($(uname_S),Darwin) + # Darwin (nothing to do) +else +ifeq ($(uname_S),AIX) + # AIX + FINAL_LDFLAGS+= -Wl,-bexpall + FINAL_LIBS+= -pthread -lcrypt -lbsd + +else + # All the other OSes (notably Linux) + FINAL_LDFLAGS+= -rdynamic + FINAL_LIBS+= -pthread +endif endif +endif +# Include paths to dependencies +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src ifeq ($(MALLOC),tcmalloc) - ALLOC_LINK=-ltcmalloc - ALLOC_FLAGS=-DUSE_TCMALLOC + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc endif ifeq ($(MALLOC),tcmalloc_minimal) - ALLOC_LINK=-ltcmalloc_minimal - ALLOC_FLAGS=-DUSE_TCMALLOC + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc_minimal endif ifeq ($(MALLOC),jemalloc) - ALLOC_LINK=../deps/jemalloc/lib/libjemalloc.a -ldl - ALLOC_FLAGS=-DUSE_JEMALLOC -I../deps/jemalloc/include - DEPENDENCY_TARGETS+= jemalloc + DEPENDENCY_TARGETS+= jemalloc + FINAL_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include + FINAL_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl endif -CCLINK+= $(ALLOC_LINK) -CFLAGS+= $(ALLOC_FLAGS) -CCOPT= $(CFLAGS) $(ARCH) $(PROF) - -PREFIX= /usr/local -INSTALL_BIN= $(PREFIX)/bin -INSTALL= cp -pf +REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) +REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) +REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) CCCOLOR="\033[34m" LINKCOLOR="\033[34;1m" @@ -69,202 +106,149 @@ MAKECOLOR="\033[32;1m" ENDCOLOR="\033[0m" ifndef V -QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR); -QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); +QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR) 1>&2; +QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; +QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o -BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o -CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o -CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o -CHECKAOFOBJ = redis-check-aof.o - -PRGNAME = redis-server -BENCHPRGNAME = redis-benchmark -CLIPRGNAME = redis-cli -CHECKDUMPPRGNAME = redis-check-dump -CHECKAOFPRGNAME = redis-check-aof - -all: redis-benchmark redis-cli redis-check-dump redis-check-aof redis-server +REDIS_SERVER_NAME=redis-server +REDIS_SENTINEL_NAME=redis-sentinel +REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o hyperloglog.o latency.o sparkline.o +REDIS_CLI_NAME=redis-cli +REDIS_CLI_OBJ=anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o +REDIS_BENCHMARK_NAME=redis-benchmark +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o +REDIS_CHECK_DUMP_NAME=redis-check-dump +REDIS_CHECK_DUMP_OBJ=redis-check-dump.o lzf_c.o lzf_d.o crc64.o +REDIS_CHECK_AOF_NAME=redis-check-aof +REDIS_CHECK_AOF_OBJ=redis-check-aof.o + +all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" - @echo "Hint: To run 'make test' is a good idea ;)" + @echo "Hint: It's a good idea to run 'make test' ;)" @echo "" +.PHONY: all + # Deps (use make dep to generate this) -adlist.o: adlist.c adlist.h zmalloc.h -ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c -ae_epoll.o: ae_epoll.c -ae_kqueue.o: ae_kqueue.c -ae_select.o: ae_select.c -anet.o: anet.c fmacros.h anet.h -aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h bio.h -cluster.o: cluster.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h sha1.h -dict.o: dict.c fmacros.h dict.h zmalloc.h -endianconv.o: endianconv.c -intset.o: intset.c intset.h zmalloc.h endianconv.h -lzf_c.o: lzf_c.c lzfP.h -lzf_d.o: lzf_d.c lzfP.h -multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -pqsort.o: pqsort.c -pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h lzf.h -redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ - ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h -redis-check-aof.o: redis-check-aof.c fmacros.h config.h -redis-check-dump.o: redis-check-dump.c lzf.h -redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ - sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h -redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ - bio.h asciilogo.h -release.o: release.c release.h -replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ - sha1.h rand.h -rio.o: rio.c sds.h -sds.o: sds.c sds.h zmalloc.h -sha1.o: sha1.c sha1.h config.h -slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ - slowlog.h -sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h pqsort.h -syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h -util.o: util.c fmacros.h util.h -ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h -zipmap.o: zipmap.c zmalloc.h endianconv.h -zmalloc.o: zmalloc.c config.h zmalloc.h - -# Clean local objects when ARCH is different -ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) -.make-arch: clean -else -.make-arch: -endif +include Makefile.dep -.make-arch: - -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS) ARCH="$(ARCH)") - -(echo $(ARCH) > .make-arch) +dep: + $(REDIS_CC) -MM *.c > Makefile.dep + +.PHONY: dep + +persist-settings: distclean + echo STD=$(STD) >> .make-settings + echo WARN=$(WARN) >> .make-settings + echo OPT=$(OPT) >> .make-settings + echo MALLOC=$(MALLOC) >> .make-settings + echo CFLAGS=$(CFLAGS) >> .make-settings + echo LDFLAGS=$(LDFLAGS) >> .make-settings + echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings + echo REDIS_LDFLAGS=$(REDIS_LDFLAGS) >> .make-settings + echo PREV_FINAL_CFLAGS=$(FINAL_CFLAGS) >> .make-settings + echo PREV_FINAL_LDFLAGS=$(FINAL_LDFLAGS) >> .make-settings + -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) + +.PHONY: persist-settings + +# Prerequisites target +.make-prerequisites: + @touch $@ -# Clean local objects when allocator changes -ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc'), $(MALLOC)) -.make-malloc: clean -else -.make-malloc: +# Clean everything, persist settings and build dependencies if anything changed +ifneq ($(strip $(PREV_FINAL_CFLAGS)), $(strip $(FINAL_CFLAGS))) +.make-prerequisites: persist-settings endif -.make-malloc: - -(echo $(MALLOC) > .make-malloc) - -# Union of make-prerequisites -.make-prerequisites: .make-arch .make-malloc - @touch $@ - -redis-server: .make-prerequisites $(OBJ) - $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) +ifneq ($(strip $(PREV_FINAL_LDFLAGS)), $(strip $(FINAL_LDFLAGS))) +.make-prerequisites: persist-settings +endif -redis-benchmark: .make-prerequisites $(BENCHOBJ) - $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) +# redis-server +$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a $(FINAL_LIBS) -redis-benchmark.o: redis-benchmark.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< +# redis-sentinel +$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME) + $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) -redis-cli: .make-prerequisites $(CLIOBJ) - $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) +# redis-cli +$(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) -redis-cli.o: redis-cli.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< +# redis-benchmark +$(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) -redis-check-dump: .make-prerequisites $(CHECKDUMPOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) +# redis-check-dump +$(REDIS_CHECK_DUMP_NAME): $(REDIS_CHECK_DUMP_OBJ) + $(REDIS_LD) -o $@ $^ $(FINAL_LIBS) -redis-check-aof: .make-prerequisites $(CHECKAOFOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) +# redis-check-aof +$(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ) + $(REDIS_LD) -o $@ $^ $(FINAL_LIBS) -# Because the jemalloc.h header is generated as a part of the jemalloc build -# process, building it should complete before building any other object. Instead of -# depending on a single artifact, simply build all dependencies first. +# Because the jemalloc.h header is generated as a part of the jemalloc build, +# building it should complete before building any other object. Instead of +# depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< - -.PHONY: all clean distclean + $(REDIS_CC) -c $< clean: - rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov + rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html + +.PHONY: clean distclean: clean -(cd ../deps && $(MAKE) distclean) - -(rm -f .make-arch .make-malloc) + -(rm -f .make-*) -dep: - $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise +.PHONY: distclean -test: redis-server redis-check-aof +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) @(cd ..; ./runtest) -bench: - ./redis-benchmark +test-sentinel: $(REDIS_SENTINEL_NAME) + @(cd ..; ./runtest-sentinel) -log: - git log '--pretty=format:%ad %s (%cn)' --date=short > ../Changelog +check: test + +lcov: + $(MAKE) gcov + @(set -e; cd ..; ./runtest --clients 1) + @geninfo -o redis.info . + @genhtml --legend -o lcov-html redis.info + +.PHONY: lcov + +bench: $(REDIS_BENCHMARK_NAME) + ./$(REDIS_BENCHMARK_NAME) 32bit: @echo "" @echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386" @echo "" - $(MAKE) ARCH="-m32" JEMALLOC_CFLAGS='CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32"' - -gprof: - $(MAKE) PROF="-pg" + $(MAKE) CFLAGS="-m32" LDFLAGS="-m32" gcov: - $(MAKE) PROF="-fprofile-arcs -ftest-coverage" + $(MAKE) REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage" noopt: - $(MAKE) OPTIMIZATION="" + $(MAKE) OPTIMIZATION="-O0" -32bitgprof: - $(MAKE) PROF="-pg" ARCH="-arch i386" +valgrind: + $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" src/help.h: @../utils/generate-command-help.rb > help.h install: all - mkdir -p $(INSTALL_BIN) - $(INSTALL) $(PRGNAME) $(INSTALL_BIN) - $(INSTALL) $(BENCHPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CLIPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CHECKDUMPPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CHECKAOFPRGNAME) $(INSTALL_BIN) + @mkdir -p $(INSTALL_BIN) + $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) + $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) + $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) + $(REDIS_INSTALL) $(REDIS_CHECK_DUMP_NAME) $(INSTALL_BIN) + $(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) diff --git a/src/Makefile.dep b/src/Makefile.dep new file mode 100644 index 00000000000..1f24232fd8d --- /dev/null +++ b/src/Makefile.dep @@ -0,0 +1,132 @@ +adlist.o: adlist.c adlist.h zmalloc.h +ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c ae_select.c ae_evport.c ae_epoll.c +ae_epoll.o: ae_epoll.c +ae_evport.o: ae_evport.c +ae_kqueue.o: ae_kqueue.c +ae_select.o: ae_select.c +anet.o: anet.c fmacros.h anet.h +aof.o: aof.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + bio.h +bio.o: bio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + bio.h +bitops.o: bitops.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +config.o: config.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +crc64.o: crc64.c +db.o: db.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +debug.o: debug.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + sha1.h crc64.h bio.h +dict.o: dict.c fmacros.h dict.h zmalloc.h redisassert.h +endianconv.o: endianconv.c +hyperloglog.o: hyperloglog.c redis.h fmacros.h config.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h \ + latency.h sparkline.h rdb.h rio.h +intset.o: intset.c intset.h zmalloc.h endianconv.h config.h +latency.o: latency.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +lzf_c.o: lzf_c.c lzfP.h +lzf_d.o: lzf_d.c lzfP.h +memtest.o: memtest.c config.h +migrate.o: migrate.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + endianconv.h +multi.o: multi.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +networking.o: networking.c redis.h fmacros.h config.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h \ + latency.h sparkline.h rdb.h rio.h +notify.o: notify.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +object.o: object.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +pqsort.o: pqsort.c +pubsub.o: pubsub.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +rand.o: rand.c +rdb.o: rdb.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + lzf.h zipmap.h endianconv.h +redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ + ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h +redis-check-aof.o: redis-check-aof.c fmacros.h config.h +redis-check-dump.o: redis-check-dump.c lzf.h crc64.h +redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ + sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h anet.h ae.h +redis.o: redis.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + slowlog.h bio.h asciilogo.h +release.o: release.c release.h version.h crc64.h +replication.o: replication.c redis.h fmacros.h config.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h \ + latency.h sparkline.h rdb.h rio.h +rio.o: rio.c fmacros.h rio.h sds.h util.h crc64.h config.h redis.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h latency.h sparkline.h \ + rdb.h +scripting.o: scripting.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + sha1.h rand.h ../deps/lua/src/lauxlib.h ../deps/lua/src/lualib.h +sds.o: sds.c sds.h zmalloc.h +sentinel.o: sentinel.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + ../deps/hiredis/hiredis.h ../deps/hiredis/async.h +setproctitle.o: setproctitle.c +sha1.o: sha1.c sha1.h config.h +slowlog.o: slowlog.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + slowlog.h +sort.o: sort.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h \ + pqsort.h +sparkline.o: sparkline.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +syncio.o: syncio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +t_hash.o: t_hash.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +t_list.o: t_list.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +t_set.o: t_set.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +t_string.o: t_string.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +t_zset.o: t_zset.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h latency.h sparkline.h rdb.h rio.h +util.o: util.c fmacros.h util.h sds.h +ziplist.o: ziplist.c zmalloc.h util.h sds.h ziplist.h endianconv.h \ + config.h redisassert.h +zipmap.o: zipmap.c zmalloc.h endianconv.h config.h +zmalloc.o: zmalloc.c config.h zmalloc.h diff --git a/src/adlist.c b/src/adlist.c index 51ba03bd5e0..b4cc785be18 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -71,7 +71,7 @@ void listRelease(list *list) zfree(list); } -/* Add a new node to the list, to head, contaning the specified 'value' +/* Add a new node to the list, to head, containing the specified 'value' * pointer as value. * * On error, NULL is returned and no operation is performed (i.e. the @@ -97,7 +97,7 @@ list *listAddNodeHead(list *list, void *value) return list; } -/* Add a new node to the list, to tail, contaning the specified 'value' +/* Add a new node to the list, to tail, containing the specified 'value' * pointer as value. * * On error, NULL is returned and no operation is performed (i.e. the @@ -178,7 +178,7 @@ void listDelNode(list *list, listNode *node) listIter *listGetIterator(list *list, int direction) { listIter *iter; - + if ((iter = zmalloc(sizeof(*iter))) == NULL) return NULL; if (direction == AL_START_HEAD) iter->next = list->head; @@ -308,7 +308,7 @@ listNode *listSearchKey(list *list, void *key) /* Return the element at the specified zero-based index * where 0 is the head, 1 is the element next to head * and so on. Negative integers are used in order to count - * from the tail, -1 is the last element, -2 the penultimante + * from the tail, -1 is the last element, -2 the penultimate * and so on. If the index is out of range NULL is returned. */ listNode *listIndex(list *list, long index) { listNode *n; @@ -323,3 +323,19 @@ listNode *listIndex(list *list, long index) { } return n; } + +/* Rotate the list removing the tail node and inserting it to the head. */ +void listRotate(list *list) { + listNode *tail = list->tail; + + if (listLength(list) <= 1) return; + + /* Detach current tail */ + list->tail = tail->prev; + list->tail->next = NULL; + /* Move it as head */ + list->head->prev = tail; + tail->prev = NULL; + tail->next = list->head; + list->head = tail; +} diff --git a/src/adlist.h b/src/adlist.h index 36dba1ff324..be322552f50 100644 --- a/src/adlist.h +++ b/src/adlist.h @@ -1,6 +1,6 @@ /* adlist.h - A generic doubly linked list implementation * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -84,6 +84,7 @@ listNode *listSearchKey(list *list, void *key); listNode *listIndex(list *list, long index); void listRewind(list *list, listIter *li); void listRewindTail(list *list, listIter *li); +void listRotate(list *list); /* Directions for iterators */ #define AL_START_HEAD 0 diff --git a/src/ae.c b/src/ae.c index 4099b12598f..63a1ab4eb2c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -35,7 +35,10 @@ #include #include #include +#include #include +#include +#include #include "ae.h" #include "zmalloc.h" @@ -43,13 +46,17 @@ /* Include the best multiplexing layer supported by this system. * The following should be ordered by performances, descending. */ -#ifdef HAVE_EPOLL -#include "ae_epoll.c" +#ifdef HAVE_EVPORT +#include "ae_evport.c" #else - #ifdef HAVE_KQUEUE - #include "ae_kqueue.c" + #ifdef HAVE_EPOLL + #include "ae_epoll.c" #else - #include "ae_select.c" + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif #endif #endif @@ -62,6 +69,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; eventLoop->setsize = setsize; + eventLoop->lastTime = time(NULL); eventLoop->timeEventHead = NULL; eventLoop->timeEventNextId = 0; eventLoop->stop = 0; @@ -83,6 +91,36 @@ aeEventLoop *aeCreateEventLoop(int setsize) { return NULL; } +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + int i; + + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->setsize = setsize; + + /* Make sure that if we created new slots, they are initialized with + * an AE_NONE mask. */ + for (i = eventLoop->maxfd+1; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return AE_OK; +} + void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); @@ -97,7 +135,10 @@ void aeStop(aeEventLoop *eventLoop) { int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc, void *clientData) { - if (fd >= eventLoop->setsize) return AE_ERR; + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } aeFileEvent *fe = &eventLoop->events[fd]; if (aeApiAddEvent(eventLoop, fd, mask) == -1) @@ -115,8 +156,9 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { if (fd >= eventLoop->setsize) return; aeFileEvent *fe = &eventLoop->events[fd]; - if (fe->mask == AE_NONE) return; + + aeApiDelEvent(eventLoop, fd, mask); fe->mask = fe->mask & (~mask); if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { /* Update the max fd */ @@ -126,7 +168,6 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) if (eventLoop->events[j].mask != AE_NONE) break; eventLoop->maxfd = j; } - aeApiDelEvent(eventLoop, fd, mask); } int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { @@ -231,6 +272,24 @@ static int processTimeEvents(aeEventLoop *eventLoop) { int processed = 0; aeTimeEvent *te; long long maxId; + time_t now = time(NULL); + + /* If the system clock is moved to the future, and then set back to the + * right value, time events may be delayed in a random way. Often this + * means that scheduled operations will not be performed soon enough. + * + * Here we try to detect system clock skews, and force all the time + * events to be processed ASAP when this happens: the idea is that + * processing events earlier is less dangerous than delaying them + * indefinitely, and practice suggests it is. */ + if (now < eventLoop->lastTime) { + te = eventLoop->timeEventHead; + while(te) { + te->when_sec = 0; + te = te->next; + } + } + eventLoop->lastTime = now; te = eventLoop->timeEventHead; maxId = eventLoop->timeEventNextId-1; @@ -280,7 +339,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) { /* Process every pending time event, then every pending file event * (that may be registered by time event callbacks just processed). * Without special flags the function sleeps until some file event - * fires, or when the next time event occurrs (if any). + * fires, or when the next time event occurs (if any). * * If flags is 0, the function does nothing and returns. * if flags has AE_ALL_EVENTS set, all the kind of events are processed. @@ -327,7 +386,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (tvp->tv_usec < 0) tvp->tv_usec = 0; } else { /* If we have to check for events but need to return - * ASAP because of AE_DONT_WAIT we need to se the timeout + * ASAP because of AE_DONT_WAIT we need to set the timeout * to zero */ if (flags & AE_DONT_WAIT) { tv.tv_sec = tv.tv_usec = 0; @@ -366,24 +425,22 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) return processed; /* return the number of processed file/time events */ } -/* Wait for millseconds until the given file descriptor becomes +/* Wait for milliseconds until the given file descriptor becomes * writable/readable/exception */ int aeWait(int fd, int mask, long long milliseconds) { - struct timeval tv; - fd_set rfds, wfds, efds; + struct pollfd pfd; int retmask = 0, retval; - tv.tv_sec = milliseconds/1000; - tv.tv_usec = (milliseconds%1000)*1000; - FD_ZERO(&rfds); - FD_ZERO(&wfds); - FD_ZERO(&efds); - - if (mask & AE_READABLE) FD_SET(fd,&rfds); - if (mask & AE_WRITABLE) FD_SET(fd,&wfds); - if ((retval = select(fd+1, &rfds, &wfds, &efds, &tv)) > 0) { - if (FD_ISSET(fd,&rfds)) retmask |= AE_READABLE; - if (FD_ISSET(fd,&wfds)) retmask |= AE_WRITABLE; + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; return retmask; } else { return retval; diff --git a/src/ae.h b/src/ae.h index e1dccfc76a6..15ca1b5e750 100644 --- a/src/ae.h +++ b/src/ae.h @@ -2,7 +2,7 @@ * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated * it in form of a library for easy reuse. * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -88,6 +88,7 @@ typedef struct aeEventLoop { int maxfd; /* highest file descriptor currently registered */ int setsize; /* max number of file descriptors tracked */ long long timeEventNextId; + time_t lastTime; /* Used to detect system clock skew */ aeFileEvent *events; /* Registered events */ aeFiredEvent *fired; /* Fired events */ aeTimeEvent *timeEventHead; @@ -113,5 +114,7 @@ int aeWait(int fd, int mask, long long milliseconds); void aeMain(aeEventLoop *eventLoop); char *aeGetApiName(void); void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); #endif diff --git a/src/ae_epoll.c b/src/ae_epoll.c index cac10d67f7c..da9c7b90672 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -1,6 +1,33 @@ /* Linux epoll(2) based ae.c module - * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com - * Released under the BSD license. See the COPYING file for more info. */ + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include @@ -18,7 +45,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } - state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */ + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ if (state->epfd == -1) { zfree(state->events); zfree(state); @@ -28,6 +55,13 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { aeApiState *state = eventLoop->apidata; @@ -89,6 +123,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE; eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].mask = mask; } diff --git a/src/ae_evport.c b/src/ae_evport.c new file mode 100644 index 00000000000..5c317becb6f --- /dev/null +++ b/src/ae_evport.c @@ -0,0 +1,320 @@ +/* ae.c module for illumos event ports. + * + * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include +#include + +#include +#include + +#include + +static int evport_debug = 0; + +/* + * This file implements the ae API using event ports, present on Solaris-based + * systems since Solaris 10. Using the event port interface, we associate file + * descriptors with the port. Each association also includes the set of poll(2) + * events that the consumer is interested in (e.g., POLLIN and POLLOUT). + * + * There's one tricky piece to this implementation: when we return events via + * aeApiPoll, the corresponding file descriptors become dissociated from the + * port. This is necessary because poll events are level-triggered, so if the + * fd didn't become dissociated, it would immediately fire another event since + * the underlying state hasn't changed yet. We must re-associate the file + * descriptor, but only after we know that our caller has actually read from it. + * The ae API does not tell us exactly when that happens, but we do know that + * it must happen by the time aeApiPoll is called again. Our solution is to + * keep track of the last fds returned by aeApiPoll and re-associate them next + * time aeApiPoll is invoked. + * + * To summarize, in this module, each fd association is EITHER (a) represented + * only via the in-kernel association OR (b) represented by pending_fds and + * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, + * and only until we enter aeApiPoll again (at which point we restore the + * in-kernel association). + */ +#define MAX_EVENT_BATCHSZ 512 + +typedef struct aeApiState { + int portfd; /* event port */ + int npending; /* # of pending fds */ + int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ + int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + int i; + aeApiState *state = zmalloc(sizeof(aeApiState)); + if (!state) return -1; + + state->portfd = port_create(); + if (state->portfd == -1) { + zfree(state); + return -1; + } + + state->npending = 0; + + for (i = 0; i < MAX_EVENT_BATCHSZ; i++) { + state->pending_fds[i] = -1; + state->pending_masks[i] = AE_NONE; + } + + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Nothing to resize here. */ + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->portfd); + zfree(state); +} + +static int aeApiLookupPending(aeApiState *state, int fd) { + int i; + + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == fd) + return (i); + } + + return (-1); +} + +/* + * Helper function to invoke port_associate for the given fd and mask. + */ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) { + int events = 0; + int rv, err; + + if (mask & AE_READABLE) + events |= POLLIN; + if (mask & AE_WRITABLE) + events |= POLLOUT; + + if (evport_debug) + fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); + + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, + (void *)(uintptr_t)mask); + err = errno; + + if (evport_debug) + fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err)); + + if (rv == -1) { + fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err)); + + if (err == EAGAIN) + fprintf(stderr, "aeApiAssociate: event port limit exceeded."); + } + + return rv; +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); + + /* + * Since port_associate's "events" argument replaces any existing events, we + * must be sure to include whatever events are already associated when + * we call port_associate() again. + */ + fullmask = mask | eventLoop->events[fd].mask; + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + /* + * This fd was recently returned from aeApiPoll. It should be safe to + * assume that the consumer has processed that poll event, but we play + * it safer by simply updating pending_mask. The fd will be + * re-associated as usual when aeApiPoll is called again. + */ + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); + state->pending_masks[pfd] |= fullmask; + return 0; + } + + return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask)); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask, pfd; + + if (evport_debug) + fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); + + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { + if (evport_debug) + fprintf(stderr, "deleting event from pending fd %d\n", fd); + + /* + * This fd was just returned from aeApiPoll, so it's not currently + * associated with the port. All we need to do is update + * pending_mask appropriately. + */ + state->pending_masks[pfd] &= ~mask; + + if (state->pending_masks[pfd] == AE_NONE) + state->pending_fds[pfd] = -1; + + return; + } + + /* + * The fd is currently associated with the port. Like with the add case + * above, we must look at the full mask for the file descriptor before + * updating that association. We don't have a good way of knowing what the + * events are without looking into the eventLoop state directly. We rely on + * the fact that our caller has already updated the mask in the eventLoop. + */ + + fullmask = eventLoop->events[fd].mask; + if (fullmask == AE_NONE) { + /* + * We're removing *all* events, so use port_dissociate to remove the + * association completely. Failure here indicates a bug. + */ + if (evport_debug) + fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd); + + if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) { + perror("aeApiDelEvent: port_dissociate"); + abort(); /* will not return */ + } + } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd, + fullmask) != 0) { + /* + * ENOMEM is a potentially transient condition, but the kernel won't + * generally return it unless things are really bad. EAGAIN indicates + * we've reached an resource limit, for which it doesn't make sense to + * retry (counter-intuitively). All other errors indicate a bug. In any + * of these cases, the best we can do is to abort. + */ + abort(); /* will not return */ + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + struct timespec timeout, *tsp; + int mask, i; + uint_t nevents; + port_event_t event[MAX_EVENT_BATCHSZ]; + + /* + * If we've returned fd events before, we must re-associate them with the + * port now, before calling port_get(). See the block comment at the top of + * this file for an explanation of why. + */ + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == -1) + /* This fd has since been deleted. */ + continue; + + if (aeApiAssociate("aeApiPoll", state->portfd, + state->pending_fds[i], state->pending_masks[i]) != 0) { + /* See aeApiDelEvent for why this case is fatal. */ + abort(); + } + + state->pending_masks[i] = AE_NONE; + state->pending_fds[i] = -1; + } + + state->npending = 0; + + if (tvp != NULL) { + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + tsp = &timeout; + } else { + tsp = NULL; + } + + /* + * port_getn can return with errno == ETIME having returned some events (!). + * So if we get ETIME, we check nevents, too. + */ + nevents = 1; + if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents, + tsp) == -1 && (errno != ETIME || nevents == 0)) { + if (errno == ETIME || errno == EINTR) + return 0; + + /* Any other error indicates a bug. */ + perror("aeApiPoll: port_get"); + abort(); + } + + state->npending = nevents; + + for (i = 0; i < nevents; i++) { + mask = 0; + if (event[i].portev_events & POLLIN) + mask |= AE_READABLE; + if (event[i].portev_events & POLLOUT) + mask |= AE_WRITABLE; + + eventLoop->fired[i].fd = event[i].portev_object; + eventLoop->fired[i].mask = mask; + + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", + (int)event[i].portev_object, mask); + + state->pending_fds[i] = event[i].portev_object; + state->pending_masks[i] = (uintptr_t)event[i].portev_user; + } + + return nevents; +} + +static char *aeApiName(void) { + return "evport"; +} diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index e91a254d524..6796f4ceb59 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -1,6 +1,33 @@ /* Kqueue(2)-based ae.c module + * * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com - * Released under the BSD license. See the COPYING file for more info. */ + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include #include @@ -27,8 +54,14 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return -1; } eventLoop->apidata = state; - - return 0; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + return 0; } static void aeApiFree(aeEventLoop *eventLoop) { @@ -42,7 +75,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { aeApiState *state = eventLoop->apidata; struct kevent ke; - + if (mask & AE_READABLE) { EV_SET(&ke, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; @@ -85,16 +118,16 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (retval > 0) { int j; - + numevents = retval; for(j = 0; j < numevents; j++) { int mask = 0; struct kevent *e = state->events+j; - + if (e->filter == EVFILT_READ) mask |= AE_READABLE; if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; - eventLoop->fired[j].fd = e->ident; - eventLoop->fired[j].mask = mask; + eventLoop->fired[j].fd = e->ident; + eventLoop->fired[j].mask = mask; } } return numevents; diff --git a/src/ae_select.c b/src/ae_select.c index 43f5867f369..e2b7a9e8aca 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -1,6 +1,33 @@ -/* Select()-based ae.c module - * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com - * Released under the BSD license. See the COPYING file for more info. */ +/* Select()-based ae.c module. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include @@ -21,6 +48,12 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) return -1; + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { zfree(eventLoop->apidata); } diff --git a/src/anet.c b/src/anet.c index ba4e6cce89b..732816308d4 100644 --- a/src/anet.c +++ b/src/anet.c @@ -1,6 +1,6 @@ /* anet.c -- Basic TCP socket stuff made a bit less boring * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -57,28 +58,89 @@ static void anetSetError(char *err, const char *fmt, ...) va_end(ap); } -int anetNonBlock(char *err, int fd) -{ +int anetSetBlock(char *err, int fd, int non_block) { int flags; - /* Set the socket nonblocking. + /* Set the socket blocking (if non_block is zero) or non-blocking. * Note that fcntl(2) for F_GETFL and F_SETFL can't be * interrupted by a signal. */ if ((flags = fcntl(fd, F_GETFL)) == -1) { anetSetError(err, "fcntl(F_GETFL): %s", strerror(errno)); return ANET_ERR; } - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) { + + if (non_block) + flags |= O_NONBLOCK; + else + flags &= ~O_NONBLOCK; + + if (fcntl(fd, F_SETFL, flags) == -1) { anetSetError(err, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno)); return ANET_ERR; } return ANET_OK; } -int anetTcpNoDelay(char *err, int fd) +int anetNonBlock(char *err, int fd) { + return anetSetBlock(err,fd,1); +} + +int anetBlock(char *err, int fd) { + return anetSetBlock(err,fd,0); +} + +/* Set TCP keep alive option to detect dead peers. The interval option + * is only used for Linux as we are using Linux-specific APIs to set + * the probe send time, interval, and count. */ +int anetKeepAlive(char *err, int fd, int interval) { - int yes = 1; - if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) + int val = 1; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1) + { + anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); + return ANET_ERR; + } + +#ifdef __linux__ + /* Default settings are more or less garbage, with the keepalive time + * set to 7200 by default on Linux. Modify settings to make the feature + * actually useful. */ + + /* Send first probe after interval. */ + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPIDLE: %s\n", strerror(errno)); + return ANET_ERR; + } + + /* Send next probes after the specified interval. Note that we set the + * delay as interval / 3, as we send three probes before detecting + * an error (see the next setsockopt call). */ + val = interval/3; + if (val == 0) val = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); + return ANET_ERR; + } + + /* Consider the socket in error state after three we send three ACK + * probes without getting a reply. */ + val = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); + return ANET_ERR; + } +#else + ((void) interval); /* Avoid unused var warning for non Linux systems. */ +#endif + + return ANET_OK; +} + +static int anetSetTcpNoDelay(char *err, int fd, int val) +{ + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) == -1) { anetSetError(err, "setsockopt TCP_NODELAY: %s", strerror(errno)); return ANET_ERR; @@ -86,6 +148,17 @@ int anetTcpNoDelay(char *err, int fd) return ANET_OK; } +int anetEnableTcpNoDelay(char *err, int fd) +{ + return anetSetTcpNoDelay(err, fd, 1); +} + +int anetDisableTcpNoDelay(char *err, int fd) +{ + return anetSetTcpNoDelay(err, fd, 0); +} + + int anetSetSendBuffer(char *err, int fd, int buffsize) { if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) @@ -106,36 +179,84 @@ int anetTcpKeepAlive(char *err, int fd) return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf) +/* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetSendTimeout(char *err, int fd, long long ms) { + struct timeval tv; + + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_SNDTIMEO: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to + * do the actual work. It resolves the hostname "host" and set the string + * representation of the IP address into the buffer pointed by "ipbuf". + * + * If flags is set to ANET_IP_ONLY the function only resolves hostnames + * that are actually already IPv4 or IPv6 addresses. This turns the function + * into a validating / normalizing function. */ +int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, + int flags) { - struct sockaddr_in sa; + struct addrinfo hints, *info; + int rv; - sa.sin_family = AF_INET; - if (inet_aton(host, &sa.sin_addr) == 0) { - struct hostent *he; + memset(&hints,0,sizeof(hints)); + if (flags & ANET_IP_ONLY) hints.ai_flags = AI_NUMERICHOST; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; /* specify socktype to avoid dups */ - he = gethostbyname(host); - if (he == NULL) { - anetSetError(err, "can't resolve: %s", host); - return ANET_ERR; - } - memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr)); + if ((rv = getaddrinfo(host, NULL, &hints, &info)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; + } + if (info->ai_family == AF_INET) { + struct sockaddr_in *sa = (struct sockaddr_in *)info->ai_addr; + inet_ntop(AF_INET, &(sa->sin_addr), ipbuf, ipbuf_len); + } else { + struct sockaddr_in6 *sa = (struct sockaddr_in6 *)info->ai_addr; + inet_ntop(AF_INET6, &(sa->sin6_addr), ipbuf, ipbuf_len); + } + + freeaddrinfo(info); + return ANET_OK; +} + +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { + return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); +} + +int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { + return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); +} + +static int anetSetReuseAddr(char *err, int fd) { + int yes = 1; + /* Make sure connection-intensive things like the redis benckmark + * will be able to close/open sockets a zillion of times */ + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1) { + anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); + return ANET_ERR; } - strcpy(ipbuf,inet_ntoa(sa.sin_addr)); return ANET_OK; } static int anetCreateSocket(char *err, int domain) { - int s, on = 1; + int s; if ((s = socket(domain, SOCK_STREAM, 0)) == -1) { anetSetError(err, "creating socket: %s", strerror(errno)); return ANET_ERR; } - /* Make sure connection-intensive things like the redis benckmark + /* Make sure connection-intensive things like the redis benchmark * will be able to close/open sockets a zillion of times */ - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { - anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); + if (anetSetReuseAddr(err,s) == ANET_ERR) { + close(s); return ANET_ERR; } return s; @@ -145,38 +266,52 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONBLOCK 1 static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) { - int s; - struct sockaddr_in sa; + int s = ANET_ERR, rv; + char portstr[6]; /* strlen("65535") + 1; */ + struct addrinfo hints, *servinfo, *p; - if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) - return ANET_ERR; + snprintf(portstr,sizeof(portstr),"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; - sa.sin_family = AF_INET; - sa.sin_port = htons(port); - if (inet_aton(addr, &sa.sin_addr) == 0) { - struct hostent *he; - - he = gethostbyname(addr); - if (he == NULL) { - anetSetError(err, "can't resolve: %s", addr); + if ((rv = getaddrinfo(addr,portstr,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; + } + for (p = servinfo; p != NULL; p = p->ai_next) { + /* Try to create the socket and to connect it. + * If we fail in the socket() call, or on connect(), we retry with + * the next entry in servinfo. */ + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (flags & ANET_CONNECT_NONBLOCK && anetNonBlock(err,s) != ANET_OK) + goto error; + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { + /* If the socket is non-blocking, it is ok for connect() to + * return an EINPROGRESS error here. */ + if (errno == EINPROGRESS && flags & ANET_CONNECT_NONBLOCK) + goto end; close(s); - return ANET_ERR; + s = ANET_ERR; + continue; } - memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr)); - } - if (flags & ANET_CONNECT_NONBLOCK) { - if (anetNonBlock(err,s) != ANET_OK) - return ANET_ERR; + + /* If we ended an iteration of the for loop without errors, we + * have a connected socket. Let's return to the caller. */ + goto end; } - if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { - if (errno == EINPROGRESS && - flags & ANET_CONNECT_NONBLOCK) - return s; + if (p == NULL) + anetSetError(err, "creating socket: %s", strerror(errno)); - anetSetError(err, "connect: %s", strerror(errno)); +error: + if (s != ANET_ERR) { close(s); - return ANET_ERR; + s = ANET_ERR; } +end: + freeaddrinfo(servinfo); return s; } @@ -256,13 +391,14 @@ int anetWrite(int fd, char *buf, int count) return totlen; } -static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { +static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) { if (bind(s,sa,len) == -1) { anetSetError(err, "bind: %s", strerror(errno)); close(s); return ANET_ERR; } - if (listen(s, 511) == -1) { /* the magic 511 constant is from nginx */ + + if (listen(s, backlog) == -1) { anetSetError(err, "listen: %s", strerror(errno)); close(s); return ANET_ERR; @@ -270,29 +406,64 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { return ANET_OK; } -int anetTcpServer(char *err, int port, char *bindaddr) -{ - int s; - struct sockaddr_in sa; - - if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) - return ANET_ERR; - - memset(&sa,0,sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_port = htons(port); - sa.sin_addr.s_addr = htonl(INADDR_ANY); - if (bindaddr && inet_aton(bindaddr, &sa.sin_addr) == 0) { - anetSetError(err, "invalid bind address"); +static int anetV6Only(char *err, int s) { + int yes = 1; + if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { + anetSetError(err, "setsockopt: %s", strerror(errno)); close(s); return ANET_ERR; } - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) + return ANET_OK; +} + +static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog) +{ + int s, rv; + char _port[6]; /* strlen("65535") */ + struct addrinfo hints, *servinfo, *p; + + snprintf(_port,6,"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = af; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ + + if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); return ANET_ERR; + } + for (p = servinfo; p != NULL; p = p->ai_next) { + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + + if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) goto error; + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (anetListen(err,s,p->ai_addr,p->ai_addrlen,backlog) == ANET_ERR) goto error; + goto end; + } + if (p == NULL) { + anetSetError(err, "unable to bind socket"); + goto error; + } + +error: + s = ANET_ERR; +end: + freeaddrinfo(servinfo); return s; } -int anetUnixServer(char *err, char *path, mode_t perm) +int anetTcpServer(char *err, int port, char *bindaddr, int backlog) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET, backlog); +} + +int anetTcp6Server(char *err, int port, char *bindaddr, int backlog) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET6, backlog); +} + +int anetUnixServer(char *err, char *path, mode_t perm, int backlog) { int s; struct sockaddr_un sa; @@ -303,7 +474,7 @@ int anetUnixServer(char *err, char *path, mode_t perm) memset(&sa,0,sizeof(sa)); sa.sun_family = AF_LOCAL; strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) + if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa),backlog) == ANET_ERR) return ANET_ERR; if (perm) chmod(sa.sun_path, perm); @@ -327,15 +498,22 @@ static int anetGenericAccept(char *err, int s, struct sockaddr *sa, socklen_t *l return fd; } -int anetTcpAccept(char *err, int s, char *ip, int *port) { +int anetTcpAccept(char *err, int s, char *ip, size_t ip_len, int *port) { int fd; - struct sockaddr_in sa; + struct sockaddr_storage sa; socklen_t salen = sizeof(sa); - if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR) + if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == -1) return ANET_ERR; - if (ip) strcpy(ip,inet_ntoa(sa.sin_addr)); - if (port) *port = ntohs(sa.sin_port); + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } return fd; } @@ -343,23 +521,52 @@ int anetUnixAccept(char *err, int s) { int fd; struct sockaddr_un sa; socklen_t salen = sizeof(sa); - if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR) + if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == -1) return ANET_ERR; return fd; } -int anetPeerToString(int fd, char *ip, int *port) { - struct sockaddr_in sa; +int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { + struct sockaddr_storage sa; socklen_t salen = sizeof(sa); if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) { - *port = 0; + if (port) *port = 0; + ip[0] = '?'; + ip[1] = '\0'; + return -1; + } + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } + return 0; +} + +int anetSockName(int fd, char *ip, size_t ip_len, int *port) { + struct sockaddr_storage sa; + socklen_t salen = sizeof(sa); + + if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { + if (port) *port = 0; ip[0] = '?'; ip[1] = '\0'; return -1; } - if (ip) strcpy(ip,inet_ntoa(sa.sin_addr)); - if (port) *port = ntohs(sa.sin_port); + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } return 0; } diff --git a/src/anet.h b/src/anet.h index 406c578326c..d78b1e7a04e 100644 --- a/src/anet.h +++ b/src/anet.h @@ -1,6 +1,6 @@ /* anet.c -- Basic TCP socket stuff made a bit less boring * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,24 +35,39 @@ #define ANET_ERR -1 #define ANET_ERR_LEN 256 -#if defined(__sun) +/* Flags used with certain functions. */ +#define ANET_NONE 0 +#define ANET_IP_ONLY (1<<0) + +#if defined(__sun) || defined(_AIX) #define AF_LOCAL AF_UNIX #endif +#ifdef _AIX +#undef ip_len +#endif + int anetTcpConnect(char *err, char *addr, int port); int anetTcpNonBlockConnect(char *err, char *addr, int port); int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf); -int anetTcpServer(char *err, int port, char *bindaddr); -int anetUnixServer(char *err, char *path, mode_t perm); -int anetTcpAccept(char *err, int serversock, char *ip, int *port); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); +int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); +int anetTcpServer(char *err, int port, char *bindaddr, int backlog); +int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); +int anetUnixServer(char *err, char *path, mode_t perm, int backlog); +int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); -int anetTcpNoDelay(char *err, int fd); +int anetBlock(char *err, int fd); +int anetEnableTcpNoDelay(char *err, int fd); +int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); -int anetPeerToString(int fd, char *ip, int *port); +int anetSendTimeout(char *err, int fd, long long ms); +int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); +int anetKeepAlive(char *err, int fd, int interval); +int anetSockName(int fd, char *ip, size_t ip_len, int *port); #endif diff --git a/src/aof.c b/src/aof.c index 64cd76d3d76..e0d7019006d 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1,3 +1,32 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" #include "bio.h" #include "rio.h" @@ -12,6 +41,121 @@ void aofUpdateCurrentSize(void); +/* ---------------------------------------------------------------------------- + * AOF rewrite buffer implementation. + * + * The following code implement a simple buffer used in order to accumulate + * changes while the background process is rewriting the AOF file. + * + * We only need to append, but can't just use realloc with a large block + * because 'huge' reallocs are not always handled as one could expect + * (via remapping of pages at OS level) but may involve copying data. + * + * For this reason we use a list of blocks, every block is + * AOF_RW_BUF_BLOCK_SIZE bytes. + * ------------------------------------------------------------------------- */ + +#define AOF_RW_BUF_BLOCK_SIZE (1024*1024*10) /* 10 MB per block */ + +typedef struct aofrwblock { + unsigned long used, free; + char buf[AOF_RW_BUF_BLOCK_SIZE]; +} aofrwblock; + +/* This function free the old AOF rewrite buffer if needed, and initialize + * a fresh new one. It tests for server.aof_rewrite_buf_blocks equal to NULL + * so can be used for the first initialization as well. */ +void aofRewriteBufferReset(void) { + if (server.aof_rewrite_buf_blocks) + listRelease(server.aof_rewrite_buf_blocks); + + server.aof_rewrite_buf_blocks = listCreate(); + listSetFreeMethod(server.aof_rewrite_buf_blocks,zfree); +} + +/* Return the current size of the AOF rewrite buffer. */ +unsigned long aofRewriteBufferSize(void) { + listNode *ln = listLast(server.aof_rewrite_buf_blocks); + aofrwblock *block = ln ? ln->value : NULL; + + if (block == NULL) return 0; + unsigned long size = + (listLength(server.aof_rewrite_buf_blocks)-1) * AOF_RW_BUF_BLOCK_SIZE; + size += block->used; + return size; +} + +/* Append data to the AOF rewrite buffer, allocating new blocks if needed. */ +void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { + listNode *ln = listLast(server.aof_rewrite_buf_blocks); + aofrwblock *block = ln ? ln->value : NULL; + + while(len) { + /* If we already got at least an allocated block, try appending + * at least some piece into it. */ + if (block) { + unsigned long thislen = (block->free < len) ? block->free : len; + if (thislen) { /* The current block is not already full. */ + memcpy(block->buf+block->used, s, thislen); + block->used += thislen; + block->free -= thislen; + s += thislen; + len -= thislen; + } + } + + if (len) { /* First block to allocate, or need another block. */ + int numblocks; + + block = zmalloc(sizeof(*block)); + block->free = AOF_RW_BUF_BLOCK_SIZE; + block->used = 0; + listAddNodeTail(server.aof_rewrite_buf_blocks,block); + + /* Log every time we cross more 10 or 100 blocks, respectively + * as a notice or warning. */ + numblocks = listLength(server.aof_rewrite_buf_blocks); + if (((numblocks+1) % 10) == 0) { + int level = ((numblocks+1) % 100) == 0 ? REDIS_WARNING : + REDIS_NOTICE; + redisLog(level,"Background AOF buffer size: %lu MB", + aofRewriteBufferSize()/(1024*1024)); + } + } + } +} + +/* Write the buffer (possibly composed of multiple blocks) into the specified + * fd. If a short write or any other error happens -1 is returned, + * otherwise the number of bytes written is returned. */ +ssize_t aofRewriteBufferWrite(int fd) { + listNode *ln; + listIter li; + ssize_t count = 0; + + listRewind(server.aof_rewrite_buf_blocks,&li); + while((ln = listNext(&li))) { + aofrwblock *block = listNodeValue(ln); + ssize_t nwritten; + + if (block->used) { + nwritten = write(fd,block->buf,block->used); + if (nwritten != (ssize_t)block->used) { + if (nwritten == 0) errno = EIO; + return -1; + } + count += nwritten; + } + } + return count; +} + +/* ---------------------------------------------------------------------------- + * AOF file implementation + * ------------------------------------------------------------------------- */ + +/* Starts a background task that performs fsync() against the specified + * file descriptor (the one of the AOF file) in another thread. */ void aof_background_fsync(int fd) { bioCreateBackgroundJob(REDIS_BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); } @@ -33,20 +177,20 @@ void stopAppendOnly(void) { redisLog(REDIS_NOTICE,"Killing running AOF rewrite child: %ld", (long) server.aof_child_pid); - if (kill(server.aof_child_pid,SIGKILL) != -1) + if (kill(server.aof_child_pid,SIGUSR1) != -1) wait3(&statloc,0,NULL); /* reset the buffer accumulating changes while the child saves */ - sdsfree(server.aof_rewrite_buf); - server.aof_rewrite_buf = sdsempty(); + aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; + server.aof_rewrite_time_start = -1; } } /* Called when the user switches from "appendonly no" to "appendonly yes" * at runtime using the CONFIG command. */ int startAppendOnly(void) { - server.aof_last_fsync = time(NULL); + server.aof_last_fsync = server.unixtime; server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT,0644); redisAssert(server.aof_state == REDIS_AOF_OFF); if (server.aof_fd == -1) { @@ -58,7 +202,7 @@ int startAppendOnly(void) { redisLog(REDIS_WARNING,"Redis needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error."); return REDIS_ERR; } - /* We correctly switched on AOF, now wait for the rerwite to be complete + /* We correctly switched on AOF, now wait for the rewrite to be complete * in order to append data on disk. */ server.aof_state = REDIS_AOF_WAIT_REWRITE; return REDIS_OK; @@ -82,9 +226,11 @@ int startAppendOnly(void) { * * However if force is set to 1 we'll write regardless of the background * fsync. */ +#define AOF_WRITE_LOG_ERROR_RATE 30 /* Seconds between errors logging. */ void flushAppendOnlyFile(int force) { ssize_t nwritten; int sync_in_progress = 0; + mstime_t latency; if (sdslen(server.aof_buf) == 0) return; @@ -97,7 +243,7 @@ void flushAppendOnlyFile(int force) { * the write for a couple of seconds. */ if (sync_in_progress) { if (server.aof_flush_postponed_start == 0) { - /* No previous write postponinig, remember that we are + /* No previous write postponing, remember that we are * postponing the flush and return. */ server.aof_flush_postponed_start = server.unixtime; return; @@ -108,34 +254,107 @@ void flushAppendOnlyFile(int force) { } /* Otherwise fall trough, and go write since we can't wait * over two seconds. */ + server.aof_delayed_fsync++; redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis."); } } - /* If you are following this code path, then we are going to write so - * set reset the postponed flush sentinel to zero. */ - server.aof_flush_postponed_start = 0; - /* We want to perform a single write. This should be guaranteed atomic * at least if the filesystem we are writing is a real physical one. * While this will save us against the server being killed I don't think * there is much to do about the whole server stopping for power problems * or alike */ + + latencyStartMonitor(latency); nwritten = write(server.aof_fd,server.aof_buf,sdslen(server.aof_buf)); + latencyEndMonitor(latency); + /* We want to capture different events for delayed writes: + * when the delay happens with a pending fsync, or with a saving child + * active, and when the above two conditions are missing. + * We also use an additional event name to save all samples which is + * useful for graphing / monitoring purposes. */ + if (sync_in_progress) { + latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); + } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + latencyAddSampleIfNeeded("aof-write-active-child",latency); + } else { + latencyAddSampleIfNeeded("aof-write-alone",latency); + } + latencyAddSampleIfNeeded("aof-write",latency); + + /* We performed the write so reset the postponed flush sentinel to zero. */ + server.aof_flush_postponed_start = 0; + if (nwritten != (signed)sdslen(server.aof_buf)) { - /* Ooops, we are in troubles. The best thing to do for now is - * aborting instead of giving the illusion that everything is - * working as expected. */ + static time_t last_write_error_log = 0; + int can_log = 0; + + /* Limit logging rate to 1 line per AOF_WRITE_LOG_ERROR_RATE seconds. */ + if ((server.unixtime - last_write_error_log) > AOF_WRITE_LOG_ERROR_RATE) { + can_log = 1; + last_write_error_log = server.unixtime; + } + + /* Log the AOF write error and record the error code. */ if (nwritten == -1) { - redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno)); + if (can_log) { + redisLog(REDIS_WARNING,"Error writing to the AOF file: %s", + strerror(errno)); + server.aof_last_write_errno = errno; + } } else { - redisLog(REDIS_WARNING,"Exiting on short write while writing to " - "the append-only file: %s (nwritten=%ld, " - "expected=%ld)", - strerror(errno), - (long)nwritten, - (long)sdslen(server.aof_buf)); + if (can_log) { + redisLog(REDIS_WARNING,"Short write while writing to " + "the AOF file: (nwritten=%lld, " + "expected=%lld)", + (long long)nwritten, + (long long)sdslen(server.aof_buf)); + } + + if (ftruncate(server.aof_fd, server.aof_current_size) == -1) { + if (can_log) { + redisLog(REDIS_WARNING, "Could not remove short write " + "from the append-only file. Redis may refuse " + "to load the AOF the next time it starts. " + "ftruncate: %s", strerror(errno)); + } + } else { + /* If the ftruncate() succeeded we can set nwritten to + * -1 since there is no longer partial data into the AOF. */ + nwritten = -1; + } + server.aof_last_write_errno = ENOSPC; + } + + /* Handle the AOF write error. */ + if (server.aof_fsync == AOF_FSYNC_ALWAYS) { + /* We can't recover when the fsync policy is ALWAYS since the + * reply for the client is already in the output buffers, and we + * have the contract with the user that on acknowledged write data + * is synced on disk. */ + redisLog(REDIS_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting..."); + exit(1); + } else { + /* Recover from failed write leaving data into the buffer. However + * set an error to stop accepting writes as long as the error + * condition is not cleared. */ + server.aof_last_write_status = REDIS_ERR; + + /* Trim the sds buffer if there was a partial write, and there + * was no way to undo it with ftruncate(2). */ + if (nwritten > 0) { + server.aof_current_size += nwritten; + sdsrange(server.aof_buf,nwritten,-1); + } + return; /* We'll try again on the next call... */ + } + } else { + /* Successful write(2). If AOF was in error state, restore the + * OK state and log the event. */ + if (server.aof_last_write_status == REDIS_ERR) { + redisLog(REDIS_WARNING, + "AOF write error looks solved, Redis can write again."); + server.aof_last_write_status = REDIS_OK; } - exit(1); } server.aof_current_size += nwritten; @@ -158,7 +377,10 @@ void flushAppendOnlyFile(int force) { if (server.aof_fsync == AOF_FSYNC_ALWAYS) { /* aof_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ + latencyStartMonitor(latency); aof_fsync(server.aof_fd); /* Let's try to get this data on the disk */ + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-fsync-always",latency); server.aof_last_fsync = server.unixtime; } else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC && server.unixtime > server.aof_last_fsync)) { @@ -203,7 +425,7 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r long long when; robj *argv[3]; - /* Make sure we can use strtol */ + /* Make sure we can use strtoll */ seconds = getDecodedObject(seconds); when = strtoll(seconds->ptr,NULL,10); /* Convert argument into milliseconds for EXPIRE, SETEX, EXPIREAT */ @@ -233,8 +455,8 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a sds buf = sdsempty(); robj *tmpargv[3]; - /* The DB this command was targetting is not the same as the last command - * we appendend. To issue a SELECT command is needed. */ + /* The DB this command was targeting is not the same as the last command + * we appended. To issue a SELECT command is needed. */ if (dictid != server.aof_selected_db) { char seldb[64]; @@ -274,11 +496,15 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a * in a buffer, so that when the child process will do its work we * can append the differences to the new append only file. */ if (server.aof_child_pid != -1) - server.aof_rewrite_buf = sdscatlen(server.aof_rewrite_buf,buf,sdslen(buf)); + aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf)); sdsfree(buf); } +/* ---------------------------------------------------------------------------- + * AOF loading + * ------------------------------------------------------------------------- */ + /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ struct redisClient *createFakeClient(void) { @@ -286,7 +512,9 @@ struct redisClient *createFakeClient(void) { selectDb(c,0); c->fd = -1; + c->name = NULL; c->querybuf = sdsempty(); + c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; c->bufpos = 0; @@ -298,12 +526,21 @@ struct redisClient *createFakeClient(void) { c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); - listSetFreeMethod(c->reply,decrRefCount); + c->peerid = NULL; + listSetFreeMethod(c->reply,decrRefCountVoid); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); return c; } +void freeFakeClientArgv(struct redisClient *c) { + int j; + + for (j = 0; j < c->argc; j++) + decrRefCount(c->argv[j]); + zfree(c->argv); +} + void freeFakeClient(struct redisClient *c) { sdsfree(c->querybuf); listRelease(c->reply); @@ -312,7 +549,7 @@ void freeFakeClient(struct redisClient *c) { zfree(c); } -/* Replay the append log file. On error REDIS_OK is returned. On non fatal +/* Replay the append log file. On success REDIS_OK is returned. On non fatal * error (the append only file is zero-length) REDIS_ERR is returned. On * fatal error an error message is logged and the program exists. */ int loadAppendOnlyFile(char *filename) { @@ -321,6 +558,7 @@ int loadAppendOnlyFile(char *filename) { struct redis_stat sb; int old_aof_state = server.aof_state; long loops = 0; + off_t valid_up_to = 0; /* Offset of the latest well-formed command loaded. */ if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) { server.aof_current_size = 0; @@ -351,7 +589,7 @@ int loadAppendOnlyFile(char *filename) { /* Serve the clients from time to time */ if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); - aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); + processEventsWhileBlocked(); } if (fgets(buf,sizeof(buf),fp) == NULL) { @@ -361,29 +599,45 @@ int loadAppendOnlyFile(char *filename) { goto readerr; } if (buf[0] != '*') goto fmterr; + if (buf[1] == '\0') goto readerr; argc = atoi(buf+1); if (argc < 1) goto fmterr; argv = zmalloc(sizeof(robj*)*argc); + fakeClient->argc = argc; + fakeClient->argv = argv; + for (j = 0; j < argc; j++) { - if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr; + if (fgets(buf,sizeof(buf),fp) == NULL) { + fakeClient->argc = j; /* Free up to j-1. */ + freeFakeClientArgv(fakeClient); + goto readerr; + } if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); argsds = sdsnewlen(NULL,len); - if (len && fread(argsds,len,1,fp) == 0) goto fmterr; + if (len && fread(argsds,len,1,fp) == 0) { + sdsfree(argsds); + fakeClient->argc = j; /* Free up to j-1. */ + freeFakeClientArgv(fakeClient); + goto readerr; + } argv[j] = createObject(REDIS_STRING,argsds); - if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */ + if (fread(buf,2,1,fp) == 0) { + fakeClient->argc = j+1; /* Free up to j. */ + freeFakeClientArgv(fakeClient); + goto readerr; /* discard CRLF */ + } } /* Command lookup */ cmd = lookupCommand(argv[0]->ptr); if (!cmd) { - redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr); + redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", (char*)argv[0]->ptr); exit(1); } + /* Run the command in the context of a fake client */ - fakeClient->argc = argc; - fakeClient->argv = argv; cmd->proc(fakeClient); /* The fake client should not have a reply */ @@ -393,15 +647,15 @@ int loadAppendOnlyFile(char *filename) { /* Clean up. Command code may have changed argv/argc so we use the * argv/argc of the client instead of the local variables. */ - for (j = 0; j < fakeClient->argc; j++) - decrRefCount(fakeClient->argv[j]); - zfree(fakeClient->argv); + freeFakeClientArgv(fakeClient); + if (server.aof_load_truncated) valid_up_to = ftello(fp); } /* This point can only be reached when EOF is reached without errors. * If the client is in the middle of a MULTI/EXEC, log error and quit. */ - if (fakeClient->flags & REDIS_MULTI) goto readerr; + if (fakeClient->flags & REDIS_MULTI) goto uxeof; +loaded_ok: /* DB loaded, cleanup and return REDIS_OK to the caller. */ fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; @@ -410,18 +664,49 @@ int loadAppendOnlyFile(char *filename) { server.aof_rewrite_base_size = server.aof_current_size; return REDIS_OK; -readerr: - if (feof(fp)) { - redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file"); - } else { +readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ + if (!feof(fp)) { redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); + exit(1); } + +uxeof: /* Unexpected AOF end of file. */ + if (server.aof_load_truncated) { + redisLog(REDIS_WARNING,"!!! Warning: short read while loading the AOF file !!!"); + redisLog(REDIS_WARNING,"!!! Truncating the AOF at offset %llu !!!", + (unsigned long long) valid_up_to); + if (valid_up_to == -1 || truncate(filename,valid_up_to) == -1) { + if (valid_up_to == -1) { + redisLog(REDIS_WARNING,"Last valid command offset is invalid"); + } else { + redisLog(REDIS_WARNING,"Error truncating the AOF file: %s", + strerror(errno)); + } + } else { + /* Make sure the AOF file descriptor points to the end of the + * file after the truncate call. */ + if (server.aof_fd != -1 && lseek(server.aof_fd,0,SEEK_END) == -1) { + redisLog(REDIS_WARNING,"Can't seek the end of the AOF file: %s", + strerror(errno)); + } else { + redisLog(REDIS_WARNING, + "AOF loaded anyway because aof-load-truncated is enabled"); + goto loaded_ok; + } + } + } + redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); -fmterr: + +fmterr: /* Format error. */ redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); exit(1); } +/* ---------------------------------------------------------------------------- + * AOF rewrite + * ------------------------------------------------------------------------- */ + /* Delegate writing an object to writing a bulk string or bulk long long. * This is not placed in rio.c since that adds the redis.h dependency. */ int rioWriteBulkObject(rio *r, robj *obj) { @@ -609,7 +894,7 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { return 1; } -/* Write either the key or the value of the currently selected item of an hash. +/* Write either the key or the value of the currently selected item of a hash. * The 'hi' argument passes a valid Redis hash iterator. * The 'what' filed specifies if to write a key or a value and can be * either REDIS_HASH_KEY or REDIS_HASH_VALUE. @@ -693,6 +978,8 @@ int rewriteAppendOnlyFile(char *filename) { } rioInitWithFile(&aof,fp); + if (server.aof_rewrite_incremental_fsync) + rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES); for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; redisDb *db = server.db+j; @@ -720,6 +1007,9 @@ int rewriteAppendOnlyFile(char *filename) { expiretime = getExpire(db,&key); + /* If this key is already expired skip it */ + if (expiretime != -1 && expiretime < now) continue; + /* Save the key and associated value */ if (o->type == REDIS_STRING) { /* Emit a SET command */ @@ -742,20 +1032,19 @@ int rewriteAppendOnlyFile(char *filename) { /* Save the expire time */ if (expiretime != -1) { char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n"; - /* If this key is already expired skip it */ - if (expiretime < now) continue; if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr; } } dictReleaseIterator(di); + di = NULL; } /* Make sure data will not remain on the OS's output buffers */ - fflush(fp); - aof_fsync(fileno(fp)); - fclose(fp); + if (fflush(fp) == EOF) goto werr; + if (fsync(fileno(fp)) == -1) goto werr; + if (fclose(fp) == EOF) goto werr; /* Use RENAME to make sure the DB file is changed atomically only * if the generate DB file is ok. */ @@ -768,9 +1057,9 @@ int rewriteAppendOnlyFile(char *filename) { return REDIS_OK; werr: + redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); - redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); if (di) dictReleaseIterator(di); return REDIS_ERR; } @@ -797,17 +1086,26 @@ int rewriteAppendOnlyFileBackground(void) { char tmpfile[256]; /* Child */ - if (server.ipfd > 0) close(server.ipfd); - if (server.sofd > 0) close(server.sofd); + closeListeningSockets(0); + redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) { - _exit(0); + size_t private_dirty = zmalloc_get_private_dirty(); + + if (private_dirty) { + redisLog(REDIS_NOTICE, + "AOF rewrite: %zu MB of memory used by copy-on-write", + private_dirty/(1024*1024)); + } + exitFromChild(0); } else { - _exit(1); + exitFromChild(1); } } else { /* Parent */ server.stat_fork_time = ustime()-start; + server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ + latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { redisLog(REDIS_WARNING, "Can't rewrite append only file in background: fork: %s", @@ -817,6 +1115,7 @@ int rewriteAppendOnlyFileBackground(void) { redisLog(REDIS_NOTICE, "Background append only file rewriting started by pid %d",childpid); server.aof_rewrite_scheduled = 0; + server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the @@ -824,6 +1123,7 @@ int rewriteAppendOnlyFileBackground(void) { * accumulated by the parent into server.aof_rewrite_buf will start * with a SELECT statement and it will be safe to merge. */ server.aof_selected_db = -1; + replicationScriptCacheFlush(); return REDIS_OK; } return REDIS_OK; /* unreached */ @@ -849,19 +1149,23 @@ void aofRemoveTempFile(pid_t childpid) { unlink(tmpfile); } -/* Update the server.aof_current_size filed explicitly using stat(2) +/* Update the server.aof_current_size field explicitly using stat(2) * to check the size of the file. This is useful after a rewrite or after * a restart, normally the size is updated just adding the write length * to the current length, that is much faster. */ void aofUpdateCurrentSize(void) { struct redis_stat sb; + mstime_t latency; + latencyStartMonitor(latency); if (redis_fstat(server.aof_fd,&sb) == -1) { redisLog(REDIS_WARNING,"Unable to obtain the AOF file length. stat: %s", strerror(errno)); } else { server.aof_current_size = sb.st_size; } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-fstat",latency); } /* A background append only file rewriting (BGREWRITEAOF) terminated its work. @@ -869,15 +1173,16 @@ void aofUpdateCurrentSize(void) { void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (!bysignal && exitcode == 0) { int newfd, oldfd; - int nwritten; char tmpfile[256]; long long now = ustime(); + mstime_t latency; redisLog(REDIS_NOTICE, "Background AOF rewrite terminated with success"); /* Flush the differences accumulated by the parent to the * rewritten AOF. */ + latencyStartMonitor(latency); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int)server.aof_child_pid); newfd = open(tmpfile,O_WRONLY|O_APPEND); @@ -887,21 +1192,17 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { goto cleanup; } - nwritten = write(newfd,server.aof_rewrite_buf,sdslen(server.aof_rewrite_buf)); - if (nwritten != (signed)sdslen(server.aof_rewrite_buf)) { - if (nwritten == -1) { - redisLog(REDIS_WARNING, - "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); - } else { - redisLog(REDIS_WARNING, - "Short write trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); - } + if (aofRewriteBufferWrite(newfd) == -1) { + redisLog(REDIS_WARNING, + "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); close(newfd); goto cleanup; } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency); redisLog(REDIS_NOTICE, - "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", nwritten); + "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", aofRewriteBufferSize()); /* The only remaining thing to do is to rename the temporary file to * the configured file and switch the file descriptor used to do AOF @@ -944,6 +1245,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { /* Rename the temporary file. This will not unlink the target file if * it exists, because we reference it with "oldfd". */ + latencyStartMonitor(latency); if (rename(tmpfile,server.aof_filename) == -1) { redisLog(REDIS_WARNING, "Error trying to rename the temporary AOF file: %s", strerror(errno)); @@ -951,6 +1253,8 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (oldfd != -1) close(oldfd); goto cleanup; } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rename",latency); if (server.aof_fd == -1) { /* AOF disabled, we don't need to set the AOF file descriptor @@ -974,6 +1278,8 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { server.aof_buf = sdsempty(); } + server.aof_lastbgrewrite_status = REDIS_OK; + redisLog(REDIS_NOTICE, "Background AOF rewrite finished successfully"); /* Change state from WAIT_REWRITE to ON if needed */ if (server.aof_state == REDIS_AOF_WAIT_REWRITE) @@ -985,18 +1291,23 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { redisLog(REDIS_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); } else if (!bysignal && exitcode != 0) { + server.aof_lastbgrewrite_status = REDIS_ERR; + redisLog(REDIS_WARNING, "Background AOF rewrite terminated with error"); } else { + server.aof_lastbgrewrite_status = REDIS_ERR; + redisLog(REDIS_WARNING, "Background AOF rewrite terminated by signal %d", bysignal); } cleanup: - sdsfree(server.aof_rewrite_buf); - server.aof_rewrite_buf = sdsempty(); + aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; + server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start; + server.aof_rewrite_time_start = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ if (server.aof_state == REDIS_AOF_WAIT_REWRITE) server.aof_rewrite_scheduled = 1; diff --git a/src/asciilogo.h b/src/asciilogo.h index 878a107006b..83c538b54e5 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -1,3 +1,32 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" diff --git a/src/bio.c b/src/bio.c index eaac8e40d55..4bd5a17c613 100644 --- a/src/bio.c +++ b/src/bio.c @@ -26,11 +26,42 @@ * * Currently there is no way for the creator of the job to be notified about * the completion of the operation, this will only be added when/if needed. + * + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + #include "redis.h" #include "bio.h" +static pthread_t bio_threads[REDIS_BIO_NUM_OPS]; static pthread_mutex_t bio_mutex[REDIS_BIO_NUM_OPS]; static pthread_cond_t bio_condvar[REDIS_BIO_NUM_OPS]; static list *bio_jobs[REDIS_BIO_NUM_OPS]; @@ -43,7 +74,7 @@ static list *bio_jobs[REDIS_BIO_NUM_OPS]; static unsigned long long bio_pending[REDIS_BIO_NUM_OPS]; /* This structure represents a background Job. It is only used locally to this - * file as the API deos not expose the internals at all. */ + * file as the API does not expose the internals at all. */ struct bio_job { time_t time; /* Time at which the job was created. */ /* Job specific arguments pointers. If we need to pass more than three @@ -88,6 +119,7 @@ void bioInit(void) { redisLog(REDIS_WARNING,"Fatal: Can't initialize Background Jobs."); exit(1); } + bio_threads[j] = thread; } } @@ -108,9 +140,22 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { void *bioProcessBackgroundJobs(void *arg) { struct bio_job *job; unsigned long type = (unsigned long) arg; + sigset_t sigset; + + /* Make the thread killable at any time, so that bioKillThreads() + * can work reliably. */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); - pthread_detach(pthread_self()); pthread_mutex_lock(&bio_mutex[type]); + /* Block SIGALRM so we are sure that only the main thread will + * receive the watchdog signal. */ + sigemptyset(&sigset); + sigaddset(&sigset, SIGALRM); + if (pthread_sigmask(SIG_BLOCK, &sigset, NULL)) + redisLog(REDIS_WARNING, + "Warning: can't mask SIGALRM in bio.c thread: %s", strerror(errno)); + while(1) { listNode *ln; @@ -153,56 +198,23 @@ unsigned long long bioPendingJobsOfType(int type) { return val; } -#if 0 /* We don't use the following code for now, and bioWaitPendingJobsLE - probably needs a rewrite using conditional variables instead of the - current implementation. */ - - -/* Wait until the number of pending jobs of the specified type are - * less or equal to the specified number. - * - * This function may block for long time, it should only be used to perform - * the following tasks: - * - * 1) To avoid that the main thread is pushing jobs of a given time so fast - * that the background thread can't process them at the same speed. - * So before creating a new job of a given type the main thread should - * call something like: bioWaitPendingJobsLE(job_type,10000); - * 2) In order to perform special operations that make it necessary to be sure - * no one is touching shared resourced in the background. - */ -void bioWaitPendingJobsLE(int type, unsigned long long num) { - unsigned long long iteration = 0; +/* Kill the running bio threads in an unclean way. This function should be + * used only when it's critical to stop the threads for some reason. + * Currently Redis does this only on crash (for instance on SIGSEGV) in order + * to perform a fast memory check without other threads messing with memory. */ +void bioKillThreads(void) { + int err, j; - /* We poll the jobs queue aggressively to start, and gradually relax - * the polling speed if it is going to take too much time. */ - while(1) { - iteration++; - if (iteration > 1000 && iteration <= 10000) { - usleep(100); - } else if (iteration > 10000) { - usleep(1000); + for (j = 0; j < REDIS_BIO_NUM_OPS; j++) { + if (pthread_cancel(bio_threads[j]) == 0) { + if ((err = pthread_join(bio_threads[j],NULL)) != 0) { + redisLog(REDIS_WARNING, + "Bio thread for job type #%d can be joined: %s", + j, strerror(err)); + } else { + redisLog(REDIS_WARNING, + "Bio thread for job type #%d terminated",j); + } } - if (bioPendingJobsOfType(type) <= num) break; } } - -/* Return the older job of the specified type. */ -time_t bioOlderJobOfType(int type) { - time_t time; - listNode *ln; - struct bio_job *job; - - pthread_mutex_lock(&bio_mutex[type]); - ln = listFirst(bio_jobs[type]); - if (ln == NULL) { - pthread_mutex_unlock(&bio_mutex[type]); - return 0; - } - job = ln->value; - time = job->time; - pthread_mutex_unlock(&bio_mutex[type]); - return time; -} - -#endif diff --git a/src/bio.h b/src/bio.h index 22a9b33e16c..85f03ad1af3 100644 --- a/src/bio.h +++ b/src/bio.h @@ -1,9 +1,39 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + /* Exported API */ void bioInit(void); void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); unsigned long long bioPendingJobsOfType(int type); void bioWaitPendingJobsLE(int type, unsigned long long num); time_t bioOlderJobOfType(int type); +void bioKillThreads(void); /* Background job opcodes */ #define REDIS_BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */ diff --git a/src/bitops.c b/src/bitops.c new file mode 100644 index 00000000000..209f9b0fcff --- /dev/null +++ b/src/bitops.c @@ -0,0 +1,588 @@ +/* Bit operations. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "redis.h" + +/* ----------------------------------------------------------------------------- + * Helpers and low level bit functions. + * -------------------------------------------------------------------------- */ + +/* This helper function used by GETBIT / SETBIT parses the bit offset argument + * making sure an error is returned if it is negative or if it overflows + * Redis 512 MB limit for the string value. */ +static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { + long long loffset; + char *err = "bit offset is not an integer or out of range"; + + if (getLongLongFromObjectOrReply(c,o,&loffset,err) != REDIS_OK) + return REDIS_ERR; + + /* Limit offset to 512MB in bytes */ + if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) + { + addReplyError(c,err); + return REDIS_ERR; + } + + *offset = (size_t)loffset; + return REDIS_OK; +} + +/* Count number of bits set in the binary array pointed by 's' and long + * 'count' bytes. The implementation of this function is required to + * work with a input string length up to 512 MB. */ +size_t redisPopcount(void *s, long count) { + size_t bits = 0; + unsigned char *p = s; + uint32_t *p4; + static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; + + /* Count initial bytes not aligned to 32 bit. */ + while((unsigned long)p & 3 && count) { + bits += bitsinbyte[*p++]; + count--; + } + + /* Count bits 16 bytes at a time */ + p4 = (uint32_t*)p; + while(count>=16) { + uint32_t aux1, aux2, aux3, aux4; + + aux1 = *p4++; + aux2 = *p4++; + aux3 = *p4++; + aux4 = *p4++; + count -= 16; + + aux1 = aux1 - ((aux1 >> 1) & 0x55555555); + aux1 = (aux1 & 0x33333333) + ((aux1 >> 2) & 0x33333333); + aux2 = aux2 - ((aux2 >> 1) & 0x55555555); + aux2 = (aux2 & 0x33333333) + ((aux2 >> 2) & 0x33333333); + aux3 = aux3 - ((aux3 >> 1) & 0x55555555); + aux3 = (aux3 & 0x33333333) + ((aux3 >> 2) & 0x33333333); + aux4 = aux4 - ((aux4 >> 1) & 0x55555555); + aux4 = (aux4 & 0x33333333) + ((aux4 >> 2) & 0x33333333); + bits += ((((aux1 + (aux1 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux2 + (aux2 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux3 + (aux3 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux4 + (aux4 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24); + } + /* Count the remaining bytes. */ + p = (unsigned char*)p4; + while(count--) bits += bitsinbyte[*p++]; + return bits; +} + +/* Return the position of the first bit set to one (if 'bit' is 1) or + * zero (if 'bit' is 0) in the bitmap starting at 's' and long 'count' bytes. + * + * The function is guaranteed to return a value >= 0 if 'bit' is 0 since if + * no zero bit is found, it returns count*8 assuming the string is zero + * padded on the right. However if 'bit' is 1 it is possible that there is + * not a single set bit in the bitmap. In this special case -1 is returned. */ +long redisBitpos(void *s, unsigned long count, int bit) { + unsigned long *l; + unsigned char *c; + unsigned long skipval, word = 0, one; + long pos = 0; /* Position of bit, to return to the caller. */ + unsigned long j; + + /* Process whole words first, seeking for first word that is not + * all ones or all zeros respectively if we are lookig for zeros + * or ones. This is much faster with large strings having contiguous + * blocks of 1 or 0 bits compared to the vanilla bit per bit processing. + * + * Note that if we start from an address that is not aligned + * to sizeof(unsigned long) we consume it byte by byte until it is + * aligned. */ + + /* Skip initial bits not aligned to sizeof(unsigned long) byte by byte. */ + skipval = bit ? 0 : UCHAR_MAX; + c = (unsigned char*) s; + while((unsigned long)c & (sizeof(*l)-1) && count) { + if (*c != skipval) break; + c++; + count--; + pos += 8; + } + + /* Skip bits with full word step. */ + skipval = bit ? 0 : ULONG_MAX; + l = (unsigned long*) c; + while (count >= sizeof(*l)) { + if (*l != skipval) break; + l++; + count -= sizeof(*l); + pos += sizeof(*l)*8; + } + + /* Load bytes into "word" considering the first byte as the most significant + * (we basically consider it as written in big endian, since we consider the + * string as a set of bits from left to right, with the first bit at position + * zero. + * + * Note that the loading is designed to work even when the bytes left + * (count) are less than a full word. We pad it with zero on the right. */ + c = (unsigned char*)l; + for (j = 0; j < sizeof(*l); j++) { + word <<= 8; + if (count) { + word |= *c; + c++; + count--; + } + } + + /* Special case: + * If bits in the string are all zero and we are looking for one, + * return -1 to signal that there is not a single "1" in the whole + * string. This can't happen when we are looking for "0" as we assume + * that the right of the string is zero padded. */ + if (bit == 1 && word == 0) return -1; + + /* Last word left, scan bit by bit. The first thing we need is to + * have a single "1" set in the most significant position in an + * unsigned long. We don't know the size of the long so we use a + * simple trick. */ + one = ULONG_MAX; /* All bits set to 1.*/ + one >>= 1; /* All bits set to 1 but the MSB. */ + one = ~one; /* All bits set to 0 but the MSB. */ + + while(one) { + if (((one & word) != 0) == bit) return pos; + pos++; + one >>= 1; + } + + /* If we reached this point, there is a bug in the algorithm, since + * the case of no match is handled as a special case before. */ + redisPanic("End of redisBitpos() reached."); + return 0; /* Just to avoid warnings. */ +} + +/* ----------------------------------------------------------------------------- + * Bits related string commands: GETBIT, SETBIT, BITCOUNT, BITOP. + * -------------------------------------------------------------------------- */ + +#define BITOP_AND 0 +#define BITOP_OR 1 +#define BITOP_XOR 2 +#define BITOP_NOT 3 + +/* SETBIT key offset bitvalue */ +void setbitCommand(redisClient *c) { + robj *o; + char *err = "bit is not an integer or out of range"; + size_t bitoffset; + int byte, bit; + int byteval, bitval; + long on; + + if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) + return; + + if (getLongFromObjectOrReply(c,c->argv[3],&on,err) != REDIS_OK) + return; + + /* Bits can only be set or cleared... */ + if (on & ~1) { + addReplyError(c,err); + return; + } + + o = lookupKeyWrite(c->db,c->argv[1]); + if (o == NULL) { + o = createObject(REDIS_STRING,sdsempty()); + dbAdd(c->db,c->argv[1],o); + } else { + if (checkType(c,o,REDIS_STRING)) return; + o = dbUnshareStringValue(c->db,c->argv[1],o); + } + + /* Grow sds value to the right length if necessary */ + byte = bitoffset >> 3; + o->ptr = sdsgrowzero(o->ptr,byte+1); + + /* Get current values */ + byteval = ((uint8_t*)o->ptr)[byte]; + bit = 7 - (bitoffset & 0x7); + bitval = byteval & (1 << bit); + + /* Update byte with new bit value and return original value */ + byteval &= ~(1 << bit); + byteval |= ((on & 0x1) << bit); + ((uint8_t*)o->ptr)[byte] = byteval; + signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"setbit",c->argv[1],c->db->id); + server.dirty++; + addReply(c, bitval ? shared.cone : shared.czero); +} + +/* GETBIT key offset */ +void getbitCommand(redisClient *c) { + robj *o; + char llbuf[32]; + size_t bitoffset; + size_t byte, bit; + size_t bitval = 0; + + if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) + return; + + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,o,REDIS_STRING)) return; + + byte = bitoffset >> 3; + bit = 7 - (bitoffset & 0x7); + if (o->encoding != REDIS_ENCODING_RAW) { + if (byte < (size_t)ll2string(llbuf,sizeof(llbuf),(long)o->ptr)) + bitval = llbuf[byte] & (1 << bit); + } else { + if (byte < sdslen(o->ptr)) + bitval = ((uint8_t*)o->ptr)[byte] & (1 << bit); + } + + addReply(c, bitval ? shared.cone : shared.czero); +} + +/* BITOP op_name target_key src_key1 src_key2 src_key3 ... src_keyN */ +void bitopCommand(redisClient *c) { + char *opname = c->argv[1]->ptr; + robj *o, *targetkey = c->argv[2]; + unsigned long op, j, numkeys; + robj **objects; /* Array of source objects. */ + unsigned char **src; /* Array of source strings pointers. */ + unsigned long *len, maxlen = 0; /* Array of length of src strings, + and max len. */ + unsigned long minlen = 0; /* Min len among the input keys. */ + unsigned char *res = NULL; /* Resulting string. */ + + /* Parse the operation name. */ + if ((opname[0] == 'a' || opname[0] == 'A') && !strcasecmp(opname,"and")) + op = BITOP_AND; + else if((opname[0] == 'o' || opname[0] == 'O') && !strcasecmp(opname,"or")) + op = BITOP_OR; + else if((opname[0] == 'x' || opname[0] == 'X') && !strcasecmp(opname,"xor")) + op = BITOP_XOR; + else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not")) + op = BITOP_NOT; + else { + addReply(c,shared.syntaxerr); + return; + } + + /* Sanity check: NOT accepts only a single key argument. */ + if (op == BITOP_NOT && c->argc != 4) { + addReplyError(c,"BITOP NOT must be called with a single source key."); + return; + } + + /* Lookup keys, and store pointers to the string objects into an array. */ + numkeys = c->argc - 3; + src = zmalloc(sizeof(unsigned char*) * numkeys); + len = zmalloc(sizeof(long) * numkeys); + objects = zmalloc(sizeof(robj*) * numkeys); + for (j = 0; j < numkeys; j++) { + o = lookupKeyRead(c->db,c->argv[j+3]); + /* Handle non-existing keys as empty strings. */ + if (o == NULL) { + objects[j] = NULL; + src[j] = NULL; + len[j] = 0; + minlen = 0; + continue; + } + /* Return an error if one of the keys is not a string. */ + if (checkType(c,o,REDIS_STRING)) { + unsigned long i; + for (i = 0; i < j; i++) { + if (objects[i]) + decrRefCount(objects[i]); + } + zfree(src); + zfree(len); + zfree(objects); + return; + } + objects[j] = getDecodedObject(o); + src[j] = objects[j]->ptr; + len[j] = sdslen(objects[j]->ptr); + if (len[j] > maxlen) maxlen = len[j]; + if (j == 0 || len[j] < minlen) minlen = len[j]; + } + + /* Compute the bit operation, if at least one string is not empty. */ + if (maxlen) { + res = (unsigned char*) sdsnewlen(NULL,maxlen); + unsigned char output, byte; + unsigned long i; + + /* Fast path: as far as we have data for all the input bitmaps we + * can take a fast path that performs much better than the + * vanilla algorithm. */ + j = 0; + if (minlen && numkeys <= 16) { + unsigned long *lp[16]; + unsigned long *lres = (unsigned long*) res; + + /* Note: sds pointer is always aligned to 8 byte boundary. */ + memcpy(lp,src,sizeof(unsigned long*)*numkeys); + memcpy(res,src[0],minlen); + + /* Different branches per different operations for speed (sorry). */ + if (op == BITOP_AND) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] &= lp[i][0]; + lres[1] &= lp[i][1]; + lres[2] &= lp[i][2]; + lres[3] &= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_OR) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] |= lp[i][0]; + lres[1] |= lp[i][1]; + lres[2] |= lp[i][2]; + lres[3] |= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_XOR) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] ^= lp[i][0]; + lres[1] ^= lp[i][1]; + lres[2] ^= lp[i][2]; + lres[3] ^= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_NOT) { + while(minlen >= sizeof(unsigned long)*4) { + lres[0] = ~lres[0]; + lres[1] = ~lres[1]; + lres[2] = ~lres[2]; + lres[3] = ~lres[3]; + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } + } + + /* j is set to the next byte to process by the previous loop. */ + for (; j < maxlen; j++) { + output = (len[0] <= j) ? 0 : src[0][j]; + if (op == BITOP_NOT) output = ~output; + for (i = 1; i < numkeys; i++) { + byte = (len[i] <= j) ? 0 : src[i][j]; + switch(op) { + case BITOP_AND: output &= byte; break; + case BITOP_OR: output |= byte; break; + case BITOP_XOR: output ^= byte; break; + } + } + res[j] = output; + } + } + for (j = 0; j < numkeys; j++) { + if (objects[j]) + decrRefCount(objects[j]); + } + zfree(src); + zfree(len); + zfree(objects); + + /* Store the computed value into the target key */ + if (maxlen) { + o = createObject(REDIS_STRING,res); + setKey(c->db,targetkey,o); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",targetkey,c->db->id); + decrRefCount(o); + } else if (dbDelete(c->db,targetkey)) { + signalModifiedKey(c->db,targetkey); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",targetkey,c->db->id); + } + server.dirty++; + addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ +} + +/* BITCOUNT key [start end] */ +void bitcountCommand(redisClient *c) { + robj *o; + long start, end, strlen; + unsigned char *p; + char llbuf[32]; + + /* Lookup, check for type, and return 0 for non existing keys. */ + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,o,REDIS_STRING)) return; + + /* Set the 'p' pointer to the string, that can be just a stack allocated + * array if our string was integer encoded. */ + if (o->encoding == REDIS_ENCODING_INT) { + p = (unsigned char*) llbuf; + strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr); + } else { + p = (unsigned char*) o->ptr; + strlen = sdslen(o->ptr); + } + + /* Parse start/end range if any. */ + if (c->argc == 4) { + if (getLongFromObjectOrReply(c,c->argv[2],&start,NULL) != REDIS_OK) + return; + if (getLongFromObjectOrReply(c,c->argv[3],&end,NULL) != REDIS_OK) + return; + /* Convert negative indexes */ + if (start < 0) start = strlen+start; + if (end < 0) end = strlen+end; + if (start < 0) start = 0; + if (end < 0) end = 0; + if (end >= strlen) end = strlen-1; + } else if (c->argc == 2) { + /* The whole string. */ + start = 0; + end = strlen-1; + } else { + /* Syntax error. */ + addReply(c,shared.syntaxerr); + return; + } + + /* Precondition: end >= 0 && end < strlen, so the only condition where + * zero can be returned is: start > end. */ + if (start > end) { + addReply(c,shared.czero); + } else { + long bytes = end-start+1; + + addReplyLongLong(c,redisPopcount(p+start,bytes)); + } +} + +/* BITPOS key bit [start [end]] */ +void bitposCommand(redisClient *c) { + robj *o; + long bit, start, end, strlen; + unsigned char *p; + char llbuf[32]; + int end_given = 0; + + /* Parse the bit argument to understand what we are looking for, set + * or clear bits. */ + if (getLongFromObjectOrReply(c,c->argv[2],&bit,NULL) != REDIS_OK) + return; + if (bit != 0 && bit != 1) { + addReplyError(c, "The bit argument must be 1 or 0."); + return; + } + + /* If the key does not exist, from our point of view it is an infinite + * array of 0 bits. If the user is looking for the fist clear bit return 0, + * If the user is looking for the first set bit, return -1. */ + if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { + addReplyLongLong(c, bit ? -1 : 0); + return; + } + if (checkType(c,o,REDIS_STRING)) return; + + /* Set the 'p' pointer to the string, that can be just a stack allocated + * array if our string was integer encoded. */ + if (o->encoding == REDIS_ENCODING_INT) { + p = (unsigned char*) llbuf; + strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr); + } else { + p = (unsigned char*) o->ptr; + strlen = sdslen(o->ptr); + } + + /* Parse start/end range if any. */ + if (c->argc == 4 || c->argc == 5) { + if (getLongFromObjectOrReply(c,c->argv[3],&start,NULL) != REDIS_OK) + return; + if (c->argc == 5) { + if (getLongFromObjectOrReply(c,c->argv[4],&end,NULL) != REDIS_OK) + return; + end_given = 1; + } else { + end = strlen-1; + } + /* Convert negative indexes */ + if (start < 0) start = strlen+start; + if (end < 0) end = strlen+end; + if (start < 0) start = 0; + if (end < 0) end = 0; + if (end >= strlen) end = strlen-1; + } else if (c->argc == 3) { + /* The whole string. */ + start = 0; + end = strlen-1; + } else { + /* Syntax error. */ + addReply(c,shared.syntaxerr); + return; + } + + /* For empty ranges (start > end) we return -1 as an empty range does + * not contain a 0 nor a 1. */ + if (start > end) { + addReplyLongLong(c, -1); + } else { + long bytes = end-start+1; + long pos = redisBitpos(p+start,bytes,bit); + + /* If we are looking for clear bits, and the user specified an exact + * range with start-end, we can't consider the right of the range as + * zero padded (as we do when no explicit end is given). + * + * So if redisBitpos() returns the first bit outside the range, + * we return -1 to the caller, to mean, in the specified range there + * is not a single "0" bit. */ + if (end_given && bit == 0 && pos == bytes*8) { + addReplyLongLong(c,-1); + return; + } + if (pos != -1) pos += start*8; /* Adjust for the bytes we skipped. */ + addReplyLongLong(c,pos); + } +} diff --git a/src/cluster.c b/src/cluster.c deleted file mode 100644 index f76e8ff5cf8..00000000000 --- a/src/cluster.c +++ /dev/null @@ -1,1762 +0,0 @@ -#include "redis.h" - -#include -#include -#include - -void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterSendPing(clusterLink *link, int type); -void clusterSendFail(char *nodename); -void clusterUpdateState(void); -int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(void); -clusterNode *clusterLookupNode(char *name); -int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); -int clusterAddSlot(clusterNode *n, int slot); - -/* ----------------------------------------------------------------------------- - * Initialization - * -------------------------------------------------------------------------- */ - -int clusterLoadConfig(char *filename) { - FILE *fp = fopen(filename,"r"); - char *line; - int maxline, j; - - if (fp == NULL) return REDIS_ERR; - - /* Parse the file. Note that single liens of the cluster config file can - * be really long as they include all the hash slots of the node. - * This means in the worst possible case REDIS_CLUSTER_SLOTS/2 integers. - * To simplify we allocate 1024+REDIS_CLUSTER_SLOTS*16 bytes per line. */ - maxline = 1024+REDIS_CLUSTER_SLOTS*16; - line = zmalloc(maxline); - while(fgets(line,maxline,fp) != NULL) { - int argc; - sds *argv = sdssplitargs(line,&argc); - clusterNode *n, *master; - char *p, *s; - - /* Create this node if it does not exist */ - n = clusterLookupNode(argv[0]); - if (!n) { - n = createClusterNode(argv[0],0); - clusterAddNode(n); - } - /* Address and port */ - if ((p = strchr(argv[1],':')) == NULL) goto fmterr; - *p = '\0'; - memcpy(n->ip,argv[1],strlen(argv[1])+1); - n->port = atoi(p+1); - - /* Parse flags */ - p = s = argv[2]; - while(p) { - p = strchr(s,','); - if (p) *p = '\0'; - if (!strcasecmp(s,"myself")) { - redisAssert(server.cluster.myself == NULL); - server.cluster.myself = n; - n->flags |= REDIS_NODE_MYSELF; - } else if (!strcasecmp(s,"master")) { - n->flags |= REDIS_NODE_MASTER; - } else if (!strcasecmp(s,"slave")) { - n->flags |= REDIS_NODE_SLAVE; - } else if (!strcasecmp(s,"fail?")) { - n->flags |= REDIS_NODE_PFAIL; - } else if (!strcasecmp(s,"fail")) { - n->flags |= REDIS_NODE_FAIL; - } else if (!strcasecmp(s,"handshake")) { - n->flags |= REDIS_NODE_HANDSHAKE; - } else if (!strcasecmp(s,"noaddr")) { - n->flags |= REDIS_NODE_NOADDR; - } else if (!strcasecmp(s,"noflags")) { - /* nothing to do */ - } else { - redisPanic("Unknown flag in redis cluster config file"); - } - if (p) s = p+1; - } - - /* Get master if any. Set the master and populate master's - * slave list. */ - if (argv[3][0] != '-') { - master = clusterLookupNode(argv[3]); - if (!master) { - master = createClusterNode(argv[3],0); - clusterAddNode(master); - } - n->slaveof = master; - clusterNodeAddSlave(master,n); - } - - /* Set ping sent / pong received timestamps */ - if (atoi(argv[4])) n->ping_sent = time(NULL); - if (atoi(argv[5])) n->pong_received = time(NULL); - - /* Populate hash slots served by this instance. */ - for (j = 7; j < argc; j++) { - int start, stop; - - if (argv[j][0] == '[') { - /* Here we handle migrating / importing slots */ - int slot; - char direction; - clusterNode *cn; - - p = strchr(argv[j],'-'); - redisAssert(p != NULL); - *p = '\0'; - direction = p[1]; /* Either '>' or '<' */ - slot = atoi(argv[j]+1); - p += 3; - cn = clusterLookupNode(p); - if (!cn) { - cn = createClusterNode(p,0); - clusterAddNode(cn); - } - if (direction == '>') { - server.cluster.migrating_slots_to[slot] = cn; - } else { - server.cluster.importing_slots_from[slot] = cn; - } - continue; - } else if ((p = strchr(argv[j],'-')) != NULL) { - *p = '\0'; - start = atoi(argv[j]); - stop = atoi(p+1); - } else { - start = stop = atoi(argv[j]); - } - while(start <= stop) clusterAddSlot(n, start++); - } - - sdssplitargs_free(argv,argc); - } - zfree(line); - fclose(fp); - - /* Config sanity check */ - redisAssert(server.cluster.myself != NULL); - redisLog(REDIS_NOTICE,"Node configuration loaded, I'm %.40s", - server.cluster.myself->name); - clusterUpdateState(); - return REDIS_OK; - -fmterr: - redisLog(REDIS_WARNING,"Unrecovarable error: corrupted cluster config file."); - fclose(fp); - exit(1); -} - -/* Cluster node configuration is exactly the same as CLUSTER NODES output. - * - * This function writes the node config and returns 0, on error -1 - * is returned. */ -int clusterSaveConfig(void) { - sds ci = clusterGenNodesDescription(); - int fd; - - if ((fd = open(server.cluster.configfile,O_WRONLY|O_CREAT|O_TRUNC,0644)) - == -1) goto err; - if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; - close(fd); - sdsfree(ci); - return 0; - -err: - sdsfree(ci); - return -1; -} - -void clusterSaveConfigOrDie(void) { - if (clusterSaveConfig() == -1) { - redisLog(REDIS_WARNING,"Fatal: can't update cluster config file."); - exit(1); - } -} - -void clusterInit(void) { - int saveconf = 0; - - server.cluster.myself = NULL; - server.cluster.state = REDIS_CLUSTER_FAIL; - server.cluster.nodes = dictCreate(&clusterNodesDictType,NULL); - server.cluster.node_timeout = 15; - memset(server.cluster.migrating_slots_to,0, - sizeof(server.cluster.migrating_slots_to)); - memset(server.cluster.importing_slots_from,0, - sizeof(server.cluster.importing_slots_from)); - memset(server.cluster.slots,0, - sizeof(server.cluster.slots)); - if (clusterLoadConfig(server.cluster.configfile) == REDIS_ERR) { - /* No configuration found. We will just use the random name provided - * by the createClusterNode() function. */ - server.cluster.myself = createClusterNode(NULL,REDIS_NODE_MYSELF); - redisLog(REDIS_NOTICE,"No cluster configuration found, I'm %.40s", - server.cluster.myself->name); - clusterAddNode(server.cluster.myself); - saveconf = 1; - } - if (saveconf) clusterSaveConfigOrDie(); - /* We need a listening TCP port for our cluster messaging needs */ - server.cfd = anetTcpServer(server.neterr, - server.port+REDIS_CLUSTER_PORT_INCR, server.bindaddr); - if (server.cfd == -1) { - redisLog(REDIS_WARNING, "Opening cluster TCP port: %s", server.neterr); - exit(1); - } - if (aeCreateFileEvent(server.el, server.cfd, AE_READABLE, - clusterAcceptHandler, NULL) == AE_ERR) oom("creating file event"); - server.cluster.slots_to_keys = zslCreate(); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER communication link - * -------------------------------------------------------------------------- */ - -clusterLink *createClusterLink(clusterNode *node) { - clusterLink *link = zmalloc(sizeof(*link)); - link->sndbuf = sdsempty(); - link->rcvbuf = sdsempty(); - link->node = node; - link->fd = -1; - return link; -} - -/* Free a cluster link, but does not free the associated node of course. - * Just this function will make sure that the original node associated - * with this link will have the 'link' field set to NULL. */ -void freeClusterLink(clusterLink *link) { - if (link->fd != -1) { - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); - aeDeleteFileEvent(server.el, link->fd, AE_READABLE); - } - sdsfree(link->sndbuf); - sdsfree(link->rcvbuf); - if (link->node) - link->node->link = NULL; - close(link->fd); - zfree(link); -} - -void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - int cport, cfd; - char cip[128]; - clusterLink *link; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - REDIS_NOTUSED(privdata); - - cfd = anetTcpAccept(server.neterr, fd, cip, &cport); - if (cfd == AE_ERR) { - redisLog(REDIS_VERBOSE,"Accepting cluster node: %s", server.neterr); - return; - } - redisLog(REDIS_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* We need to create a temporary node in order to read the incoming - * packet in a valid contest. This node will be released once we - * read the packet and reply. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(server.el,cfd,AE_READABLE,clusterReadHandler,link); -} - -/* ----------------------------------------------------------------------------- - * Key space handling - * -------------------------------------------------------------------------- */ - -/* We have 4096 hash slots. The hash slot of a given key is obtained - * as the least significant 12 bits of the crc16 of the key. */ -unsigned int keyHashSlot(char *key, int keylen) { - return crc16(key,keylen) & 0x0FFF; -} - -/* ----------------------------------------------------------------------------- - * CLUSTER node API - * -------------------------------------------------------------------------- */ - -/* Create a new cluster node, with the specified flags. - * If "nodename" is NULL this is considered a first handshake and a random - * node name is assigned to this node (it will be fixed later when we'll - * receive the first pong). - * - * The node is created and returned to the user, but it is not automatically - * added to the nodes hash table. */ -clusterNode *createClusterNode(char *nodename, int flags) { - clusterNode *node = zmalloc(sizeof(*node)); - - if (nodename) - memcpy(node->name, nodename, REDIS_CLUSTER_NAMELEN); - else - getRandomHexChars(node->name, REDIS_CLUSTER_NAMELEN); - node->flags = flags; - memset(node->slots,0,sizeof(node->slots)); - node->numslaves = 0; - node->slaves = NULL; - node->slaveof = NULL; - node->ping_sent = node->pong_received = 0; - node->configdigest = NULL; - node->configdigest_ts = 0; - node->link = NULL; - return node; -} - -int clusterNodeRemoveSlave(clusterNode *master, clusterNode *slave) { - int j; - - for (j = 0; j < master->numslaves; j++) { - if (master->slaves[j] == slave) { - memmove(master->slaves+j,master->slaves+(j+1), - (master->numslaves-1)-j); - master->numslaves--; - return REDIS_OK; - } - } - return REDIS_ERR; -} - -int clusterNodeAddSlave(clusterNode *master, clusterNode *slave) { - int j; - - /* If it's already a slave, don't add it again. */ - for (j = 0; j < master->numslaves; j++) - if (master->slaves[j] == slave) return REDIS_ERR; - master->slaves = zrealloc(master->slaves, - sizeof(clusterNode*)*(master->numslaves+1)); - master->slaves[master->numslaves] = slave; - master->numslaves++; - return REDIS_OK; -} - -void clusterNodeResetSlaves(clusterNode *n) { - zfree(n->slaves); - n->numslaves = 0; -} - -void freeClusterNode(clusterNode *n) { - sds nodename; - - nodename = sdsnewlen(n->name, REDIS_CLUSTER_NAMELEN); - redisAssert(dictDelete(server.cluster.nodes,nodename) == DICT_OK); - sdsfree(nodename); - if (n->slaveof) clusterNodeRemoveSlave(n->slaveof, n); - if (n->link) freeClusterLink(n->link); - zfree(n); -} - -/* Add a node to the nodes hash table */ -int clusterAddNode(clusterNode *node) { - int retval; - - retval = dictAdd(server.cluster.nodes, - sdsnewlen(node->name,REDIS_CLUSTER_NAMELEN), node); - return (retval == DICT_OK) ? REDIS_OK : REDIS_ERR; -} - -/* Node lookup by name */ -clusterNode *clusterLookupNode(char *name) { - sds s = sdsnewlen(name, REDIS_CLUSTER_NAMELEN); - struct dictEntry *de; - - de = dictFind(server.cluster.nodes,s); - sdsfree(s); - if (de == NULL) return NULL; - return dictGetVal(de); -} - -/* This is only used after the handshake. When we connect a given IP/PORT - * as a result of CLUSTER MEET we don't have the node name yet, so we - * pick a random one, and will fix it when we receive the PONG request using - * this function. */ -void clusterRenameNode(clusterNode *node, char *newname) { - int retval; - sds s = sdsnewlen(node->name, REDIS_CLUSTER_NAMELEN); - - redisLog(REDIS_DEBUG,"Renaming node %.40s into %.40s", - node->name, newname); - retval = dictDelete(server.cluster.nodes, s); - sdsfree(s); - redisAssert(retval == DICT_OK); - memcpy(node->name, newname, REDIS_CLUSTER_NAMELEN); - clusterAddNode(node); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER messages exchange - PING/PONG and gossip - * -------------------------------------------------------------------------- */ - -/* Process the gossip section of PING or PONG packets. - * Note that this function assumes that the packet is already sanity-checked - * by the caller, not in the content of the gossip section, but in the - * length. */ -void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { - uint16_t count = ntohs(hdr->count); - clusterMsgDataGossip *g = (clusterMsgDataGossip*) hdr->data.ping.gossip; - clusterNode *sender = link->node ? link->node : clusterLookupNode(hdr->sender); - - while(count--) { - sds ci = sdsempty(); - uint16_t flags = ntohs(g->flags); - clusterNode *node; - - if (flags == 0) ci = sdscat(ci,"noflags,"); - if (flags & REDIS_NODE_MYSELF) ci = sdscat(ci,"myself,"); - if (flags & REDIS_NODE_MASTER) ci = sdscat(ci,"master,"); - if (flags & REDIS_NODE_SLAVE) ci = sdscat(ci,"slave,"); - if (flags & REDIS_NODE_PFAIL) ci = sdscat(ci,"fail?,"); - if (flags & REDIS_NODE_FAIL) ci = sdscat(ci,"fail,"); - if (flags & REDIS_NODE_HANDSHAKE) ci = sdscat(ci,"handshake,"); - if (flags & REDIS_NODE_NOADDR) ci = sdscat(ci,"noaddr,"); - if (ci[sdslen(ci)-1] == ',') ci[sdslen(ci)-1] = ' '; - - redisLog(REDIS_DEBUG,"GOSSIP %.40s %s:%d %s", - g->nodename, - g->ip, - ntohs(g->port), - ci); - sdsfree(ci); - - /* Update our state accordingly to the gossip sections */ - node = clusterLookupNode(g->nodename); - if (node != NULL) { - /* We already know this node. Let's start updating the last - * time PONG figure if it is newer than our figure. - * Note that it's not a problem if we have a PING already - * in progress against this node. */ - if (node->pong_received < (signed) ntohl(g->pong_received)) { - redisLog(REDIS_DEBUG,"Node pong_received updated by gossip"); - node->pong_received = ntohl(g->pong_received); - } - /* Mark this node as FAILED if we think it is possibly failing - * and another node also thinks it's failing. */ - if (node->flags & REDIS_NODE_PFAIL && - (flags & (REDIS_NODE_FAIL|REDIS_NODE_PFAIL))) - { - redisLog(REDIS_NOTICE,"Received a PFAIL acknowledge from node %.40s, marking node %.40s as FAIL!", hdr->sender, node->name); - node->flags &= ~REDIS_NODE_PFAIL; - node->flags |= REDIS_NODE_FAIL; - /* Broadcast the failing node name to everybody */ - clusterSendFail(node->name); - clusterUpdateState(); - clusterSaveConfigOrDie(); - } - } else { - /* If it's not in NOADDR state and we don't have it, we - * start an handshake process against this IP/PORT pairs. - * - * Note that we require that the sender of this gossip message - * is a well known node in our cluster, otherwise we risk - * joining another cluster. */ - if (sender && !(flags & REDIS_NODE_NOADDR)) { - clusterNode *newnode; - - redisLog(REDIS_DEBUG,"Adding the new node"); - newnode = createClusterNode(NULL,REDIS_NODE_HANDSHAKE); - memcpy(newnode->ip,g->ip,sizeof(g->ip)); - newnode->port = ntohs(g->port); - clusterAddNode(newnode); - } - } - - /* Next node */ - g++; - } -} - -/* IP -> string conversion. 'buf' is supposed to at least be 16 bytes. */ -void nodeIp2String(char *buf, clusterLink *link) { - struct sockaddr_in sa; - socklen_t salen = sizeof(sa); - - if (getpeername(link->fd, (struct sockaddr*) &sa, &salen) == -1) - redisPanic("getpeername() failed."); - strncpy(buf,inet_ntoa(sa.sin_addr),sizeof(link->node->ip)); -} - - -/* Update the node address to the IP address that can be extracted - * from link->fd, and at the specified port. */ -void nodeUpdateAddress(clusterNode *node, clusterLink *link, int port) { - /* TODO */ -} - -/* When this function is called, there is a packet to process starting - * at node->rcvbuf. Releasing the buffer is up to the caller, so this - * function should just handle the higher level stuff of processing the - * packet, modifying the cluster state if needed. - * - * The function returns 1 if the link is still valid after the packet - * was processed, otherwise 0 if the link was freed since the packet - * processing lead to some inconsistency error (for instance a PONG - * received from the wrong sender ID). */ -int clusterProcessPacket(clusterLink *link) { - clusterMsg *hdr = (clusterMsg*) link->rcvbuf; - uint32_t totlen = ntohl(hdr->totlen); - uint16_t type = ntohs(hdr->type); - clusterNode *sender; - - redisLog(REDIS_DEBUG,"--- Processing packet of type %d, %lu bytes", - type, (unsigned long) totlen); - - /* Perform sanity checks */ - if (totlen < 8) return 1; - if (totlen > sdslen(link->rcvbuf)) return 1; - if (type == CLUSTERMSG_TYPE_PING || type == CLUSTERMSG_TYPE_PONG || - type == CLUSTERMSG_TYPE_MEET) - { - uint16_t count = ntohs(hdr->count); - uint32_t explen; /* expected length of this packet */ - - explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - explen += (sizeof(clusterMsgDataGossip)*count); - if (totlen != explen) return 1; - } - if (type == CLUSTERMSG_TYPE_FAIL) { - uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - - explen += sizeof(clusterMsgDataFail); - if (totlen != explen) return 1; - } - if (type == CLUSTERMSG_TYPE_PUBLISH) { - uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - - explen += sizeof(clusterMsgDataPublish) + - ntohl(hdr->data.publish.msg.channel_len) + - ntohl(hdr->data.publish.msg.message_len); - if (totlen != explen) return 1; - } - - /* Ready to process the packet. Dispatch by type. */ - sender = clusterLookupNode(hdr->sender); - if (type == CLUSTERMSG_TYPE_PING || type == CLUSTERMSG_TYPE_MEET) { - int update_config = 0; - redisLog(REDIS_DEBUG,"Ping packet received: %p", link->node); - - /* Add this node if it is new for us and the msg type is MEET. - * In this stage we don't try to add the node with the right - * flags, slaveof pointer, and so forth, as this details will be - * resolved when we'll receive PONGs from the server. */ - if (!sender && type == CLUSTERMSG_TYPE_MEET) { - clusterNode *node; - - node = createClusterNode(NULL,REDIS_NODE_HANDSHAKE); - nodeIp2String(node->ip,link); - node->port = ntohs(hdr->port); - clusterAddNode(node); - update_config = 1; - } - - /* Get info from the gossip section */ - clusterProcessGossipSection(hdr,link); - - /* Anyway reply with a PONG */ - clusterSendPing(link,CLUSTERMSG_TYPE_PONG); - - /* Update config if needed */ - if (update_config) clusterSaveConfigOrDie(); - } else if (type == CLUSTERMSG_TYPE_PONG) { - int update_state = 0; - int update_config = 0; - - redisLog(REDIS_DEBUG,"Pong packet received: %p", link->node); - if (link->node) { - if (link->node->flags & REDIS_NODE_HANDSHAKE) { - /* If we already have this node, try to change the - * IP/port of the node with the new one. */ - if (sender) { - redisLog(REDIS_WARNING, - "Handshake error: we already know node %.40s, updating the address if needed.", sender->name); - nodeUpdateAddress(sender,link,ntohs(hdr->port)); - freeClusterNode(link->node); /* will free the link too */ - return 0; - } - - /* First thing to do is replacing the random name with the - * right node name if this was an handshake stage. */ - clusterRenameNode(link->node, hdr->sender); - redisLog(REDIS_DEBUG,"Handshake with node %.40s completed.", - link->node->name); - link->node->flags &= ~REDIS_NODE_HANDSHAKE; - update_config = 1; - } else if (memcmp(link->node->name,hdr->sender, - REDIS_CLUSTER_NAMELEN) != 0) - { - /* If the reply has a non matching node ID we - * disconnect this node and set it as not having an associated - * address. */ - redisLog(REDIS_DEBUG,"PONG contains mismatching sender ID"); - link->node->flags |= REDIS_NODE_NOADDR; - freeClusterLink(link); - update_config = 1; - /* FIXME: remove this node if we already have it. - * - * If we already have it but the IP is different, use - * the new one if the old node is in FAIL, PFAIL, or NOADDR - * status... */ - return 0; - } - } - /* Update our info about the node */ - if (link->node) link->node->pong_received = time(NULL); - - /* Update master/slave info */ - if (sender) { - if (!memcmp(hdr->slaveof,REDIS_NODE_NULL_NAME, - sizeof(hdr->slaveof))) - { - sender->flags &= ~REDIS_NODE_SLAVE; - sender->flags |= REDIS_NODE_MASTER; - sender->slaveof = NULL; - } else { - clusterNode *master = clusterLookupNode(hdr->slaveof); - - sender->flags &= ~REDIS_NODE_MASTER; - sender->flags |= REDIS_NODE_SLAVE; - if (sender->numslaves) clusterNodeResetSlaves(sender); - if (master) clusterNodeAddSlave(master,sender); - } - } - - /* Update our info about served slots if this new node is serving - * slots that are not served from our point of view. */ - if (sender && sender->flags & REDIS_NODE_MASTER) { - int newslots, j; - - newslots = - memcmp(sender->slots,hdr->myslots,sizeof(hdr->myslots)) != 0; - memcpy(sender->slots,hdr->myslots,sizeof(hdr->myslots)); - if (newslots) { - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (clusterNodeGetSlotBit(sender,j)) { - if (server.cluster.slots[j] == sender) continue; - if (server.cluster.slots[j] == NULL || - server.cluster.slots[j]->flags & REDIS_NODE_FAIL) - { - server.cluster.slots[j] = sender; - update_state = update_config = 1; - } - } - } - } - } - - /* Get info from the gossip section */ - clusterProcessGossipSection(hdr,link); - - /* Update the cluster state if needed */ - if (update_state) clusterUpdateState(); - if (update_config) clusterSaveConfigOrDie(); - } else if (type == CLUSTERMSG_TYPE_FAIL && sender) { - clusterNode *failing; - - failing = clusterLookupNode(hdr->data.fail.about.nodename); - if (failing && !(failing->flags & (REDIS_NODE_FAIL|REDIS_NODE_MYSELF))) - { - redisLog(REDIS_NOTICE, - "FAIL message received from %.40s about %.40s", - hdr->sender, hdr->data.fail.about.nodename); - failing->flags |= REDIS_NODE_FAIL; - failing->flags &= ~REDIS_NODE_PFAIL; - clusterUpdateState(); - clusterSaveConfigOrDie(); - } - } else if (type == CLUSTERMSG_TYPE_PUBLISH) { - robj *channel, *message; - uint32_t channel_len, message_len; - - /* Don't bother creating useless objects if there are no Pub/Sub subscribers. */ - if (dictSize(server.pubsub_channels) || listLength(server.pubsub_patterns)) { - channel_len = ntohl(hdr->data.publish.msg.channel_len); - message_len = ntohl(hdr->data.publish.msg.message_len); - channel = createStringObject( - (char*)hdr->data.publish.msg.bulk_data,channel_len); - message = createStringObject( - (char*)hdr->data.publish.msg.bulk_data+channel_len, message_len); - pubsubPublishMessage(channel,message); - decrRefCount(channel); - decrRefCount(message); - } - } else { - redisLog(REDIS_WARNING,"Received unknown packet type: %d", type); - } - return 1; -} - -/* This function is called when we detect the link with this node is lost. - We set the node as no longer connected. The Cluster Cron will detect - this connection and will try to get it connected again. - - Instead if the node is a temporary node used to accept a query, we - completely free the node on error. */ -void handleLinkIOError(clusterLink *link) { - freeClusterLink(link); -} - -/* Send data. This is handled using a trivial send buffer that gets - * consumed by write(). We don't try to optimize this for speed too much - * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - clusterLink *link = (clusterLink*) privdata; - ssize_t nwritten; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - - nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf)); - if (nwritten <= 0) { - redisLog(REDIS_NOTICE,"I/O error writing to node link: %s", - strerror(errno)); - handleLinkIOError(link); - return; - } - link->sndbuf = sdsrange(link->sndbuf,nwritten,-1); - if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); -} - -/* Read data. Try to read the first field of the header first to check the - * full length of the packet. When a whole packet is in memory this function - * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[1024]; - ssize_t nread; - clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; - int readlen; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - -again: - if (sdslen(link->rcvbuf) >= 4) { - hdr = (clusterMsg*) link->rcvbuf; - readlen = ntohl(hdr->totlen) - sdslen(link->rcvbuf); - } else { - readlen = 4 - sdslen(link->rcvbuf); - } - - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* Just no data */ - - if (nread <= 0) { - /* I/O error... */ - redisLog(REDIS_NOTICE,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); - handleLinkIOError(link); - return; - } else { - /* Read data and recast the pointer to the new buffer. */ - link->rcvbuf = sdscatlen(link->rcvbuf,buf,nread); - hdr = (clusterMsg*) link->rcvbuf; - } - - /* Total length obtained? read the payload now instead of burning - * cycles waiting for a new event to fire. */ - if (sdslen(link->rcvbuf) == 4) goto again; - - /* Whole packet in memory? We can process it. */ - if (sdslen(link->rcvbuf) == ntohl(hdr->totlen)) { - if (clusterProcessPacket(link)) { - sdsfree(link->rcvbuf); - link->rcvbuf = sdsempty(); - } - } -} - -/* Put stuff into the send buffer. */ -void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { - if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE, - clusterWriteHandler,link); - - link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); -} - -/* Send a message to all the nodes with a reliable link */ -void clusterBroadcastMessage(void *buf, size_t len) { - dictIterator *di; - dictEntry *de; - - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - if (!node->link) continue; - if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; - clusterSendMessage(node->link,buf,len); - } - dictReleaseIterator(di); -} - -/* Build the message header */ -void clusterBuildMessageHdr(clusterMsg *hdr, int type) { - int totlen = 0; - - memset(hdr,0,sizeof(*hdr)); - hdr->type = htons(type); - memcpy(hdr->sender,server.cluster.myself->name,REDIS_CLUSTER_NAMELEN); - memcpy(hdr->myslots,server.cluster.myself->slots, - sizeof(hdr->myslots)); - memset(hdr->slaveof,0,REDIS_CLUSTER_NAMELEN); - if (server.cluster.myself->slaveof != NULL) { - memcpy(hdr->slaveof,server.cluster.myself->slaveof->name, - REDIS_CLUSTER_NAMELEN); - } - hdr->port = htons(server.port); - hdr->state = server.cluster.state; - memset(hdr->configdigest,0,32); /* FIXME: set config digest */ - - if (type == CLUSTERMSG_TYPE_FAIL) { - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += sizeof(clusterMsgDataFail); - } - hdr->totlen = htonl(totlen); - /* For PING, PONG, and MEET, fixing the totlen field is up to the caller */ -} - -/* Send a PING or PONG packet to the specified node, making sure to add enough - * gossip informations. */ -void clusterSendPing(clusterLink *link, int type) { - unsigned char buf[1024]; - clusterMsg *hdr = (clusterMsg*) buf; - int gossipcount = 0, totlen; - /* freshnodes is the number of nodes we can still use to populate the - * gossip section of the ping packet. Basically we start with the nodes - * we have in memory minus two (ourself and the node we are sending the - * message to). Every time we add a node we decrement the counter, so when - * it will drop to <= zero we know there is no more gossip info we can - * send. */ - int freshnodes = dictSize(server.cluster.nodes)-2; - - if (link->node && type == CLUSTERMSG_TYPE_PING) - link->node->ping_sent = time(NULL); - clusterBuildMessageHdr(hdr,type); - - /* Populate the gossip fields */ - while(freshnodes > 0 && gossipcount < 3) { - struct dictEntry *de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetVal(de); - clusterMsgDataGossip *gossip; - int j; - - /* Not interesting to gossip about ourself. - * Nor to send gossip info about HANDSHAKE state nodes (zero info). */ - if (this == server.cluster.myself || - this->flags & REDIS_NODE_HANDSHAKE) { - freshnodes--; /* otherwise we may loop forever. */ - continue; - } - - /* Check if we already added this node */ - for (j = 0; j < gossipcount; j++) { - if (memcmp(hdr->data.ping.gossip[j].nodename,this->name, - REDIS_CLUSTER_NAMELEN) == 0) break; - } - if (j != gossipcount) continue; - - /* Add it */ - freshnodes--; - gossip = &(hdr->data.ping.gossip[gossipcount]); - memcpy(gossip->nodename,this->name,REDIS_CLUSTER_NAMELEN); - gossip->ping_sent = htonl(this->ping_sent); - gossip->pong_received = htonl(this->pong_received); - memcpy(gossip->ip,this->ip,sizeof(this->ip)); - gossip->port = htons(this->port); - gossip->flags = htons(this->flags); - gossipcount++; - } - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += (sizeof(clusterMsgDataGossip)*gossipcount); - hdr->count = htons(gossipcount); - hdr->totlen = htonl(totlen); - clusterSendMessage(link,buf,totlen); -} - -/* Send a PUBLISH message. - * - * If link is NULL, then the message is broadcasted to the whole cluster. */ -void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[4096], *payload; - clusterMsg *hdr = (clusterMsg*) buf; - uint32_t totlen; - uint32_t channel_len, message_len; - - channel = getDecodedObject(channel); - message = getDecodedObject(message); - channel_len = sdslen(channel->ptr); - message_len = sdslen(message->ptr); - - clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_PUBLISH); - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += sizeof(clusterMsgDataPublish) + channel_len + message_len; - - hdr->data.publish.msg.channel_len = htonl(channel_len); - hdr->data.publish.msg.message_len = htonl(message_len); - hdr->totlen = htonl(totlen); - - /* Try to use the local buffer if possible */ - if (totlen < sizeof(buf)) { - payload = buf; - } else { - payload = zmalloc(totlen); - hdr = (clusterMsg*) payload; - memcpy(payload,hdr,sizeof(hdr)); - } - memcpy(hdr->data.publish.msg.bulk_data,channel->ptr,sdslen(channel->ptr)); - memcpy(hdr->data.publish.msg.bulk_data+sdslen(channel->ptr), - message->ptr,sdslen(message->ptr)); - - if (link) - clusterSendMessage(link,payload,totlen); - else - clusterBroadcastMessage(payload,totlen); - - decrRefCount(channel); - decrRefCount(message); - if (payload != buf) zfree(payload); -} - -/* Send a FAIL message to all the nodes we are able to contact. - * The FAIL message is sent when we detect that a node is failing - * (REDIS_NODE_PFAIL) and we also receive a gossip confirmation of this: - * we switch the node state to REDIS_NODE_FAIL and ask all the other - * nodes to do the same ASAP. */ -void clusterSendFail(char *nodename) { - unsigned char buf[1024]; - clusterMsg *hdr = (clusterMsg*) buf; - - clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); - memcpy(hdr->data.fail.about.nodename,nodename,REDIS_CLUSTER_NAMELEN); - clusterBroadcastMessage(buf,ntohl(hdr->totlen)); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER Pub/Sub support - * - * For now we do very little, just propagating PUBLISH messages across the whole - * cluster. In the future we'll try to get smarter and avoiding propagating those - * messages to hosts without receives for a given channel. - * -------------------------------------------------------------------------- */ -void clusterPropagatePublish(robj *channel, robj *message) { - clusterSendPublish(NULL, channel, message); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER cron job - * -------------------------------------------------------------------------- */ - -/* This is executed 1 time every second */ -void clusterCron(void) { - dictIterator *di; - dictEntry *de; - int j; - time_t min_ping_sent = 0; - clusterNode *min_ping_node = NULL; - - /* Check if we have disconnected nodes and reestablish the connection. */ - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; - if (node->link == NULL) { - int fd; - clusterLink *link; - - fd = anetTcpNonBlockConnect(server.neterr, node->ip, - node->port+REDIS_CLUSTER_PORT_INCR); - if (fd == -1) continue; - link = createClusterLink(node); - link->fd = fd; - node->link = link; - aeCreateFileEvent(server.el,link->fd,AE_READABLE,clusterReadHandler,link); - /* If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - clusterSendPing(link, node->flags & REDIS_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~REDIS_NODE_MEET; - - redisLog(REDIS_NOTICE,"Connecting with Node %.40s at %s:%d", node->name, node->ip, node->port+REDIS_CLUSTER_PORT_INCR); - } - } - dictReleaseIterator(di); - - /* Ping some random node. Check a few random nodes and ping the one with - * the oldest ping_sent time */ - for (j = 0; j < 5; j++) { - de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetVal(de); - - if (this->link == NULL) continue; - if (this->flags & (REDIS_NODE_MYSELF|REDIS_NODE_HANDSHAKE)) continue; - if (min_ping_node == NULL || min_ping_sent > this->ping_sent) { - min_ping_node = this; - min_ping_sent = this->ping_sent; - } - } - if (min_ping_node) { - redisLog(REDIS_DEBUG,"Pinging node %40s", min_ping_node->name); - clusterSendPing(min_ping_node->link, CLUSTERMSG_TYPE_PING); - } - - /* Iterate nodes to check if we need to flag something as failing */ - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - int delay; - - if (node->flags & - (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR|REDIS_NODE_HANDSHAKE)) - continue; - /* Check only if we already sent a ping and did not received - * a reply yet. */ - if (node->ping_sent == 0 || - node->ping_sent <= node->pong_received) continue; - - delay = time(NULL) - node->pong_received; - if (delay < server.cluster.node_timeout) { - /* The PFAIL condition can be reversed without external - * help if it is not transitive (that is, if it does not - * turn into a FAIL state). - * - * The FAIL condition is also reversible if there are no slaves - * for this host, so no slave election should be in progress. - * - * TODO: consider all the implications of resurrecting a - * FAIL node. */ - if (node->flags & REDIS_NODE_PFAIL) { - node->flags &= ~REDIS_NODE_PFAIL; - } else if (node->flags & REDIS_NODE_FAIL && !node->numslaves) { - node->flags &= ~REDIS_NODE_FAIL; - clusterUpdateState(); - } - } else { - /* Timeout reached. Set the noad se possibly failing if it is - * not already in this state. */ - if (!(node->flags & (REDIS_NODE_PFAIL|REDIS_NODE_FAIL))) { - redisLog(REDIS_DEBUG,"*** NODE %.40s possibly failing", - node->name); - node->flags |= REDIS_NODE_PFAIL; - } - } - } - dictReleaseIterator(di); -} - -/* ----------------------------------------------------------------------------- - * Slots management - * -------------------------------------------------------------------------- */ - -/* Set the slot bit and return the old value. */ -int clusterNodeSetSlotBit(clusterNode *n, int slot) { - off_t byte = slot/8; - int bit = slot&7; - int old = (n->slots[byte] & (1<slots[byte] |= 1<slots[byte] & (1<slots[byte] &= ~(1<slots[byte] & (1<flags & (REDIS_NODE_FAIL)) - { - ok = 0; - break; - } - } - if (ok) { - if (server.cluster.state == REDIS_CLUSTER_NEEDHELP) { - server.cluster.state = REDIS_CLUSTER_NEEDHELP; - } else { - server.cluster.state = REDIS_CLUSTER_OK; - } - } else { - server.cluster.state = REDIS_CLUSTER_FAIL; - } -} - -/* ----------------------------------------------------------------------------- - * CLUSTER command - * -------------------------------------------------------------------------- */ - -sds clusterGenNodesDescription(void) { - sds ci = sdsempty(); - dictIterator *di; - dictEntry *de; - int j, start; - - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - /* Node coordinates */ - ci = sdscatprintf(ci,"%.40s %s:%d ", - node->name, - node->ip, - node->port); - - /* Flags */ - if (node->flags == 0) ci = sdscat(ci,"noflags,"); - if (node->flags & REDIS_NODE_MYSELF) ci = sdscat(ci,"myself,"); - if (node->flags & REDIS_NODE_MASTER) ci = sdscat(ci,"master,"); - if (node->flags & REDIS_NODE_SLAVE) ci = sdscat(ci,"slave,"); - if (node->flags & REDIS_NODE_PFAIL) ci = sdscat(ci,"fail?,"); - if (node->flags & REDIS_NODE_FAIL) ci = sdscat(ci,"fail,"); - if (node->flags & REDIS_NODE_HANDSHAKE) ci =sdscat(ci,"handshake,"); - if (node->flags & REDIS_NODE_NOADDR) ci = sdscat(ci,"noaddr,"); - if (ci[sdslen(ci)-1] == ',') ci[sdslen(ci)-1] = ' '; - - /* Slave of... or just "-" */ - if (node->slaveof) - ci = sdscatprintf(ci,"%.40s ",node->slaveof->name); - else - ci = sdscatprintf(ci,"- "); - - /* Latency from the POV of this node, link status */ - ci = sdscatprintf(ci,"%ld %ld %s", - (long) node->ping_sent, - (long) node->pong_received, - (node->link || node->flags & REDIS_NODE_MYSELF) ? - "connected" : "disconnected"); - - /* Slots served by this instance */ - start = -1; - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - int bit; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == REDIS_CLUSTER_SLOTS-1)) { - if (j == REDIS_CLUSTER_SLOTS-1) j++; - - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); - } - start = -1; - } - } - - /* Just for MYSELF node we also dump info about slots that - * we are migrating to other instances or importing from other - * instances. */ - if (node->flags & REDIS_NODE_MYSELF) { - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (server.cluster.migrating_slots_to[j]) { - ci = sdscatprintf(ci," [%d->-%.40s]",j, - server.cluster.migrating_slots_to[j]->name); - } else if (server.cluster.importing_slots_from[j]) { - ci = sdscatprintf(ci," [%d-<-%.40s]",j, - server.cluster.importing_slots_from[j]->name); - } - } - } - ci = sdscatlen(ci,"\n",1); - } - dictReleaseIterator(di); - return ci; -} - -int getSlotOrReply(redisClient *c, robj *o) { - long long slot; - - if (getLongLongFromObject(o,&slot) != REDIS_OK || - slot < 0 || slot > REDIS_CLUSTER_SLOTS) - { - addReplyError(c,"Invalid or out of range slot"); - return -1; - } - return (int) slot; -} - -void clusterCommand(redisClient *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } - - if (!strcasecmp(c->argv[1]->ptr,"meet") && c->argc == 4) { - clusterNode *n; - struct sockaddr_in sa; - long port; - - /* Perform sanity checks on IP/port */ - if (inet_aton(c->argv[2]->ptr,&sa.sin_addr) == 0) { - addReplyError(c,"Invalid IP address in MEET"); - return; - } - if (getLongFromObjectOrReply(c, c->argv[3], &port, NULL) != REDIS_OK || - port < 0 || port > (65535-REDIS_CLUSTER_PORT_INCR)) - { - addReplyError(c,"Invalid TCP port specified"); - return; - } - - /* Finally add the node to the cluster with a random name, this - * will get fixed in the first handshake (ping/pong). */ - n = createClusterNode(NULL,REDIS_NODE_HANDSHAKE|REDIS_NODE_MEET); - strncpy(n->ip,inet_ntoa(sa.sin_addr),sizeof(n->ip)); - n->port = port; - clusterAddNode(n); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { - robj *o; - sds ci = clusterGenNodesDescription(); - - o = createObject(REDIS_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); - } else if ((!strcasecmp(c->argv[1]->ptr,"addslots") || - !strcasecmp(c->argv[1]->ptr,"delslots")) && c->argc >= 3) - { - /* CLUSTER ADDSLOTS [slot] ... */ - /* CLUSTER DELSLOTS [slot] ... */ - int j, slot; - unsigned char *slots = zmalloc(REDIS_CLUSTER_SLOTS); - int del = !strcasecmp(c->argv[1]->ptr,"delslots"); - - memset(slots,0,REDIS_CLUSTER_SLOTS); - /* Check that all the arguments are parsable and that all the - * slots are not already busy. */ - for (j = 2; j < c->argc; j++) { - if ((slot = getSlotOrReply(c,c->argv[j])) == -1) { - zfree(slots); - return; - } - if (del && server.cluster.slots[slot] == NULL) { - addReplyErrorFormat(c,"Slot %d is already unassigned", slot); - zfree(slots); - return; - } else if (!del && server.cluster.slots[slot]) { - addReplyErrorFormat(c,"Slot %d is already busy", slot); - zfree(slots); - return; - } - if (slots[slot]++ == 1) { - addReplyErrorFormat(c,"Slot %d specified multiple times", - (int)slot); - zfree(slots); - return; - } - } - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (slots[j]) { - int retval; - - /* If this slot was set as importing we can clear this - * state as now we are the real owner of the slot. */ - if (server.cluster.importing_slots_from[j]) - server.cluster.importing_slots_from[j] = NULL; - - retval = del ? clusterDelSlot(j) : - clusterAddSlot(server.cluster.myself,j); - redisAssertWithInfo(c,NULL,retval == REDIS_OK); - } - } - zfree(slots); - clusterUpdateState(); - clusterSaveConfigOrDie(); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"setslot") && c->argc >= 4) { - /* SETSLOT 10 MIGRATING */ - /* SETSLOT 10 IMPORTING */ - /* SETSLOT 10 STABLE */ - /* SETSLOT 10 NODE */ - int slot; - clusterNode *n; - - if ((slot = getSlotOrReply(c,c->argv[2])) == -1) return; - - if (!strcasecmp(c->argv[3]->ptr,"migrating") && c->argc == 5) { - if (server.cluster.slots[slot] != server.cluster.myself) { - addReplyErrorFormat(c,"I'm not the owner of hash slot %u",slot); - return; - } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { - addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[4]->ptr); - return; - } - server.cluster.migrating_slots_to[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"importing") && c->argc == 5) { - if (server.cluster.slots[slot] == server.cluster.myself) { - addReplyErrorFormat(c, - "I'm already the owner of hash slot %u",slot); - return; - } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { - addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[3]->ptr); - return; - } - server.cluster.importing_slots_from[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"stable") && c->argc == 4) { - /* CLUSTER SETSLOT STABLE */ - server.cluster.importing_slots_from[slot] = NULL; - server.cluster.migrating_slots_to[slot] = NULL; - } else if (!strcasecmp(c->argv[3]->ptr,"node") && c->argc == 5) { - /* CLUSTER SETSLOT NODE */ - clusterNode *n = clusterLookupNode(c->argv[4]->ptr); - - if (!n) addReplyErrorFormat(c,"Unknown node %s", - (char*)c->argv[4]->ptr); - /* If this hash slot was served by 'myself' before to switch - * make sure there are no longer local keys for this hash slot. */ - if (server.cluster.slots[slot] == server.cluster.myself && - n != server.cluster.myself) - { - int numkeys; - robj **keys; - - keys = zmalloc(sizeof(robj*)*1); - numkeys = GetKeysInSlot(slot, keys, 1); - zfree(keys); - if (numkeys != 0) { - addReplyErrorFormat(c, "Can't assign hashslot %d to a different node while I still hold keys for this hash slot.", slot); - return; - } - } - /* If this node was the slot owner and the slot was marked as - * migrating, assigning the slot to another node will clear - * the migratig status. */ - if (server.cluster.slots[slot] == server.cluster.myself && - server.cluster.migrating_slots_to[slot]) - server.cluster.migrating_slots_to[slot] = NULL; - - /* If this node was importing this slot, assigning the slot to - * itself also clears the importing status. */ - if (n == server.cluster.myself && server.cluster.importing_slots_from[slot]) - server.cluster.importing_slots_from[slot] = NULL; - - clusterDelSlot(slot); - clusterAddSlot(n,slot); - } else { - addReplyError(c,"Invalid CLUSTER SETSLOT action or number of arguments"); - return; - } - clusterSaveConfigOrDie(); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) { - char *statestr[] = {"ok","fail","needhelp"}; - int slots_assigned = 0, slots_ok = 0, slots_pfail = 0, slots_fail = 0; - int j; - - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - clusterNode *n = server.cluster.slots[j]; - - if (n == NULL) continue; - slots_assigned++; - if (n->flags & REDIS_NODE_FAIL) { - slots_fail++; - } else if (n->flags & REDIS_NODE_PFAIL) { - slots_pfail++; - } else { - slots_ok++; - } - } - - sds info = sdscatprintf(sdsempty(), - "cluster_state:%s\r\n" - "cluster_slots_assigned:%d\r\n" - "cluster_slots_ok:%d\r\n" - "cluster_slots_pfail:%d\r\n" - "cluster_slots_fail:%d\r\n" - "cluster_known_nodes:%lu\r\n" - , statestr[server.cluster.state], - slots_assigned, - slots_ok, - slots_pfail, - slots_fail, - dictSize(server.cluster.nodes) - ); - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); - } else if (!strcasecmp(c->argv[1]->ptr,"keyslot") && c->argc == 3) { - sds key = c->argv[2]->ptr; - - addReplyLongLong(c,keyHashSlot(key,sdslen(key))); - } else if (!strcasecmp(c->argv[1]->ptr,"getkeysinslot") && c->argc == 4) { - long long maxkeys, slot; - unsigned int numkeys, j; - robj **keys; - - if (getLongLongFromObjectOrReply(c,c->argv[2],&slot,NULL) != REDIS_OK) - return; - if (getLongLongFromObjectOrReply(c,c->argv[3],&maxkeys,NULL) != REDIS_OK) - return; - if (slot < 0 || slot >= REDIS_CLUSTER_SLOTS || maxkeys < 0 || - maxkeys > 1024*1024) { - addReplyError(c,"Invalid slot or number of keys"); - return; - } - - keys = zmalloc(sizeof(robj*)*maxkeys); - numkeys = GetKeysInSlot(slot, keys, maxkeys); - addReplyMultiBulkLen(c,numkeys); - for (j = 0; j < numkeys; j++) addReplyBulk(c,keys[j]); - zfree(keys); - } else { - addReplyError(c,"Wrong CLUSTER subcommand or number of arguments"); - } -} - -/* ----------------------------------------------------------------------------- - * RESTORE and MIGRATE commands - * -------------------------------------------------------------------------- */ - -/* RESTORE key ttl serialized-value */ -void restoreCommand(redisClient *c) { - long ttl; - rio payload; - int type; - robj *obj; - - /* Make sure this key does not already exist here... */ - if (lookupKeyWrite(c->db,c->argv[1]) != NULL) { - addReplyError(c,"Target key name is busy."); - return; - } - - /* Check if the TTL value makes sense */ - if (getLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { - return; - } else if (ttl < 0) { - addReplyError(c,"Invalid TTL value, must be >= 0"); - return; - } - - rioInitWithBuffer(&payload,c->argv[3]->ptr); - if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload)) == NULL)) - { - addReplyError(c,"Bad data format"); - return; - } - - /* Create the key and set the TTL if any */ - dbAdd(c->db,c->argv[1],obj); - if (ttl) setExpire(c->db,c->argv[1],time(NULL)+ttl); - signalModifiedKey(c->db,c->argv[1]); - addReply(c,shared.ok); - server.dirty++; -} - -/* MIGRATE host port key dbid timeout */ -void migrateCommand(redisClient *c) { - int fd; - long timeout; - long dbid; - time_t ttl; - robj *o; - rio cmd, payload; - - /* Sanity check */ - if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK) - return; - if (getLongFromObjectOrReply(c,c->argv[4],&dbid,NULL) != REDIS_OK) - return; - if (timeout <= 0) timeout = 1; - - /* Check if the key is here. If not we reply with success as there is - * nothing to migrate (for instance the key expired in the meantime), but - * we include such information in the reply string. */ - if ((o = lookupKeyRead(c->db,c->argv[3])) == NULL) { - addReplySds(c,sdsnew("+NOKEY\r\n")); - return; - } - - /* Connect */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); - if (fd == -1) { - addReplyErrorFormat(c,"Can't connect to target node: %s", - server.neterr); - return; - } - if ((aeWait(fd,AE_WRITABLE,timeout*1000) & AE_WRITABLE) == 0) { - addReplyError(c,"Timeout connecting to the client"); - return; - } - - rioInitWithBuffer(&cmd,sdsempty()); - redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); - - ttl = getExpire(c->db,c->argv[3]); - redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); - redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl)); - - /* Finally the last argument that is the serailized object payload - * in the form: . */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o) != -1); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr))); - sdsfree(payload.io.buffer.ptr); - - /* Tranfer the query to the other node in 64K chunks. */ - { - sds buf = cmd.io.buffer.ptr; - size_t pos = 0, towrite; - int nwritten = 0; - - while ((towrite = sdslen(buf)-pos) > 0) { - towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(fd,buf+nwritten,towrite,timeout); - if (nwritten != (signed)towrite) goto socket_wr_err; - pos += nwritten; - } - } - - /* Read back the reply. */ - { - char buf1[1024]; - char buf2[1024]; - - /* Read the two replies */ - if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0) - goto socket_rd_err; - if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0) - goto socket_rd_err; - if (buf1[0] == '-' || buf2[0] == '-') { - addReplyErrorFormat(c,"Target instance replied with error: %s", - (buf1[0] == '-') ? buf1+1 : buf2+1); - } else { - robj *aux; - - dbDelete(c->db,c->argv[3]); - signalModifiedKey(c->db,c->argv[3]); - addReply(c,shared.ok); - server.dirty++; - - /* Translate MIGRATE as DEL for replication/AOF. */ - aux = createStringObject("DEL",3); - rewriteClientCommandVector(c,2,aux,c->argv[3]); - decrRefCount(aux); - } - } - - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; - -socket_wr_err: - redisLog(REDIS_NOTICE,"Can't write to target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.", - strerror(errno)); - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; - -socket_rd_err: - redisLog(REDIS_NOTICE,"Can't read from target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.", - strerror(errno)); - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; -} - -/* DUMP keyname - * DUMP is actually not used by Redis Cluster but it is the obvious - * complement of RESTORE and can be useful for different applications. */ -void dumpCommand(redisClient *c) { - robj *o, *dumpobj; - rio payload; - - /* Check if the key is here. */ - if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { - addReply(c,shared.nullbulk); - return; - } - - /* Serialize the object in a RDB-like format. It consist of an object type - * byte followed by the serialized object. This is understood by RESTORE. */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o)); - - /* Transfer to the client */ - dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); - return; -} - -/* The ASKING command is required after a -ASK redirection. - * The client should issue ASKING before to actualy send the command to - * the target instance. See the Redis Cluster specification for more - * information. */ -void askingCommand(redisClient *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } - c->flags |= REDIS_ASKING; - addReply(c,shared.ok); -} - -/* ----------------------------------------------------------------------------- - * Cluster functions related to serving / redirecting clients - * -------------------------------------------------------------------------- */ - -/* Return the pointer to the cluster node that is able to serve the query - * as all the keys belong to hash slots for which the node is in charge. - * - * If the returned node should be used only for this request, the *ask - * integer is set to '1', otherwise to '0'. This is used in order to - * let the caller know if we should reply with -MOVED or with -ASK. - * - * If the request contains more than a single key NULL is returned, - * however a request with more then a key argument where the key is always - * the same is valid, like in: RPOPLPUSH mylist mylist.*/ -clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask) { - clusterNode *n = NULL; - robj *firstkey = NULL; - multiState *ms, _ms; - multiCmd mc; - int i, slot = 0; - - /* We handle all the cases as if they were EXEC commands, so we have - * a common code path for everything */ - if (cmd->proc == execCommand) { - /* If REDIS_MULTI flag is not set EXEC is just going to return an - * error. */ - if (!(c->flags & REDIS_MULTI)) return server.cluster.myself; - ms = &c->mstate; - } else { - /* In order to have a single codepath create a fake Multi State - * structure if the client is not in MULTI/EXEC state, this way - * we have a single codepath below. */ - ms = &_ms; - _ms.commands = &mc; - _ms.count = 1; - mc.argv = argv; - mc.argc = argc; - mc.cmd = cmd; - } - - /* Check that all the keys are the same key, and get the slot and - * node for this key. */ - for (i = 0; i < ms->count; i++) { - struct redisCommand *mcmd; - robj **margv; - int margc, *keyindex, numkeys, j; - - mcmd = ms->commands[i].cmd; - margc = ms->commands[i].argc; - margv = ms->commands[i].argv; - - keyindex = getKeysFromCommand(mcmd,margv,margc,&numkeys, - REDIS_GETKEYS_ALL); - for (j = 0; j < numkeys; j++) { - if (firstkey == NULL) { - /* This is the first key we see. Check what is the slot - * and node. */ - firstkey = margv[keyindex[j]]; - - slot = keyHashSlot((char*)firstkey->ptr, sdslen(firstkey->ptr)); - n = server.cluster.slots[slot]; - redisAssertWithInfo(c,firstkey,n != NULL); - } else { - /* If it is not the first key, make sure it is exactly - * the same key as the first we saw. */ - if (!equalStringObjects(firstkey,margv[keyindex[j]])) { - decrRefCount(firstkey); - getKeysFreeResult(keyindex); - return NULL; - } - } - } - getKeysFreeResult(keyindex); - } - if (ask) *ask = 0; /* This is the default. Set to 1 if needed later. */ - /* No key at all in command? then we can serve the request - * without redirections. */ - if (n == NULL) return server.cluster.myself; - if (hashslot) *hashslot = slot; - /* This request is about a slot we are migrating into another instance? - * Then we need to check if we have the key. If we have it we can reply. - * If instead is a new key, we pass the request to the node that is - * receiving the slot. */ - if (n == server.cluster.myself && - server.cluster.migrating_slots_to[slot] != NULL) - { - if (lookupKeyRead(&server.db[0],firstkey) == NULL) { - if (ask) *ask = 1; - return server.cluster.migrating_slots_to[slot]; - } - } - /* Handle the case in which we are receiving this hash slot from - * another instance, so we'll accept the query even if in the table - * it is assigned to a different node, but only if the client - * issued an ASKING command before. */ - if (server.cluster.importing_slots_from[slot] != NULL && - c->flags & REDIS_ASKING) { - return server.cluster.myself; - } - /* It's not a -ASK case. Base case: just return the right node. */ - return n; -} diff --git a/src/config.c b/src/config.c index 533a2a572a1..38e9dcf5d16 100644 --- a/src/config.c +++ b/src/config.c @@ -1,5 +1,60 @@ +/* Configuration file parsing and CONFIG GET/SET commands implementation. + * + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" +#include +#include + +static struct { + const char *name; + const int value; +} validSyslogFacilities[] = { + {"user", LOG_USER}, + {"local0", LOG_LOCAL0}, + {"local1", LOG_LOCAL1}, + {"local2", LOG_LOCAL2}, + {"local3", LOG_LOCAL3}, + {"local4", LOG_LOCAL4}, + {"local5", LOG_LOCAL5}, + {"local6", LOG_LOCAL6}, + {"local7", LOG_LOCAL7}, + {NULL, 0} +}; + +clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_TYPE_COUNT] = { + {0, 0, 0}, /* normal */ + {1024*1024*256, 1024*1024*64, 60}, /* slave */ + {1024*1024*32, 1024*1024*8, 60} /* pubsub */ +}; + /*----------------------------------------------------------------------------- * Config file parsing *----------------------------------------------------------------------------*/ @@ -17,7 +72,7 @@ void appendServerSaveParams(time_t seconds, int changes) { server.saveparamslen++; } -void resetServerSaveParams() { +void resetServerSaveParams(void) { zfree(server.saveparams); server.saveparams = NULL; server.saveparamslen = 0; @@ -37,11 +92,21 @@ void loadServerConfigFromString(char *config) { linenum = i+1; lines[i] = sdstrim(lines[i]," \t\r\n"); - /* Skip comments and blank lines*/ + /* Skip comments and blank lines */ if (lines[i][0] == '#' || lines[i][0] == '\0') continue; /* Split into arguments */ argv = sdssplitargs(lines[i],&argc); + if (argv == NULL) { + err = "Unbalanced quotes in configuration line"; + goto loaderr; + } + + /* Skip this line if the resulting command vector is empty. */ + if (argc == 0) { + sdsfreesplitres(argv,argc); + continue; + } sdstolower(argv[0]); /* Execute config directives */ @@ -50,13 +115,30 @@ void loadServerConfigFromString(char *config) { if (server.maxidletime < 0) { err = "Invalid timeout value"; goto loaderr; } + } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { + server.tcpkeepalive = atoi(argv[1]); + if (server.tcpkeepalive < 0) { + err = "Invalid tcp-keepalive value"; goto loaderr; + } } else if (!strcasecmp(argv[0],"port") && argc == 2) { server.port = atoi(argv[1]); if (server.port < 0 || server.port > 65535) { err = "Invalid port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"bind") && argc == 2) { - server.bindaddr = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { + server.tcp_backlog = atoi(argv[1]); + if (server.tcp_backlog < 0) { + err = "Invalid backlog value"; goto loaderr; + } + } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + int j, addresses = argc-1; + + if (addresses > REDIS_BINDADDR_MAX) { + err = "Too many bind addresses specified"; goto loaderr; + } + for (j = 0; j < addresses; j++) + server.bindaddr[j] = zstrdup(argv[j+1]); + server.bindaddr_count = addresses; } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { server.unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { @@ -94,12 +176,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; + zfree(server.logfile); server.logfile = zstrdup(argv[1]); - if (!strcasecmp(server.logfile,"stdout")) { - zfree(server.logfile); - server.logfile = NULL; - } - if (server.logfile) { + if (server.logfile[0] != '\0') { /* Test if we are able to open the file. The server will not * be able to abort just for this problem later... */ logfp = fopen(server.logfile,"a"); @@ -118,21 +197,6 @@ void loadServerConfigFromString(char *config) { if (server.syslog_ident) zfree(server.syslog_ident); server.syslog_ident = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - struct { - const char *name; - const int value; - } validSyslogFacilities[] = { - {"user", LOG_USER}, - {"local0", LOG_LOCAL0}, - {"local1", LOG_LOCAL1}, - {"local2", LOG_LOCAL2}, - {"local3", LOG_LOCAL3}, - {"local4", LOG_LOCAL4}, - {"local5", LOG_LOCAL5}, - {"local6", LOG_LOCAL6}, - {"local7", LOG_LOCAL7}, - {NULL, 0} - }; int i; for (i = 0; validSyslogFacilities[i].name; i++) { @@ -155,6 +219,9 @@ void loadServerConfigFromString(char *config) { loadServerConfig(argv[1],NULL); } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { server.maxclients = atoi(argv[1]); + if (server.maxclients < 1) { + err = "Invalid max clients limit"; goto loaderr; + } } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { server.maxmemory = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { @@ -196,16 +263,51 @@ void loadServerConfigFromString(char *config) { err = "repl-timeout must be 1 or greater"; goto loaderr; } + } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { + if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } + } else if (!strcasecmp(argv[0],"repl-diskless-sync") && argc==2) { + if ((server.repl_diskless_sync = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } + } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { + server.repl_diskless_sync_delay = atoi(argv[1]); + if (server.repl_diskless_sync_delay < 0) { + err = "repl-diskless-sync-delay can't be negative"; + goto loaderr; + } + } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { + long long size = memtoll(argv[1],NULL); + if (size <= 0) { + err = "repl-backlog-size must be 1 or greater."; + goto loaderr; + } + resizeReplicationBacklog(size); + } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { + server.repl_backlog_time_limit = atoi(argv[1]); + if (server.repl_backlog_time_limit < 0) { + err = "repl-backlog-ttl can't be negative "; + goto loaderr; + } } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { - server.masterauth = zstrdup(argv[1]); + server.masterauth = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"slave-serve-stale-data") && argc == 2) { if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-read-only") && argc == 2) { + if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"rdbchecksum") && argc == 2) { + if ((server.rdb_checksum = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"activerehashing") && argc == 2) { if ((server.activerehashing = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -214,6 +316,10 @@ void loadServerConfigFromString(char *config) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"hz") && argc == 2) { + server.hz = atoi(argv[1]); + if (server.hz < REDIS_MIN_HZ) server.hz = REDIS_MIN_HZ; + if (server.hz > REDIS_MAX_HZ) server.hz = REDIS_MAX_HZ; } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { int yes; @@ -222,6 +328,10 @@ void loadServerConfigFromString(char *config) { } server.aof_state = yes ? REDIS_AOF_ON : REDIS_AOF_OFF; } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { + if (!pathIsBaseName(argv[1])) { + err = "appendfilename can't be a path, just a filename"; + goto loaderr; + } zfree(server.aof_filename); server.aof_filename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"no-appendfsync-on-rewrite") @@ -252,12 +362,31 @@ void loadServerConfigFromString(char *config) { argc == 2) { server.aof_rewrite_min_size = memtoll(argv[1],NULL); + } else if (!strcasecmp(argv[0],"aof-rewrite-incremental-fsync") && + argc == 2) + { + if ((server.aof_rewrite_incremental_fsync = + yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } + } else if (!strcasecmp(argv[0],"aof-load-truncated") && argc == 2) { + if ((server.aof_load_truncated = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { + if (strlen(argv[1]) > REDIS_AUTHPASS_MAX_LEN) { + err = "Password is longer than REDIS_AUTHPASS_MAX_LEN"; + goto loaderr; + } server.requirepass = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { zfree(server.pidfile); server.pidfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { + if (!pathIsBaseName(argv[1])) { + err = "dbfilename can't be a path, just a filename"; + goto loaderr; + } zfree(server.rdb_filename); server.rdb_filename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { @@ -274,6 +403,8 @@ void loadServerConfigFromString(char *config) { server.zset_max_ziplist_entries = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) { server.zset_max_ziplist_value = memtoll(argv[1], NULL); + } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) { + server.hll_sparse_max_bytes = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) { struct redisCommand *cmd = lookupCommand(argv[1]); int retval; @@ -283,7 +414,7 @@ void loadServerConfigFromString(char *config) { goto loaderr; } - /* If the target command name is the emtpy string we just + /* If the target command name is the empty string we just * remove it from the command table. */ retval = dictDelete(server.commands, argv[1]); redisAssert(retval == DICT_OK); @@ -298,25 +429,26 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((server.cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { - zfree(server.cluster.configfile); - server.cluster.configfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { server.slowlog_log_slower_than = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"latency-monitor-threshold") && + argc == 2) + { + server.latency_monitor_threshold = strtoll(argv[1],NULL,10); + if (server.latency_monitor_threshold < 0) { + err = "The latency threshold can't be negative"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { server.slowlog_max_len = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { - int class = getClientLimitClassByName(argv[1]); + int class = getClientTypeByName(argv[1]); unsigned long long hard, soft; int soft_seconds; @@ -328,7 +460,7 @@ void loadServerConfigFromString(char *config) { soft = memtoll(argv[3],NULL); soft_seconds = atoi(argv[4]); if (soft_seconds < 0) { - err = "Negative number of seconds in soft limt is invalid"; + err = "Negative number of seconds in soft limit is invalid"; goto loaderr; } server.client_obuf_limits[class].hard_limit_bytes = hard; @@ -339,6 +471,37 @@ void loadServerConfigFromString(char *config) { if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { + server.slave_priority = atoi(argv[1]); + } else if (!strcasecmp(argv[0],"min-slaves-to-write") && argc == 2) { + server.repl_min_slaves_to_write = atoi(argv[1]); + if (server.repl_min_slaves_to_write < 0) { + err = "Invalid value for min-slaves-to-write."; goto loaderr; + } + } else if (!strcasecmp(argv[0],"min-slaves-max-lag") && argc == 2) { + server.repl_min_slaves_max_lag = atoi(argv[1]); + if (server.repl_min_slaves_max_lag < 0) { + err = "Invalid value for min-slaves-max-lag."; goto loaderr; + } + } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { + int flags = keyspaceEventsStringToFlags(argv[1]); + + if (flags == -1) { + err = "Invalid event class character. Use 'g$lshzxeA'."; + goto loaderr; + } + server.notify_keyspace_events = flags; + } else if (!strcasecmp(argv[0],"sentinel")) { + /* argc == 1 is handled by main() as we need to enter the sentinel + * mode ASAP. */ + if (argc != 1) { + if (!server.sentinel_mode) { + err = "sentinel directive while not in sentinel mode"; + goto loaderr; + } + err = sentinelHandleConfiguration(argv+1,argc-1); + if (err) goto loaderr; + } } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -360,7 +523,7 @@ void loadServerConfigFromString(char *config) { * in the 'options' string to the config file before loading. * * Both filename and options can be NULL, in such a case are considered - * emtpy. This way loadServerConfig can be used to just load a file or + * empty. This way loadServerConfig can be used to just load a file or * just load a string. */ void loadServerConfig(char *filename, char *options) { sds config = sdsempty(); @@ -393,7 +556,7 @@ void loadServerConfig(char *filename, char *options) { } /*----------------------------------------------------------------------------- - * CONFIG command for remote configuration + * CONFIG SET implementation *----------------------------------------------------------------------------*/ void configSetCommand(redisClient *c) { @@ -404,19 +567,60 @@ void configSetCommand(redisClient *c) { o = c->argv[3]; if (!strcasecmp(c->argv[2]->ptr,"dbfilename")) { + if (!pathIsBaseName(o->ptr)) { + addReplyError(c, "dbfilename can't be a path, just a filename"); + return; + } zfree(server.rdb_filename); server.rdb_filename = zstrdup(o->ptr); } else if (!strcasecmp(c->argv[2]->ptr,"requirepass")) { + if (sdslen(o->ptr) > REDIS_AUTHPASS_MAX_LEN) goto badfmt; zfree(server.requirepass); server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"masterauth")) { zfree(server.masterauth); - server.masterauth = zstrdup(o->ptr); + server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.maxmemory = ll; - if (server.maxmemory) freeMemoryIfNeeded(); + if (server.maxmemory) { + if (server.maxmemory < zmalloc_used_memory()) { + redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); + } + freeMemoryIfNeeded(); + } + } else if (!strcasecmp(c->argv[2]->ptr,"maxclients")) { + int orig_value = server.maxclients; + + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 1) goto badfmt; + + /* Try to check if the OS is capable of supporting so many FDs. */ + server.maxclients = ll; + if (ll > orig_value) { + adjustOpenFilesLimit(); + if (server.maxclients != ll) { + addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); + server.maxclients = orig_value; + return; + } + if ((unsigned int) aeGetSetSize(server.el) < + server.maxclients + REDIS_EVENTLOOP_FDSET_INCR) + { + if (aeResizeSetSize(server.el, + server.maxclients + REDIS_EVENTLOOP_FDSET_INCR) == AE_ERR) + { + addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); + server.maxclients = orig_value; + return; + } + } + } + } else if (!strcasecmp(c->argv[2]->ptr,"hz")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.hz = ll; + if (server.hz < REDIS_MIN_HZ) server.hz = REDIS_MIN_HZ; + if (server.hz > REDIS_MAX_HZ) server.hz = REDIS_MAX_HZ; } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory-policy")) { if (!strcasecmp(o->ptr,"volatile-lru")) { server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU; @@ -441,6 +645,10 @@ void configSetCommand(redisClient *c) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0 || ll > LONG_MAX) goto badfmt; server.maxidletime = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"tcp-keepalive")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0 || ll > INT_MAX) goto badfmt; + server.tcpkeepalive = ll; } else if (!strcasecmp(c->argv[2]->ptr,"appendfsync")) { if (!strcasecmp(o->ptr,"no")) { server.aof_fsync = AOF_FSYNC_NO; @@ -475,6 +683,16 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"auto-aof-rewrite-min-size")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.aof_rewrite_min_size = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"aof-rewrite-incremental-fsync")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.aof_rewrite_incremental_fsync = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"aof-load-truncated")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.aof_load_truncated = yn; } else if (!strcasecmp(c->argv[2]->ptr,"save")) { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -514,6 +732,16 @@ void configSetCommand(redisClient *c) { if (yn == -1) goto badfmt; server.repl_serve_stale_data = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"slave-read-only")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.repl_slave_ro = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"activerehashing")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.activerehashing = yn; } else if (!strcasecmp(c->argv[2]->ptr,"dir")) { if (chdir((char*)o->ptr) == -1) { addReplyErrorFormat(c,"Changing directory: %s", strerror(errno)); @@ -540,6 +768,9 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-value")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.zset_max_ziplist_value = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"hll-sparse-max-bytes")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.hll_sparse_max_bytes = ll; } else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.lua_time_limit = ll; @@ -549,6 +780,9 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"slowlog-max-len")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.slowlog_max_len = (unsigned)ll; + } else if (!strcasecmp(c->argv[2]->ptr,"latency-monitor-threshold")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.latency_monitor_threshold = ll; } else if (!strcasecmp(c->argv[2]->ptr,"loglevel")) { if (!strcasecmp(o->ptr,"warning")) { server.verbosity = REDIS_WARNING; @@ -579,7 +813,7 @@ void configSetCommand(redisClient *c) { long val; if ((j % 4) == 0) { - if (getClientLimitClassByName(v[j]) == -1) { + if (getClientTypeByName(v[j]) == -1) { sdsfreesplitres(v,vlen); goto badfmt; } @@ -597,7 +831,7 @@ void configSetCommand(redisClient *c) { unsigned long long hard, soft; int soft_seconds; - class = getClientLimitClassByName(v[j]); + class = getClientTypeByName(v[j]); hard = strtoll(v[j+1],NULL,10); soft = strtoll(v[j+2],NULL,10); soft_seconds = strtoll(v[j+3],NULL,10); @@ -618,6 +852,56 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"repl-timeout")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; server.repl_timeout = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-backlog-size")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; + resizeReplicationBacklog(ll); + } else if (!strcasecmp(c->argv[2]->ptr,"repl-backlog-ttl")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.repl_backlog_time_limit = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"watchdog-period")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + if (ll) + enableWatchdog(ll); + else + disableWatchdog(); + } else if (!strcasecmp(c->argv[2]->ptr,"rdbcompression")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.rdb_compression = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"notify-keyspace-events")) { + int flags = keyspaceEventsStringToFlags(o->ptr); + + if (flags == -1) goto badfmt; + server.notify_keyspace_events = flags; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-disable-tcp-nodelay")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.repl_disable_tcp_nodelay = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-diskless-sync")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.repl_diskless_sync = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-diskless-sync-delay")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.repl_diskless_sync_delay = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.slave_priority = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-to-write")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.repl_min_slaves_to_write = ll; + refreshGoodSlavesCount(); + } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-max-lag")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.repl_min_slaves_max_lag = ll; + refreshGoodSlavesCount(); } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -632,6 +916,10 @@ void configSetCommand(redisClient *c) { (char*)c->argv[2]->ptr); } +/*----------------------------------------------------------------------------- + * CONFIG GET implementation + *----------------------------------------------------------------------------*/ + #define config_get_string_field(_name,_var) do { \ if (stringmatch(pattern,_name,0)) { \ addReplyBulkCString(c,_name); \ @@ -668,8 +956,7 @@ void configGetCommand(redisClient *c) { /* String values */ config_get_string_field("dbfilename",server.rdb_filename); config_get_string_field("requirepass",server.requirepass); - config_get_string_field("masterauth",server.requirepass); - config_get_string_field("bind",server.bindaddr); + config_get_string_field("masterauth",server.masterauth); config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); config_get_string_field("pidfile",server.pidfile); @@ -678,6 +965,7 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("maxmemory",server.maxmemory); config_get_numerical_field("maxmemory-samples",server.maxmemory_samples); config_get_numerical_field("timeout",server.maxidletime); + config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); config_get_numerical_field("auto-aof-rewrite-percentage", server.aof_rewrite_perc); config_get_numerical_field("auto-aof-rewrite-min-size", @@ -696,27 +984,51 @@ void configGetCommand(redisClient *c) { server.zset_max_ziplist_entries); config_get_numerical_field("zset-max-ziplist-value", server.zset_max_ziplist_value); + config_get_numerical_field("hll-sparse-max-bytes", + server.hll_sparse_max_bytes); config_get_numerical_field("lua-time-limit",server.lua_time_limit); config_get_numerical_field("slowlog-log-slower-than", server.slowlog_log_slower_than); + config_get_numerical_field("latency-monitor-threshold", + server.latency_monitor_threshold); config_get_numerical_field("slowlog-max-len", server.slowlog_max_len); config_get_numerical_field("port",server.port); + config_get_numerical_field("tcp-backlog",server.tcp_backlog); config_get_numerical_field("databases",server.dbnum); config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); config_get_numerical_field("repl-timeout",server.repl_timeout); + config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); + config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); config_get_numerical_field("maxclients",server.maxclients); + config_get_numerical_field("watchdog-period",server.watchdog_period); + config_get_numerical_field("slave-priority",server.slave_priority); + config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); + config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); + config_get_numerical_field("hz",server.hz); + config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); /* Bool (yes/no) values */ config_get_bool_field("no-appendfsync-on-rewrite", server.aof_no_fsync_on_rewrite); config_get_bool_field("slave-serve-stale-data", server.repl_serve_stale_data); + config_get_bool_field("slave-read-only", + server.repl_slave_ro); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); config_get_bool_field("rdbcompression", server.rdb_compression); + config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); + config_get_bool_field("repl-disable-tcp-nodelay", + server.repl_disable_tcp_nodelay); + config_get_bool_field("repl-diskless-sync", + server.repl_diskless_sync); + config_get_bool_field("aof-rewrite-incremental-fsync", + server.aof_rewrite_incremental_fsync); + config_get_bool_field("aof-load-truncated", + server.aof_load_truncated); /* Everything we can't handle with macros follows. */ @@ -769,8 +1081,8 @@ void configGetCommand(redisClient *c) { int j; for (j = 0; j < server.saveparamslen; j++) { - buf = sdscatprintf(buf,"%ld %d", - server.saveparams[j].seconds, + buf = sdscatprintf(buf,"%jd %d", + (intmax_t)server.saveparams[j].seconds, server.saveparams[j].changes); if (j != server.saveparamslen-1) buf = sdscatlen(buf," ",1); @@ -798,13 +1110,13 @@ void configGetCommand(redisClient *c) { sds buf = sdsempty(); int j; - for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) { + for (j = 0; j < REDIS_CLIENT_TYPE_COUNT; j++) { buf = sdscatprintf(buf,"%s %llu %llu %ld", - getClientLimitClassName(j), + getClientTypeName(j), server.client_obuf_limits[j].hard_limit_bytes, server.client_obuf_limits[j].soft_limit_bytes, (long) server.client_obuf_limits[j].soft_limit_seconds); - if (j != REDIS_CLIENT_LIMIT_NUM_CLASSES-1) + if (j != REDIS_CLIENT_TYPE_COUNT-1) buf = sdscatlen(buf," ",1); } addReplyBulkCString(c,"client-output-buffer-limit"); @@ -831,9 +1143,678 @@ void configGetCommand(redisClient *c) { addReplyBulkCString(c,buf); matches++; } + if (stringmatch(pattern,"notify-keyspace-events",0)) { + robj *flagsobj = createObject(REDIS_STRING, + keyspaceEventsFlagsToString(server.notify_keyspace_events)); + + addReplyBulkCString(c,"notify-keyspace-events"); + addReplyBulk(c,flagsobj); + decrRefCount(flagsobj); + matches++; + } + if (stringmatch(pattern,"bind",0)) { + sds aux = sdsjoin(server.bindaddr,server.bindaddr_count," "); + + addReplyBulkCString(c,"bind"); + addReplyBulkCString(c,aux); + sdsfree(aux); + matches++; + } setDeferredMultiBulkLength(c,replylen,matches*2); } +/*----------------------------------------------------------------------------- + * CONFIG REWRITE implementation + *----------------------------------------------------------------------------*/ + +#define REDIS_CONFIG_REWRITE_SIGNATURE "# Generated by CONFIG REWRITE" + +/* We use the following dictionary type to store where a configuration + * option is mentioned in the old configuration file, so it's + * like "maxmemory" -> list of line numbers (first line is zero). */ +unsigned int dictSdsCaseHash(const void *key); +int dictSdsKeyCaseCompare(void *privdata, const void *key1, const void *key2); +void dictSdsDestructor(void *privdata, void *val); +void dictListDestructor(void *privdata, void *val); + +/* Sentinel config rewriting is implemented inside sentinel.c by + * rewriteConfigSentinelOption(). */ +void rewriteConfigSentinelOption(struct rewriteConfigState *state); + +dictType optionToLineDictType = { + dictSdsCaseHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCaseCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + dictListDestructor /* val destructor */ +}; + +dictType optionSetDictType = { + dictSdsCaseHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCaseCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL /* val destructor */ +}; + +/* The config rewrite state. */ +struct rewriteConfigState { + dict *option_to_line; /* Option -> list of config file lines map */ + dict *rewritten; /* Dictionary of already processed options */ + int numlines; /* Number of lines in current config */ + sds *lines; /* Current lines as an array of sds strings */ + int has_tail; /* True if we already added directives that were + not present in the original config file. */ +}; + +/* Append the new line to the current configuration state. */ +void rewriteConfigAppendLine(struct rewriteConfigState *state, sds line) { + state->lines = zrealloc(state->lines, sizeof(char*) * (state->numlines+1)); + state->lines[state->numlines++] = line; +} + +/* Populate the option -> list of line numbers map. */ +void rewriteConfigAddLineNumberToOption(struct rewriteConfigState *state, sds option, int linenum) { + list *l = dictFetchValue(state->option_to_line,option); + + if (l == NULL) { + l = listCreate(); + dictAdd(state->option_to_line,sdsdup(option),l); + } + listAddNodeTail(l,(void*)(long)linenum); +} + +/* Add the specified option to the set of processed options. + * This is useful as only unused lines of processed options will be blanked + * in the config file, while options the rewrite process does not understand + * remain untouched. */ +void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, char *option) { + sds opt = sdsnew(option); + + if (dictAdd(state->rewritten,opt,NULL) != DICT_OK) sdsfree(opt); +} + +/* Read the old file, split it into lines to populate a newly created + * config rewrite state, and return it to the caller. + * + * If it is impossible to read the old file, NULL is returned. + * If the old file does not exist at all, an empty state is returned. */ +struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { + FILE *fp = fopen(path,"r"); + struct rewriteConfigState *state = zmalloc(sizeof(*state)); + char buf[REDIS_CONFIGLINE_MAX+1]; + int linenum = -1; + + if (fp == NULL && errno != ENOENT) return NULL; + + state->option_to_line = dictCreate(&optionToLineDictType,NULL); + state->rewritten = dictCreate(&optionSetDictType,NULL); + state->numlines = 0; + state->lines = NULL; + state->has_tail = 0; + if (fp == NULL) return state; + + /* Read the old file line by line, populate the state. */ + while(fgets(buf,REDIS_CONFIGLINE_MAX+1,fp) != NULL) { + int argc; + sds *argv; + sds line = sdstrim(sdsnew(buf),"\r\n\t "); + + linenum++; /* Zero based, so we init at -1 */ + + /* Handle comments and empty lines. */ + if (line[0] == '#' || line[0] == '\0') { + if (!state->has_tail && !strcmp(line,REDIS_CONFIG_REWRITE_SIGNATURE)) + state->has_tail = 1; + rewriteConfigAppendLine(state,line); + continue; + } + + /* Not a comment, split into arguments. */ + argv = sdssplitargs(line,&argc); + if (argv == NULL) { + /* Apparently the line is unparsable for some reason, for + * instance it may have unbalanced quotes. Load it as a + * comment. */ + sds aux = sdsnew("# ??? "); + aux = sdscatsds(aux,line); + sdsfree(line); + rewriteConfigAppendLine(state,aux); + continue; + } + + sdstolower(argv[0]); /* We only want lowercase config directives. */ + + /* Now we populate the state according to the content of this line. + * Append the line and populate the option -> line numbers map. */ + rewriteConfigAppendLine(state,line); + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + + sdsfreesplitres(argv,argc); + } + fclose(fp); + return state; +} + +/* Rewrite the specified configuration option with the new "line". + * It progressively uses lines of the file that were already used for the same + * configuration option in the old version of the file, removing that line from + * the map of options -> line numbers. + * + * If there are lines associated with a given configuration option and + * "force" is non-zero, the line is appended to the configuration file. + * Usually "force" is true when an option has not its default value, so it + * must be rewritten even if not present previously. + * + * The first time a line is appended into a configuration file, a comment + * is added to show that starting from that point the config file was generated + * by CONFIG REWRITE. + * + * "line" is either used, or freed, so the caller does not need to free it + * in any way. */ +void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sds line, int force) { + sds o = sdsnew(option); + list *l = dictFetchValue(state->option_to_line,o); + + rewriteConfigMarkAsProcessed(state,option); + + if (!l && !force) { + /* Option not used previously, and we are not forced to use it. */ + sdsfree(line); + sdsfree(o); + return; + } + + if (l) { + listNode *ln = listFirst(l); + int linenum = (long) ln->value; + + /* There are still lines in the old configuration file we can reuse + * for this option. Replace the line with the new one. */ + listDelNode(l,ln); + if (listLength(l) == 0) dictDelete(state->option_to_line,o); + sdsfree(state->lines[linenum]); + state->lines[linenum] = line; + } else { + /* Append a new line. */ + if (!state->has_tail) { + rewriteConfigAppendLine(state, + sdsnew(REDIS_CONFIG_REWRITE_SIGNATURE)); + state->has_tail = 1; + } + rewriteConfigAppendLine(state,line); + } + sdsfree(o); +} + +/* Write the long long 'bytes' value as a string in a way that is parsable + * inside redis.conf. If possible uses the GB, MB, KB notation. */ +int rewriteConfigFormatMemory(char *buf, size_t len, long long bytes) { + int gb = 1024*1024*1024; + int mb = 1024*1024; + int kb = 1024; + + if (bytes && (bytes % gb) == 0) { + return snprintf(buf,len,"%lldgb",bytes/gb); + } else if (bytes && (bytes % mb) == 0) { + return snprintf(buf,len,"%lldmb",bytes/mb); + } else if (bytes && (bytes % kb) == 0) { + return snprintf(buf,len,"%lldkb",bytes/kb); + } else { + return snprintf(buf,len,"%lld",bytes); + } +} + +/* Rewrite a simple "option-name " configuration option. */ +void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { + char buf[64]; + int force = value != defvalue; + sds line; + + rewriteConfigFormatMemory(buf,sizeof(buf),value); + line = sdscatprintf(sdsempty(),"%s %s",option,buf); + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite a yes/no option. */ +void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %s",option, + value ? "yes" : "no"); + + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite a string option. */ +void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { + int force = 1; + sds line; + + /* String options set to NULL need to be not present at all in the + * configuration file to be set to NULL again at the next reboot. */ + if (value == NULL) { + rewriteConfigMarkAsProcessed(state,option); + return; + } + + /* Set force to zero if the value is set to its default. */ + if (defvalue && strcmp(value,defvalue) == 0) force = 0; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, value, strlen(value)); + + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite a numerical (long long range) option. */ +void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); + + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite a octal option. */ +void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %o",option,value); + + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite an enumeration option, after the "value" every enum/value pair + * is specified, terminated by NULL. After NULL the default value is + * specified. See how the function is used for more information. */ +void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, ...) { + va_list ap; + char *enum_name, *matching_name = NULL; + int enum_val, def_val, force; + sds line; + + va_start(ap, value); + while(1) { + enum_name = va_arg(ap,char*); + enum_val = va_arg(ap,int); + if (enum_name == NULL) { + def_val = enum_val; + break; + } + if (value == enum_val) matching_name = enum_name; + } + va_end(ap); + + force = value != def_val; + line = sdscatprintf(sdsempty(),"%s %s",option,matching_name); + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite the syslog-facility option. */ +void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { + int value = server.syslog_facility, j; + int force = value != LOG_LOCAL0; + char *name = NULL, *option = "syslog-facility"; + sds line; + + for (j = 0; validSyslogFacilities[j].name; j++) { + if (validSyslogFacilities[j].value == value) { + name = (char*) validSyslogFacilities[j].name; + break; + } + } + line = sdscatprintf(sdsempty(),"%s %s",option,name); + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite the save option. */ +void rewriteConfigSaveOption(struct rewriteConfigState *state) { + int j; + sds line; + + /* Note that if there are no save parameters at all, all the current + * config line with "save" will be detected as orphaned and deleted, + * resulting into no RDB persistence as expected. */ + for (j = 0; j < server.saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + (long) server.saveparams[j].seconds, server.saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } + /* Mark "save" as processed in case server.saveparamslen is zero. */ + rewriteConfigMarkAsProcessed(state,"save"); +} + +/* Rewrite the dir option, always using absolute paths.*/ +void rewriteConfigDirOption(struct rewriteConfigState *state) { + char cwd[1024]; + + if (getcwd(cwd,sizeof(cwd)) == NULL) { + rewriteConfigMarkAsProcessed(state,"dir"); + return; /* no rewrite on error. */ + } + rewriteConfigStringOption(state,"dir",cwd,NULL); +} + +/* Rewrite the slaveof option. */ +void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { + char *option = "slaveof"; + sds line; + + /* If this is a master, we want all the slaveof config options + * in the file to be removed. */ + if (server.masterhost == NULL) { + rewriteConfigMarkAsProcessed(state,"slaveof"); + return; + } + line = sdscatprintf(sdsempty(),"%s %s %d", option, + server.masterhost, server.masterport); + rewriteConfigRewriteLine(state,option,line,1); +} + +/* Rewrite the notify-keyspace-events option. */ +void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { + int force = server.notify_keyspace_events != 0; + char *option = "notify-keyspace-events"; + sds line, flags; + + flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, flags, sdslen(flags)); + sdsfree(flags); + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Rewrite the client-output-buffer-limit option. */ +void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state) { + int j; + char *option = "client-output-buffer-limit"; + + for (j = 0; j < REDIS_CLIENT_TYPE_COUNT; j++) { + int force = (server.client_obuf_limits[j].hard_limit_bytes != + clientBufferLimitsDefaults[j].hard_limit_bytes) || + (server.client_obuf_limits[j].soft_limit_bytes != + clientBufferLimitsDefaults[j].soft_limit_bytes) || + (server.client_obuf_limits[j].soft_limit_seconds != + clientBufferLimitsDefaults[j].soft_limit_seconds); + sds line; + char hard[64], soft[64]; + + rewriteConfigFormatMemory(hard,sizeof(hard), + server.client_obuf_limits[j].hard_limit_bytes); + rewriteConfigFormatMemory(soft,sizeof(soft), + server.client_obuf_limits[j].soft_limit_bytes); + + line = sdscatprintf(sdsempty(),"%s %s %s %s %ld", + option, getClientTypeName(j), hard, soft, + (long) server.client_obuf_limits[j].soft_limit_seconds); + rewriteConfigRewriteLine(state,option,line,force); + } +} + +/* Rewrite the bind option. */ +void rewriteConfigBindOption(struct rewriteConfigState *state) { + int force = 1; + sds line, addresses; + char *option = "bind"; + + /* Nothing to rewrite if we don't have bind addresses. */ + if (server.bindaddr_count == 0) { + rewriteConfigMarkAsProcessed(state,option); + return; + } + + /* Rewrite as bind ... */ + addresses = sdsjoin(server.bindaddr,server.bindaddr_count," "); + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatsds(line, addresses); + sdsfree(addresses); + + rewriteConfigRewriteLine(state,option,line,force); +} + +/* Glue together the configuration lines in the current configuration + * rewrite state into a single string, stripping multiple empty lines. */ +sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { + sds content = sdsempty(); + int j, was_empty = 0; + + for (j = 0; j < state->numlines; j++) { + /* Every cluster of empty lines is turned into a single empty line. */ + if (sdslen(state->lines[j]) == 0) { + if (was_empty) continue; + was_empty = 1; + } else { + was_empty = 0; + } + content = sdscatsds(content,state->lines[j]); + content = sdscatlen(content,"\n",1); + } + return content; +} + +/* Free the configuration rewrite state. */ +void rewriteConfigReleaseState(struct rewriteConfigState *state) { + sdsfreesplitres(state->lines,state->numlines); + dictRelease(state->option_to_line); + dictRelease(state->rewritten); + zfree(state); +} + +/* At the end of the rewrite process the state contains the remaining + * map between "option name" => "lines in the original config file". + * Lines used by the rewrite process were removed by the function + * rewriteConfigRewriteLine(), all the other lines are "orphaned" and + * should be replaced by empty lines. + * + * This function does just this, iterating all the option names and + * blanking all the lines still associated. */ +void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { + dictIterator *di = dictGetIterator(state->option_to_line); + dictEntry *de; + + while((de = dictNext(di)) != NULL) { + list *l = dictGetVal(de); + sds option = dictGetKey(de); + + /* Don't blank lines about options the rewrite process + * don't understand. */ + if (dictFind(state->rewritten,option) == NULL) { + redisLog(REDIS_DEBUG,"Not rewritten option: %s", option); + continue; + } + + while(listLength(l)) { + listNode *ln = listFirst(l); + int linenum = (long) ln->value; + + sdsfree(state->lines[linenum]); + state->lines[linenum] = sdsempty(); + listDelNode(l,ln); + } + } + dictReleaseIterator(di); +} + +/* This function overwrites the old configuration file with the new content. + * + * 1) The old file length is obtained. + * 2) If the new content is smaller, padding is added. + * 3) A single write(2) call is used to replace the content of the file. + * 4) Later the file is truncated to the length of the new content. + * + * This way we are sure the file is left in a consistent state even if the + * process is stopped between any of the four operations. + * + * The function returns 0 on success, otherwise -1 is returned and errno + * set accordingly. */ +int rewriteConfigOverwriteFile(char *configfile, sds content) { + int retval = 0; + int fd = open(configfile,O_RDWR|O_CREAT,0644); + int content_size = sdslen(content), padding = 0; + struct stat sb; + sds content_padded; + + /* 1) Open the old file (or create a new one if it does not + * exist), get the size. */ + if (fd == -1) return -1; /* errno set by open(). */ + if (fstat(fd,&sb) == -1) { + close(fd); + return -1; /* errno set by fstat(). */ + } + + /* 2) Pad the content at least match the old file size. */ + content_padded = sdsdup(content); + if (content_size < sb.st_size) { + /* If the old file was bigger, pad the content with + * a newline plus as many "#" chars as required. */ + padding = sb.st_size - content_size; + content_padded = sdsgrowzero(content_padded,sb.st_size); + content_padded[content_size] = '\n'; + memset(content_padded+content_size+1,'#',padding-1); + } + + /* 3) Write the new content using a single write(2). */ + if (write(fd,content_padded,strlen(content_padded)) == -1) { + retval = -1; + goto cleanup; + } + + /* 4) Truncate the file to the right length if we used padding. */ + if (padding) { + if (ftruncate(fd,content_size) == -1) { + /* Non critical error... */ + } + } + +cleanup: + sdsfree(content_padded); + close(fd); + return retval; +} + +/* Rewrite the configuration file at "path". + * If the configuration file already exists, we try at best to retain comments + * and overall structure. + * + * Configuration parameters that are at their default value, unless already + * explicitly included in the old configuration file, are not rewritten. + * + * On error -1 is returned and errno is set accordingly, otherwise 0. */ +int rewriteConfig(char *path) { + struct rewriteConfigState *state; + sds newcontent; + int retval; + + /* Step 1: read the old config into our rewrite state. */ + if ((state = rewriteConfigReadOldFile(path)) == NULL) return -1; + + /* Step 2: rewrite every single option, replacing or appending it inside + * the rewrite state. */ + + rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); + rewriteConfigStringOption(state,"pidfile",server.pidfile,REDIS_DEFAULT_PID_FILE); + rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); + rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,REDIS_TCP_BACKLOG); + rewriteConfigBindOption(state); + rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); + rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,REDIS_DEFAULT_UNIX_SOCKET_PERM); + rewriteConfigNumericalOption(state,"timeout",server.maxidletime,REDIS_MAXIDLETIME); + rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,REDIS_DEFAULT_TCP_KEEPALIVE); + rewriteConfigEnumOption(state,"loglevel",server.verbosity, + "debug", REDIS_DEBUG, + "verbose", REDIS_VERBOSE, + "notice", REDIS_NOTICE, + "warning", REDIS_WARNING, + NULL, REDIS_DEFAULT_VERBOSITY); + rewriteConfigStringOption(state,"logfile",server.logfile,REDIS_DEFAULT_LOGFILE); + rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,REDIS_DEFAULT_SYSLOG_ENABLED); + rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,REDIS_DEFAULT_SYSLOG_IDENT); + rewriteConfigSyslogfacilityOption(state); + rewriteConfigSaveOption(state); + rewriteConfigNumericalOption(state,"databases",server.dbnum,REDIS_DEFAULT_DBNUM); + rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR); + rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,REDIS_DEFAULT_RDB_COMPRESSION); + rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,REDIS_DEFAULT_RDB_CHECKSUM); + rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,REDIS_DEFAULT_RDB_FILENAME); + rewriteConfigDirOption(state); + rewriteConfigSlaveofOption(state); + rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); + rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA); + rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,REDIS_DEFAULT_SLAVE_READ_ONLY); + rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,REDIS_REPL_PING_SLAVE_PERIOD); + rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,REDIS_REPL_TIMEOUT); + rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); + rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); + rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); + rewriteConfigYesNoOption(state,"repl-diskless-sync",server.repl_diskless_sync,REDIS_DEFAULT_REPL_DISKLESS_SYNC); + rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,REDIS_DEFAULT_REPL_DISKLESS_SYNC_DELAY); + rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); + rewriteConfigNumericalOption(state,"min-slaves-to-write",server.repl_min_slaves_to_write,REDIS_DEFAULT_MIN_SLAVES_TO_WRITE); + rewriteConfigNumericalOption(state,"min-slaves-max-lag",server.repl_min_slaves_max_lag,REDIS_DEFAULT_MIN_SLAVES_MAX_LAG); + rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); + rewriteConfigNumericalOption(state,"maxclients",server.maxclients,REDIS_MAX_CLIENTS); + rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,REDIS_DEFAULT_MAXMEMORY); + rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy, + "volatile-lru", REDIS_MAXMEMORY_VOLATILE_LRU, + "allkeys-lru", REDIS_MAXMEMORY_ALLKEYS_LRU, + "volatile-random", REDIS_MAXMEMORY_VOLATILE_RANDOM, + "allkeys-random", REDIS_MAXMEMORY_ALLKEYS_RANDOM, + "volatile-ttl", REDIS_MAXMEMORY_VOLATILE_TTL, + "noeviction", REDIS_MAXMEMORY_NO_EVICTION, + NULL, REDIS_DEFAULT_MAXMEMORY_POLICY); + rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,REDIS_DEFAULT_MAXMEMORY_SAMPLES); + rewriteConfigYesNoOption(state,"appendonly",server.aof_state != REDIS_AOF_OFF,0); + rewriteConfigStringOption(state,"appendfilename",server.aof_filename,REDIS_DEFAULT_AOF_FILENAME); + rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync, + "everysec", AOF_FSYNC_EVERYSEC, + "always", AOF_FSYNC_ALWAYS, + "no", AOF_FSYNC_NO, + NULL, REDIS_DEFAULT_AOF_FSYNC); + rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE); + rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,REDIS_AOF_REWRITE_PERC); + rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,REDIS_AOF_REWRITE_MIN_SIZE); + rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,REDIS_LUA_TIME_LIMIT); + rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,REDIS_SLOWLOG_LOG_SLOWER_THAN); + rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,REDIS_DEFAULT_LATENCY_MONITOR_THRESHOLD); + rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,REDIS_SLOWLOG_MAX_LEN); + rewriteConfigNotifykeyspaceeventsOption(state); + rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,REDIS_HASH_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,REDIS_HASH_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"list-max-ziplist-entries",server.list_max_ziplist_entries,REDIS_LIST_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"list-max-ziplist-value",server.list_max_ziplist_value,REDIS_LIST_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); + rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES); + rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING); + rewriteConfigClientoutputbufferlimitOption(state); + rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ); + rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); + rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,REDIS_DEFAULT_AOF_LOAD_TRUNCATED); + if (server.sentinel_mode) rewriteConfigSentinelOption(state); + + /* Step 3: remove all the orphaned lines in the old file, that is, lines + * that were used by a config option and are no longer used, like in case + * of multiple "save" options or duplicated options. */ + rewriteConfigRemoveOrphaned(state); + + /* Step 4: generate a new configuration file from the modified state + * and write it into the original file. */ + newcontent = rewriteConfigGetContentFromState(state); + retval = rewriteConfigOverwriteFile(server.configfile,newcontent); + + sdsfree(newcontent); + rewriteConfigReleaseState(state); + return retval; +} + +/*----------------------------------------------------------------------------- + * CONFIG command entry point + *----------------------------------------------------------------------------*/ + void configCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"set")) { if (c->argc != 4) goto badarity; @@ -843,16 +1824,25 @@ void configCommand(redisClient *c) { configGetCommand(c); } else if (!strcasecmp(c->argv[1]->ptr,"resetstat")) { if (c->argc != 2) goto badarity; - server.stat_keyspace_hits = 0; - server.stat_keyspace_misses = 0; - server.stat_numcommands = 0; - server.stat_numconnections = 0; - server.stat_expiredkeys = 0; + resetServerStats(); resetCommandTableStats(); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"rewrite")) { + if (c->argc != 2) goto badarity; + if (server.configfile == NULL) { + addReplyError(c,"The server is running without a config file"); + return; + } + if (rewriteConfig(server.configfile) == -1) { + redisLog(REDIS_WARNING,"CONFIG REWRITE failed: %s", strerror(errno)); + addReplyErrorFormat(c,"Rewriting config file: %s", strerror(errno)); + } else { + redisLog(REDIS_WARNING,"CONFIG REWRITE executed with success."); + addReply(c,shared.ok); + } } else { addReplyError(c, - "CONFIG subcommand must be one of GET, SET, RESETSTAT"); + "CONFIG subcommand must be one of GET, SET, RESETSTAT, REWRITE"); } return; diff --git a/src/config.h b/src/config.h index 6a69364a971..1f2919ed21a 100644 --- a/src/config.h +++ b/src/config.h @@ -1,3 +1,32 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __CONFIG_H #define __CONFIG_H @@ -16,7 +45,10 @@ /* Test for proc filesystem */ #ifdef __linux__ -#define HAVE_PROCFS 1 +#define HAVE_PROC_STAT 1 +#define HAVE_PROC_MAPS 1 +#define HAVE_PROC_SMAPS 1 +#define HAVE_PROC_SOMAXCONN 1 #endif /* Test for task_info() */ @@ -25,7 +57,7 @@ #endif /* Test for backtrace() */ -#if defined(__APPLE__) || defined(__linux__) || defined(__sun) +#if defined(__APPLE__) || defined(__linux__) #define HAVE_BACKTRACE 1 #endif @@ -38,6 +70,13 @@ #define HAVE_KQUEUE 1 #endif +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#endif +#endif + /* Define aof_fsync to fdatasync() in Linux and fsync() for all the rest */ #ifdef __linux__ #define aof_fsync fdatasync @@ -45,6 +84,42 @@ #define aof_fsync fsync #endif +/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use + * the plain fsync() call. */ +#ifdef __linux__ +#include +#include +#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) +#if (LINUX_VERSION_CODE >= 0x020611 && __GLIBC_PREREQ(2, 6)) +#define HAVE_SYNC_FILE_RANGE 1 +#endif +#else +#if (LINUX_VERSION_CODE >= 0x020611) +#define HAVE_SYNC_FILE_RANGE 1 +#endif +#endif +#endif + +#ifdef HAVE_SYNC_FILE_RANGE +#define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE) +#else +#define rdb_fsync_range(fd,off,size) fsync(fd) +#endif + +/* Check if we can use setproctitle(). + * BSD systems have support for it, we provide an implementation for + * Linux and osx. */ +#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__) +#define USE_SETPROCTITLE +#endif + +#if (defined __linux || defined __APPLE__) +#define USE_SETPROCTITLE +#define INIT_SETPROCTITLE_REPLACEMENT +void spt_init(int argc, char *argv[]); +void setproctitle(const char *fmt, ...); +#endif + /* Byte ordering detection */ #include /* This will likely define BYTE_ORDER */ @@ -59,10 +134,11 @@ #define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ #define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ -#if defined(vax) || defined(ns32000) || defined(sun386) || defined(__i386__) || \ - defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ - defined(__alpha__) || defined(__alpha) -#define BYTE_ORDER LITTLE_ENDIAN +#if defined(__i386__) || defined(__x86_64__) || defined(__amd64__) || \ + defined(vax) || defined(ns32000) || defined(sun386) || \ + defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ + defined(__alpha__) || defined(__alpha) +#define BYTE_ORDER LITTLE_ENDIAN #endif #if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ @@ -78,13 +154,27 @@ #endif /* BSD */ #endif /* BYTE_ORDER */ -#if defined(__BYTE_ORDER) && !defined(BYTE_ORDER) +/* Sometimes after including an OS-specific header that defines the + * endianess we end with __BYTE_ORDER but not with BYTE_ORDER that is what + * the Redis code uses. In this case let's define everything without the + * underscores. */ +#ifndef BYTE_ORDER +#ifdef __BYTE_ORDER +#if defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif #if (__BYTE_ORDER == __LITTLE_ENDIAN) #define BYTE_ORDER LITTLE_ENDIAN #else #define BYTE_ORDER BIG_ENDIAN #endif #endif +#endif +#endif #if !defined(BYTE_ORDER) || \ (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN) @@ -96,12 +186,16 @@ #error "Undefined or invalid BYTE_ORDER" #endif -#if (__i386 || __amd64) && __GNUC__ +#if (__i386 || __amd64 || __powerpc__) && __GNUC__ #define GNUC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -#if GNUC_VERSION >= 40100 +#if defined(__clang__) +#define HAVE_ATOMIC +#endif +#if (defined(__GLIBC__) && defined(__GLIBC_PREREQ)) +#if (GNUC_VERSION >= 40100 && __GLIBC_PREREQ(2, 6)) #define HAVE_ATOMIC #endif #endif - +#endif #endif diff --git a/src/crc16.c b/src/crc16.c deleted file mode 100644 index 6ba7f2448b1..00000000000 --- a/src/crc16.c +++ /dev/null @@ -1,87 +0,0 @@ -#include "redis.h" - -/* - * Copyright 2001-2010 Georges Menie (www.menie.org) - * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) - * All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the University of California, Berkeley nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* CRC16 implementation acording to CCITT standards. - * - * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the - * following parameters: - * - * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" - * Width : 16 bit - * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) - * Initialization : 0000 - * Reflect Input byte : False - * Reflect Output CRC : False - * Xor constant to output CRC : 0000 - * Output for "123456789" : 31C3 - */ - -static const uint16_t crc16tab[256]= { - 0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7, - 0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef, - 0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6, - 0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de, - 0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485, - 0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d, - 0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4, - 0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc, - 0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823, - 0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b, - 0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12, - 0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a, - 0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41, - 0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49, - 0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70, - 0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78, - 0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f, - 0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067, - 0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e, - 0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256, - 0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d, - 0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405, - 0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c, - 0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634, - 0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab, - 0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3, - 0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a, - 0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92, - 0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9, - 0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1, - 0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8, - 0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0 -}; - -uint16_t crc16(const char *buf, int len) { - int counter; - uint16_t crc = 0; - for (counter = 0; counter < len; counter++) - crc = (crc<<8) ^ crc16tab[((crc>>8) ^ *buf++)&0x00FF]; - return crc; -} diff --git a/src/crc64.c b/src/crc64.c new file mode 100644 index 00000000000..ecdba90e045 --- /dev/null +++ b/src/crc64.c @@ -0,0 +1,191 @@ +/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. + * + * Specification of this CRC64 variant follows: + * Name: crc-64-jones + * Width: 64 bites + * Poly: 0xad93d23594c935a9 + * Reflected In: True + * Xor_In: 0xffffffffffffffff + * Reflected_Out: True + * Xor_Out: 0x0 + * Check("123456789"): 0xe9c6d914c4b8d9ca + * + * Copyright (c) 2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#include + +static const uint64_t crc64_tab[256] = { + UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), + UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), + UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), + UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), + UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), + UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), + UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), + UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), + UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), + UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), + UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), + UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), + UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), + UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), + UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), + UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), + UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), + UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), + UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), + UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), + UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), + UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), + UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), + UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), + UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), + UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), + UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), + UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), + UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), + UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), + UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), + UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), + UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), + UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), + UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), + UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), + UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), + UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), + UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), + UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), + UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), + UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), + UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), + UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), + UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), + UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), + UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), + UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), + UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), + UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), + UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), + UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), + UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), + UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), + UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), + UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), + UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), + UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), + UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), + UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), + UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), + UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), + UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), + UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), + UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), + UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), + UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), + UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), + UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), + UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), + UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), + UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), + UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), + UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), + UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), + UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), + UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), + UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), + UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), + UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), + UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), + UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), + UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), + UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), + UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), + UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), + UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), + UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), + UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), + UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), + UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), + UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), + UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), + UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), + UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), + UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), + UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), + UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), + UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), + UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), + UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), + UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), + UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), + UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), + UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), + UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), + UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), + UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), + UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), + UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), + UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), + UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), + UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), + UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), + UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), + UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), + UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), + UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), + UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), + UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), + UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), + UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), + UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), + UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), + UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), + UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), + UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), + UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), +}; + +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { + uint64_t j; + + for (j = 0; j < l; j++) { + uint8_t byte = s[j]; + crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8); + } + return crc; +} + +/* Test main */ +#ifdef TEST_MAIN +#include +int main(void) { + printf("e9c6d914c4b8d9ca == %016llx\n", + (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); + return 0; +} +#endif diff --git a/src/crc64.h b/src/crc64.h new file mode 100644 index 00000000000..ab375d3f4f7 --- /dev/null +++ b/src/crc64.h @@ -0,0 +1,8 @@ +#ifndef CRC64_H +#define CRC64_H + +#include + +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); + +#endif diff --git a/src/db.c b/src/db.c index a0775af9650..18b09e77ef7 100644 --- a/src/db.c +++ b/src/db.c @@ -1,3 +1,32 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" #include @@ -10,34 +39,12 @@ void SlotToKeyDel(robj *key); * C-level DB API *----------------------------------------------------------------------------*/ -/* Important notes on lookup and disk store. - * - * When disk store is enabled on lookup we can have different cases. - * - * a) The key is in memory: - * - If the key is not in IO_SAVEINPROG state we can access it. - * As if it's just IO_SAVE this means we have the key in the IO queue - * but can't be accessed by the IO thread (it requires to be - * translated into an IO Job by the cache cron function.) - * - If the key is in IO_SAVEINPROG we can't touch the key and have - * to blocking wait completion of operations. - * b) The key is not in memory: - * - If it's marked as non existing on disk as well (negative cache) - * we don't need to perform the disk access. - * - if the key MAY EXIST, but is not in memory, and it is marked as IO_SAVE - * then the key can only be a deleted one. As IO_SAVE keys are never - * evicted (dirty state), so the only possibility is that key was deleted. - * - if the key MAY EXIST we need to blocking load it. - * We check that the key is not in IO_SAVEINPROG state before accessing - * the disk object. If it is in this state, we wait. - */ - robj *lookupKey(redisDb *db, robj *key) { dictEntry *de = dictFind(db->dict,key->ptr); if (de) { robj *val = dictGetVal(de); - /* Update the access time for the aging algorithm. + /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) @@ -78,7 +85,7 @@ robj *lookupKeyWriteOrReply(redisClient *c, robj *key, robj *reply) { } /* Add the key to the DB. It's up to the caller to increment the reference - * counte of the value if needed. + * counter of the value if needed. * * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { @@ -86,7 +93,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) { int retval = dictAdd(db->dict, copy, val); redisAssertWithInfo(NULL,key,retval == REDIS_OK); - if (server.cluster_enabled) SlotToKeyAdd(key); + if (val->type == REDIS_LIST) signalListAsReady(db, key); } /* Overwrite an existing key with a new value. Incrementing the reference @@ -96,7 +103,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) { * The program is aborted if the key was not already present. */ void dbOverwrite(redisDb *db, robj *key, robj *val) { struct dictEntry *de = dictFind(db->dict,key->ptr); - + redisAssertWithInfo(NULL,key,de != NULL); dictReplace(db->dict, key->ptr, val); } @@ -154,23 +161,58 @@ int dbDelete(redisDb *db, robj *key) { * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); if (dictDelete(db->dict,key->ptr) == DICT_OK) { - if (server.cluster_enabled) SlotToKeyDel(key); return 1; } else { return 0; } } -/* Empty the whole database. - * If diskstore is enabled this function will just flush the in-memory cache. */ -long long emptyDb() { +/* Prepare the string object stored at 'key' to be modified destructively + * to implement commands like SETBIT or APPEND. + * + * An object is usually ready to be modified unless one of the two conditions + * are true: + * + * 1) The object 'o' is shared (refcount > 1), we don't want to affect + * other users. + * 2) The object encoding is not "RAW". + * + * If the object is found in one of the above conditions (or both) by the + * function, an unshared / not-encoded copy of the string object is stored + * at 'key' in the specified 'db'. Otherwise the object 'o' itself is + * returned. + * + * USAGE: + * + * The object 'o' is what the caller already obtained by looking up 'key' + * in 'db', the usage pattern looks like this: + * + * o = lookupKeyWrite(db,key); + * if (checkType(c,o,REDIS_STRING)) return; + * o = dbUnshareStringValue(db,key,o); + * + * At this point the caller is ready to modify the object, for example + * using an sdscat() call to append some data, or anything else. + */ +robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { + redisAssert(o->type == REDIS_STRING); + if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) { + robj *decoded = getDecodedObject(o); + o = createStringObject(decoded->ptr, sdslen(decoded->ptr)); + decrRefCount(decoded); + dbOverwrite(db,key,o); + } + return o; +} + +long long emptyDb(void(callback)(void*)) { int j; long long removed = 0; for (j = 0; j < server.dbnum; j++) { removed += dictSize(server.db[j].dict); - dictEmpty(server.db[j].dict); - dictEmpty(server.db[j].expires); + dictEmpty(server.db[j].dict,callback); + dictEmpty(server.db[j].expires,callback); } return removed; } @@ -206,17 +248,17 @@ void signalFlushedDb(int dbid) { void flushdbCommand(redisClient *c) { server.dirty += dictSize(c->db->dict); signalFlushedDb(c->db->id); - dictEmpty(c->db->dict); - dictEmpty(c->db->expires); + dictEmpty(c->db->dict,NULL); + dictEmpty(c->db->expires,NULL); addReply(c,shared.ok); } void flushallCommand(redisClient *c) { signalFlushedDb(-1); - server.dirty += emptyDb(); + server.dirty += emptyDb(NULL); addReply(c,shared.ok); if (server.rdb_child_pid != -1) { - kill(server.rdb_child_pid,SIGKILL); + kill(server.rdb_child_pid,SIGUSR1); rdbRemoveTempFile(server.rdb_child_pid); } if (server.saveparamslen > 0) { @@ -233,8 +275,11 @@ void delCommand(redisClient *c) { int deleted = 0, j; for (j = 1; j < c->argc; j++) { + expireIfNeeded(c->db,c->argv[j]); if (dbDelete(c->db,c->argv[j])) { signalModifiedKey(c->db,c->argv[j]); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC, + "del",c->argv[j],c->db->id); server.dirty++; deleted++; } @@ -252,12 +297,12 @@ void existsCommand(redisClient *c) { } void selectCommand(redisClient *c) { - int id = atoi(c->argv[1]->ptr); + long id; - if (server.cluster_enabled && id != 0) { - addReplyError(c,"SELECT is not allowed in cluster mode"); + if (getLongFromObjectOrReply(c, c->argv[1], &id, + "invalid DB index") != REDIS_OK) return; - } + if (selectDb(c,id) == REDIS_ERR) { addReplyError(c,"invalid DB index"); } else { @@ -285,7 +330,7 @@ void keysCommand(redisClient *c) { unsigned long numkeys = 0; void *replylen = addDeferredMultiBulkLength(c); - di = dictGetIterator(c->db->dict); + di = dictGetSafeIterator(c->db->dict); allkeys = (pattern[0] == '*' && pattern[1] == '\0'); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); @@ -304,6 +349,251 @@ void keysCommand(redisClient *c) { setDeferredMultiBulkLength(c,replylen,numkeys); } +/* This callback is used by scanGenericCommand in order to collect elements + * returned by the dictionary iterator into a list. */ +void scanCallback(void *privdata, const dictEntry *de) { + void **pd = (void**) privdata; + list *keys = pd[0]; + robj *o = pd[1]; + robj *key, *val = NULL; + + if (o == NULL) { + sds sdskey = dictGetKey(de); + key = createStringObject(sdskey, sdslen(sdskey)); + } else if (o->type == REDIS_SET) { + key = dictGetKey(de); + incrRefCount(key); + } else if (o->type == REDIS_HASH) { + key = dictGetKey(de); + incrRefCount(key); + val = dictGetVal(de); + incrRefCount(val); + } else if (o->type == REDIS_ZSET) { + key = dictGetKey(de); + incrRefCount(key); + val = createStringObjectFromLongDouble(*(double*)dictGetVal(de),0); + } else { + redisPanic("Type not handled in SCAN callback."); + } + + listAddNodeTail(keys, key); + if (val) listAddNodeTail(keys, val); +} + +/* Try to parse a SCAN cursor stored at object 'o': + * if the cursor is valid, store it as unsigned integer into *cursor and + * returns REDIS_OK. Otherwise return REDIS_ERR and send an error to the + * client. */ +int parseScanCursorOrReply(redisClient *c, robj *o, unsigned long *cursor) { + char *eptr; + + /* Use strtoul() because we need an *unsigned* long, so + * getLongLongFromObject() does not cover the whole cursor space. */ + errno = 0; + *cursor = strtoul(o->ptr, &eptr, 10); + if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || errno == ERANGE) + { + addReplyError(c, "invalid cursor"); + return REDIS_ERR; + } + return REDIS_OK; +} + +/* This command implements SCAN, HSCAN and SSCAN commands. + * If object 'o' is passed, then it must be a Hash or Set object, otherwise + * if 'o' is NULL the command will operate on the dictionary associated with + * the current database. + * + * When 'o' is not NULL the function assumes that the first argument in + * the client arguments vector is a key so it skips it before iterating + * in order to parse options. + * + * In the case of a Hash object the function returns both the field and value + * of every element on the Hash. */ +void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor) { + int i, j; + list *keys = listCreate(); + listNode *node, *nextnode; + long count = 10; + sds pat; + int patlen, use_pattern = 0; + dict *ht; + + /* Object must be NULL (to iterate keys names), or the type of the object + * must be Set, Sorted Set, or Hash. */ + redisAssert(o == NULL || o->type == REDIS_SET || o->type == REDIS_HASH || + o->type == REDIS_ZSET); + + /* Set i to the first option argument. The previous one is the cursor. */ + i = (o == NULL) ? 2 : 3; /* Skip the key argument if needed. */ + + /* Step 1: Parse options. */ + while (i < c->argc) { + j = c->argc - i; + if (!strcasecmp(c->argv[i]->ptr, "count") && j >= 2) { + if (getLongFromObjectOrReply(c, c->argv[i+1], &count, NULL) + != REDIS_OK) + { + goto cleanup; + } + + if (count < 1) { + addReply(c,shared.syntaxerr); + goto cleanup; + } + + i += 2; + } else if (!strcasecmp(c->argv[i]->ptr, "match") && j >= 2) { + pat = c->argv[i+1]->ptr; + patlen = sdslen(pat); + + /* The pattern always matches if it is exactly "*", so it is + * equivalent to disabling it. */ + use_pattern = !(pat[0] == '*' && patlen == 1); + + i += 2; + } else { + addReply(c,shared.syntaxerr); + goto cleanup; + } + } + + /* Step 2: Iterate the collection. + * + * Note that if the object is encoded with a ziplist, intset, or any other + * representation that is not a hash table, we are sure that it is also + * composed of a small number of elements. So to avoid taking state we + * just return everything inside the object in a single call, setting the + * cursor to zero to signal the end of the iteration. */ + + /* Handle the case of a hash table. */ + ht = NULL; + if (o == NULL) { + ht = c->db->dict; + } else if (o->type == REDIS_SET && o->encoding == REDIS_ENCODING_HT) { + ht = o->ptr; + } else if (o->type == REDIS_HASH && o->encoding == REDIS_ENCODING_HT) { + ht = o->ptr; + count *= 2; /* We return key / value for this type. */ + } else if (o->type == REDIS_ZSET && o->encoding == REDIS_ENCODING_SKIPLIST) { + zset *zs = o->ptr; + ht = zs->dict; + count *= 2; /* We return key / value for this type. */ + } + + if (ht) { + void *privdata[2]; + /* We set the max number of iterations to ten times the specified + * COUNT, so if the hash table is in a pathological state (very + * sparsely populated) we avoid to block too much time at the cost + * of returning no or very few elements. */ + long maxiterations = count*10; + + /* We pass two pointers to the callback: the list to which it will + * add new elements, and the object containing the dictionary so that + * it is possible to fetch more data in a type-dependent way. */ + privdata[0] = keys; + privdata[1] = o; + do { + cursor = dictScan(ht, cursor, scanCallback, privdata); + } while (cursor && + maxiterations-- && + listLength(keys) < (unsigned long)count); + } else if (o->type == REDIS_SET) { + int pos = 0; + int64_t ll; + + while(intsetGet(o->ptr,pos++,&ll)) + listAddNodeTail(keys,createStringObjectFromLongLong(ll)); + cursor = 0; + } else if (o->type == REDIS_HASH || o->type == REDIS_ZSET) { + unsigned char *p = ziplistIndex(o->ptr,0); + unsigned char *vstr; + unsigned int vlen; + long long vll; + + while(p) { + ziplistGet(p,&vstr,&vlen,&vll); + listAddNodeTail(keys, + (vstr != NULL) ? createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll)); + p = ziplistNext(o->ptr,p); + } + cursor = 0; + } else { + redisPanic("Not handled encoding in SCAN."); + } + + /* Step 3: Filter elements. */ + node = listFirst(keys); + while (node) { + robj *kobj = listNodeValue(node); + nextnode = listNextNode(node); + int filter = 0; + + /* Filter element if it does not match the pattern. */ + if (!filter && use_pattern) { + if (kobj->encoding == REDIS_ENCODING_INT) { + char buf[REDIS_LONGSTR_SIZE]; + int len; + + redisAssert(kobj->encoding == REDIS_ENCODING_INT); + len = ll2string(buf,sizeof(buf),(long)kobj->ptr); + if (!stringmatchlen(pat, patlen, buf, len, 0)) filter = 1; + } else { + if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) + filter = 1; + } + } + + /* Filter element if it is an expired key. */ + if (!filter && o == NULL && expireIfNeeded(c->db, kobj)) filter = 1; + + /* Remove the element and its associted value if needed. */ + if (filter) { + decrRefCount(kobj); + listDelNode(keys, node); + } + + /* If this is a hash or a sorted set, we have a flat list of + * key-value elements, so if this element was filtered, remove the + * value, or skip it if it was not filtered: we only match keys. */ + if (o && (o->type == REDIS_ZSET || o->type == REDIS_HASH)) { + node = nextnode; + nextnode = listNextNode(node); + if (filter) { + kobj = listNodeValue(node); + decrRefCount(kobj); + listDelNode(keys, node); + } + } + node = nextnode; + } + + /* Step 4: Reply to the client. */ + addReplyMultiBulkLen(c, 2); + addReplyBulkLongLong(c,cursor); + + addReplyMultiBulkLen(c, listLength(keys)); + while ((node = listFirst(keys)) != NULL) { + robj *kobj = listNodeValue(node); + addReplyBulk(c, kobj); + decrRefCount(kobj); + listDelNode(keys, node); + } + +cleanup: + listSetFreeMethod(keys,decrRefCountVoid); + listRelease(keys); +} + +/* The SCAN command completely relies on scanGenericCommand. */ +void scanCommand(redisClient *c) { + unsigned long cursor; + if (parseScanCursorOrReply(c,c->argv[1],&cursor) == REDIS_ERR) return; + scanGenericCommand(c,NULL,cursor); +} + void dbsizeCommand(redisClient *c) { addReplyLongLong(c,dictSize(c->db->dict)); } @@ -348,6 +638,14 @@ void shutdownCommand(redisClient *c) { return; } } + /* When SHUTDOWN is called while the server is loading a dataset in + * memory we need to make sure no attempt is performed to save + * the dataset on shutdown (otherwise it could overwrite the current DB + * with half-read data). + * + * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */ + if (server.loading || server.sentinel_mode) + flags = (flags & ~REDIS_SHUTDOWN_SAVE) | REDIS_SHUTDOWN_NOSAVE; if (prepareForShutdown(flags) == REDIS_OK) exit(0); addReplyError(c,"Errors trying to SHUTDOWN. Check logs."); } @@ -373,7 +671,8 @@ void renameGenericCommand(redisClient *c, int nx) { addReply(c,shared.czero); return; } - /* Overwrite: delete the old key before creating the new one with the same name. */ + /* Overwrite: delete the old key before creating the new one + * with the same name. */ dbDelete(c->db,c->argv[2]); } dbAdd(c->db,c->argv[2],o); @@ -381,6 +680,10 @@ void renameGenericCommand(redisClient *c, int nx) { dbDelete(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[2]); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"rename_from", + c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"rename_to", + c->argv[2],c->db->id); server.dirty++; addReply(c,nx ? shared.cone : shared.ok); } @@ -397,16 +700,16 @@ void moveCommand(redisClient *c) { robj *o; redisDb *src, *dst; int srcid; - - if (server.cluster_enabled) { - addReplyError(c,"MOVE is not allowed in cluster mode"); - return; - } + long long dbid, expire; /* Obtain source and target DB pointers */ src = c->db; srcid = c->db->id; - if (selectDb(c,atoi(c->argv[2]->ptr)) == REDIS_ERR) { + + if (getLongLongFromObject(c->argv[2],&dbid) == REDIS_ERR || + dbid < INT_MIN || dbid > INT_MAX || + selectDb(c,dbid) == REDIS_ERR) + { addReply(c,shared.outofrangeerr); return; } @@ -426,6 +729,7 @@ void moveCommand(redisClient *c) { addReply(c,shared.czero); return; } + expire = getExpire(c->db,c->argv[1]); /* Return zero if the key already exists in the target DB */ if (lookupKeyWrite(dst,c->argv[1]) != NULL) { @@ -433,6 +737,7 @@ void moveCommand(redisClient *c) { return; } dbAdd(dst,c->argv[1],o); + if (expire != -1) setExpire(dst,c->argv[1],expire); incrRefCount(o); /* OK! key moved, free the entry in the source DB */ @@ -495,38 +800,45 @@ void propagateExpire(redisDb *db, robj *key) { if (server.aof_state != REDIS_AOF_OFF) feedAppendOnlyFile(server.delCommand,db->id,argv,2); - if (listLength(server.slaves)) - replicationFeedSlaves(server.slaves,db->id,argv,2); + replicationFeedSlaves(server.slaves,db->id,argv,2); decrRefCount(argv[0]); decrRefCount(argv[1]); } int expireIfNeeded(redisDb *db, robj *key) { - long long when = getExpire(db,key); + mstime_t when = getExpire(db,key); + mstime_t now; if (when < 0) return 0; /* No expire for this key */ /* Don't expire anything while loading. It will be done later. */ if (server.loading) return 0; + /* If we are in the context of a Lua script, we claim that time is + * blocked to when the Lua script started. This way a key can expire + * only the first time it is accessed and not in the middle of the + * script execution, making propagation to slaves / AOF consistent. + * See issue #1525 on Github for more information. */ + now = server.lua_caller ? server.lua_time_start : mstime(); + /* If we are running in the context of a slave, return ASAP: * the slave key expiration is controlled by the master that will * send us synthesized DEL operations for expired keys. * - * Still we try to return the right information to the caller, + * Still we try to return the right information to the caller, * that is, 0 if we think the key should be still valid, 1 if * we think the key is expired at this time. */ - if (server.masterhost != NULL) { - return time(NULL) > when; - } + if (server.masterhost != NULL) return now > when; /* Return when this key has not expired */ - if (mstime() <= when) return 0; + if (now <= when) return 0; /* Delete the key */ server.stat_expiredkeys++; propagateExpire(db,key); + notifyKeyspaceEvent(REDIS_NOTIFY_EXPIRED, + "expired",key,db->id); return dbDelete(db,key); } @@ -534,37 +846,36 @@ int expireIfNeeded(redisDb *db, robj *key) { * Expires Commands *----------------------------------------------------------------------------*/ -/* Given an string object return true if it contains exactly the "ms" - * or "MS" string. This is used in order to check if the last argument - * of EXPIRE, EXPIREAT or TTL is "ms" to switch into millisecond input/output */ -int stringObjectEqualsMs(robj *a) { - char *arg = a->ptr; - return tolower(arg[0]) == 'm' && tolower(arg[1]) == 's' && arg[2] == '\0'; -} - -void expireGenericCommand(redisClient *c, long long offset, int unit) { - dictEntry *de; +/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT + * and PEXPIREAT. Because the commad second argument may be relative or absolute + * the "basetime" argument is used to signal what the base time is (either 0 + * for *AT variants of the command, or the current time for relative expires). + * + * unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for + * the argv[2] parameter. The basetime is always specified in milliseconds. */ +void expireGenericCommand(redisClient *c, long long basetime, int unit) { robj *key = c->argv[1], *param = c->argv[2]; - long long milliseconds; + long long when; /* unix time in milliseconds when the key will expire. */ - if (getLongLongFromObjectOrReply(c, param, &milliseconds, NULL) != REDIS_OK) + if (getLongLongFromObjectOrReply(c, param, &when, NULL) != REDIS_OK) return; - if (unit == UNIT_SECONDS) milliseconds *= 1000; - milliseconds -= offset; + if (unit == UNIT_SECONDS) when *= 1000; + when += basetime; - de = dictFind(c->db->dict,key->ptr); - if (de == NULL) { + /* No key, return zero. */ + if (lookupKeyRead(c->db,key) == NULL) { addReply(c,shared.czero); return; } + /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past * should never be executed as a DEL when load the AOF or in the context * of a slave instance. * * Instead we take the other branch of the IF statement setting an expire * (possibly in the past) and wait for an explicit DEL from the master. */ - if (milliseconds <= 0 && !server.loading && !server.masterhost) { + if (when <= mstime() && !server.loading && !server.masterhost) { robj *aux; redisAssertWithInfo(c,key,dbDelete(c->db,key)); @@ -575,41 +886,49 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { rewriteClientCommandVector(c,2,aux,key); decrRefCount(aux); signalModifiedKey(c->db,key); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",key,c->db->id); addReply(c, shared.cone); return; } else { - long long when = mstime()+milliseconds; setExpire(c->db,key,when); addReply(c,shared.cone); signalModifiedKey(c->db,key); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"expire",key,c->db->id); server.dirty++; return; } } void expireCommand(redisClient *c) { - expireGenericCommand(c,0,UNIT_SECONDS); + expireGenericCommand(c,mstime(),UNIT_SECONDS); } void expireatCommand(redisClient *c) { - expireGenericCommand(c,mstime(),UNIT_SECONDS); + expireGenericCommand(c,0,UNIT_SECONDS); } void pexpireCommand(redisClient *c) { - expireGenericCommand(c,0,UNIT_MILLISECONDS); + expireGenericCommand(c,mstime(),UNIT_MILLISECONDS); } void pexpireatCommand(redisClient *c) { - expireGenericCommand(c,mstime(),UNIT_MILLISECONDS); + expireGenericCommand(c,0,UNIT_MILLISECONDS); } void ttlGenericCommand(redisClient *c, int output_ms) { long long expire, ttl = -1; + /* If the key does not exist at all, return -2 */ + if (lookupKeyRead(c->db,c->argv[1]) == NULL) { + addReplyLongLong(c,-2); + return; + } + /* The key exists. Return -1 if it has no expire, or the actual + * TTL value otherwise. */ expire = getExpire(c->db,c->argv[1]); if (expire != -1) { ttl = expire-mstime(); - if (ttl < 0) ttl = -1; + if (ttl < 0) ttl = 0; } if (ttl == -1) { addReplyLongLong(c,-1); @@ -714,35 +1033,3 @@ int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *num *numkeys = num; return keys; } - -/* Slot to Key API. This is used by Redis Cluster in order to obtain in - * a fast way a key that belongs to a specified hash slot. This is useful - * while rehashing the cluster. */ -void SlotToKeyAdd(robj *key) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); - - zslInsert(server.cluster.slots_to_keys,hashslot,key); - incrRefCount(key); -} - -void SlotToKeyDel(robj *key) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); - - zslDelete(server.cluster.slots_to_keys,hashslot,key); -} - -unsigned int GetKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count) { - zskiplistNode *n; - zrangespec range; - int j = 0; - - range.min = range.max = hashslot; - range.minex = range.maxex = 0; - - n = zslFirstInRange(server.cluster.slots_to_keys, range); - while(n && n->score == hashslot && count--) { - keys[j++] = n->obj; - n = n->level[0].forward; - } - return j; -} diff --git a/src/debug.c b/src/debug.c index a355df05f6d..ad1fcf71d17 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1,5 +1,35 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" #include "sha1.h" /* SHA1 is used for DEBUG DIGEST */ +#include "crc64.h" #include #include @@ -7,12 +37,20 @@ #ifdef HAVE_BACKTRACE #include #include +#include +#include "bio.h" #endif /* HAVE_BACKTRACE */ +#ifdef __CYGWIN__ +#ifndef SA_ONSTACK +#define SA_ONSTACK 0x08000000 +#endif +#endif + /* ================================= Debugging ============================== */ /* Compute the sha1 of string at 's' with 'len' bytes long. - * The SHA1 is then xored againt the string pointed by digest. + * The SHA1 is then xored against the string pointed by digest. * Since xor is commutative, this operation is used in order to * "add" digests relative to unordered elements. * @@ -37,7 +75,7 @@ void xorObjectDigest(unsigned char *digest, robj *o) { } /* This function instead of just computing the SHA1 and xoring it - * against diget, also perform the digest of "digest" itself and + * against digest, also perform the digest of "digest" itself and * replace the old value with the new one. * * So the final digest will be: @@ -105,7 +143,6 @@ void computeDatasetDigest(unsigned char *final) { mixDigest(digest,key,sdslen(key)); - /* Make sure the key is loaded if VM is active */ o = dictGetVal(de); aux = htonl(o->type); @@ -218,6 +255,10 @@ void computeDatasetDigest(unsigned char *final) { void debugCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"segfault")) { *((char*)-1) = 'x'; + } else if (!strcasecmp(c->argv[1]->ptr,"oom")) { + void *ptr = zmalloc(ULONG_MAX); /* Should trigger an out of memory. */ + zfree(ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"assert")) { if (c->argc >= 3) c->argv[2] = tryObjectEncoding(c->argv[2]); redisAssertWithInfo(c,c->argv[0],1 == 2); @@ -226,7 +267,7 @@ void debugCommand(redisClient *c) { addReply(c,shared.err); return; } - emptyDb(); + emptyDb(NULL); if (rdbLoad(server.rdb_filename) != REDIS_OK) { addReplyError(c,"Error trying to load the RDB dump"); return; @@ -234,7 +275,7 @@ void debugCommand(redisClient *c) { redisLog(REDIS_WARNING,"DB reloaded by DEBUG RELOAD"); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { - emptyDb(); + emptyDb(NULL); if (loadAppendOnlyFile(server.aof_filename) != REDIS_OK) { addReply(c,shared.err); return; @@ -261,15 +302,41 @@ void debugCommand(redisClient *c) { (void*)val, val->refcount, strenc, (long long) rdbSavedObjectLen(val), val->lru, estimateObjectIdleTime(val)); - } else if (!strcasecmp(c->argv[1]->ptr,"populate") && c->argc == 3) { + } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { + dictEntry *de; + robj *val; + sds key; + + if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) { + addReply(c,shared.nokeyerr); + return; + } + val = dictGetVal(de); + key = dictGetKey(de); + + if (val->type != REDIS_STRING || val->encoding != REDIS_ENCODING_RAW) { + addReplyError(c,"Not an sds encoded string."); + } else { + addReplyStatusFormat(c, + "key_sds_len:%lld, key_sds_avail:%lld, " + "val_sds_len:%lld, val_sds_avail:%lld", + (long long) sdslen(key), + (long long) sdsavail(key), + (long long) sdslen(val->ptr), + (long long) sdsavail(val->ptr)); + } + } else if (!strcasecmp(c->argv[1]->ptr,"populate") && + (c->argc == 3 || c->argc == 4)) { long keys, j; robj *key, *val; char buf[128]; if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != REDIS_OK) return; + dictExpand(c->db->dict,keys); for (j = 0; j < keys; j++) { - snprintf(buf,sizeof(buf),"key:%lu",j); + snprintf(buf,sizeof(buf),"%s:%lu", + (c->argc == 3) ? "key" : (char*)c->argv[3]->ptr, j); key = createStringObject(buf,strlen(buf)); if (lookupKeyRead(c->db,key) != NULL) { decrRefCount(key); @@ -294,12 +361,27 @@ void debugCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { double dtime = strtod(c->argv[2]->ptr,NULL); long long utime = dtime*1000000; + struct timespec tv; - usleep(utime); + tv.tv_sec = utime / 1000000; + tv.tv_nsec = (utime % 1000000) * 1000; + nanosleep(&tv, NULL); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"set-active-expire") && + c->argc == 3) + { + server.active_expire_enabled = atoi(c->argv[2]->ptr); + addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"error") && c->argc == 3) { + sds errstr = sdsnewlen("-",1); + + errstr = sdscatsds(errstr,c->argv[2]->ptr); + errstr = sdsmapchars(errstr,"\n\r"," ",2); /* no newlines in errors. */ + errstr = sdscatlen(errstr,"\r\n",2); + addReplySds(c,errstr); } else { - addReplyError(c, - "Syntax error, try DEBUG [SEGFAULT|OBJECT |SWAPIN |SWAPOUT |RELOAD]"); + addReplyErrorFormat(c, "Unknown DEBUG subcommand or wrong number of arguments for '%s'", + (char*)c->argv[1]->ptr); } } @@ -349,9 +431,12 @@ void redisLogObjectDebugInfo(robj *o) { redisLog(REDIS_WARNING,"Object encoding: %d", o->encoding); redisLog(REDIS_WARNING,"Object refcount: %d", o->refcount); if (o->type == REDIS_STRING && o->encoding == REDIS_ENCODING_RAW) { - redisLog(REDIS_WARNING,"Object raw string len: %d", sdslen(o->ptr)); - if (sdslen(o->ptr) < 4096) - redisLog(REDIS_WARNING,"Object raw string content: \"%s\"", (char*)o->ptr); + redisLog(REDIS_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); + if (sdslen(o->ptr) < 4096) { + sds repr = sdscatrepr(sdsempty(),o->ptr,sdslen(o->ptr)); + redisLog(REDIS_WARNING,"Object raw string content: %s", repr); + sdsfree(repr); + } } else if (o->type == REDIS_LIST) { redisLog(REDIS_WARNING,"List length: %d", (int) listTypeLength(o)); } else if (o->type == REDIS_SET) { @@ -399,30 +484,31 @@ void bugReportStart(void) { #ifdef HAVE_BACKTRACE static void *getMcontextEip(ucontext_t *uc) { -#if defined(__FreeBSD__) - return (void*) uc->uc_mcontext.mc_eip; -#elif defined(__dietlibc__) - return (void*) uc->uc_mcontext.eip; -#elif defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) - #if __x86_64__ +#if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) + /* OSX < 10.6 */ + #if defined(__x86_64__) return (void*) uc->uc_mcontext->__ss.__rip; - #elif __i386__ + #elif defined(__i386__) return (void*) uc->uc_mcontext->__ss.__eip; - #else + #else return (void*) uc->uc_mcontext->__ss.__srr0; - #endif + #endif #elif defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6) - #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) + /* OSX >= 10.6 */ + #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) return (void*) uc->uc_mcontext->__ss.__rip; - #else + #else return (void*) uc->uc_mcontext->__ss.__eip; - #endif -#elif defined(__i386__) + #endif +#elif defined(__linux__) + /* Linux */ + #if defined(__i386__) return (void*) uc->uc_mcontext.gregs[14]; /* Linux 32 */ -#elif defined(__X86_64__) || defined(__x86_64__) + #elif defined(__X86_64__) || defined(__x86_64__) return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */ -#elif defined(__ia64__) /* Linux IA64 */ + #elif defined(__ia64__) /* Linux IA64 */ return (void*) uc->uc_mcontext.sc_ip; + #endif #else return NULL; #endif @@ -431,17 +517,23 @@ static void *getMcontextEip(ucontext_t *uc) { void logStackContent(void **sp) { int i; for (i = 15; i >= 0; i--) { + unsigned long addr = (unsigned long) sp+i; + unsigned long val = (unsigned long) sp[i]; + if (sizeof(long) == 4) - redisLog(REDIS_WARNING, "(%08lx) -> %08lx", sp+i, sp[i]); + redisLog(REDIS_WARNING, "(%08lx) -> %08lx", addr, val); else - redisLog(REDIS_WARNING, "(%016lx) -> %016lx", sp+i, sp[i]); + redisLog(REDIS_WARNING, "(%016lx) -> %016lx", addr, val); } } void logRegisters(ucontext_t *uc) { redisLog(REDIS_WARNING, "--- REGISTERS"); + +/* OSX */ #if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6) - #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) + /* OSX AMD64 */ + #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) redisLog(REDIS_WARNING, "\n" "RAX:%016lx RBX:%016lx\nRCX:%016lx RDX:%016lx\n" @@ -449,81 +541,86 @@ void logRegisters(ucontext_t *uc) { "R8 :%016lx R9 :%016lx\nR10:%016lx R11:%016lx\n" "R12:%016lx R13:%016lx\nR14:%016lx R15:%016lx\n" "RIP:%016lx EFL:%016lx\nCS :%016lx FS:%016lx GS:%016lx", - uc->uc_mcontext->__ss.__rax, - uc->uc_mcontext->__ss.__rbx, - uc->uc_mcontext->__ss.__rcx, - uc->uc_mcontext->__ss.__rdx, - uc->uc_mcontext->__ss.__rdi, - uc->uc_mcontext->__ss.__rsi, - uc->uc_mcontext->__ss.__rbp, - uc->uc_mcontext->__ss.__rsp, - uc->uc_mcontext->__ss.__r8, - uc->uc_mcontext->__ss.__r9, - uc->uc_mcontext->__ss.__r10, - uc->uc_mcontext->__ss.__r11, - uc->uc_mcontext->__ss.__r12, - uc->uc_mcontext->__ss.__r13, - uc->uc_mcontext->__ss.__r14, - uc->uc_mcontext->__ss.__r15, - uc->uc_mcontext->__ss.__rip, - uc->uc_mcontext->__ss.__rflags, - uc->uc_mcontext->__ss.__cs, - uc->uc_mcontext->__ss.__fs, - uc->uc_mcontext->__ss.__gs + (unsigned long) uc->uc_mcontext->__ss.__rax, + (unsigned long) uc->uc_mcontext->__ss.__rbx, + (unsigned long) uc->uc_mcontext->__ss.__rcx, + (unsigned long) uc->uc_mcontext->__ss.__rdx, + (unsigned long) uc->uc_mcontext->__ss.__rdi, + (unsigned long) uc->uc_mcontext->__ss.__rsi, + (unsigned long) uc->uc_mcontext->__ss.__rbp, + (unsigned long) uc->uc_mcontext->__ss.__rsp, + (unsigned long) uc->uc_mcontext->__ss.__r8, + (unsigned long) uc->uc_mcontext->__ss.__r9, + (unsigned long) uc->uc_mcontext->__ss.__r10, + (unsigned long) uc->uc_mcontext->__ss.__r11, + (unsigned long) uc->uc_mcontext->__ss.__r12, + (unsigned long) uc->uc_mcontext->__ss.__r13, + (unsigned long) uc->uc_mcontext->__ss.__r14, + (unsigned long) uc->uc_mcontext->__ss.__r15, + (unsigned long) uc->uc_mcontext->__ss.__rip, + (unsigned long) uc->uc_mcontext->__ss.__rflags, + (unsigned long) uc->uc_mcontext->__ss.__cs, + (unsigned long) uc->uc_mcontext->__ss.__fs, + (unsigned long) uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__rsp); - #else + #else + /* OSX x86 */ redisLog(REDIS_WARNING, "\n" "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n" "EDI:%08lx ESI:%08lx EBP:%08lx ESP:%08lx\n" "SS:%08lx EFL:%08lx EIP:%08lx CS :%08lx\n" "DS:%08lx ES:%08lx FS :%08lx GS :%08lx", - uc->uc_mcontext->__ss.__eax, - uc->uc_mcontext->__ss.__ebx, - uc->uc_mcontext->__ss.__ecx, - uc->uc_mcontext->__ss.__edx, - uc->uc_mcontext->__ss.__edi, - uc->uc_mcontext->__ss.__esi, - uc->uc_mcontext->__ss.__ebp, - uc->uc_mcontext->__ss.__esp, - uc->uc_mcontext->__ss.__ss, - uc->uc_mcontext->__ss.__eflags, - uc->uc_mcontext->__ss.__eip, - uc->uc_mcontext->__ss.__cs, - uc->uc_mcontext->__ss.__ds, - uc->uc_mcontext->__ss.__es, - uc->uc_mcontext->__ss.__fs, - uc->uc_mcontext->__ss.__gs + (unsigned long) uc->uc_mcontext->__ss.__eax, + (unsigned long) uc->uc_mcontext->__ss.__ebx, + (unsigned long) uc->uc_mcontext->__ss.__ecx, + (unsigned long) uc->uc_mcontext->__ss.__edx, + (unsigned long) uc->uc_mcontext->__ss.__edi, + (unsigned long) uc->uc_mcontext->__ss.__esi, + (unsigned long) uc->uc_mcontext->__ss.__ebp, + (unsigned long) uc->uc_mcontext->__ss.__esp, + (unsigned long) uc->uc_mcontext->__ss.__ss, + (unsigned long) uc->uc_mcontext->__ss.__eflags, + (unsigned long) uc->uc_mcontext->__ss.__eip, + (unsigned long) uc->uc_mcontext->__ss.__cs, + (unsigned long) uc->uc_mcontext->__ss.__ds, + (unsigned long) uc->uc_mcontext->__ss.__es, + (unsigned long) uc->uc_mcontext->__ss.__fs, + (unsigned long) uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__esp); - #endif -#elif defined(__i386__) + #endif +/* Linux */ +#elif defined(__linux__) + /* Linux x86 */ + #if defined(__i386__) redisLog(REDIS_WARNING, "\n" "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n" "EDI:%08lx ESI:%08lx EBP:%08lx ESP:%08lx\n" "SS :%08lx EFL:%08lx EIP:%08lx CS:%08lx\n" "DS :%08lx ES :%08lx FS :%08lx GS:%08lx", - uc->uc_mcontext.gregs[11], - uc->uc_mcontext.gregs[8], - uc->uc_mcontext.gregs[10], - uc->uc_mcontext.gregs[9], - uc->uc_mcontext.gregs[4], - uc->uc_mcontext.gregs[5], - uc->uc_mcontext.gregs[6], - uc->uc_mcontext.gregs[7], - uc->uc_mcontext.gregs[18], - uc->uc_mcontext.gregs[17], - uc->uc_mcontext.gregs[14], - uc->uc_mcontext.gregs[15], - uc->uc_mcontext.gregs[3], - uc->uc_mcontext.gregs[2], - uc->uc_mcontext.gregs[1], - uc->uc_mcontext.gregs[0] + (unsigned long) uc->uc_mcontext.gregs[11], + (unsigned long) uc->uc_mcontext.gregs[8], + (unsigned long) uc->uc_mcontext.gregs[10], + (unsigned long) uc->uc_mcontext.gregs[9], + (unsigned long) uc->uc_mcontext.gregs[4], + (unsigned long) uc->uc_mcontext.gregs[5], + (unsigned long) uc->uc_mcontext.gregs[6], + (unsigned long) uc->uc_mcontext.gregs[7], + (unsigned long) uc->uc_mcontext.gregs[18], + (unsigned long) uc->uc_mcontext.gregs[17], + (unsigned long) uc->uc_mcontext.gregs[14], + (unsigned long) uc->uc_mcontext.gregs[15], + (unsigned long) uc->uc_mcontext.gregs[3], + (unsigned long) uc->uc_mcontext.gregs[2], + (unsigned long) uc->uc_mcontext.gregs[1], + (unsigned long) uc->uc_mcontext.gregs[0] ); logStackContent((void**)uc->uc_mcontext.gregs[7]); -#elif defined(__X86_64__) || defined(__x86_64__) + #elif defined(__X86_64__) || defined(__x86_64__) + /* Linux AMD64 */ redisLog(REDIS_WARNING, "\n" "RAX:%016lx RBX:%016lx\nRCX:%016lx RDX:%016lx\n" @@ -531,37 +628,183 @@ void logRegisters(ucontext_t *uc) { "R8 :%016lx R9 :%016lx\nR10:%016lx R11:%016lx\n" "R12:%016lx R13:%016lx\nR14:%016lx R15:%016lx\n" "RIP:%016lx EFL:%016lx\nCSGSFS:%016lx", - uc->uc_mcontext.gregs[13], - uc->uc_mcontext.gregs[11], - uc->uc_mcontext.gregs[14], - uc->uc_mcontext.gregs[12], - uc->uc_mcontext.gregs[8], - uc->uc_mcontext.gregs[9], - uc->uc_mcontext.gregs[10], - uc->uc_mcontext.gregs[15], - uc->uc_mcontext.gregs[0], - uc->uc_mcontext.gregs[1], - uc->uc_mcontext.gregs[2], - uc->uc_mcontext.gregs[3], - uc->uc_mcontext.gregs[4], - uc->uc_mcontext.gregs[5], - uc->uc_mcontext.gregs[6], - uc->uc_mcontext.gregs[7], - uc->uc_mcontext.gregs[16], - uc->uc_mcontext.gregs[17], - uc->uc_mcontext.gregs[18] + (unsigned long) uc->uc_mcontext.gregs[13], + (unsigned long) uc->uc_mcontext.gregs[11], + (unsigned long) uc->uc_mcontext.gregs[14], + (unsigned long) uc->uc_mcontext.gregs[12], + (unsigned long) uc->uc_mcontext.gregs[8], + (unsigned long) uc->uc_mcontext.gregs[9], + (unsigned long) uc->uc_mcontext.gregs[10], + (unsigned long) uc->uc_mcontext.gregs[15], + (unsigned long) uc->uc_mcontext.gregs[0], + (unsigned long) uc->uc_mcontext.gregs[1], + (unsigned long) uc->uc_mcontext.gregs[2], + (unsigned long) uc->uc_mcontext.gregs[3], + (unsigned long) uc->uc_mcontext.gregs[4], + (unsigned long) uc->uc_mcontext.gregs[5], + (unsigned long) uc->uc_mcontext.gregs[6], + (unsigned long) uc->uc_mcontext.gregs[7], + (unsigned long) uc->uc_mcontext.gregs[16], + (unsigned long) uc->uc_mcontext.gregs[17], + (unsigned long) uc->uc_mcontext.gregs[18] ); logStackContent((void**)uc->uc_mcontext.gregs[15]); + #endif #else redisLog(REDIS_WARNING, " Dumping of registers not supported for this OS/arch"); #endif } -void sigsegvHandler(int sig, siginfo_t *info, void *secret) { +/* Logs the stack trace using the backtrace() call. This function is designed + * to be called from signal handlers safely. */ +void logStackTrace(ucontext_t *uc) { void *trace[100]; - char **messages = NULL; - int i, trace_size = 0; + int trace_size = 0, fd; + int log_to_stdout = server.logfile[0] == '\0'; + + /* Open the log file in append mode. */ + fd = log_to_stdout ? + STDOUT_FILENO : + open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644); + if (fd == -1) return; + + /* Generate the stack trace */ + trace_size = backtrace(trace, 100); + + /* overwrite sigaction with caller's address */ + if (getMcontextEip(uc) != NULL) + trace[1] = getMcontextEip(uc); + + /* Write symbols to log file */ + backtrace_symbols_fd(trace, trace_size, fd); + + /* Cleanup */ + if (!log_to_stdout) close(fd); +} + +/* Log information about the "current" client, that is, the client that is + * currently being served by Redis. May be NULL if Redis is not serving a + * client right now. */ +void logCurrentClient(void) { + if (server.current_client == NULL) return; + + redisClient *cc = server.current_client; + sds client; + int j; + + redisLog(REDIS_WARNING, "--- CURRENT CLIENT INFO"); + client = catClientInfoString(sdsempty(),cc); + redisLog(REDIS_WARNING,"client: %s", client); + sdsfree(client); + for (j = 0; j < cc->argc; j++) { + robj *decoded; + + decoded = getDecodedObject(cc->argv[j]); + redisLog(REDIS_WARNING,"argv[%d]: '%s'", j, (char*)decoded->ptr); + decrRefCount(decoded); + } + /* Check if the first argument, usually a key, is found inside the + * selected DB, and if so print info about the associated object. */ + if (cc->argc >= 1) { + robj *val, *key; + dictEntry *de; + + key = getDecodedObject(cc->argv[1]); + de = dictFind(cc->db->dict, key->ptr); + if (de) { + val = dictGetVal(de); + redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", (char*)key->ptr); + redisLogObjectDebugInfo(val); + } + decrRefCount(key); + } +} + +#if defined(HAVE_PROC_MAPS) +void memtest_non_destructive_invert(void *addr, size_t size); +void memtest_non_destructive_swap(void *addr, size_t size); +#define MEMTEST_MAX_REGIONS 128 + +int memtest_test_linux_anonymous_maps(void) { + FILE *fp = fopen("/proc/self/maps","r"); + char line[1024]; + size_t start_addr, end_addr, size; + size_t start_vect[MEMTEST_MAX_REGIONS]; + size_t size_vect[MEMTEST_MAX_REGIONS]; + int regions = 0, j; + uint64_t crc1 = 0, crc2 = 0, crc3 = 0; + + while(fgets(line,sizeof(line),fp) != NULL) { + char *start, *end, *p = line; + + start = p; + p = strchr(p,'-'); + if (!p) continue; + *p++ = '\0'; + end = p; + p = strchr(p,' '); + if (!p) continue; + *p++ = '\0'; + if (strstr(p,"stack") || + strstr(p,"vdso") || + strstr(p,"vsyscall")) continue; + if (!strstr(p,"00:00")) continue; + if (!strstr(p,"rw")) continue; + + start_addr = strtoul(start,NULL,16); + end_addr = strtoul(end,NULL,16); + size = end_addr-start_addr; + + start_vect[regions] = start_addr; + size_vect[regions] = size; + printf("Testing %lx %lu\n", (unsigned long) start_vect[regions], + (unsigned long) size_vect[regions]); + regions++; + } + + /* Test all the regions as an unique sequential region. + * 1) Take the CRC64 of the memory region. */ + for (j = 0; j < regions; j++) { + crc1 = crc64(crc1,(void*)start_vect[j],size_vect[j]); + } + + /* 2) Invert bits, swap adjacent words, swap again, invert bits. + * This is the error amplification step. */ + for (j = 0; j < regions; j++) + memtest_non_destructive_invert((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_invert((void*)start_vect[j],size_vect[j]); + + /* 3) Take the CRC64 sum again. */ + for (j = 0; j < regions; j++) + crc2 = crc64(crc2,(void*)start_vect[j],size_vect[j]); + + /* 4) Swap + Swap again */ + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + + /* 5) Take the CRC64 sum again. */ + for (j = 0; j < regions; j++) + crc3 = crc64(crc3,(void*)start_vect[j],size_vect[j]); + + /* NOTE: It is very important to close the file descriptor only now + * because closing it before may result into unmapping of some memory + * region that we are testing. */ + fclose(fp); + + /* If the two CRC are not the same, we trapped a memory error. */ + return crc1 != crc2 || crc2 != crc3; +} +#endif + +void sigsegvHandler(int sig, siginfo_t *info, void *secret) { ucontext_t *uc = (ucontext_t*) secret; sds infostring, clients; struct sigaction act; @@ -570,21 +813,17 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { bugReportStart(); redisLog(REDIS_WARNING, " Redis %s crashed by signal: %d", REDIS_VERSION, sig); + if (sig == SIGSEGV) { + redisLog(REDIS_WARNING, + " SIGSEGV caused by address: %p", (void*)info->si_addr); + } redisLog(REDIS_WARNING, " Failed assertion: %s (%s:%d)", server.assert_failed, server.assert_file, server.assert_line); - /* Generate the stack trace */ - trace_size = backtrace(trace, 100); - - /* overwrite sigaction with caller's address */ - if (getMcontextEip(uc) != NULL) { - trace[1] = getMcontextEip(uc); - } - messages = backtrace_symbols(trace, trace_size); + /* Log the stack trace */ redisLog(REDIS_WARNING, "--- STACK TRACE"); - for (i=1; iargc; j++) { - robj *decoded; + sdsfree(infostring); + sdsfree(clients); - decoded = getDecodedObject(cc->argv[j]); - redisLog(REDIS_WARNING,"argv[%d]: '%s'", j, (char*)decoded->ptr); - decrRefCount(decoded); - } - /* Check if the first argument, usually a key, is found inside the - * selected DB, and if so print info about the associated object. */ - if (cc->argc >= 1) { - robj *val, *key; - dictEntry *de; - - key = getDecodedObject(cc->argv[1]); - de = dictFind(cc->db->dict, key->ptr); - if (de) { - val = dictGetVal(de); - redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", key->ptr); - redisLogObjectDebugInfo(val); - } - decrRefCount(key); - } - } + /* Log the current client */ + logCurrentClient(); /* Log dump of processor registers */ logRegisters(uc); +#if defined(HAVE_PROC_MAPS) + /* Test memory */ + redisLog(REDIS_WARNING, "--- FAST MEMORY TEST"); + bioKillThreads(); + if (memtest_test_linux_anonymous_maps()) { + redisLog(REDIS_WARNING, + "!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!"); + } else { + redisLog(REDIS_WARNING, + "Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible."); + } +#endif + redisLog(REDIS_WARNING, "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n" -" Please report the crash opening an issue on github:\n\n" -" http://github.com/antirez/redis/issues\n\n" +" Please report the crash by opening an issue on github:\n\n" +" http://github.com/antirez/redis/issues\n\n" +" Suspect RAM error? Use redis-server --test-memory to verify it.\n\n" ); /* free(messages); Don't call free() with possibly corrupted memory. */ if (server.daemonize) unlink(server.pidfile); @@ -645,11 +868,105 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Make sure we exit with the right signal at the end. So for instance * the core will be dumped if enabled. */ sigemptyset (&act.sa_mask); - /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction - * is used. Otherwise, sa_handler is used */ act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; act.sa_handler = SIG_DFL; sigaction (sig, &act, NULL); kill(getpid(),sig); } #endif /* HAVE_BACKTRACE */ + +/* ==================== Logging functions for debugging ===================== */ + +void redisLogHexDump(int level, char *descr, void *value, size_t len) { + char buf[65], *b; + unsigned char *v = value; + char charset[] = "0123456789abcdef"; + + redisLog(level,"%s (hexdump):", descr); + b = buf; + while(len) { + b[0] = charset[(*v)>>4]; + b[1] = charset[(*v)&0xf]; + b[2] = '\0'; + b += 2; + len--; + v++; + if (b-buf == 64 || len == 0) { + redisLogRaw(level|REDIS_LOG_RAW,buf); + b = buf; + } + } + redisLogRaw(level|REDIS_LOG_RAW,"\n"); +} + +/* =========================== Software Watchdog ============================ */ +#include + +void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) { +#ifdef HAVE_BACKTRACE + ucontext_t *uc = (ucontext_t*) secret; +#endif + REDIS_NOTUSED(info); + REDIS_NOTUSED(sig); + + redisLogFromHandler(REDIS_WARNING,"\n--- WATCHDOG TIMER EXPIRED ---"); +#ifdef HAVE_BACKTRACE + logStackTrace(uc); +#else + redisLogFromHandler(REDIS_WARNING,"Sorry: no support for backtrace()."); +#endif + redisLogFromHandler(REDIS_WARNING,"--------\n"); +} + +/* Schedule a SIGALRM delivery after the specified period in milliseconds. + * If a timer is already scheduled, this function will re-schedule it to the + * specified time. If period is 0 the current timer is disabled. */ +void watchdogScheduleSignal(int period) { + struct itimerval it; + + /* Will stop the timer if period is 0. */ + it.it_value.tv_sec = period/1000; + it.it_value.tv_usec = (period%1000)*1000; + /* Don't automatically restart. */ + it.it_interval.tv_sec = 0; + it.it_interval.tv_usec = 0; + setitimer(ITIMER_REAL, &it, NULL); +} + +/* Enable the software watchdog with the specified period in milliseconds. */ +void enableWatchdog(int period) { + int min_period; + + if (server.watchdog_period == 0) { + struct sigaction act; + + /* Watchdog was actually disabled, so we have to setup the signal + * handler. */ + sigemptyset(&act.sa_mask); + act.sa_flags = SA_ONSTACK | SA_SIGINFO; + act.sa_sigaction = watchdogSignalHandler; + sigaction(SIGALRM, &act, NULL); + } + /* If the configured period is smaller than twice the timer period, it is + * too short for the software watchdog to work reliably. Fix it now + * if needed. */ + min_period = (1000/server.hz)*2; + if (period < min_period) period = min_period; + watchdogScheduleSignal(period); /* Adjust the current timer. */ + server.watchdog_period = period; +} + +/* Disable the software watchdog. */ +void disableWatchdog(void) { + struct sigaction act; + if (server.watchdog_period == 0) return; /* Already disabled. */ + watchdogScheduleSignal(0); /* Stop the current timer. */ + + /* Set the signal handler to SIG_IGN, this will also remove pending + * signals from the queue. */ + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_IGN; + sigaction(SIGALRM, &act, NULL); + server.watchdog_period = 0; +} diff --git a/src/dict.c b/src/dict.c index 53e16be0ff2..7be34994284 100644 --- a/src/dict.c +++ b/src/dict.c @@ -5,7 +5,7 @@ * tables of power of two in size are used, collisions are handled by * chaining. See the source code for more information... :) * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,13 +39,13 @@ #include #include #include -#include #include #include #include #include "dict.h" #include "zmalloc.h" +#include "redisassert.h" /* Using dictEnableResize() / dictDisableResize() we make possible to * enable/disable resizing of the hash table as needed. This is very important @@ -53,7 +53,7 @@ * around when there is a child performing saving operations. * * Note that even when dict_can_resize is set to 0, not all resizes are - * prevented: an hash table is still allowed to grow if the ratio between + * prevented: a hash table is still allowed to grow if the ratio between * the number of elements and the buckets > dict_force_resize_ratio. */ static int dict_can_resize = 1; static unsigned int dict_force_resize_ratio = 5; @@ -62,7 +62,7 @@ static unsigned int dict_force_resize_ratio = 5; static int _dictExpandIfNeeded(dict *ht); static unsigned long _dictNextPower(unsigned long size); -static int _dictKeyIndex(dict *ht, const void *key); +static long _dictKeyIndex(dict *ht, const void *key); static int _dictInit(dict *ht, dictType *type, void *privDataPtr); /* -------------------------- hash functions -------------------------------- */ @@ -79,35 +79,74 @@ unsigned int dictIntHashFunction(unsigned int key) return key; } -/* Identity hash function for integer keys */ -unsigned int dictIdentityHashFunction(unsigned int key) -{ - return key; -} - -static int dict_hash_function_seed = 5381; +static uint32_t dict_hash_function_seed = 5381; -void dictSetHashFunctionSeed(unsigned int seed) { +void dictSetHashFunctionSeed(uint32_t seed) { dict_hash_function_seed = seed; } -unsigned int dictGetHashFunctionSeed(void) { +uint32_t dictGetHashFunctionSeed(void) { return dict_hash_function_seed; } -/* Generic hash function (a popular one from Bernstein). - * I tested a few and this was the best. */ -unsigned int dictGenHashFunction(const unsigned char *buf, int len) { - unsigned int hash = dict_hash_function_seed; +/* MurmurHash2, by Austin Appleby + * Note - This code makes a few assumptions about how your machine behaves - + * 1. We can read a 4-byte value from any address without crashing + * 2. sizeof(int) == 4 + * + * And it has a few limitations - + * + * 1. It will not work incrementally. + * 2. It will not produce the same results on little-endian and big-endian + * machines. + */ - while (len--) - hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */ - return hash; +uint64_t dictGenHashFunction(const void *key, int len) { + /* 'm' and 'r' are mixing constants generated offline. + They're not really 'magic', they just happen to work well. */ + uint32_t seed = dict_hash_function_seed; + const uint32_t m = 0x5bd1e995; + const int r = 24; + + /* Initialize the hash to a 'random' value */ + uint32_t h = seed ^ len; + + /* Mix 4 bytes at a time into the hash */ + const unsigned char *data = (const unsigned char *)key; + + while(len >= 4) { + uint32_t k = *(uint32_t*)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + /* Handle the last few bytes of the input array */ + switch(len) { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; h *= m; + }; + + /* Do a few final mixes of the hash to ensure the last few + * bytes are well-incorporated. */ + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return (unsigned int)h; } -/* And a case insensitive version */ -unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) { - unsigned int hash = dict_hash_function_seed; +/* And a case insensitive hash function (based on djb hash) */ +uint64_t dictGenCaseHashFunction(const unsigned char *buf, int len) { + unsigned int hash = (unsigned int)dict_hash_function_seed; while (len--) hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */ @@ -116,8 +155,8 @@ unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) { /* ----------------------------- API implementation ------------------------- */ -/* Reset an hashtable already initialized with ht_init(). - * NOTE: This function should only called by ht_destroy(). */ +/* Reset a hash table already initialized with ht_init(). + * NOTE: This function should only be called by ht_destroy(). */ static void _dictReset(dictht *ht) { ht->table = NULL; @@ -150,7 +189,7 @@ int _dictInit(dict *d, dictType *type, } /* Resize the table to the minimal size that contains all the elements, - * but with the invariant of a USER/BUCKETS ratio near to <= 1 */ + * but with the invariant of a USED/BUCKETS ratio near to <= 1 */ int dictResize(dict *d) { int minimal; @@ -162,18 +201,18 @@ int dictResize(dict *d) return dictExpand(d, minimal); } -/* Expand or create the hashtable */ +/* Expand or create the hash table */ int dictExpand(dict *d, unsigned long size) { - dictht n; /* the new hashtable */ + dictht n; /* the new hash table */ unsigned long realsize = _dictNextPower(size); /* the size is invalid if it is smaller than the number of - * elements already inside the hashtable */ + * elements already inside the hash table */ if (dictIsRehashing(d) || d->ht[0].used > size) return DICT_ERR; - /* Allocate the new hashtable and initialize all pointers to NULL */ + /* Allocate the new hash table and initialize all pointers to NULL */ n.size = realsize; n.sizemask = realsize-1; n.table = zcalloc(realsize*sizeof(dictEntry*)); @@ -195,7 +234,7 @@ int dictExpand(dict *d, unsigned long size) /* Performs N steps of incremental rehashing. Returns 1 if there are still * keys to move from the old to the new hash table, otherwise 0 is returned. * Note that a rehashing step consists in moving a bucket (that may have more - * thank one key as we use chaining) from the old to the new hash table. */ + * than one key as we use chaining) from the old to the new hash table. */ int dictRehash(dict *d, int n) { if (!dictIsRehashing(d)) return 0; @@ -213,12 +252,12 @@ int dictRehash(dict *d, int n) { /* Note that rehashidx can't overflow as we are sure there are more * elements because ht[0].used != 0 */ - assert(d->ht[0].size > (unsigned)d->rehashidx); + assert(d->ht[0].size > (unsigned long)d->rehashidx); while(d->ht[0].table[d->rehashidx] == NULL) d->rehashidx++; de = d->ht[0].table[d->rehashidx]; /* Move all the keys in this bucket from the old to the new hash HT */ while(de) { - unsigned int h; + uint64_t h; nextde = de->next; /* Get the index in the new hash table */ @@ -280,7 +319,7 @@ int dictAdd(dict *d, void *key, void *val) * a value returns the dictEntry structure to the user, that will make * sure to fill the value field as he wishes. * - * This function is also directly expoed to user API to be called + * This function is also directly exposed to user API to be called * mainly in order to store non-pointers inside the hash value, example: * * entry = dictAddRaw(dict,mykey); @@ -293,7 +332,7 @@ int dictAdd(dict *d, void *key, void *val) */ dictEntry *dictAddRaw(dict *d, void *key) { - int index; + long index; dictEntry *entry; dictht *ht; @@ -356,7 +395,7 @@ dictEntry *dictReplaceRaw(dict *d, void *key) { /* Search and remove an element */ static int dictGenericDelete(dict *d, const void *key, int nofree) { - unsigned int h, idx; + uint64_t h, idx; dictEntry *he, *prevHe; int table; @@ -400,14 +439,15 @@ int dictDeleteNoFree(dict *ht, const void *key) { } /* Destroy an entire dictionary */ -int _dictClear(dict *d, dictht *ht) -{ +int _dictClear(dict *d, dictht *ht, void(callback)(void *)) { unsigned long i; /* Free all the elements */ for (i = 0; i < ht->size && ht->used > 0; i++) { dictEntry *he, *nextHe; + if (callback && (i & 65535) == 0) callback(d->privdata); + if ((he = ht->table[i]) == NULL) continue; while(he) { nextHe = he->next; @@ -428,15 +468,15 @@ int _dictClear(dict *d, dictht *ht) /* Clear & Release the hash table */ void dictRelease(dict *d) { - _dictClear(d,&d->ht[0]); - _dictClear(d,&d->ht[1]); + _dictClear(d,&d->ht[0],NULL); + _dictClear(d,&d->ht[1],NULL); zfree(d); } dictEntry *dictFind(dict *d, const void *key) { dictEntry *he; - unsigned int h, idx, table; + uint64_t h, idx, table; if (d->ht[0].size == 0) return NULL; /* We don't have a table at all */ if (dictIsRehashing(d)) _dictRehashStep(d); @@ -461,6 +501,44 @@ void *dictFetchValue(dict *d, const void *key) { return he ? dictGetVal(he) : NULL; } +/* A fingerprint is a 64 bit number that represents the state of the dictionary + * at a given time, it's just a few dict properties xored together. + * When an unsafe iterator is initialized, we get the dict fingerprint, and check + * the fingerprint again when the iterator is released. + * If the two fingerprints are different it means that the user of the iterator + * performed forbidden operations against the dictionary while iterating. */ +long long dictFingerprint(dict *d) { + long long integers[6], hash = 0; + int j; + + integers[0] = (long) d->ht[0].table; + integers[1] = d->ht[0].size; + integers[2] = d->ht[0].used; + integers[3] = (long) d->ht[1].table; + integers[4] = d->ht[1].size; + integers[5] = d->ht[1].used; + + /* We hash N integers by summing every successive integer with the integer + * hashing of the previous sum. Basically: + * + * Result = hash(hash(hash(int1)+int2)+int3) ... + * + * This way the same set of integers in a different order will (likely) hash + * to a different number. */ + for (j = 0; j < 6; j++) { + hash += integers[j]; + /* For the hashing step we use Tomas Wang's 64 bit integer hash. */ + hash = (~hash) + (hash << 21); // hash = (hash << 21) - hash - 1; + hash = hash ^ (hash >> 24); + hash = (hash + (hash << 3)) + (hash << 8); // hash * 265 + hash = hash ^ (hash >> 14); + hash = (hash + (hash << 2)) + (hash << 4); // hash * 21 + hash = hash ^ (hash >> 28); + hash = hash + (hash << 31); + } + return hash; +} + dictIterator *dictGetIterator(dict *d) { dictIterator *iter = zmalloc(sizeof(*iter)); @@ -486,10 +564,14 @@ dictEntry *dictNext(dictIterator *iter) while (1) { if (iter->entry == NULL) { dictht *ht = &iter->d->ht[iter->table]; - if (iter->safe && iter->index == -1 && iter->table == 0) - iter->d->iterators++; + if (iter->index == -1 && iter->table == 0) { + if (iter->safe) + iter->d->iterators++; + else + iter->fingerprint = dictFingerprint(iter->d); + } iter->index++; - if (iter->index >= (signed) ht->size) { + if (iter->index >= (long) ht->size) { if (dictIsRehashing(iter->d) && iter->table == 0) { iter->table++; iter->index = 0; @@ -514,8 +596,12 @@ dictEntry *dictNext(dictIterator *iter) void dictReleaseIterator(dictIterator *iter) { - if (iter->safe && !(iter->index == -1 && iter->table == 0)) - iter->d->iterators--; + if (!(iter->index == -1 && iter->table == 0)) { + if (iter->safe) + iter->d->iterators--; + else + assert(iter->fingerprint == dictFingerprint(iter->d)); + } zfree(iter); } @@ -524,7 +610,7 @@ void dictReleaseIterator(dictIterator *iter) dictEntry *dictGetRandomKey(dict *d) { dictEntry *he, *orighe; - unsigned int h; + unsigned long h; int listlen, listele; if (dictSize(d) == 0) return NULL; @@ -558,6 +644,173 @@ dictEntry *dictGetRandomKey(dict *d) return he; } +/* Function to reverse bits. Algorithm from: + * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */ +static unsigned long rev(unsigned long v) { + unsigned long s = 8 * sizeof(v); // bit size; must be power of 2 + unsigned long mask = ~0; + while ((s >>= 1) > 0) { + mask ^= (mask << s); + v = ((v >> s) & mask) | ((v << s) & ~mask); + } + return v; +} + +/* dictScan() is used to iterate over the elements of a dictionary. + * + * Iterating works the following way: + * + * 1) Initially you call the function using a cursor (v) value of 0. + * 2) The function performs one step of the iteration, and returns the + * new cursor value you must use in the next call. + * 3) When the returned cursor is 0, the iteration is complete. + * + * The function guarantees all elements present in the + * dictionary get returned between the start and end of the iteration. + * However it is possible some elements get returned multiple times. + * + * For every element returned, the callback argument 'fn' is + * called with 'privdata' as first argument and the dictionary entry + * 'de' as second argument. + * + * HOW IT WORKS. + * + * The iteration algorithm was designed by Pieter Noordhuis. + * The main idea is to increment a cursor starting from the higher order + * bits. That is, instead of incrementing the cursor normally, the bits + * of the cursor are reversed, then the cursor is incremented, and finally + * the bits are reversed again. + * + * This strategy is needed because the hash table may be resized between + * iteration calls. + * + * dict.c hash tables are always power of two in size, and they + * use chaining, so the position of an element in a given table is given + * by computing the bitwise AND between Hash(key) and SIZE-1 + * (where SIZE-1 is always the mask that is equivalent to taking the rest + * of the division between the Hash of the key and SIZE). + * + * For example if the current hash table size is 16, the mask is + * (in binary) 1111. The position of a key in the hash table will always be + * the last four bits of the hash output, and so forth. + * + * WHAT HAPPENS IF THE TABLE CHANGES IN SIZE? + * + * If the hash table grows, elements can go anywhere in one multiple of + * the old bucket: for example let's say we already iterated with + * a 4 bit cursor 1100 (the mask is 1111 because hash table size = 16). + * + * If the hash table will be resized to 64 elements, then the new mask will + * be 111111. The new buckets you obtain by substituting in ??1100 + * with either 0 or 1 can be targeted only by keys we already visited + * when scanning the bucket 1100 in the smaller hash table. + * + * By iterating the higher bits first, because of the inverted counter, the + * cursor does not need to restart if the table size gets bigger. It will + * continue iterating using cursors without '1100' at the end, and also + * without any other combination of the final 4 bits already explored. + * + * Similarly when the table size shrinks over time, for example going from + * 16 to 8, if a combination of the lower three bits (the mask for size 8 + * is 111) were already completely explored, it would not be visited again + * because we are sure we tried, for example, both 0111 and 1111 (all the + * variations of the higher bit) so we don't need to test it again. + * + * WAIT... YOU HAVE *TWO* TABLES DURING REHASHING! + * + * Yes, this is true, but we always iterate the smaller table first, then + * we test all the expansions of the current cursor into the larger + * table. For example if the current cursor is 101 and we also have a + * larger table of size 16, we also test (0)101 and (1)101 inside the larger + * table. This reduces the problem back to having only one table, where + * the larger one, if it exists, is just an expansion of the smaller one. + * + * LIMITATIONS + * + * This iterator is completely stateless, and this is a huge advantage, + * including no additional memory used. + * + * The disadvantages resulting from this design are: + * + * 1) It is possible we return elements more than once. However this is usually + * easy to deal with in the application level. + * 2) The iterator must return multiple elements per call, as it needs to always + * return all the keys chained in a given bucket, and all the expansions, so + * we are sure we don't miss keys moving during rehashing. + * 3) The reverse cursor is somewhat hard to understand at first, but this + * comment is supposed to help. + */ +unsigned long dictScan(dict *d, + unsigned long v, + dictScanFunction *fn, + void *privdata) +{ + dictht *t0, *t1; + const dictEntry *de; + unsigned long m0, m1; + + if (dictSize(d) == 0) return 0; + + if (!dictIsRehashing(d)) { + t0 = &(d->ht[0]); + m0 = t0->sizemask; + + /* Emit entries at cursor */ + de = t0->table[v & m0]; + while (de) { + fn(privdata, de); + de = de->next; + } + + } else { + t0 = &d->ht[0]; + t1 = &d->ht[1]; + + /* Make sure t0 is the smaller and t1 is the bigger table */ + if (t0->size > t1->size) { + t0 = &d->ht[1]; + t1 = &d->ht[0]; + } + + m0 = t0->sizemask; + m1 = t1->sizemask; + + /* Emit entries at cursor */ + de = t0->table[v & m0]; + while (de) { + fn(privdata, de); + de = de->next; + } + + /* Iterate over indices in larger table that are the expansion + * of the index pointed to by the cursor in the smaller table */ + do { + /* Emit entries at cursor */ + de = t1->table[v & m1]; + while (de) { + fn(privdata, de); + de = de->next; + } + + /* Increment bits not covered by the smaller mask */ + v = (((v | m0) + 1) & ~m0) | (v & m0); + + /* Continue while bits covered by mask difference is non-zero */ + } while (v & (m0 ^ m1)); + } + + /* Set unmasked bits so incrementing the reversed cursor + * operates on the masked bits of the smaller table */ + v |= ~m0; + + /* Increment the reverse cursor */ + v = rev(v); + v++; + v = rev(v); + + return v; +} + /* ------------------------- private functions ------------------------------ */ /* Expand the hash table if needed */ @@ -566,7 +819,7 @@ static int _dictExpandIfNeeded(dict *d) /* Incremental rehashing already in progress. Return. */ if (dictIsRehashing(d)) return DICT_OK; - /* If the hash table is empty expand it to the intial size. */ + /* If the hash table is empty expand it to the initial size. */ if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE); /* If we reached the 1:1 ratio, and we are allowed to resize the hash @@ -577,8 +830,7 @@ static int _dictExpandIfNeeded(dict *d) (dict_can_resize || d->ht[0].used/d->ht[0].size > dict_force_resize_ratio)) { - return dictExpand(d, ((d->ht[0].size > d->ht[0].used) ? - d->ht[0].size : d->ht[0].used)*2); + return dictExpand(d, d->ht[0].used*2); } return DICT_OK; } @@ -597,17 +849,18 @@ static unsigned long _dictNextPower(unsigned long size) } /* Returns the index of a free slot that can be populated with - * an hash entry for the given 'key'. + * a hash entry for the given 'key'. * If the key already exists, -1 is returned. * * Note that if we are in the process of rehashing the hash table, the * index is always returned in the context of the second (new) hash table. */ -static int _dictKeyIndex(dict *d, const void *key) + +static long _dictKeyIndex(dict *d, const void *key) { - unsigned int h, idx, table; + unsigned long h, idx, table; dictEntry *he; - /* Expand the hashtable if needed */ + /* Expand the hash table if needed */ if (_dictExpandIfNeeded(d) == DICT_ERR) return -1; /* Compute the key hash value */ @@ -626,13 +879,28 @@ static int _dictKeyIndex(dict *d, const void *key) return idx; } -void dictEmpty(dict *d) { - _dictClear(d,&d->ht[0]); - _dictClear(d,&d->ht[1]); +void dictEmpty(dict *d, void(callback)(void*)) { + _dictClear(d,&d->ht[0],callback); + _dictClear(d,&d->ht[1],callback); d->rehashidx = -1; d->iterators = 0; } +void dictEnableResize(void) { + dict_can_resize = 1; +} + +void dictDisableResize(void) { + dict_can_resize = 0; +} + +#if 0 + +/* The following is code that we don't use for Redis currently, but that is part +of the library. */ + +/* ----------------------- Debugging ------------------------*/ + #define DICT_STATS_VECTLEN 50 static void _dictPrintStatsHt(dictht *ht) { unsigned long i, slots = 0, chainlen, maxchainlen = 0; @@ -686,20 +954,6 @@ void dictPrintStats(dict *d) { } } -void dictEnableResize(void) { - dict_can_resize = 1; -} - -void dictDisableResize(void) { - dict_can_resize = 0; -} - -#if 0 - -/* The following are just example hash table types implementations. - * Not useful for Redis so they are commented out. - */ - /* ----------------------- StringCopy Hash Table Type ------------------------*/ static unsigned int _dictStringCopyHTHashFunction(const void *key) diff --git a/src/dict.h b/src/dict.h index 5f856953544..cbe27aeaaea 100644 --- a/src/dict.h +++ b/src/dict.h @@ -1,11 +1,11 @@ /* Hash Tables Implementation. * - * This file implements in memory hash tables with insert/del/replace/find/ - * get-random-element operations. Hash tables will auto resize if needed + * This file implements in-memory hash tables with insert/del/replace/find/ + * get-random-element operations. Hash tables will auto-resize if needed * tables of power of two in size are used, collisions are handled by * chaining. See the source code for more information... :) * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,6 +50,7 @@ typedef struct dictEntry { void *val; uint64_t u64; int64_t s64; + double d; } v; struct dictEntry *next; } dictEntry; @@ -76,20 +77,25 @@ typedef struct dict { dictType *type; void *privdata; dictht ht[2]; - int rehashidx; /* rehashing not in progress if rehashidx == -1 */ + long rehashidx; /* rehashing not in progress if rehashidx == -1 */ int iterators; /* number of iterators currently running */ } dict; -/* If safe is set to 1 this is a safe iteartor, that means, you can call +/* If safe is set to 1 this is a safe iterator, that means, you can call * dictAdd, dictFind, and other functions against the dictionary even while * iterating. Otherwise it is a non safe iterator, and only dictNext() * should be called while iterating. */ typedef struct dictIterator { dict *d; - int table, index, safe; + long index; + int table, safe; dictEntry *entry, *nextEntry; + /* unsafe iterator fingerprint for misuse detection. */ + long long fingerprint; } dictIterator; +typedef void (dictScanFunction)(void *privdata, const dictEntry *de); + /* This is the initial size of every hash table */ #define DICT_HT_INITIAL_SIZE 4 @@ -111,6 +117,9 @@ typedef struct dictIterator { #define dictSetUnsignedIntegerVal(entry, _val_) \ do { entry->v.u64 = _val_; } while(0) +#define dictSetDoubleVal(entry, _val_) \ + do { entry->v.d = _val_; } while(0) + #define dictFreeKey(d, entry) \ if ((d)->type->keyDestructor) \ (d)->type->keyDestructor((d)->privdata, (entry)->key) @@ -132,9 +141,10 @@ typedef struct dictIterator { #define dictGetVal(he) ((he)->v.val) #define dictGetSignedIntegerVal(he) ((he)->v.s64) #define dictGetUnsignedIntegerVal(he) ((he)->v.u64) +#define dictGetDoubleVal(he) ((he)->v.d) #define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size) #define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used) -#define dictIsRehashing(ht) ((ht)->rehashidx != -1) +#define dictIsRehashing(d) ((d)->rehashidx != -1) /* API */ dict *dictCreate(dictType *type, void *privDataPtr); @@ -155,15 +165,16 @@ dictEntry *dictNext(dictIterator *iter); void dictReleaseIterator(dictIterator *iter); dictEntry *dictGetRandomKey(dict *d); void dictPrintStats(dict *d); -unsigned int dictGenHashFunction(const unsigned char *buf, int len); -unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len); -void dictEmpty(dict *d); +uint64_t dictGenHashFunction(const void *key, int len); +uint64_t dictGenCaseHashFunction(const unsigned char *buf, int len); +void dictEmpty(dict *d, void(callback)(void*)); void dictEnableResize(void); void dictDisableResize(void); int dictRehash(dict *d, int n); int dictRehashMilliseconds(dict *d, int ms); void dictSetHashFunctionSeed(unsigned int initval); unsigned int dictGetHashFunctionSeed(void); +unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, void *privdata); /* Hash table types */ extern dictType dictTypeHeapStringCopyKey; diff --git a/src/endianconv.h b/src/endianconv.h index f76e0e6b3eb..1f15b739323 100644 --- a/src/endianconv.h +++ b/src/endianconv.h @@ -28,11 +28,12 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. - */ + */ #ifndef __ENDIANCONV_H #define __ENDIANCONV_H +#include "config.h" #include void memrev16(void *p); diff --git a/src/fmacros.h b/src/fmacros.h index 866a9afa816..6e56c759dd5 100644 --- a/src/fmacros.h +++ b/src/fmacros.h @@ -1,14 +1,60 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef _REDIS_FMACRO_H #define _REDIS_FMACRO_H #define _BSD_SOURCE +#if defined(__linux__) +#define _GNU_SOURCE +#define _DEFAULT_SOURCE +#endif + +#if defined(_AIX) +#define _ALL_SOURCE +#endif + #if defined(__linux__) || defined(__OpenBSD__) #define _XOPEN_SOURCE 700 -#else +/* + * On NetBSD, _XOPEN_SOURCE undefines _NETBSD_SOURCE and + * thus hides inet_aton etc. + */ +#elif !defined(__NetBSD__) #define _XOPEN_SOURCE #endif +#if defined(__sun) +#define _POSIX_C_SOURCE 199506L +#endif + #define _LARGEFILE_SOURCE #define _FILE_OFFSET_BITS 64 diff --git a/src/help.h b/src/help.h index 51613c9b3b4..8395c525bb7 100644 --- a/src/help.h +++ b/src/help.h @@ -1,4 +1,4 @@ -/* Automatically generated by generate-command-help.rb, do not edit. */ +/* Automatically generated by utils/generate-command-help.rb, do not edit. */ #ifndef __REDIS_HELP_H #define __REDIS_HELP_H @@ -13,7 +13,9 @@ static char *commandGroups[] = { "pubsub", "transactions", "connection", - "server" + "server", + "scripting", + "hyperloglog" }; struct commandHelp { @@ -27,612 +29,802 @@ struct commandHelp { "key value", "Append a value to a key", 1, - "1.3.3" }, + "2.0.0" }, { "AUTH", "password", "Authenticate to the server", 8, - "0.08" }, + "1.0.0" }, { "BGREWRITEAOF", "-", "Asynchronously rewrite the append-only file", 9, - "1.07" }, + "1.0.0" }, { "BGSAVE", "-", "Asynchronously save the dataset to disk", 9, - "0.07" }, + "1.0.0" }, + { "BITCOUNT", + "key [start] [end]", + "Count set bits in a string", + 1, + "2.6.0" }, + { "BITOP", + "operation destkey key [key ...]", + "Perform bitwise operations between strings", + 1, + "2.6.0" }, + { "BITPOS", + "key bit [start] [end]", + "Find first bit set or clear in a string", + 1, + "2.8.7" }, { "BLPOP", "key [key ...] timeout", "Remove and get the first element in a list, or block until one is available", 2, - "1.3.1" }, + "2.0.0" }, { "BRPOP", "key [key ...] timeout", "Remove and get the last element in a list, or block until one is available", 2, - "1.3.1" }, + "2.0.0" }, { "BRPOPLPUSH", "source destination timeout", "Pop a value from a list, push it to another list and return it; or block until one is available", 2, - "2.1.7" }, + "2.2.0" }, + { "CLIENT GETNAME", + "-", + "Get the current connection name", + 9, + "2.6.9" }, + { "CLIENT KILL", + "ip:port", + "Kill the connection of a client", + 9, + "2.4.0" }, + { "CLIENT LIST", + "-", + "Get the list of client connections", + 9, + "2.4.0" }, + { "CLIENT PAUSE", + "timeout", + "Stop processing commands from clients for some time", + 9, + "2.9.50" }, + { "CLIENT SETNAME", + "connection-name", + "Set the current connection name", + 9, + "2.6.9" }, { "CONFIG GET", "parameter", "Get the value of a configuration parameter", 9, - "2.0" }, + "2.0.0" }, { "CONFIG RESETSTAT", "-", "Reset the stats returned by INFO", 9, - "2.0" }, + "2.0.0" }, + { "CONFIG REWRITE", + "-", + "Rewrite the configuration file with the in memory configuration", + 9, + "2.8.0" }, { "CONFIG SET", "parameter value", "Set a configuration parameter to the given value", 9, - "2.0" }, + "2.0.0" }, { "DBSIZE", "-", "Return the number of keys in the selected database", 9, - "0.07" }, + "1.0.0" }, { "DEBUG OBJECT", "key", "Get debugging information about a key", 9, - "0.101" }, + "1.0.0" }, { "DEBUG SEGFAULT", "-", "Make the server crash", 9, - "0.101" }, + "1.0.0" }, { "DECR", "key", "Decrement the integer value of a key by one", 1, - "0.07" }, + "1.0.0" }, { "DECRBY", "key decrement", "Decrement the integer value of a key by the given number", 1, - "0.07" }, + "1.0.0" }, { "DEL", "key [key ...]", "Delete a key", 0, - "0.07" }, + "1.0.0" }, { "DISCARD", "-", "Discard all commands issued after MULTI", 7, - "1.3.3" }, + "2.0.0" }, + { "DUMP", + "key", + "Return a serialized version of the value stored at the specified key.", + 0, + "2.6.0" }, { "ECHO", "message", "Echo the given string", 8, - "0.07" }, + "1.0.0" }, + { "EVAL", + "script numkeys key [key ...] arg [arg ...]", + "Execute a Lua script server side", + 10, + "2.6.0" }, + { "EVALSHA", + "sha1 numkeys key [key ...] arg [arg ...]", + "Execute a Lua script server side", + 10, + "2.6.0" }, { "EXEC", "-", "Execute all commands issued after MULTI", 7, - "1.1.95" }, + "1.2.0" }, { "EXISTS", "key", "Determine if a key exists", - 9, - "0.07" }, + 0, + "1.0.0" }, { "EXPIRE", "key seconds", "Set a key's time to live in seconds", 0, - "0.09" }, + "1.0.0" }, { "EXPIREAT", "key timestamp", "Set the expiration for a key as a UNIX timestamp", 0, - "1.1" }, + "1.2.0" }, { "FLUSHALL", "-", "Remove all keys from all databases", 9, - "0.07" }, + "1.0.0" }, { "FLUSHDB", "-", "Remove all keys from the current database", 9, - "0.07" }, + "1.0.0" }, { "GET", "key", "Get the value of a key", 1, - "0.07" }, + "1.0.0" }, { "GETBIT", "key offset", "Returns the bit value at offset in the string value stored at key", 1, - "2.1.8" }, + "2.2.0" }, + { "GETRANGE", + "key start end", + "Get a substring of the string stored at a key", + 1, + "2.4.0" }, { "GETSET", "key value", "Set the string value of a key and return its old value", 1, - "0.091" }, + "1.0.0" }, { "HDEL", - "key field", - "Delete a hash field", + "key field [field ...]", + "Delete one or more hash fields", 5, - "1.3.10" }, + "2.0.0" }, { "HEXISTS", "key field", "Determine if a hash field exists", 5, - "1.3.10" }, + "2.0.0" }, { "HGET", "key field", "Get the value of a hash field", 5, - "1.3.10" }, + "2.0.0" }, { "HGETALL", "key", "Get all the fields and values in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HINCRBY", "key field increment", "Increment the integer value of a hash field by the given number", 5, - "1.3.10" }, + "2.0.0" }, + { "HINCRBYFLOAT", + "key field increment", + "Increment the float value of a hash field by the given amount", + 5, + "2.6.0" }, { "HKEYS", "key", "Get all the fields in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HLEN", "key", "Get the number of fields in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HMGET", "key field [field ...]", "Get the values of all the given hash fields", 5, - "1.3.10" }, + "2.0.0" }, { "HMSET", "key field value [field value ...]", "Set multiple hash fields to multiple values", 5, - "1.3.8" }, + "2.0.0" }, + { "HSCAN", + "key cursor [MATCH pattern] [COUNT count]", + "Incrementally iterate hash fields and associated values", + 5, + "2.8.0" }, { "HSET", "key field value", "Set the string value of a hash field", 5, - "1.3.10" }, + "2.0.0" }, { "HSETNX", "key field value", "Set the value of a hash field, only if the field does not exist", 5, - "1.3.8" }, + "2.0.0" }, { "HVALS", "key", "Get all the values in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "INCR", "key", "Increment the integer value of a key by one", 1, - "0.07" }, + "1.0.0" }, { "INCRBY", "key increment", - "Increment the integer value of a key by the given number", + "Increment the integer value of a key by the given amount", 1, - "0.07" }, + "1.0.0" }, + { "INCRBYFLOAT", + "key increment", + "Increment the float value of a key by the given amount", + 1, + "2.6.0" }, { "INFO", - "-", + "[section]", "Get information and statistics about the server", 9, - "0.07" }, + "1.0.0" }, { "KEYS", "pattern", "Find all keys matching the given pattern", 0, - "0.07" }, + "1.0.0" }, { "LASTSAVE", "-", "Get the UNIX time stamp of the last successful save to disk", 9, - "0.07" }, + "1.0.0" }, { "LINDEX", "key index", "Get an element from a list by its index", 2, - "0.07" }, + "1.0.0" }, { "LINSERT", "key BEFORE|AFTER pivot value", "Insert an element before or after another element in a list", 2, - "2.1.1" }, + "2.2.0" }, { "LLEN", "key", "Get the length of a list", 2, - "0.07" }, + "1.0.0" }, { "LPOP", "key", "Remove and get the first element in a list", 2, - "0.07" }, + "1.0.0" }, { "LPUSH", - "key value", - "Prepend a value to a list", + "key value [value ...]", + "Prepend one or multiple values to a list", 2, - "0.07" }, + "1.0.0" }, { "LPUSHX", "key value", "Prepend a value to a list, only if the list exists", 2, - "2.1.1" }, + "2.2.0" }, { "LRANGE", "key start stop", "Get a range of elements from a list", 2, - "0.07" }, + "1.0.0" }, { "LREM", "key count value", "Remove elements from a list", 2, - "0.07" }, + "1.0.0" }, { "LSET", "key index value", "Set the value of an element in a list by its index", 2, - "0.07" }, + "1.0.0" }, { "LTRIM", "key start stop", "Trim a list to the specified range", 2, - "0.07" }, + "1.0.0" }, { "MGET", "key [key ...]", "Get the values of all the given keys", 1, - "0.07" }, + "1.0.0" }, + { "MIGRATE", + "host port key destination-db timeout [COPY] [REPLACE]", + "Atomically transfer a key from a Redis instance to another one.", + 0, + "2.6.0" }, { "MONITOR", "-", "Listen for all requests received by the server in real time", 9, - "0.07" }, + "1.0.0" }, { "MOVE", "key db", "Move a key to another database", 0, - "0.07" }, + "1.0.0" }, { "MSET", "key value [key value ...]", "Set multiple keys to multiple values", 1, - "1.001" }, + "1.0.1" }, { "MSETNX", "key value [key value ...]", "Set multiple keys to multiple values, only if none of the keys exist", 1, - "1.001" }, + "1.0.1" }, { "MULTI", "-", "Mark the start of a transaction block", 7, - "1.1.95" }, + "1.2.0" }, + { "OBJECT", + "subcommand [arguments [arguments ...]]", + "Inspect the internals of Redis objects", + 0, + "2.2.3" }, { "PERSIST", "key", "Remove the expiration from a key", 0, - "2.1.2" }, + "2.2.0" }, + { "PEXPIRE", + "key milliseconds", + "Set a key's time to live in milliseconds", + 0, + "2.6.0" }, + { "PEXPIREAT", + "key milliseconds-timestamp", + "Set the expiration for a key as a UNIX timestamp specified in milliseconds", + 0, + "2.6.0" }, + { "PFADD", + "key element [element ...]", + "Adds the specified elements to the specified HyperLogLog.", + 11, + "2.8.9" }, + { "PFCOUNT", + "key [key ...]", + "Return the approximated cardinality of the set(s) observed by the HyperLogLog at key(s).", + 11, + "2.8.9" }, + { "PFMERGE", + "destkey sourcekey [sourcekey ...]", + "Merge N different HyperLogLogs into a single one.", + 11, + "2.8.9" }, { "PING", "-", "Ping the server", 8, - "0.07" }, + "1.0.0" }, + { "PSETEX", + "key milliseconds value", + "Set the value and expiration in milliseconds of a key", + 1, + "2.6.0" }, { "PSUBSCRIBE", - "pattern", + "pattern [pattern ...]", "Listen for messages published to channels matching the given patterns", 6, - "1.3.8" }, + "2.0.0" }, + { "PTTL", + "key", + "Get the time to live for a key in milliseconds", + 0, + "2.6.0" }, { "PUBLISH", "channel message", "Post a message to a channel", 6, - "1.3.8" }, + "2.0.0" }, + { "PUBSUB", + "subcommand [argument [argument ...]]", + "Inspect the state of the Pub/Sub subsystem", + 6, + "2.8.0" }, { "PUNSUBSCRIBE", "[pattern [pattern ...]]", "Stop listening for messages posted to channels matching the given patterns", 6, - "1.3.8" }, + "2.0.0" }, { "QUIT", "-", "Close the connection", 8, - "0.07" }, + "1.0.0" }, { "RANDOMKEY", "-", "Return a random key from the keyspace", 0, - "0.07" }, + "1.0.0" }, { "RENAME", "key newkey", "Rename a key", 0, - "0.07" }, + "1.0.0" }, { "RENAMENX", "key newkey", "Rename a key, only if the new key does not exist", 0, - "0.07" }, + "1.0.0" }, + { "RESTORE", + "key ttl serialized-value", + "Create a key using the provided serialized value, previously obtained using DUMP.", + 0, + "2.6.0" }, { "RPOP", "key", "Remove and get the last element in a list", 2, - "0.07" }, + "1.0.0" }, { "RPOPLPUSH", "source destination", "Remove the last element in a list, append it to another list and return it", 2, - "1.1" }, + "1.2.0" }, { "RPUSH", - "key value", - "Append a value to a list", + "key value [value ...]", + "Append one or multiple values to a list", 2, - "0.07" }, + "1.0.0" }, { "RPUSHX", "key value", "Append a value to a list, only if the list exists", 2, - "2.1.1" }, + "2.2.0" }, { "SADD", - "key member", - "Add a member to a set", + "key member [member ...]", + "Add one or more members to a set", 3, - "0.07" }, + "1.0.0" }, { "SAVE", "-", "Synchronously save the dataset to disk", 9, - "0.07" }, + "1.0.0" }, + { "SCAN", + "cursor [MATCH pattern] [COUNT count]", + "Incrementally iterate the keys space", + 0, + "2.8.0" }, { "SCARD", "key", "Get the number of members in a set", 3, - "0.07" }, + "1.0.0" }, + { "SCRIPT EXISTS", + "script [script ...]", + "Check existence of scripts in the script cache.", + 10, + "2.6.0" }, + { "SCRIPT FLUSH", + "-", + "Remove all the scripts from the script cache.", + 10, + "2.6.0" }, + { "SCRIPT KILL", + "-", + "Kill the script currently in execution.", + 10, + "2.6.0" }, + { "SCRIPT LOAD", + "script", + "Load the specified Lua script into the script cache.", + 10, + "2.6.0" }, { "SDIFF", "key [key ...]", "Subtract multiple sets", 3, - "0.100" }, + "1.0.0" }, { "SDIFFSTORE", "destination key [key ...]", "Subtract multiple sets and store the resulting set in a key", 3, - "0.100" }, + "1.0.0" }, { "SELECT", "index", "Change the selected database for the current connection", 8, - "0.07" }, + "1.0.0" }, { "SET", - "key value", + "key value [EX seconds] [PX milliseconds] [NX|XX]", "Set the string value of a key", 1, - "0.07" }, + "1.0.0" }, { "SETBIT", "key offset value", "Sets or clears the bit at offset in the string value stored at key", 1, - "2.1.8" }, + "2.2.0" }, { "SETEX", "key seconds value", "Set the value and expiration of a key", 1, - "1.3.10" }, + "2.0.0" }, { "SETNX", "key value", "Set the value of a key, only if the key does not exist", 1, - "0.07" }, + "1.0.0" }, { "SETRANGE", "key offset value", "Overwrite part of a string at key starting at the specified offset", 1, - "2.1.8" }, + "2.2.0" }, { "SHUTDOWN", - "-", + "[NOSAVE] [SAVE]", "Synchronously save the dataset to disk and then shut down the server", 9, - "0.07" }, + "1.0.0" }, { "SINTER", "key [key ...]", "Intersect multiple sets", 3, - "0.07" }, + "1.0.0" }, { "SINTERSTORE", "destination key [key ...]", "Intersect multiple sets and store the resulting set in a key", 3, - "0.07" }, + "1.0.0" }, { "SISMEMBER", "key member", "Determine if a given value is a member of a set", 3, - "0.07" }, + "1.0.0" }, { "SLAVEOF", "host port", "Make the server a slave of another instance, or promote it as master", 9, - "0.100" }, + "1.0.0" }, + { "SLOWLOG", + "subcommand [argument]", + "Manages the Redis slow queries log", + 9, + "2.2.12" }, { "SMEMBERS", "key", "Get all the members in a set", 3, - "0.07" }, + "1.0.0" }, { "SMOVE", "source destination member", "Move a member from one set to another", 3, - "0.091" }, + "1.0.0" }, { "SORT", "key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]", "Sort the elements in a list, set or sorted set", 0, - "0.07" }, + "1.0.0" }, { "SPOP", "key", "Remove and return a random member from a set", 3, - "0.101" }, + "1.0.0" }, { "SRANDMEMBER", - "key", - "Get a random member from a set", + "key [count]", + "Get one or multiple random members from a set", 3, - "1.001" }, + "1.0.0" }, { "SREM", - "key member", - "Remove a member from a set", + "key member [member ...]", + "Remove one or more members from a set", 3, - "0.07" }, + "1.0.0" }, + { "SSCAN", + "key cursor [MATCH pattern] [COUNT count]", + "Incrementally iterate Set elements", + 3, + "2.8.0" }, { "STRLEN", "key", "Get the length of the value stored in a key", 1, - "2.1.2" }, + "2.2.0" }, { "SUBSCRIBE", - "channel", + "channel [channel ...]", "Listen for messages published to the given channels", 6, - "1.3.8" }, - { "SUBSTR", - "key start end", - "Get a substring of the string stored at a key", - 1, - "1.3.4" }, + "2.0.0" }, { "SUNION", "key [key ...]", "Add multiple sets", 3, - "0.091" }, + "1.0.0" }, { "SUNIONSTORE", "destination key [key ...]", "Add multiple sets and store the resulting set in a key", 3, - "0.091" }, + "1.0.0" }, { "SYNC", "-", "Internal command used for replication", 9, - "0.07" }, + "1.0.0" }, + { "TIME", + "-", + "Return the current server time", + 9, + "2.6.0" }, { "TTL", "key", "Get the time to live for a key", 0, - "0.100" }, + "1.0.0" }, { "TYPE", "key", "Determine the type stored at key", 0, - "0.07" }, + "1.0.0" }, { "UNSUBSCRIBE", "[channel [channel ...]]", "Stop listening for messages posted to the given channels", 6, - "1.3.8" }, + "2.0.0" }, { "UNWATCH", "-", "Forget about all watched keys", 7, - "2.1.0" }, + "2.2.0" }, { "WATCH", "key [key ...]", "Watch the given keys to determine execution of the MULTI/EXEC block", 7, - "2.1.0" }, + "2.2.0" }, { "ZADD", - "key score member", - "Add a member to a sorted set, or update its score if it already exists", + "key score member [score member ...]", + "Add one or more members to a sorted set, or update its score if it already exists", 4, - "1.1" }, + "1.2.0" }, { "ZCARD", "key", "Get the number of members in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZCOUNT", "key min max", "Count the members in a sorted set with scores within the given values", 4, - "1.3.3" }, + "2.0.0" }, { "ZINCRBY", "key increment member", "Increment the score of a member in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZINTERSTORE", "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Intersect multiple sorted sets and store the resulting sorted set in a new key", 4, - "1.3.10" }, + "2.0.0" }, + { "ZLEXCOUNT", + "key min max", + "Count the number of members in a sorted set between a given lexicographical range", + 4, + "2.8.9" }, { "ZRANGE", "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index", 4, - "1.1" }, + "1.2.0" }, + { "ZRANGEBYLEX", + "key min max [LIMIT offset count]", + "Return a range of members in a sorted set, by lexicographical range", + 4, + "2.8.9" }, { "ZRANGEBYSCORE", "key min max [WITHSCORES] [LIMIT offset count]", "Return a range of members in a sorted set, by score", 4, - "1.050" }, + "1.0.5" }, { "ZRANK", "key member", "Determine the index of a member in a sorted set", 4, - "1.3.4" }, + "2.0.0" }, { "ZREM", - "key member", - "Remove a member from a sorted set", + "key member [member ...]", + "Remove one or more members from a sorted set", 4, - "1.1" }, + "1.2.0" }, + { "ZREMRANGEBYLEX", + "key min max", + "Remove all members in a sorted set between the given lexicographical range", + 4, + "2.8.9" }, { "ZREMRANGEBYRANK", "key start stop", "Remove all members in a sorted set within the given indexes", 4, - "1.3.4" }, + "2.0.0" }, { "ZREMRANGEBYSCORE", "key min max", "Remove all members in a sorted set within the given scores", 4, - "1.1" }, + "1.2.0" }, { "ZREVRANGE", "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index, with scores ordered from high to low", 4, - "1.1" }, + "1.2.0" }, { "ZREVRANGEBYSCORE", "key max min [WITHSCORES] [LIMIT offset count]", "Return a range of members in a sorted set, by score, with scores ordered from high to low", 4, - "2.1.6" }, + "2.2.0" }, { "ZREVRANK", "key member", "Determine the index of a member in a sorted set, with scores ordered from high to low", 4, - "1.3.4" }, + "2.0.0" }, + { "ZSCAN", + "key cursor [MATCH pattern] [COUNT count]", + "Incrementally iterate sorted sets elements and associated scores", + 4, + "2.8.0" }, { "ZSCORE", "key member", "Get the score associated with the given member in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZUNIONSTORE", "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Add multiple sorted sets and store the resulting sorted set in a new key", 4, - "1.3.10" } + "2.0.0" } }; #endif diff --git a/src/hyperloglog.c b/src/hyperloglog.c new file mode 100644 index 00000000000..a6cfdc7428c --- /dev/null +++ b/src/hyperloglog.c @@ -0,0 +1,1556 @@ +/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation. + * This file implements the algorithm and the exported Redis commands. + * + * Copyright (c) 2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "redis.h" + +#include +#include + +/* The Redis HyperLogLog implementation is based on the following ideas: + * + * * The use of a 64 bit hash function as proposed in [1], in order to don't + * limited to cardinalities up to 10^9, at the cost of just 1 additional + * bit per register. + * * The use of 16384 6-bit registers for a great level of accuracy, using + * a total of 12k per key. + * * The use of the Redis string data type. No new type is introduced. + * * No attempt is made to compress the data structure as in [1]. Also the + * algorithm used is the original HyperLogLog Algorithm as in [2], with + * the only difference that a 64 bit hash function is used, so no correction + * is performed for values near 2^32 as in [1]. + * + * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic + * Engineering of a State of The Art Cardinality Estimation Algorithm. + * + * [2] P. Flajolet, Éric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The + * analysis of a near-optimal cardinality estimation algorithm. + * + * Redis uses two representations: + * + * 1) A "dense" representation where every entry is represented by + * a 6-bit integer. + * 2) A "sparse" representation using run length compression suitable + * for representing HyperLogLogs with many registers set to 0 in + * a memory efficient way. + * + * + * HLL header + * === + * + * Both the dense and sparse representation have a 16 byte header as follows: + * + * +------+---+-----+----------+ + * | HYLL | E | N/U | Cardin. | + * +------+---+-----+----------+ + * + * The first 4 bytes are a magic string set to the bytes "HYLL". + * "E" is one byte encoding, currently set to HLL_DENSE or + * HLL_SPARSE. N/U are three not used bytes. + * + * The "Cardin." field is a 64 bit integer stored in little endian format + * with the latest cardinality computed that can be reused if the data + * structure was not modified since the last computation (this is useful + * because there are high probabilities that HLLADD operations don't + * modify the actual data structure and hence the approximated cardinality). + * + * When the most significant bit in the most significant byte of the cached + * cardinality is set, it means that the data structure was modified and + * we can't reuse the cached value that must be recomputed. + * + * Dense representation + * === + * + * The dense representation used by Redis is the following: + * + * +--------+--------+--------+------// //--+ + * |11000000|22221111|33333322|55444444 .... | + * +--------+--------+--------+------// //--+ + * + * The 6 bits counters are encoded one after the other starting from the + * LSB to the MSB, and using the next bytes as needed. + * + * Sparse representation + * === + * + * The sparse representation encodes registers using a run length + * encoding composed of three opcodes, two using one byte, and one using + * of two bytes. The opcodes are called ZERO, XZERO and VAL. + * + * ZERO opcode is represented as 00xxxxxx. The 6-bit integer represented + * by the six bits 'xxxxxx', plus 1, means that there are N registers set + * to 0. This opcode can represent from 1 to 64 contiguous registers set + * to the value of 0. + * + * XZERO opcode is represented by two bytes 01xxxxxx yyyyyyyy. The 14-bit + * integer represented by the bits 'xxxxxx' as most significant bits and + * 'yyyyyyyy' as least significant bits, plus 1, means that there are N + * registers set to 0. This opcode can represent from 0 to 16384 contiguous + * registers set to the value of 0. + * + * VAL opcode is represented as 1vvvvvxx. It contains a 5-bit integer + * representing the value of a register, and a 2-bit integer representing + * the number of contiguous registers set to that value 'vvvvv'. + * To obtain the value and run length, the integers vvvvv and xx must be + * incremented by one. This opcode can represent values from 1 to 32, + * repeated from 1 to 4 times. + * + * The sparse representation can't represent registers with a value greater + * than 32, however it is very unlikely that we find such a register in an + * HLL with a cardinality where the sparse representation is still more + * memory efficient than the dense representation. When this happens the + * HLL is converted to the dense representation. + * + * The sparse representation is purely positional. For example a sparse + * representation of an empty HLL is just: XZERO:16384. + * + * An HLL having only 3 non-zero registers at position 1000, 1020, 1021 + * respectively set to 2, 3, 3, is represented by the following three + * opcodes: + * + * XZERO:1000 (Registers 0-999 are set to 0) + * VAL:2,1 (1 register set to value 2, that is register 1000) + * ZERO:19 (Registers 1001-1019 set to 0) + * VAL:3,2 (2 registers set to value 3, that is registers 1020,1021) + * XZERO:15362 (Registers 1022-16383 set to 0) + * + * In the example the sparse representation used just 7 bytes instead + * of 12k in order to represent the HLL registers. In general for low + * cardinality there is a big win in terms of space efficiency, traded + * with CPU time since the sparse representation is slower to access: + * + * The following table shows average cardinality vs bytes used, 100 + * samples per cardinality (when the set was not representable because + * of registers with too big value, the dense representation size was used + * as a sample). + * + * 100 267 + * 200 485 + * 300 678 + * 400 859 + * 500 1033 + * 600 1205 + * 700 1375 + * 800 1544 + * 900 1713 + * 1000 1882 + * 2000 3480 + * 3000 4879 + * 4000 6089 + * 5000 7138 + * 6000 8042 + * 7000 8823 + * 8000 9500 + * 9000 10088 + * 10000 10591 + * + * The dense representation uses 12288 bytes, so there is a big win up to + * a cardinality of ~2000-3000. For bigger cardinalities the constant times + * involved in updating the sparse representation is not justified by the + * memory savings. The exact maximum length of the sparse representation + * when this implementation switches to the dense representation is + * configured via the define server.hll_sparse_max_bytes. + */ + +struct hllhdr { + char magic[4]; /* "HYLL" */ + uint8_t encoding; /* HLL_DENSE or HLL_SPARSE. */ + uint8_t notused[3]; /* Reserved for future use, must be zero. */ + uint8_t card[8]; /* Cached cardinality, little endian. */ + uint8_t registers[]; /* Data bytes. */ +}; + +/* The cached cardinality MSB is used to signal validity of the cached value. */ +#define HLL_INVALIDATE_CACHE(hdr) (hdr)->card[7] |= (1<<7) +#define HLL_VALID_CACHE(hdr) (((hdr)->card[7] & (1<<7)) == 0) + +#define HLL_P 14 /* The greater is P, the smaller the error. */ +#define HLL_REGISTERS (1< 6 + * + * Right shift b0 of 'fb' bits. + * + * +--------+ + * |11000000| <- Initial value of b0 + * |00000011| <- After right shift of 6 pos. + * +--------+ + * + * Left shift b1 of bits 8-fb bits (2 bits) + * + * +--------+ + * |22221111| <- Initial value of b1 + * |22111100| <- After left shift of 2 bits. + * +--------+ + * + * OR the two bits, and finally AND with 111111 (63 in decimal) to + * clean the higher order bits we are not interested in: + * + * +--------+ + * |00000011| <- b0 right shifted + * |22111100| <- b1 left shifted + * |22111111| <- b0 OR b1 + * | 111111| <- (b0 OR b1) AND 63, our value. + * +--------+ + * + * We can try with a different example, like pos = 0. In this case + * the 6-bit counter is actually contained in a single byte. + * + * b0 = 6 * pos / 8 = 0 + * + * +--------+ + * |11000000| <- Our byte at b0 + * +--------+ + * + * fb = 6 * pos % 8 = 0 + * + * So we right shift of 0 bits (no shift in practice) and + * left shift the next byte of 8 bits, even if we don't use it, + * but this has the effect of clearing the bits so the result + * will not be affacted after the OR. + * + * ------------------------------------------------------------------------- + * + * Setting the register is a bit more complex, let's assume that 'val' + * is the value we want to set, already in the right range. + * + * We need two steps, in one we need to clear the bits, and in the other + * we need to bitwise-OR the new bits. + * + * Let's try with 'pos' = 1, so our first byte at 'b' is 0, + * + * "fb" is 6 in this case. + * + * +--------+ + * |11000000| <- Our byte at b0 + * +--------+ + * + * To create a AND-mask to clear the bits about this position, we just + * initialize the mask with the value 63, left shift it of "fs" bits, + * and finally invert the result. + * + * +--------+ + * |00111111| <- "mask" starts at 63 + * |11000000| <- "mask" after left shift of "ls" bits. + * |00111111| <- "mask" after invert. + * +--------+ + * + * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR + * it with "val" left-shifted of "ls" bits to set the new bits. + * + * Now let's focus on the next byte b1: + * + * +--------+ + * |22221111| <- Initial value of b1 + * +--------+ + * + * To build the AND mask we start again with the 63 value, right shift + * it by 8-fb bits, and invert it. + * + * +--------+ + * |00111111| <- "mask" set at 2&6-1 + * |00001111| <- "mask" after the right shift by 8-fb = 2 bits + * |11110000| <- "mask" after bitwise not. + * +--------+ + * + * Now we can mask it with b+1 to clear the old bits, and bitwise-OR + * with "val" left-shifted by "rs" bits to set the new value. + */ + +/* Note: if we access the last counter, we will also access the b+1 byte + * that is out of the array, but sds strings always have an implicit null + * term, so the byte exists, and we can skip the conditional (or the need + * to allocate 1 byte more explicitly). */ + +/* Store the value of the register at position 'regnum' into variable 'target'. + * 'p' is an array of unsigned bytes. */ +#define HLL_DENSE_GET_REGISTER(target,p,regnum) do { \ + uint8_t *_p = (uint8_t*) p; \ + unsigned long _byte = regnum*HLL_BITS/8; \ + unsigned long _fb = regnum*HLL_BITS&7; \ + unsigned long _fb8 = 8 - _fb; \ + unsigned long b0 = _p[_byte]; \ + unsigned long b1 = _p[_byte+1]; \ + target = ((b0 >> _fb) | (b1 << _fb8)) & HLL_REGISTER_MAX; \ +} while(0) + +/* Set the value of the register at position 'regnum' to 'val'. + * 'p' is an array of unsigned bytes. */ +#define HLL_DENSE_SET_REGISTER(p,regnum,val) do { \ + uint8_t *_p = (uint8_t*) p; \ + unsigned long _byte = regnum*HLL_BITS/8; \ + unsigned long _fb = regnum*HLL_BITS&7; \ + unsigned long _fb8 = 8 - _fb; \ + unsigned long _v = val; \ + _p[_byte] &= ~(HLL_REGISTER_MAX << _fb); \ + _p[_byte] |= _v << _fb; \ + _p[_byte+1] &= ~(HLL_REGISTER_MAX >> _fb8); \ + _p[_byte+1] |= _v >> _fb8; \ +} while(0) + +/* Macros to access the sparse representation. + * The macros parameter is expected to be an uint8_t pointer. */ +#define HLL_SPARSE_XZERO_BIT 0x40 /* 01xxxxxx */ +#define HLL_SPARSE_VAL_BIT 0x80 /* 1vvvvvxx */ +#define HLL_SPARSE_IS_ZERO(p) (((*(p)) & 0xc0) == 0) /* 00xxxxxx */ +#define HLL_SPARSE_IS_XZERO(p) (((*(p)) & 0xc0) == HLL_SPARSE_XZERO_BIT) +#define HLL_SPARSE_IS_VAL(p) ((*(p)) & HLL_SPARSE_VAL_BIT) +#define HLL_SPARSE_ZERO_LEN(p) (((*(p)) & 0x3f)+1) +#define HLL_SPARSE_XZERO_LEN(p) (((((*(p)) & 0x3f) << 8) | (*((p)+1)))+1) +#define HLL_SPARSE_VAL_VALUE(p) ((((*(p)) >> 2) & 0x1f)+1) +#define HLL_SPARSE_VAL_LEN(p) (((*(p)) & 0x3)+1) +#define HLL_SPARSE_VAL_MAX_VALUE 32 +#define HLL_SPARSE_VAL_MAX_LEN 4 +#define HLL_SPARSE_ZERO_MAX_LEN 64 +#define HLL_SPARSE_XZERO_MAX_LEN 16384 +#define HLL_SPARSE_VAL_SET(p,val,len) do { \ + *(p) = (((val)-1)<<2|((len)-1))|HLL_SPARSE_VAL_BIT; \ +} while(0) +#define HLL_SPARSE_ZERO_SET(p,len) do { \ + *(p) = (len)-1; \ +} while(0) +#define HLL_SPARSE_XZERO_SET(p,len) do { \ + int _l = (len)-1; \ + *(p) = (_l>>8) | HLL_SPARSE_XZERO_BIT; \ + *((p)+1) = (_l&0xff); \ +} while(0) + +/* ========================= HyperLogLog algorithm ========================= */ + +/* Our hash function is MurmurHash2, 64 bit version. + * It was modified for Redis in order to provide the same result in + * big and little endian archs (endian neutral). */ +uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) { + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + uint64_t h = seed ^ (len * m); + const uint8_t *data = (const uint8_t *)key; + const uint8_t *end = data + (len-(len&7)); + + while(data != end) { + uint64_t k; + +#if (BYTE_ORDER == LITTLE_ENDIAN) + k = *((uint64_t*)data); +#else + k = (uint64_t) data[0]; + k |= (uint64_t) data[1] << 8; + k |= (uint64_t) data[2] << 16; + k |= (uint64_t) data[3] << 24; + k |= (uint64_t) data[4] << 32; + k |= (uint64_t) data[5] << 40; + k |= (uint64_t) data[6] << 48; + k |= (uint64_t) data[7] << 56; +#endif + + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + data += 8; + } + + switch(len & 7) { + case 7: h ^= (uint64_t)data[6] << 48; + case 6: h ^= (uint64_t)data[5] << 40; + case 5: h ^= (uint64_t)data[4] << 32; + case 4: h ^= (uint64_t)data[3] << 24; + case 3: h ^= (uint64_t)data[2] << 16; + case 2: h ^= (uint64_t)data[1] << 8; + case 1: h ^= (uint64_t)data[0]; + h *= m; + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; +} + +/* Given a string element to add to the HyperLogLog, returns the length + * of the pattern 000..1 of the element hash. As a side effect 'regp' is + * set to the register index this element hashes to. */ +int hllPatLen(unsigned char *ele, size_t elesize, long *regp) { + uint64_t hash, bit, index; + int count; + + /* Count the number of zeroes starting from bit HLL_REGISTERS + * (that is a power of two corresponding to the first bit we don't use + * as index). The max run can be 64-P+1 bits. + * + * Note that the final "1" ending the sequence of zeroes must be + * included in the count, so if we find "001" the count is 3, and + * the smallest count possible is no zeroes at all, just a 1 bit + * at the first position, that is a count of 1. + * + * This may sound like inefficient, but actually in the average case + * there are high probabilities to find a 1 after a few iterations. */ + hash = MurmurHash64A(ele,elesize,0xadc83b19ULL); + index = hash & HLL_P_MASK; /* Register index. */ + hash |= ((uint64_t)1<<63); /* Make sure the loop terminates. */ + bit = HLL_REGISTERS; /* First bit not used to address the register. */ + count = 1; /* Initialized to 1 since we count the "00000...1" pattern. */ + while((hash & bit) == 0) { + count++; + bit <<= 1; + } + *regp = (int) index; + return count; +} + +/* ================== Dense representation implementation ================== */ + +/* "Add" the element in the dense hyperloglog data structure. + * Actually nothing is added, but the max 0 pattern counter of the subset + * the element belongs to is incremented if needed. + * + * 'registers' is expected to have room for HLL_REGISTERS plus an + * additional byte on the right. This requirement is met by sds strings + * automatically since they are implicitly null terminated. + * + * The function always succeed, however if as a result of the operation + * the approximated cardinality changed, 1 is returned. Otherwise 0 + * is returned. */ +int hllDenseAdd(uint8_t *registers, unsigned char *ele, size_t elesize) { + uint8_t oldcount, count; + long index; + + /* Update the register if this element produced a longer run of zeroes. */ + count = hllPatLen(ele,elesize,&index); + HLL_DENSE_GET_REGISTER(oldcount,registers,index); + if (count > oldcount) { + HLL_DENSE_SET_REGISTER(registers,index,count); + return 1; + } else { + return 0; + } +} + +/* Compute SUM(2^-reg) in the dense representation. + * PE is an array with a pre-computer table of values 2^-reg indexed by reg. + * As a side effect the integer pointed by 'ezp' is set to the number + * of zero registers. */ +double hllDenseSum(uint8_t *registers, double *PE, int *ezp) { + double E = 0; + int j, ez = 0; + + /* Redis default is to use 16384 registers 6 bits each. The code works + * with other values by modifying the defines, but for our target value + * we take a faster path with unrolled loops. */ + if (HLL_REGISTERS == 16384 && HLL_BITS == 6) { + uint8_t *r = registers; + unsigned long r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, + r10, r11, r12, r13, r14, r15; + for (j = 0; j < 1024; j++) { + /* Handle 16 registers per iteration. */ + r0 = r[0] & 63; if (r0 == 0) ez++; + r1 = (r[0] >> 6 | r[1] << 2) & 63; if (r1 == 0) ez++; + r2 = (r[1] >> 4 | r[2] << 4) & 63; if (r2 == 0) ez++; + r3 = (r[2] >> 2) & 63; if (r3 == 0) ez++; + r4 = r[3] & 63; if (r4 == 0) ez++; + r5 = (r[3] >> 6 | r[4] << 2) & 63; if (r5 == 0) ez++; + r6 = (r[4] >> 4 | r[5] << 4) & 63; if (r6 == 0) ez++; + r7 = (r[5] >> 2) & 63; if (r7 == 0) ez++; + r8 = r[6] & 63; if (r8 == 0) ez++; + r9 = (r[6] >> 6 | r[7] << 2) & 63; if (r9 == 0) ez++; + r10 = (r[7] >> 4 | r[8] << 4) & 63; if (r10 == 0) ez++; + r11 = (r[8] >> 2) & 63; if (r11 == 0) ez++; + r12 = r[9] & 63; if (r12 == 0) ez++; + r13 = (r[9] >> 6 | r[10] << 2) & 63; if (r13 == 0) ez++; + r14 = (r[10] >> 4 | r[11] << 4) & 63; if (r14 == 0) ez++; + r15 = (r[11] >> 2) & 63; if (r15 == 0) ez++; + + /* Additional parens will allow the compiler to optimize the + * code more with a loss of precision that is not very relevant + * here (floating point math is not commutative!). */ + E += (PE[r0] + PE[r1]) + (PE[r2] + PE[r3]) + (PE[r4] + PE[r5]) + + (PE[r6] + PE[r7]) + (PE[r8] + PE[r9]) + (PE[r10] + PE[r11]) + + (PE[r12] + PE[r13]) + (PE[r14] + PE[r15]); + r += 12; + } + } else { + for (j = 0; j < HLL_REGISTERS; j++) { + unsigned long reg; + + HLL_DENSE_GET_REGISTER(reg,registers,j); + if (reg == 0) { + ez++; + /* Increment E at the end of the loop. */ + } else { + E += PE[reg]; /* Precomputed 2^(-reg[j]). */ + } + } + E += ez; /* Add 2^0 'ez' times. */ + } + *ezp = ez; + return E; +} + +/* ================== Sparse representation implementation ================= */ + +/* Convert the HLL with sparse representation given as input in its dense + * representation. Both representations are represented by SDS strings, and + * the input representation is freed as a side effect. + * + * The function returns REDIS_OK if the sparse representation was valid, + * otherwise REDIS_ERR is returned if the representation was corrupted. */ +int hllSparseToDense(robj *o) { + sds sparse = o->ptr, dense; + struct hllhdr *hdr, *oldhdr = (struct hllhdr*)sparse; + int idx = 0, runlen, regval; + uint8_t *p = (uint8_t*)sparse, *end = p+sdslen(sparse); + + /* If the representation is already the right one return ASAP. */ + hdr = (struct hllhdr*) sparse; + if (hdr->encoding == HLL_DENSE) return REDIS_OK; + + /* Create a string of the right size filled with zero bytes. + * Note that the cached cardinality is set to 0 as a side effect + * that is exactly the cardinality of an empty HLL. */ + dense = sdsnewlen(NULL,HLL_DENSE_SIZE); + hdr = (struct hllhdr*) dense; + *hdr = *oldhdr; /* This will copy the magic and cached cardinality. */ + hdr->encoding = HLL_DENSE; + + /* Now read the sparse representation and set non-zero registers + * accordingly. */ + p += HLL_HDR_SIZE; + while(p < end) { + if (HLL_SPARSE_IS_ZERO(p)) { + runlen = HLL_SPARSE_ZERO_LEN(p); + idx += runlen; + p++; + } else if (HLL_SPARSE_IS_XZERO(p)) { + runlen = HLL_SPARSE_XZERO_LEN(p); + idx += runlen; + p += 2; + } else { + runlen = HLL_SPARSE_VAL_LEN(p); + regval = HLL_SPARSE_VAL_VALUE(p); + while(runlen--) { + HLL_DENSE_SET_REGISTER(hdr->registers,idx,regval); + idx++; + } + p++; + } + } + + /* If the sparse representation was valid, we expect to find idx + * set to HLL_REGISTERS. */ + if (idx != HLL_REGISTERS) { + sdsfree(dense); + return REDIS_ERR; + } + + /* Free the old representation and set the new one. */ + sdsfree(o->ptr); + o->ptr = dense; + return REDIS_OK; +} + +/* "Add" the element in the sparse hyperloglog data structure. + * Actually nothing is added, but the max 0 pattern counter of the subset + * the element belongs to is incremented if needed. + * + * The object 'o' is the String object holding the HLL. The function requires + * a reference to the object in order to be able to enlarge the string if + * needed. + * + * On success, the function returns 1 if the cardinality changed, or 0 + * if the register for this element was not updated. + * On error (if the representation is invalid) -1 is returned. + * + * As a side effect the function may promote the HLL representation from + * sparse to dense: this happens when a register requires to be set to a value + * not representable with the sparse representation, or when the resulting + * size would be greater than server.hll_sparse_max_bytes. */ +int hllSparseAdd(robj *o, unsigned char *ele, size_t elesize) { + struct hllhdr *hdr; + uint8_t oldcount, count, *sparse, *end, *p, *prev, *next; + long index, first, span; + long is_zero = 0, is_xzero = 0, is_val = 0, runlen = 0; + + /* Update the register if this element produced a longer run of zeroes. */ + count = hllPatLen(ele,elesize,&index); + + /* If the count is too big to be representable by the sparse representation + * switch to dense representation. */ + if (count > HLL_SPARSE_VAL_MAX_VALUE) goto promote; + + /* When updating a sparse representation, sometimes we may need to + * enlarge the buffer for up to 3 bytes in the worst case (XZERO split + * into XZERO-VAL-XZERO). Make sure there is enough space right now + * so that the pointers we take during the execution of the function + * will be valid all the time. */ + o->ptr = sdsMakeRoomFor(o->ptr,3); + + /* Step 1: we need to locate the opcode we need to modify to check + * if a value update is actually needed. */ + sparse = p = ((uint8_t*)o->ptr) + HLL_HDR_SIZE; + end = p + sdslen(o->ptr) - HLL_HDR_SIZE; + + first = 0; + prev = NULL; /* Points to previos opcode at the end of the loop. */ + next = NULL; /* Points to the next opcode at the end of the loop. */ + span = 0; + while(p < end) { + long oplen; + + /* Set span to the number of registers covered by this opcode. + * + * This is the most performance critical loop of the sparse + * representation. Sorting the conditionals from the most to the + * least frequent opcode in many-bytes sparse HLLs is faster. */ + oplen = 1; + if (HLL_SPARSE_IS_ZERO(p)) { + span = HLL_SPARSE_ZERO_LEN(p); + } else if (HLL_SPARSE_IS_VAL(p)) { + span = HLL_SPARSE_VAL_LEN(p); + } else { /* XZERO. */ + span = HLL_SPARSE_XZERO_LEN(p); + oplen = 2; + } + /* Break if this opcode covers the register as 'index'. */ + if (index <= first+span-1) break; + prev = p; + p += oplen; + first += span; + } + if (span == 0) return -1; /* Invalid format. */ + + next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; + if (next >= end) next = NULL; + + /* Cache current opcode type to avoid using the macro again and + * again for something that will not change. + * Also cache the run-length of the opcode. */ + if (HLL_SPARSE_IS_ZERO(p)) { + is_zero = 1; + runlen = HLL_SPARSE_ZERO_LEN(p); + } else if (HLL_SPARSE_IS_XZERO(p)) { + is_xzero = 1; + runlen = HLL_SPARSE_XZERO_LEN(p); + } else { + is_val = 1; + runlen = HLL_SPARSE_VAL_LEN(p); + } + + /* Step 2: After the loop: + * + * 'first' stores to the index of the first register covered + * by the current opcode, which is pointed by 'p'. + * + * 'next' ad 'prev' store respectively the next and previous opcode, + * or NULL if the opcode at 'p' is respectively the last or first. + * + * 'span' is set to the number of registers covered by the current + * opcode. + * + * There are different cases in order to update the data structure + * in place without generating it from scratch: + * + * A) If it is a VAL opcode already set to a value >= our 'count' + * no update is needed, regardless of the VAL run-length field. + * In this case PFADD returns 0 since no changes are performed. + * + * B) If it is a VAL opcode with len = 1 (representing only our + * register) and the value is less than 'count', we just update it + * since this is a trivial case. */ + if (is_val) { + oldcount = HLL_SPARSE_VAL_VALUE(p); + /* Case A. */ + if (oldcount >= count) return 0; + + /* Case B. */ + if (runlen == 1) { + HLL_SPARSE_VAL_SET(p,count,1); + goto updated; + } + } + + /* C) Another trivial to handle case is a ZERO opcode with a len of 1. + * We can just replace it with a VAL opcode with our value and len of 1. */ + if (is_zero && runlen == 1) { + HLL_SPARSE_VAL_SET(p,count,1); + goto updated; + } + + /* D) General case. + * + * The other cases are more complex: our register requires to be updated + * and is either currently represented by a VAL opcode with len > 1, + * by a ZERO opcode with len > 1, or by an XZERO opcode. + * + * In those cases the original opcode must be split into muliple + * opcodes. The worst case is an XZERO split in the middle resuling into + * XZERO - VAL - XZERO, so the resulting sequence max length is + * 5 bytes. + * + * We perform the split writing the new sequence into the 'new' buffer + * with 'newlen' as length. Later the new sequence is inserted in place + * of the old one, possibly moving what is on the right a few bytes + * if the new sequence is longer than the older one. */ + uint8_t seq[5], *n = seq; + int last = first+span-1; /* Last register covered by the sequence. */ + int len; + + if (is_zero || is_xzero) { + /* Handle splitting of ZERO / XZERO. */ + if (index != first) { + len = index-first; + if (len > HLL_SPARSE_ZERO_MAX_LEN) { + HLL_SPARSE_XZERO_SET(n,len); + n += 2; + } else { + HLL_SPARSE_ZERO_SET(n,len); + n++; + } + } + HLL_SPARSE_VAL_SET(n,count,1); + n++; + if (index != last) { + len = last-index; + if (len > HLL_SPARSE_ZERO_MAX_LEN) { + HLL_SPARSE_XZERO_SET(n,len); + n += 2; + } else { + HLL_SPARSE_ZERO_SET(n,len); + n++; + } + } + } else { + /* Handle splitting of VAL. */ + int curval = HLL_SPARSE_VAL_VALUE(p); + + if (index != first) { + len = index-first; + HLL_SPARSE_VAL_SET(n,curval,len); + n++; + } + HLL_SPARSE_VAL_SET(n,count,1); + n++; + if (index != last) { + len = last-index; + HLL_SPARSE_VAL_SET(n,curval,len); + n++; + } + } + + /* Step 3: substitute the new sequence with the old one. + * + * Note that we already allocated space on the sds string + * calling sdsMakeRoomFor(). */ + int seqlen = n-seq; + int oldlen = is_xzero ? 2 : 1; + int deltalen = seqlen-oldlen; + + if (deltalen > 0 && + sdslen(o->ptr)+deltalen > server.hll_sparse_max_bytes) goto promote; + if (deltalen && next) memmove(next+deltalen,next,end-next); + sdsIncrLen(o->ptr,deltalen); + memcpy(p,seq,seqlen); + end += deltalen; + +updated: + /* Step 4: Merge adjacent values if possible. + * + * The representation was updated, however the resulting representation + * may not be optimal: adjacent VAL opcodes can sometimes be merged into + * a single one. */ + p = prev ? prev : sparse; + int scanlen = 5; /* Scan up to 5 upcodes starting from prev. */ + while (p < end && scanlen--) { + if (HLL_SPARSE_IS_XZERO(p)) { + p += 2; + continue; + } else if (HLL_SPARSE_IS_ZERO(p)) { + p++; + continue; + } + /* We need two adjacent VAL opcodes to try a merge, having + * the same value, and a len that fits the VAL opcode max len. */ + if (p+1 < end && HLL_SPARSE_IS_VAL(p+1)) { + int v1 = HLL_SPARSE_VAL_VALUE(p); + int v2 = HLL_SPARSE_VAL_VALUE(p+1); + if (v1 == v2) { + int len = HLL_SPARSE_VAL_LEN(p)+HLL_SPARSE_VAL_LEN(p+1); + if (len <= HLL_SPARSE_VAL_MAX_LEN) { + HLL_SPARSE_VAL_SET(p+1,v1,len); + memmove(p,p+1,end-p); + sdsIncrLen(o->ptr,-1); + end--; + /* After a merge we reiterate without incrementing 'p' + * in order to try to merge the just merged value with + * a value on its right. */ + continue; + } + } + } + p++; + } + + /* Invalidate the cached cardinality. */ + hdr = o->ptr; + HLL_INVALIDATE_CACHE(hdr); + return 1; + +promote: /* Promote to dense representation. */ + if (hllSparseToDense(o) == REDIS_ERR) return -1; /* Corrupted HLL. */ + hdr = o->ptr; + + /* We need to call hllDenseAdd() to perform the operation after the + * conversion. However the result must be 1, since if we need to + * convert from sparse to dense a register requires to be updated. + * + * Note that this in turn means that PFADD will make sure the command + * is propagated to slaves / AOF, so if there is a sparse -> dense + * convertion, it will be performed in all the slaves as well. */ + int dense_retval = hllDenseAdd(hdr->registers, ele, elesize); + redisAssert(dense_retval == 1); + return dense_retval; +} + +/* Compute SUM(2^-reg) in the sparse representation. + * PE is an array with a pre-computer table of values 2^-reg indexed by reg. + * As a side effect the integer pointed by 'ezp' is set to the number + * of zero registers. */ +double hllSparseSum(uint8_t *sparse, int sparselen, double *PE, int *ezp, int *invalid) { + double E = 0; + int ez = 0, idx = 0, runlen, regval; + uint8_t *end = sparse+sparselen, *p = sparse; + + while(p < end) { + if (HLL_SPARSE_IS_ZERO(p)) { + runlen = HLL_SPARSE_ZERO_LEN(p); + idx += runlen; + ez += runlen; + /* Increment E at the end of the loop. */ + p++; + } else if (HLL_SPARSE_IS_XZERO(p)) { + runlen = HLL_SPARSE_XZERO_LEN(p); + idx += runlen; + ez += runlen; + /* Increment E at the end of the loop. */ + p += 2; + } else { + runlen = HLL_SPARSE_VAL_LEN(p); + regval = HLL_SPARSE_VAL_VALUE(p); + idx += runlen; + E += PE[regval]*runlen; + p++; + } + } + if (idx != HLL_REGISTERS && invalid) *invalid = 1; + E += ez; /* Add 2^0 'ez' times. */ + *ezp = ez; + return E; +} + +/* ========================= HyperLogLog Count ============================== + * This is the core of the algorithm where the approximated count is computed. + * The function uses the lower level hllDenseSum() and hllSparseSum() functions + * as helpers to compute the SUM(2^-reg) part of the computation, which is + * representation-specific, while all the rest is common. */ + +/* Implements the SUM operation for uint8_t data type which is only used + * internally as speedup for PFCOUNT with multiple keys. */ +double hllRawSum(uint8_t *registers, double *PE, int *ezp) { + double E = 0; + int j, ez = 0; + uint64_t *word = (uint64_t*) registers; + uint8_t *bytes; + + for (j = 0; j < HLL_REGISTERS/8; j++) { + if (*word == 0) { + ez += 8; + } else { + bytes = (uint8_t*) word; + if (bytes[0]) E += PE[bytes[0]]; else ez++; + if (bytes[1]) E += PE[bytes[1]]; else ez++; + if (bytes[2]) E += PE[bytes[2]]; else ez++; + if (bytes[3]) E += PE[bytes[3]]; else ez++; + if (bytes[4]) E += PE[bytes[4]]; else ez++; + if (bytes[5]) E += PE[bytes[5]]; else ez++; + if (bytes[6]) E += PE[bytes[6]]; else ez++; + if (bytes[7]) E += PE[bytes[7]]; else ez++; + } + word++; + } + E += ez; /* 2^(-reg[j]) is 1 when m is 0, add it 'ez' times for every + zero register in the HLL. */ + *ezp = ez; + return E; +} + +/* Return the approximated cardinality of the set based on the harmonic + * mean of the registers values. 'hdr' points to the start of the SDS + * representing the String object holding the HLL representation. + * + * If the sparse representation of the HLL object is not valid, the integer + * pointed by 'invalid' is set to non-zero, otherwise it is left untouched. + * + * hllCount() supports a special internal-only encoding of HLL_RAW, that + * is, hdr->registers will point to an uint8_t array of HLL_REGISTERS element. + * This is useful in order to speedup PFCOUNT when called against multiple + * keys (no need to work with 6-bit integers encoding). */ +uint64_t hllCount(struct hllhdr *hdr, int *invalid) { + double m = HLL_REGISTERS; + double E, alpha = 0.7213/(1+1.079/m); + int j, ez; /* Number of registers equal to 0. */ + + /* We precompute 2^(-reg[j]) in a small table in order to + * speedup the computation of SUM(2^-register[0..i]). */ + static int initialized = 0; + static double PE[64]; + if (!initialized) { + PE[0] = 1; /* 2^(-reg[j]) is 1 when m is 0. */ + for (j = 1; j < 64; j++) { + /* 2^(-reg[j]) is the same as 1/2^reg[j]. */ + PE[j] = 1.0/(1ULL << j); + } + initialized = 1; + } + + /* Compute SUM(2^-register[0..i]). */ + if (hdr->encoding == HLL_DENSE) { + E = hllDenseSum(hdr->registers,PE,&ez); + } else if (hdr->encoding == HLL_SPARSE) { + E = hllSparseSum(hdr->registers, + sdslen((sds)hdr)-HLL_HDR_SIZE,PE,&ez,invalid); + } else if (hdr->encoding == HLL_RAW) { + E = hllRawSum(hdr->registers,PE,&ez); + } else { + redisPanic("Unknown HyperLogLog encoding in hllCount()"); + } + + /* Muliply the inverse of E for alpha_m * m^2 to have the raw estimate. */ + E = (1/E)*alpha*m*m; + + /* Use the LINEARCOUNTING algorithm for small cardinalities. + * For larger values but up to 72000 HyperLogLog raw approximation is + * used since linear counting error starts to increase. However HyperLogLog + * shows a strong bias in the range 2.5*16384 - 72000, so we try to + * compensate for it. */ + if (E < m*2.5 && ez != 0) { + E = m*log(m/ez); /* LINEARCOUNTING() */ + } else if (m == 16384 && E < 72000) { + /* We did polynomial regression of the bias for this range, this + * way we can compute the bias for a given cardinality and correct + * according to it. Only apply the correction for P=14 that's what + * we use and the value the correction was verified with. */ + double bias = 5.9119*1.0e-18*(E*E*E*E) + -1.4253*1.0e-12*(E*E*E)+ + 1.2940*1.0e-7*(E*E) + -5.2921*1.0e-3*E+ + 83.3216; + E -= E*(bias/100); + } + /* We don't apply the correction for E > 1/30 of 2^32 since we use + * a 64 bit function and 6 bit counters. To apply the correction for + * 1/30 of 2^64 is not needed since it would require a huge set + * to approach such a value. */ + return (uint64_t) E; +} + +/* Call hllDenseAdd() or hllSparseAdd() according to the HLL encoding. */ +int hllAdd(robj *o, unsigned char *ele, size_t elesize) { + struct hllhdr *hdr = o->ptr; + switch(hdr->encoding) { + case HLL_DENSE: return hllDenseAdd(hdr->registers,ele,elesize); + case HLL_SPARSE: return hllSparseAdd(o,ele,elesize); + default: return -1; /* Invalid representation. */ + } +} + +/* Merge by computing MAX(registers[i],hll[i]) the HyperLogLog 'hll' + * with an array of uint8_t HLL_REGISTERS registers pointed by 'max'. + * + * The hll object must be already validated via isHLLObjectOrReply() + * or in some other way. + * + * If the HyperLogLog is sparse and is found to be invalid, REDIS_ERR + * is returned, otherwise the function always succeeds. */ +int hllMerge(uint8_t *max, robj *hll) { + struct hllhdr *hdr = hll->ptr; + int i; + + if (hdr->encoding == HLL_DENSE) { + uint8_t val; + + for (i = 0; i < HLL_REGISTERS; i++) { + HLL_DENSE_GET_REGISTER(val,hdr->registers,i); + if (val > max[i]) max[i] = val; + } + } else { + uint8_t *p = hll->ptr, *end = p + sdslen(hll->ptr); + long runlen, regval; + + p += HLL_HDR_SIZE; + i = 0; + while(p < end) { + if (HLL_SPARSE_IS_ZERO(p)) { + runlen = HLL_SPARSE_ZERO_LEN(p); + i += runlen; + p++; + } else if (HLL_SPARSE_IS_XZERO(p)) { + runlen = HLL_SPARSE_XZERO_LEN(p); + i += runlen; + p += 2; + } else { + runlen = HLL_SPARSE_VAL_LEN(p); + regval = HLL_SPARSE_VAL_VALUE(p); + while(runlen--) { + if (regval > max[i]) max[i] = regval; + i++; + } + p++; + } + } + if (i != HLL_REGISTERS) return REDIS_ERR; + } + return REDIS_OK; +} + +/* ========================== HyperLogLog commands ========================== */ + +/* Create an HLL object. We always create the HLL using sparse encoding. + * This will be upgraded to the dense representation as needed. */ +robj *createHLLObject(void) { + robj *o; + struct hllhdr *hdr; + sds s; + uint8_t *p; + int sparselen = HLL_HDR_SIZE + + (((HLL_REGISTERS+(HLL_SPARSE_XZERO_MAX_LEN-1)) / + HLL_SPARSE_XZERO_MAX_LEN)*2); + int aux; + + /* Populate the sparse representation with as many XZERO opcodes as + * needed to represent all the registers. */ + aux = HLL_REGISTERS; + s = sdsnewlen(NULL,sparselen); + p = (uint8_t*)s + HLL_HDR_SIZE; + while(aux) { + int xzero = HLL_SPARSE_XZERO_MAX_LEN; + if (xzero > aux) xzero = aux; + HLL_SPARSE_XZERO_SET(p,xzero); + p += 2; + aux -= xzero; + } + redisAssert((p-(uint8_t*)s) == sparselen); + + /* Create the actual object. */ + o = createObject(REDIS_STRING,s); + hdr = o->ptr; + memcpy(hdr->magic,"HYLL",4); + hdr->encoding = HLL_SPARSE; + return o; +} + +/* Check if the object is a String with a valid HLL representation. + * Return REDIS_OK if this is true, otherwise reply to the client + * with an error and return REDIS_ERR. */ +int isHLLObjectOrReply(redisClient *c, robj *o) { + struct hllhdr *hdr; + + /* Key exists, check type */ + if (checkType(c,o,REDIS_STRING)) + return REDIS_ERR; /* Error already sent. */ + + if (stringObjectLen(o) < sizeof(*hdr)) goto invalid; + hdr = o->ptr; + + /* Magic should be "HYLL". */ + if (hdr->magic[0] != 'H' || hdr->magic[1] != 'Y' || + hdr->magic[2] != 'L' || hdr->magic[3] != 'L') goto invalid; + + if (hdr->encoding > HLL_MAX_ENCODING) goto invalid; + + /* Dense representation string length should match exactly. */ + if (hdr->encoding == HLL_DENSE && + stringObjectLen(o) != HLL_DENSE_SIZE) goto invalid; + + /* All tests passed. */ + return REDIS_OK; + +invalid: + addReplySds(c, + sdsnew("-WRONGTYPE Key is not a valid " + "HyperLogLog string value.\r\n")); + return REDIS_ERR; +} + +/* PFADD var ele ele ele ... ele => :0 or :1 */ +void pfaddCommand(redisClient *c) { + robj *o = lookupKeyWrite(c->db,c->argv[1]); + struct hllhdr *hdr; + int updated = 0, j; + + if (o == NULL) { + /* Create the key with a string value of the exact length to + * hold our HLL data structure. sdsnewlen() when NULL is passed + * is guaranteed to return bytes initialized to zero. */ + o = createHLLObject(); + dbAdd(c->db,c->argv[1],o); + updated++; + } else { + if (isHLLObjectOrReply(c,o) != REDIS_OK) return; + o = dbUnshareStringValue(c->db,c->argv[1],o); + } + /* Perform the low level ADD operation for every element. */ + for (j = 2; j < c->argc; j++) { + int retval = hllAdd(o, (unsigned char*)c->argv[j]->ptr, + sdslen(c->argv[j]->ptr)); + switch(retval) { + case 1: + updated++; + break; + case -1: + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + } + hdr = o->ptr; + if (updated) { + signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); + server.dirty++; + HLL_INVALIDATE_CACHE(hdr); + } + addReply(c, updated ? shared.cone : shared.czero); +} + +/* PFCOUNT var -> approximated cardinality of set. */ +void pfcountCommand(redisClient *c) { + robj *o; + struct hllhdr *hdr; + uint64_t card; + + /* Case 1: multi-key keys, cardinality of the union. + * + * When multiple keys are specified, PFCOUNT actually computes + * the cardinality of the merge of the N HLLs specified. */ + if (c->argc > 2) { + uint8_t max[HLL_HDR_SIZE+HLL_REGISTERS], *registers; + int j; + + /* Compute an HLL with M[i] = MAX(M[i]_j). */ + memset(max,0,sizeof(max)); + hdr = (struct hllhdr*) max; + hdr->encoding = HLL_RAW; /* Special internal-only encoding. */ + registers = max + HLL_HDR_SIZE; + for (j = 1; j < c->argc; j++) { + /* Check type and size. */ + robj *o = lookupKeyRead(c->db,c->argv[j]); + if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ + if (isHLLObjectOrReply(c,o) != REDIS_OK) return; + + /* Merge with this HLL with our 'max' HHL by setting max[i] + * to MAX(max[i],hll[i]). */ + if (hllMerge(registers,o) == REDIS_ERR) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + } + + /* Compute cardinality of the resulting set. */ + addReplyLongLong(c,hllCount(hdr,NULL)); + return; + } + + /* Case 2: cardinality of the single HLL. + * + * The user specified a single key. Either return the cached value + * or compute one and update the cache. */ + o = lookupKeyRead(c->db,c->argv[1]); + if (o == NULL) { + /* No key? Cardinality is zero since no element was added, otherwise + * we would have a key as HLLADD creates it as a side effect. */ + addReply(c,shared.czero); + } else { + if (isHLLObjectOrReply(c,o) != REDIS_OK) return; + o = dbUnshareStringValue(c->db,c->argv[1],o); + + /* Check if the cached cardinality is valid. */ + hdr = o->ptr; + if (HLL_VALID_CACHE(hdr)) { + /* Just return the cached value. */ + card = (uint64_t)hdr->card[0]; + card |= (uint64_t)hdr->card[1] << 8; + card |= (uint64_t)hdr->card[2] << 16; + card |= (uint64_t)hdr->card[3] << 24; + card |= (uint64_t)hdr->card[4] << 32; + card |= (uint64_t)hdr->card[5] << 40; + card |= (uint64_t)hdr->card[6] << 48; + card |= (uint64_t)hdr->card[7] << 56; + } else { + int invalid = 0; + /* Recompute it and update the cached value. */ + card = hllCount(hdr,&invalid); + if (invalid) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + hdr->card[0] = card & 0xff; + hdr->card[1] = (card >> 8) & 0xff; + hdr->card[2] = (card >> 16) & 0xff; + hdr->card[3] = (card >> 24) & 0xff; + hdr->card[4] = (card >> 32) & 0xff; + hdr->card[5] = (card >> 40) & 0xff; + hdr->card[6] = (card >> 48) & 0xff; + hdr->card[7] = (card >> 56) & 0xff; + /* This is not considered a read-only command even if the + * data structure is not modified, since the cached value + * may be modified and given that the HLL is a Redis string + * we need to propagate the change. */ + signalModifiedKey(c->db,c->argv[1]); + server.dirty++; + } + addReplyLongLong(c,card); + } +} + +/* PFMERGE dest src1 src2 src3 ... srcN => OK */ +void pfmergeCommand(redisClient *c) { + uint8_t max[HLL_REGISTERS]; + struct hllhdr *hdr; + int j; + + /* Compute an HLL with M[i] = MAX(M[i]_j). + * We we the maximum into the max array of registers. We'll write + * it to the target variable later. */ + memset(max,0,sizeof(max)); + for (j = 1; j < c->argc; j++) { + /* Check type and size. */ + robj *o = lookupKeyRead(c->db,c->argv[j]); + if (o == NULL) continue; /* Assume empty HLL for non existing var. */ + if (isHLLObjectOrReply(c,o) != REDIS_OK) return; + + /* Merge with this HLL with our 'max' HHL by setting max[i] + * to MAX(max[i],hll[i]). */ + if (hllMerge(max,o) == REDIS_ERR) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + } + + /* Create / unshare the destination key's value if needed. */ + robj *o = lookupKeyWrite(c->db,c->argv[1]); + if (o == NULL) { + /* Create the key with a string value of the exact length to + * hold our HLL data structure. sdsnewlen() when NULL is passed + * is guaranteed to return bytes initialized to zero. */ + o = createHLLObject(); + dbAdd(c->db,c->argv[1],o); + } else { + /* If key exists we are sure it's of the right type/size + * since we checked when merging the different HLLs, so we + * don't check again. */ + o = dbUnshareStringValue(c->db,c->argv[1],o); + } + + /* Only support dense objects as destination. */ + if (hllSparseToDense(o) == REDIS_ERR) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + + /* Write the resulting HLL to the destination HLL registers and + * invalidate the cached value. */ + hdr = o->ptr; + for (j = 0; j < HLL_REGISTERS; j++) { + HLL_DENSE_SET_REGISTER(hdr->registers,j,max[j]); + } + HLL_INVALIDATE_CACHE(hdr); + + signalModifiedKey(c->db,c->argv[1]); + /* We generate an PFADD event for PFMERGE for semantical simplicity + * since in theory this is a mass-add of elements. */ + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); + server.dirty++; + addReply(c,shared.ok); +} + +/* ========================== Testing / Debugging ========================== */ + +/* PFSELFTEST + * This command performs a self-test of the HLL registers implementation. + * Something that is not easy to test from within the outside. */ +#define HLL_TEST_CYCLES 1000 +void pfselftestCommand(redisClient *c) { + unsigned int j, i; + sds bitcounters = sdsnewlen(NULL,HLL_DENSE_SIZE); + struct hllhdr *hdr = (struct hllhdr*) bitcounters, *hdr2; + robj *o = NULL; + uint8_t bytecounters[HLL_REGISTERS]; + + /* Test 1: access registers. + * The test is conceived to test that the different counters of our data + * structure are accessible and that setting their values both result in + * the correct value to be retained and not affect adjacent values. */ + for (j = 0; j < HLL_TEST_CYCLES; j++) { + /* Set the HLL counters and an array of unsigned byes of the + * same size to the same set of random values. */ + for (i = 0; i < HLL_REGISTERS; i++) { + unsigned int r = rand() & HLL_REGISTER_MAX; + + bytecounters[i] = r; + HLL_DENSE_SET_REGISTER(hdr->registers,i,r); + } + /* Check that we are able to retrieve the same values. */ + for (i = 0; i < HLL_REGISTERS; i++) { + unsigned int val; + + HLL_DENSE_GET_REGISTER(val,hdr->registers,i); + if (val != bytecounters[i]) { + addReplyErrorFormat(c, + "TESTFAILED Register %d should be %d but is %d", + i, (int) bytecounters[i], (int) val); + goto cleanup; + } + } + } + + /* Test 2: approximation error. + * The test adds unique elements and check that the estimated value + * is always reasonable bounds. + * + * We check that the error is smaller than a few times than the expected + * standard error, to make it very unlikely for the test to fail because + * of a "bad" run. + * + * The test is performed with both dense and sparse HLLs at the same + * time also verifying that the computed cardinality is the same. */ + memset(hdr->registers,0,HLL_DENSE_SIZE-HLL_HDR_SIZE); + o = createHLLObject(); + double relerr = 1.04/sqrt(HLL_REGISTERS); + int64_t checkpoint = 1; + uint64_t seed = (uint64_t)rand() | (uint64_t)rand() << 32; + uint64_t ele; + for (j = 1; j <= 10000000; j++) { + ele = j ^ seed; + hllDenseAdd(hdr->registers,(unsigned char*)&ele,sizeof(ele)); + hllAdd(o,(unsigned char*)&ele,sizeof(ele)); + + /* Make sure that for small cardinalities we use sparse + * encoding. */ + if (j == checkpoint && j < server.hll_sparse_max_bytes/2) { + hdr2 = o->ptr; + if (hdr2->encoding != HLL_SPARSE) { + addReplyError(c, "TESTFAILED sparse encoding not used"); + goto cleanup; + } + } + + /* Check that dense and sparse representations agree. */ + if (j == checkpoint && hllCount(hdr,NULL) != hllCount(o->ptr,NULL)) { + addReplyError(c, "TESTFAILED dense/sparse disagree"); + goto cleanup; + } + + /* Check error. */ + if (j == checkpoint) { + int64_t abserr = checkpoint - (int64_t)hllCount(hdr,NULL); + uint64_t maxerr = ceil(relerr*6*checkpoint); + + /* Adjust the max error we expect for cardinality 10 + * since from time to time it is statistically likely to get + * much higher error due to collision, resulting into a false + * positive. */ + if (j == 10) maxerr = 1; + + if (abserr < 0) abserr = -abserr; + if (abserr > (int64_t)maxerr) { + addReplyErrorFormat(c, + "TESTFAILED Too big error. card:%llu abserr:%llu", + (unsigned long long) checkpoint, + (unsigned long long) abserr); + goto cleanup; + } + checkpoint *= 10; + } + } + + /* Success! */ + addReply(c,shared.ok); + +cleanup: + sdsfree(bitcounters); + if (o) decrRefCount(o); +} + +/* PFDEBUG ... args ... + * Different debugging related operations about the HLL implementation. */ +void pfdebugCommand(redisClient *c) { + char *cmd = c->argv[1]->ptr; + struct hllhdr *hdr; + robj *o; + int j; + + o = lookupKeyRead(c->db,c->argv[2]); + if (o == NULL) { + addReplyError(c,"The specified key does not exist"); + return; + } + if (isHLLObjectOrReply(c,o) != REDIS_OK) return; + o = dbUnshareStringValue(c->db,c->argv[2],o); + hdr = o->ptr; + + /* PFDEBUG GETREG */ + if (!strcasecmp(cmd,"getreg")) { + if (c->argc != 3) goto arityerr; + + if (hdr->encoding == HLL_SPARSE) { + if (hllSparseToDense(o) == REDIS_ERR) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + server.dirty++; /* Force propagation on encoding change. */ + } + + hdr = o->ptr; + addReplyMultiBulkLen(c,HLL_REGISTERS); + for (j = 0; j < HLL_REGISTERS; j++) { + uint8_t val; + + HLL_DENSE_GET_REGISTER(val,hdr->registers,j); + addReplyLongLong(c,val); + } + } + /* PFDEBUG DECODE */ + else if (!strcasecmp(cmd,"decode")) { + if (c->argc != 3) goto arityerr; + + uint8_t *p = o->ptr, *end = p+sdslen(o->ptr); + sds decoded = sdsempty(); + + if (hdr->encoding != HLL_SPARSE) { + addReplyError(c,"HLL encoding is not sparse"); + return; + } + + p += HLL_HDR_SIZE; + while(p < end) { + int runlen, regval; + + if (HLL_SPARSE_IS_ZERO(p)) { + runlen = HLL_SPARSE_ZERO_LEN(p); + p++; + decoded = sdscatprintf(decoded,"z:%d ",runlen); + } else if (HLL_SPARSE_IS_XZERO(p)) { + runlen = HLL_SPARSE_XZERO_LEN(p); + p += 2; + decoded = sdscatprintf(decoded,"Z:%d ",runlen); + } else { + runlen = HLL_SPARSE_VAL_LEN(p); + regval = HLL_SPARSE_VAL_VALUE(p); + p++; + decoded = sdscatprintf(decoded,"v:%d,%d ",regval,runlen); + } + } + decoded = sdstrim(decoded," "); + addReplyBulkCBuffer(c,decoded,sdslen(decoded)); + sdsfree(decoded); + } + /* PFDEBUG ENCODING */ + else if (!strcasecmp(cmd,"encoding")) { + char *encodingstr[2] = {"dense","sparse"}; + if (c->argc != 3) goto arityerr; + + addReplyStatus(c,encodingstr[hdr->encoding]); + } + /* PFDEBUG TODENSE */ + else if (!strcasecmp(cmd,"todense")) { + int conv = 0; + if (c->argc != 3) goto arityerr; + + if (hdr->encoding == HLL_SPARSE) { + if (hllSparseToDense(o) == REDIS_ERR) { + addReplySds(c,sdsnew(invalid_hll_err)); + return; + } + conv = 1; + server.dirty++; /* Force propagation on encoding change. */ + } + addReply(c,conv ? shared.cone : shared.czero); + } else { + addReplyErrorFormat(c,"Unknown PFDEBUG subcommand '%s'", cmd); + } + return; + +arityerr: + addReplyErrorFormat(c, + "Wrong number of arguments for the '%s' subcommand",cmd); +} + diff --git a/src/intset.c b/src/intset.c index 225f0e92aea..5d894e3cd08 100644 --- a/src/intset.c +++ b/src/intset.c @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2009-2012, Pieter Noordhuis + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include #include #include @@ -103,7 +133,7 @@ static uint8_t intsetSearch(intset *is, int64_t value, uint32_t *pos) { } while(max >= min) { - mid = (min+max)/2; + mid = ((unsigned int)min + (unsigned int)max) >> 1; cur = _intsetGet(is,mid); if (value > cur) { min = mid+1; diff --git a/src/intset.h b/src/intset.h index ee4b91fa93d..bd01ff22f66 100644 --- a/src/intset.h +++ b/src/intset.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2009-2012, Pieter Noordhuis + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __INTSET_H #define __INTSET_H #include diff --git a/src/latency.c b/src/latency.c new file mode 100644 index 00000000000..ec8adea274f --- /dev/null +++ b/src/latency.c @@ -0,0 +1,613 @@ +/* The latency monitor allows to easily observe the sources of latency + * in a Redis instance using the LATENCY command. Different latency + * sources are monitored, like disk I/O, execution of commands, fork + * system call, and so forth. + * + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "redis.h" + +/* Dictionary type for latency events. */ +int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) { + REDIS_NOTUSED(privdata); + return strcmp(key1,key2) == 0; +} + +unsigned int dictStringHash(const void *key) { + return dictGenHashFunction(key, strlen(key)); +} + +void dictVanillaFree(void *privdata, void *val); + +dictType latencyTimeSeriesDictType = { + dictStringHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictStringKeyCompare, /* key compare */ + dictVanillaFree, /* key destructor */ + dictVanillaFree /* val destructor */ +}; + +/* ------------------------- Utility functions ------------------------------ */ + +#ifdef __linux__ +/* Returns 1 if Transparent Huge Pages support is enabled in the kernel. + * Otherwise (or if we are unable to check) 0 is returned. */ +int THPIsEnabled(void) { + char buf[1024]; + + FILE *fp = fopen("/sys/kernel/mm/transparent_hugepage/enabled","r"); + if (!fp) return 0; + if (fgets(buf,sizeof(buf),fp) == NULL) { + fclose(fp); + return 0; + } + fclose(fp); + return (strstr(buf,"[never]") == NULL) ? 1 : 0; +} +#endif + +/* Report the amount of AnonHugePages in smap, in bytes. If the return + * value of the function is non-zero, the process is being targeted by + * THP support, and is likely to have memory usage / latency issues. */ +int THPGetAnonHugePagesSize(void) { + return zmalloc_get_smap_bytes_by_field("AnonHugePages:"); +} + +/* ---------------------------- Latency API --------------------------------- */ + +/* Latency monitor initialization. We just need to create the dictionary + * of time series, each time serie is craeted on demand in order to avoid + * having a fixed list to maintain. */ +void latencyMonitorInit(void) { + server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); +} + +/* Add the specified sample to the specified time series "event". + * This function is usually called via latencyAddSampleIfNeeded(), that + * is a macro that only adds the sample if the latency is higher than + * server.latency_monitor_threshold. */ +void latencyAddSample(char *event, mstime_t latency) { + struct latencyTimeSeries *ts = dictFetchValue(server.latency_events,event); + time_t now = time(NULL); + int prev; + + /* Create the time series if it does not exist. */ + if (ts == NULL) { + ts = zmalloc(sizeof(*ts)); + ts->idx = 0; + ts->max = 0; + memset(ts->samples,0,sizeof(ts->samples)); + dictAdd(server.latency_events,zstrdup(event),ts); + } + + /* If the previous sample is in the same second, we update our old sample + * if this latency is > of the old one, or just return. */ + prev = (ts->idx + LATENCY_TS_LEN - 1) % LATENCY_TS_LEN; + if (ts->samples[prev].time == now) { + if (latency > ts->samples[prev].latency) + ts->samples[prev].latency = latency; + return; + } + + ts->samples[ts->idx].time = time(NULL); + ts->samples[ts->idx].latency = latency; + if (latency > ts->max) ts->max = latency; + + ts->idx++; + if (ts->idx == LATENCY_TS_LEN) ts->idx = 0; +} + +/* Reset data for the specified event, or all the events data if 'event' is + * NULL. + * + * Note: this is O(N) even when event_to_reset is not NULL because makes + * the code simpler and we have a small fixed max number of events. */ +int latencyResetEvent(char *event_to_reset) { + dictIterator *di; + dictEntry *de; + int resets = 0; + + di = dictGetSafeIterator(server.latency_events); + while((de = dictNext(di)) != NULL) { + char *event = dictGetKey(de); + + if (event_to_reset == NULL || strcasecmp(event,event_to_reset) == 0) { + dictDelete(server.latency_events, event); + resets++; + } + } + dictReleaseIterator(di); + return resets; +} + +/* ------------------------ Latency reporting (doctor) ---------------------- */ + +/* Analyze the samples avaialble for a given event and return a structure + * populate with different metrics, average, MAD, min, max, and so forth. + * Check latency.h definition of struct latenctStat for more info. + * If the specified event has no elements the structure is populate with + * zero values. */ +void analyzeLatencyForEvent(char *event, struct latencyStats *ls) { + struct latencyTimeSeries *ts = dictFetchValue(server.latency_events,event); + int j; + uint64_t sum; + + ls->all_time_high = ts ? ts->max : 0; + ls->avg = 0; + ls->min = 0; + ls->max = 0; + ls->mad = 0; + ls->samples = 0; + ls->period = 0; + if (!ts) return; + + /* First pass, populate everything but the MAD. */ + sum = 0; + for (j = 0; j < LATENCY_TS_LEN; j++) { + if (ts->samples[j].time == 0) continue; + ls->samples++; + if (ls->samples == 1) { + ls->min = ls->max = ts->samples[j].latency; + } else { + if (ls->min > ts->samples[j].latency) + ls->min = ts->samples[j].latency; + if (ls->max < ts->samples[j].latency) + ls->max = ts->samples[j].latency; + } + sum += ts->samples[j].latency; + + /* Track the oldest event time in ls->period. */ + if (ls->period == 0 || ts->samples[j].time < ls->period) + ls->period = ts->samples[j].time; + } + + /* So far avg is actually the sum of the latencies, and period is + * the oldest event time. We need to make the first an average and + * the second a range of seconds. */ + if (ls->samples) { + ls->avg = sum / ls->samples; + ls->period = time(NULL) - ls->period; + if (ls->period == 0) ls->period = 1; + } + + /* Second pass, compute MAD. */ + sum = 0; + for (j = 0; j < LATENCY_TS_LEN; j++) { + int64_t delta; + + if (ts->samples[j].time == 0) continue; + delta = (int64_t)ls->avg - ts->samples[j].latency; + if (delta < 0) delta = -delta; + sum += delta; + } + if (ls->samples) ls->mad = sum / ls->samples; +} + +/* Create a human readable report of latency events for this Redis instance. */ +sds createLatencyReport(void) { + sds report = sdsempty(); + int advise_better_vm = 0; /* Better virtual machines. */ + int advise_slowlog_enabled = 0; /* Enable slowlog. */ + int advise_slowlog_tuning = 0; /* Reconfigure slowlog. */ + int advise_slowlog_inspect = 0; /* Check your slowlog. */ + int advise_disk_contention = 0; /* Try to lower disk contention. */ + int advise_scheduler = 0; /* Intrinsic latency. */ + int advise_data_writeback = 0; /* data=writeback. */ + int advise_no_appendfsync = 0; /* don't fsync during rewrites. */ + int advise_local_disk = 0; /* Avoid remote disks. */ + int advise_ssd = 0; /* Use an SSD drive. */ + int advise_write_load_info = 0; /* Print info about AOF and write load. */ + int advise_hz = 0; /* Use higher HZ. */ + int advise_large_objects = 0; /* Deletion of large objects. */ + int advise_relax_fsync_policy = 0; /* appendfsync always is slow. */ + int advise_disable_thp = 0; /* AnonHugePages detected. */ + int advices = 0; + + /* Return ASAP if the latency engine is disabled and it looks like it + * was never enabled so far. */ + if (dictSize(server.latency_events) == 0 && + server.latency_monitor_threshold == 0) + { + report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may use \"CONFIG SET latency-monitor-threshold .\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at http://redis.io/topics/latency-monitor.\n"); + return report; + } + + /* Show all the events stats and add for each event some event-related + * comment depending on the values. */ + dictIterator *di; + dictEntry *de; + int eventnum = 0; + + di = dictGetSafeIterator(server.latency_events); + while((de = dictNext(di)) != NULL) { + char *event = dictGetKey(de); + struct latencyTimeSeries *ts = dictGetVal(de); + struct latencyStats ls; + + if (ts == NULL) continue; + eventnum++; + if (eventnum == 1) { + report = sdscat(report,"Dave, I have observed latency spikes in this Redis instance. You don't mind talking about it, do you Dave?\n\n"); + } + analyzeLatencyForEvent(event,&ls); + + report = sdscatprintf(report, + "%d. %s: %d latency spikes (average %lums, mean deviation %lums, period %.2f sec). Worst all time event %lums.", + eventnum, event, + ls.samples, + (unsigned long) ls.avg, + (unsigned long) ls.mad, + (double) ls.period/ls.samples, + (unsigned long) ts->max); + + /* Fork */ + if (!strcasecmp(event,"fork")) { + char *fork_quality; + if (server.stat_fork_rate < 10) { + fork_quality = "terrible"; + advise_better_vm = 1; + advices++; + } else if (server.stat_fork_rate < 25) { + fork_quality = "poor"; + advise_better_vm = 1; + advices++; + } else if (server.stat_fork_rate < 100) { + fork_quality = "good"; + } else { + fork_quality = "excellent"; + } + report = sdscatprintf(report, + " Fork rate is %.2f GB/sec (%s).", server.stat_fork_rate, + fork_quality); + } + + /* Potentially commands. */ + if (!strcasecmp(event,"command")) { + if (server.slowlog_log_slower_than == 0) { + advise_slowlog_enabled = 1; + advices++; + } else if (server.slowlog_log_slower_than/1000 > + server.latency_monitor_threshold) + { + advise_slowlog_tuning = 1; + advices++; + } + advise_slowlog_inspect = 1; + advise_large_objects = 1; + advices += 2; + } + + /* fast-command. */ + if (!strcasecmp(event,"fast-command")) { + advise_scheduler = 1; + advices++; + } + + /* AOF and I/O. */ + if (!strcasecmp(event,"aof-write-pending-fsync")) { + advise_local_disk = 1; + advise_disk_contention = 1; + advise_ssd = 1; + advise_data_writeback = 1; + advices += 4; + } + + if (!strcasecmp(event,"aof-write-active-child")) { + advise_no_appendfsync = 1; + advise_data_writeback = 1; + advise_ssd = 1; + advices += 3; + } + + if (!strcasecmp(event,"aof-write-alone")) { + advise_local_disk = 1; + advise_data_writeback = 1; + advise_ssd = 1; + advices += 3; + } + + if (!strcasecmp(event,"aof-fsync-always")) { + advise_relax_fsync_policy = 1; + advices++; + } + + if (!strcasecmp(event,"aof-fstat") || + !strcasecmp(event,"rdb-unlik-temp-file")) { + advise_disk_contention = 1; + advise_local_disk = 1; + advices += 2; + } + + if (!strcasecmp(event,"aof-rewrite-diff-write") || + !strcasecmp(event,"aof-rename")) { + advise_write_load_info = 1; + advise_data_writeback = 1; + advise_ssd = 1; + advise_local_disk = 1; + advices += 4; + } + + /* Expire cycle. */ + if (!strcasecmp(event,"expire-cycle")) { + advise_hz = 1; + advise_large_objects = 1; + advices += 2; + } + + /* Eviction cycle. */ + if (!strcasecmp(event,"eviction-cycle")) { + advise_large_objects = 1; + advices++; + } + + report = sdscatlen(report,"\n",1); + } + dictReleaseIterator(di); + + /* Add non event based advices. */ + if (THPGetAnonHugePagesSize() > 0) { + advise_disable_thp = 1; + advices++; + } + + if (eventnum == 0 && advices == 0) { + report = sdscat(report,"Dave, no latency spike was observed during the lifetime of this Redis instance, not in the slightest bit. I honestly think you ought to sit down calmly, take a stress pill, and think things over.\n"); + } else if (eventnum > 0 && advices == 0) { + report = sdscat(report,"\nWhile there are latency events logged, I'm not able to suggest any easy fix. Please use the Redis community to get some help, providing this report in your help request.\n"); + } else { + /* Add all the suggestions accumulated so far. */ + + /* Better VM. */ + report = sdscat(report,"\nI have a few advices for you:\n\n"); + if (advise_better_vm) { + report = sdscat(report,"- If you are using a virtual machine, consider upgrading it with a faster one using an hypervisior that provides less latency during fork() calls. Xen is known to have poor fork() performance. Even in the context of the same VM provider, certain kinds of instances can execute fork faster than others.\n"); + } + + /* Slow log. */ + if (advise_slowlog_enabled) { + report = sdscatprintf(report,"- There are latency issues with potentially slow commands you are using. Try to enable the Slow Log Redis feature using the command 'CONFIG SET slowlog-log-slower-than %llu'. If the Slow log is disabled Redis is not able to log slow commands execution for you.\n", (unsigned long long)server.latency_monitor_threshold*1000); + } + + if (advise_slowlog_tuning) { + report = sdscatprintf(report,"- Your current Slow Log configuration only logs events that are slower than your configured latency monitor threshold. Please use 'CONFIG SET slowlog-log-slower-than %llu'.\n", (unsigned long long)server.latency_monitor_threshold*1000); + } + + if (advise_slowlog_inspect) { + report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check http://redis.io/commands/slowlog for more information.\n"); + } + + /* Intrinsic latency. */ + if (advise_scheduler) { + report = sdscat(report,"- The system is slow to execute Redis code paths not containing system calls. This usually means the system does not provide Redis CPU time to run for long periods. You should try to:\n" + " 1) Lower the system load.\n" + " 2) Use a computer / VM just for Redis if you are running other softawre in the same system.\n" + " 3) Check if you have a \"noisy neighbour\" problem.\n" + " 4) Check with 'redis-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n" + " 5) Check if the problem is allocator-related by recompiling Redis with MALLOC=libc, if you are using Jemalloc. However this may create fragmentation problems.\n"); + } + + /* AOF / Disk latency. */ + if (advise_local_disk) { + report = sdscat(report,"- It is strongly advised to use local disks for persistence, especially if you are using AOF. Remote disks provided by platform-as-a-service providers are known to be slow.\n"); + } + + if (advise_ssd) { + report = sdscat(report,"- SSD disks are able to reduce fsync latency, and total time needed for snapshotting and AOF log rewriting (resulting in smaller memory usage and smaller final AOF rewrite buffer flushes). With extremely high write load SSD disks can be a good option. However Redis should perform reasonably with high load using normal disks. Use this advice as a last resort.\n"); + } + + if (advise_data_writeback) { + report = sdscat(report,"- Mounting ext3/4 filesystems with data=writeback can provide a performance boost compared to data=ordered, however this mode of operation provides less guarantees, and sometimes it can happen that after a hard crash the AOF file will have an half-written command at the end and will require to be repaired before Redis restarts.\n"); + } + + if (advise_disk_contention) { + report = sdscat(report,"- Try to lower the disk contention. This is often caused by other disk intensive processes running in the same computer (including other Redis instances).\n"); + } + + if (advise_no_appendfsync) { + report = sdscat(report,"- Assuming from the point of view of data safety this is viable in your environment, you could try to enable the 'no-appendfsync-on-rewrite' option, so that fsync will not be performed while there is a child rewriting the AOF file or producing an RDB file (the moment where there is high disk contention).\n"); + } + + if (advise_relax_fsync_policy && server.aof_fsync == AOF_FSYNC_ALWAYS) { + report = sdscat(report,"- Your fsync policy is set to 'always'. It is very hard to get good performances with such a setup, if possible try to relax the fsync policy to 'onesec'.\n"); + } + + if (advise_write_load_info) { + report = sdscat(report,"- Latency during the AOF atomic rename operation or when the final difference is flushed to the AOF file at the end of the rewrite, sometimes is caused by very high write load, causing the AOF buffer to get very large. If possible try to send less commands to accomplish the same work, or use Lua scripts to group multiple operations into a single EVALSHA call.\n"); + } + + if (advise_hz && server.hz < 100) { + report = sdscat(report,"- In order to make the Redis keys expiring process more incremental, try to set the 'hz' configuration parameter to 100 using 'CONFIG SET hz 100'.\n"); + } + + if (advise_large_objects) { + report = sdscat(report,"- Deleting, expiring or evicting (because of maxmemory policy) large objects is a blocking operation. If you have very large objects that are often deleted, expired, or evicted, try to fragment those objects into multiple smaller objects.\n"); + } + + if (advise_disable_thp) { + report = sdscat(report,"- I detected a non zero amount of anonymous huge pages used by your process. This creates very serious latency events in different conditions, especially when Redis is persisting on disk. To disable THP support use the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled', make sure to also add it into /etc/rc.local so that the command will be executed again after a reboot. Note that even if you have already disabled THP, you still need to restart the Redis process to get rid of the huge pages already created.\n"); + } + } + + return report; +} + +/* ---------------------- Latency command implementation -------------------- */ + +/* latencyCommand() helper to produce a time-delay reply for all the samples + * in memory for the specified time series. */ +void latencyCommandReplyWithSamples(redisClient *c, struct latencyTimeSeries *ts) { + void *replylen = addDeferredMultiBulkLength(c); + int samples = 0, j; + + for (j = 0; j < LATENCY_TS_LEN; j++) { + int i = (ts->idx + j) % LATENCY_TS_LEN; + + if (ts->samples[i].time == 0) continue; + addReplyMultiBulkLen(c,2); + addReplyLongLong(c,ts->samples[i].time); + addReplyLongLong(c,ts->samples[i].latency); + samples++; + } + setDeferredMultiBulkLength(c,replylen,samples); +} + +/* latencyCommand() helper to produce the reply for the LATEST subcommand, + * listing the last latency sample for every event type registered so far. */ +void latencyCommandReplyWithLatestEvents(redisClient *c) { + dictIterator *di; + dictEntry *de; + + addReplyMultiBulkLen(c,dictSize(server.latency_events)); + di = dictGetIterator(server.latency_events); + while((de = dictNext(di)) != NULL) { + char *event = dictGetKey(de); + struct latencyTimeSeries *ts = dictGetVal(de); + int last = (ts->idx + LATENCY_TS_LEN - 1) % LATENCY_TS_LEN; + + addReplyMultiBulkLen(c,4); + addReplyBulkCString(c,event); + addReplyLongLong(c,ts->samples[last].time); + addReplyLongLong(c,ts->samples[last].latency); + addReplyLongLong(c,ts->max); + } + dictReleaseIterator(di); +} + +#define LATENCY_GRAPH_COLS 80 +sds latencyCommandGenSparkeline(char *event, struct latencyTimeSeries *ts) { + int j; + struct sequence *seq = createSparklineSequence(); + sds graph = sdsempty(); + uint32_t min = 0, max = 0; + + for (j = 0; j < LATENCY_TS_LEN; j++) { + int i = (ts->idx + j) % LATENCY_TS_LEN; + int elapsed; + char buf[64]; + + if (ts->samples[i].time == 0) continue; + /* Update min and max. */ + if (seq->length == 0) { + min = max = ts->samples[i].latency; + } else { + if (ts->samples[i].latency > max) max = ts->samples[i].latency; + if (ts->samples[i].latency < min) min = ts->samples[i].latency; + } + /* Use as label the number of seconds / minutes / hours / days + * ago the event happened. */ + elapsed = time(NULL) - ts->samples[i].time; + if (elapsed < 60) + snprintf(buf,sizeof(buf),"%ds",elapsed); + else if (elapsed < 3600) + snprintf(buf,sizeof(buf),"%dm",elapsed/60); + else if (elapsed < 3600*24) + snprintf(buf,sizeof(buf),"%dh",elapsed/3600); + else + snprintf(buf,sizeof(buf),"%dd",elapsed/(3600*24)); + sparklineSequenceAddSample(seq,ts->samples[i].latency,buf); + } + + graph = sdscatprintf(graph, + "%s - high %lu ms, low %lu ms (all time high %lu ms)\n", event, + (unsigned long) max, (unsigned long) min, (unsigned long) ts->max); + for (j = 0; j < LATENCY_GRAPH_COLS; j++) + graph = sdscatlen(graph,"-",1); + graph = sdscatlen(graph,"\n",1); + graph = sparklineRender(graph,seq,LATENCY_GRAPH_COLS,4,SPARKLINE_FILL); + freeSparklineSequence(seq); + return graph; +} + +/* LATENCY command implementations. + * + * LATENCY SAMPLES: return time-latency samples for the specified event. + * LATENCY LATEST: return the latest latency for all the events classes. + * LATENCY DOCTOR: returns an human readable analysis of instance latency. + * LATENCY GRAPH: provide an ASCII graph of the latency of the specified event. + */ +void latencyCommand(redisClient *c) { + struct latencyTimeSeries *ts; + + if (!strcasecmp(c->argv[1]->ptr,"history") && c->argc == 3) { + /* LATENCY HISTORY */ + ts = dictFetchValue(server.latency_events,c->argv[2]->ptr); + if (ts == NULL) { + addReplyMultiBulkLen(c,0); + } else { + latencyCommandReplyWithSamples(c,ts); + } + } else if (!strcasecmp(c->argv[1]->ptr,"graph") && c->argc == 3) { + /* LATENCY GRAPH */ + sds graph; + dictEntry *de; + char *event; + + de = dictFind(server.latency_events,c->argv[2]->ptr); + if (de == NULL) goto nodataerr; + ts = dictGetVal(de); + event = dictGetKey(de); + + graph = latencyCommandGenSparkeline(event,ts); + addReplyBulkCString(c,graph); + sdsfree(graph); + } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { + /* LATENCY LATEST */ + latencyCommandReplyWithLatestEvents(c); + } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { + /* LATENCY DOCTOR */ + sds report = createLatencyReport(); + + addReplyBulkCBuffer(c,report,sdslen(report)); + sdsfree(report); + } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { + /* LATENCY RESET */ + if (c->argc == 2) { + addReplyLongLong(c,latencyResetEvent(NULL)); + } else { + int j, resets = 0; + + for (j = 2; j < c->argc; j++) + resets += latencyResetEvent(c->argv[j]->ptr); + addReplyLongLong(c,resets); + } + } else { + addReply(c,shared.syntaxerr); + } + return; + +nodataerr: + /* Common error when the user asks for an event we have no latency + * information about. */ + addReplyErrorFormat(c, + "No samples available for event '%s'", (char*) c->argv[2]->ptr); +} + diff --git a/src/latency.h b/src/latency.h new file mode 100644 index 00000000000..240f54b45f4 --- /dev/null +++ b/src/latency.h @@ -0,0 +1,89 @@ +/* latency.h -- latency monitor API header file + * See latency.c for more information. + * + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LATENCY_H +#define __LATENCY_H + +#define LATENCY_TS_LEN 160 /* History length for every monitored event. */ + +/* Representation of a latency sample: the sampling time and the latency + * observed in milliseconds. */ +struct latencySample { + int32_t time; /* We don't use time_t to force 4 bytes usage everywhere. */ + uint32_t latency; /* Latency in milliseconds. */ +}; + +/* The latency time series for a given event. */ +struct latencyTimeSeries { + int idx; /* Index of the next sample to store. */ + uint32_t max; /* Max latency observed for this event. */ + struct latencySample samples[LATENCY_TS_LEN]; /* Latest history. */ +}; + +/* Latency statistics structure. */ +struct latencyStats { + uint32_t all_time_high; /* Absolute max observed since latest reset. */ + uint32_t avg; /* Average of current samples. */ + uint32_t min; /* Min of current samples. */ + uint32_t max; /* Max of current samples. */ + uint32_t mad; /* Mean absolute deviation. */ + uint32_t samples; /* Number of non-zero samples. */ + time_t period; /* Number of seconds since first event and now. */ +}; + +void latencyMonitorInit(void); +void latencyAddSample(char *event, mstime_t latency); +int THPIsEnabled(void); + +/* Latency monitoring macros. */ + +/* Start monitoring an event. We just set the current time. */ +#define latencyStartMonitor(var) if (server.latency_monitor_threshold) { \ + var = mstime(); \ +} else { \ + var = 0; \ +} + +/* End monitoring an event, compute the difference with the current time + * to check the amount of time elapsed. */ +#define latencyEndMonitor(var) if (server.latency_monitor_threshold) { \ + var = mstime() - var; \ +} + +/* Add the sample only if the elapsed time is >= to the configured threshold. */ +#define latencyAddSampleIfNeeded(event,var) \ + if (server.latency_monitor_threshold && \ + (var) >= server.latency_monitor_threshold) \ + latencyAddSample((event),(var)); + +#endif /* __LATENCY_H */ diff --git a/src/lzf.h b/src/lzf.h index 919b6e6be2d..98e038f3182 100644 --- a/src/lzf.h +++ b/src/lzf.h @@ -1,16 +1,16 @@ /* * Copyright (c) 2000-2008 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -73,7 +73,7 @@ * and lzf_c.c. * */ -unsigned int +unsigned int lzf_compress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len); @@ -92,7 +92,7 @@ lzf_compress (const void *const in_data, unsigned int in_len, * * This function is very fast, about as fast as a copying loop. */ -unsigned int +unsigned int lzf_decompress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len); diff --git a/src/lzfP.h b/src/lzfP.h index d533f18292b..c9eae3f6a65 100644 --- a/src/lzfP.h +++ b/src/lzfP.h @@ -1,16 +1,16 @@ /* * Copyright (c) 2000-2007 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -93,7 +93,7 @@ /* * Avoid assigning values to errno variable? for some embedding purposes - * (linux kernel for example), this is neccessary. NOTE: this breaks + * (linux kernel for example), this is necessary. NOTE: this breaks * the documentation in lzf.h. */ #ifndef AVOID_ERRNO @@ -101,7 +101,7 @@ #endif /* - * Wether to pass the LZF_STATE variable as argument, or allocate it + * Whether to pass the LZF_STATE variable as argument, or allocate it * on the stack. For small-stack environments, define this to 1. * NOTE: this breaks the prototype in lzf.h. */ @@ -110,11 +110,11 @@ #endif /* - * Wether to add extra checks for input validity in lzf_decompress + * Whether to add extra checks for input validity in lzf_decompress * and return EINVAL if the input stream has been corrupted. This * only shields against overflowing the input buffer and will not * detect most corrupted streams. - * This check is not normally noticable on modern hardware + * This check is not normally noticeable on modern hardware * (<1% slowdown), but might slow down older cpus considerably. */ #ifndef CHECK_INPUT diff --git a/src/lzf_c.c b/src/lzf_c.c index 99dab091c62..9e031ad0bc0 100644 --- a/src/lzf_c.c +++ b/src/lzf_c.c @@ -1,16 +1,16 @@ /* * Copyright (c) 2000-2008 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO diff --git a/src/lzf_d.c b/src/lzf_d.c index e7e48c13812..6c723f5e07f 100644 --- a/src/lzf_d.c +++ b/src/lzf_d.c @@ -1,16 +1,16 @@ /* * Copyright (c) 2000-2007 Marc Alexander Lehmann - * + * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. - * + * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -52,7 +52,7 @@ #endif */ -unsigned int +unsigned int lzf_decompress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len) { diff --git a/src/memtest.c b/src/memtest.c new file mode 100644 index 00000000000..39fc4fcaad4 --- /dev/null +++ b/src/memtest.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__sun) +#include +#endif +#include "config.h" + +#if (ULONG_MAX == 4294967295UL) +#define MEMTEST_32BIT +#elif (ULONG_MAX == 18446744073709551615ULL) +#define MEMTEST_64BIT +#else +#error "ULONG_MAX value not supported." +#endif + +#ifdef MEMTEST_32BIT +#define ULONG_ONEZERO 0xaaaaaaaaUL +#define ULONG_ZEROONE 0x55555555UL +#else +#define ULONG_ONEZERO 0xaaaaaaaaaaaaaaaaUL +#define ULONG_ZEROONE 0x5555555555555555UL +#endif + +static struct winsize ws; +size_t progress_printed; /* Printed chars in screen-wide progress bar. */ +size_t progress_full; /* How many chars to write to fill the progress bar. */ + +void memtest_progress_start(char *title, int pass) { + int j; + + printf("\x1b[H\x1b[2J"); /* Cursor home, clear screen. */ + /* Fill with dots. */ + for (j = 0; j < ws.ws_col*(ws.ws_row-2); j++) printf("."); + printf("Please keep the test running several minutes per GB of memory.\n"); + printf("Also check http://www.memtest86.com/ and http://pyropus.ca/software/memtester/"); + printf("\x1b[H\x1b[2K"); /* Cursor home, clear current line. */ + printf("%s [%d]\n", title, pass); /* Print title. */ + progress_printed = 0; + progress_full = ws.ws_col*(ws.ws_row-3); + fflush(stdout); +} + +void memtest_progress_end(void) { + printf("\x1b[H\x1b[2J"); /* Cursor home, clear screen. */ +} + +void memtest_progress_step(size_t curr, size_t size, char c) { + size_t chars = ((unsigned long long)curr*progress_full)/size, j; + + for (j = 0; j < chars-progress_printed; j++) printf("%c",c); + progress_printed = chars; + fflush(stdout); +} + +/* Test that addressing is fine. Every location is populated with its own + * address, and finally verified. This test is very fast but may detect + * ASAP big issues with the memory subsystem. */ +void memtest_addressing(unsigned long *l, size_t bytes) { + unsigned long words = bytes/sizeof(unsigned long); + unsigned long j, *p; + + /* Fill */ + p = l; + for (j = 0; j < words; j++) { + *p = (unsigned long)p; + p++; + if ((j & 0xffff) == 0) memtest_progress_step(j,words*2,'A'); + } + /* Test */ + p = l; + for (j = 0; j < words; j++) { + if (*p != (unsigned long)p) { + printf("\n*** MEMORY ADDRESSING ERROR: %p contains %lu\n", + (void*) p, *p); + exit(1); + } + p++; + if ((j & 0xffff) == 0) memtest_progress_step(j+words,words*2,'A'); + } +} + +/* Fill words stepping a single page at every write, so we continue to + * touch all the pages in the smallest amount of time reducing the + * effectiveness of caches, and making it hard for the OS to transfer + * pages on the swap. */ +void memtest_fill_random(unsigned long *l, size_t bytes) { + unsigned long step = 4096/sizeof(unsigned long); + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long iwords = words/step; /* words per iteration */ + unsigned long off, w, *l1, *l2; + + assert((bytes & 4095) == 0); + for (off = 0; off < step; off++) { + l1 = l+off; + l2 = l1+words; + for (w = 0; w < iwords; w++) { +#ifdef MEMTEST_32BIT + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16); +#else + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16) | + (((unsigned long) (rand()&0xffff)) << 32) | + (((unsigned long) (rand()&0xffff)) << 48); +#endif + l1 += step; + l2 += step; + if ((w & 0xffff) == 0) + memtest_progress_step(w+iwords*off,words,'R'); + } + } +} + +/* Like memtest_fill_random() but uses the two specified values to fill + * memory, in an alternated way (v1|v2|v1|v2|...) */ +void memtest_fill_value(unsigned long *l, size_t bytes, unsigned long v1, + unsigned long v2, char sym) +{ + unsigned long step = 4096/sizeof(unsigned long); + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long iwords = words/step; /* words per iteration */ + unsigned long off, w, *l1, *l2, v; + + assert((bytes & 4095) == 0); + for (off = 0; off < step; off++) { + l1 = l+off; + l2 = l1+words; + v = (off & 1) ? v2 : v1; + for (w = 0; w < iwords; w++) { +#ifdef MEMTEST_32BIT + *l1 = *l2 = ((unsigned long) v) | + (((unsigned long) v) << 16); +#else + *l1 = *l2 = ((unsigned long) v) | + (((unsigned long) v) << 16) | + (((unsigned long) v) << 32) | + (((unsigned long) v) << 48); +#endif + l1 += step; + l2 += step; + if ((w & 0xffff) == 0) + memtest_progress_step(w+iwords*off,words,sym); + } + } +} + +void memtest_compare(unsigned long *l, size_t bytes) { + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long w, *l1, *l2; + + assert((bytes & 4095) == 0); + l1 = l; + l2 = l1+words; + for (w = 0; w < words; w++) { + if (*l1 != *l2) { + printf("\n*** MEMORY ERROR DETECTED: %p != %p (%lu vs %lu)\n", + (void*)l1, (void*)l2, *l1, *l2); + exit(1); + } + l1 ++; + l2 ++; + if ((w & 0xffff) == 0) memtest_progress_step(w,words,'='); + } +} + +void memtest_compare_times(unsigned long *m, size_t bytes, int pass, int times) { + int j; + + for (j = 0; j < times; j++) { + memtest_progress_start("Compare",pass); + memtest_compare(m,bytes); + memtest_progress_end(); + } +} + +void memtest_test(size_t megabytes, int passes) { + size_t bytes = megabytes*1024*1024; + unsigned long *m = malloc(bytes); + int pass = 0; + + if (m == NULL) { + fprintf(stderr,"Unable to allocate %zu megabytes: %s", + megabytes, strerror(errno)); + exit(1); + } + while (pass != passes) { + pass++; + + memtest_progress_start("Addressing test",pass); + memtest_addressing(m,bytes); + memtest_progress_end(); + + memtest_progress_start("Random fill",pass); + memtest_fill_random(m,bytes); + memtest_progress_end(); + memtest_compare_times(m,bytes,pass,4); + + memtest_progress_start("Solid fill",pass); + memtest_fill_value(m,bytes,0,(unsigned long)-1,'S'); + memtest_progress_end(); + memtest_compare_times(m,bytes,pass,4); + + memtest_progress_start("Checkerboard fill",pass); + memtest_fill_value(m,bytes,ULONG_ONEZERO,ULONG_ZEROONE,'C'); + memtest_progress_end(); + memtest_compare_times(m,bytes,pass,4); + } + free(m); +} + +void memtest_non_destructive_invert(void *addr, size_t size) { + volatile unsigned long *p = addr; + size_t words = size / sizeof(unsigned long); + size_t j; + + /* Invert */ + for (j = 0; j < words; j++) + p[j] = ~p[j]; +} + +void memtest_non_destructive_swap(void *addr, size_t size) { + volatile unsigned long *p = addr; + size_t words = size / sizeof(unsigned long); + size_t j; + + /* Swap */ + for (j = 0; j < words; j += 2) { + unsigned long a, b; + + a = p[j]; + b = p[j+1]; + p[j] = b; + p[j+1] = a; + } +} + +void memtest(size_t megabytes, int passes) { + if (ioctl(1, TIOCGWINSZ, &ws) == -1) { + ws.ws_col = 80; + ws.ws_row = 20; + } + memtest_test(megabytes,passes); + printf("\nYour memory passed this test.\n"); + printf("Please if you are still in doubt use the following two tools:\n"); + printf("1) memtest86: http://www.memtest86.com/\n"); + printf("2) memtester: http://pyropus.ca/software/memtester/\n"); + exit(0); +} diff --git a/src/migrate.c b/src/migrate.c new file mode 100644 index 00000000000..543b00f4dc3 --- /dev/null +++ b/src/migrate.c @@ -0,0 +1,247 @@ +#include "redis.h" +#include "endianconv.h" + +/* ----------------------------------------------------------------------------- + * DUMP, RESTORE and MIGRATE commands + * -------------------------------------------------------------------------- */ + +/* Generates a DUMP-format representation of the object 'o', adding it to the + * io stream pointed by 'rio'. This function can't fail. */ +void createDumpPayload(rio *payload, robj *o) { + unsigned char buf[2]; + uint64_t crc; + + /* Serialize the object in a RDB-like format. It consist of an object type + * byte followed by the serialized object. This is understood by RESTORE. */ + rioInitWithBuffer(payload,sdsempty()); + redisAssert(rdbSaveObjectType(payload,o)); + redisAssert(rdbSaveObject(payload,o)); + + /* Write the footer, this is how it looks like: + * ----------------+---------------------+---------------+ + * ... RDB payload | 2 bytes RDB version | 8 bytes CRC64 | + * ----------------+---------------------+---------------+ + * RDB version and CRC are both in little endian. + */ + + /* RDB version */ + buf[0] = REDIS_RDB_VERSION & 0xff; + buf[1] = (REDIS_RDB_VERSION >> 8) & 0xff; + payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,buf,2); + + /* CRC64 */ + crc = crc64(0,(unsigned char*)payload->io.buffer.ptr, + sdslen(payload->io.buffer.ptr)); + memrev64ifbe(&crc); + payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,&crc,8); +} + +/* Verify that the RDB version of the dump payload matches the one of this Redis + * instance and that the checksum is ok. + * If the DUMP payload looks valid REDIS_OK is returned, otherwise REDIS_ERR + * is returned. */ +int verifyDumpPayload(unsigned char *p, size_t len) { + unsigned char *footer; + uint16_t rdbver; + uint64_t crc; + + /* At least 2 bytes of RDB version and 8 of CRC64 should be present. */ + if (len < 10) return REDIS_ERR; + footer = p+(len-10); + + /* Verify RDB version */ + rdbver = (footer[1] << 8) | footer[0]; + if (rdbver != REDIS_RDB_VERSION) return REDIS_ERR; + + /* Verify CRC64 */ + crc = crc64(0,p,len-8); + memrev64ifbe(&crc); + return (memcmp(&crc,footer+2,8) == 0) ? REDIS_OK : REDIS_ERR; +} + +/* DUMP keyname + * DUMP is actually not used by Redis Cluster but it is the obvious + * complement of RESTORE and can be useful for different applications. */ +void dumpCommand(redisClient *c) { + robj *o, *dumpobj; + rio payload; + + /* Check if the key is here. */ + if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { + addReply(c,shared.nullbulk); + return; + } + + /* Create the DUMP encoded representation. */ + createDumpPayload(&payload,o); + + /* Transfer to the client */ + dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); + addReplyBulk(c,dumpobj); + decrRefCount(dumpobj); + return; +} + +/* RESTORE key ttl serialized-value */ +void restoreCommand(redisClient *c) { + long long ttl; + rio payload; + int type; + robj *obj; + + /* Make sure this key does not already exist here... */ + if (lookupKeyWrite(c->db,c->argv[1]) != NULL) { + addReplyError(c,"Target key name is busy."); + return; + } + + /* Check if the TTL value makes sense */ + if (getLongLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { + return; + } else if (ttl < 0) { + addReplyError(c,"Invalid TTL value, must be >= 0"); + return; + } + + /* Verify RDB version and data checksum. */ + if (verifyDumpPayload(c->argv[3]->ptr,sdslen(c->argv[3]->ptr)) == REDIS_ERR) + { + addReplyError(c,"DUMP payload version or checksum are wrong"); + return; + } + + rioInitWithBuffer(&payload,c->argv[3]->ptr); + if (((type = rdbLoadObjectType(&payload)) == -1) || + ((obj = rdbLoadObject(type,&payload)) == NULL)) + { + addReplyError(c,"Bad data format"); + return; + } + + /* Create the key and set the TTL if any */ + dbAdd(c->db,c->argv[1],obj); + if (ttl) setExpire(c->db,c->argv[1],mstime()+ttl); + signalModifiedKey(c->db,c->argv[1]); + addReply(c,shared.ok); + server.dirty++; +} + +/* MIGRATE host port key dbid timeout */ +void migrateCommand(redisClient *c) { + int fd; + long timeout; + long dbid; + long long ttl = 0, expireat; + robj *o; + rio cmd, payload; + + /* Sanity check */ + if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK) + return; + if (getLongFromObjectOrReply(c,c->argv[4],&dbid,NULL) != REDIS_OK) + return; + if (timeout <= 0) timeout = 1; + + /* Check if the key is here. If not we reply with success as there is + * nothing to migrate (for instance the key expired in the meantime), but + * we include such information in the reply string. */ + if ((o = lookupKeyRead(c->db,c->argv[3])) == NULL) { + addReplySds(c,sdsnew("+NOKEY\r\n")); + return; + } + + /* Connect */ + fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, + atoi(c->argv[2]->ptr)); + if (fd == -1) { + addReplyErrorFormat(c,"Can't connect to target node: %s", + server.neterr); + return; + } + if ((aeWait(fd,AE_WRITABLE,timeout*1000) & AE_WRITABLE) == 0) { + addReplySds(c,sdsnew("-IOERR error or timeout connecting to the client\r\n")); + return; + } + + /* Create RESTORE payload and generate the protocol to call the command. */ + rioInitWithBuffer(&cmd,sdsempty()); + redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); + + expireat = getExpire(c->db,c->argv[3]); + if (expireat != -1) { + ttl = expireat-mstime(); + if (ttl < 1) ttl = 1; + } + redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); + redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,ttl)); + + /* Finally the last argument that is the serailized object payload + * in the DUMP format. */ + createDumpPayload(&payload,o); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr, + sdslen(payload.io.buffer.ptr))); + sdsfree(payload.io.buffer.ptr); + + /* Tranfer the query to the other node in 64K chunks. */ + { + sds buf = cmd.io.buffer.ptr; + size_t pos = 0, towrite; + int nwritten = 0; + + while ((towrite = sdslen(buf)-pos) > 0) { + towrite = (towrite > (64*1024) ? (64*1024) : towrite); + nwritten = syncWrite(fd,buf+pos,towrite,timeout); + if (nwritten != (signed)towrite) goto socket_wr_err; + pos += nwritten; + } + } + + /* Read back the reply. */ + { + char buf1[1024]; + char buf2[1024]; + + /* Read the two replies */ + if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0) + goto socket_rd_err; + if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0) + goto socket_rd_err; + if (buf1[0] == '-' || buf2[0] == '-') { + addReplyErrorFormat(c,"Target instance replied with error: %s", + (buf1[0] == '-') ? buf1+1 : buf2+1); + } else { + robj *aux; + + dbDelete(c->db,c->argv[3]); + signalModifiedKey(c->db,c->argv[3]); + addReply(c,shared.ok); + server.dirty++; + + /* Translate MIGRATE as DEL for replication/AOF. */ + aux = createStringObject("DEL",3); + rewriteClientCommandVector(c,2,aux,c->argv[3]); + decrRefCount(aux); + } + } + + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; + +socket_wr_err: + addReplySds(c,sdsnew("-IOERR error or timeout writing to target instance\r\n")); + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; + +socket_rd_err: + addReplySds(c,sdsnew("-IOERR error or timeout reading from target node\r\n")); + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; +} diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index 30984160e7e..1ae95886b4a 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -1,9 +1,11 @@ #!/bin/sh GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` -GIT_DIRTY=`git diff 2> /dev/null | wc -l` +GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` +BUILD_ID=`uname -n`"-"`date +%s` test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ -(cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already uptodate +(cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already up-to-date echo "#define REDIS_GIT_SHA1 \"$GIT_SHA1\"" > release.h echo "#define REDIS_GIT_DIRTY \"$GIT_DIRTY\"" >> release.h +echo "#define REDIS_BUILD_ID \"$BUILD_ID\"" >> release.h touch release.c # Force recompile of release.c diff --git a/src/multi.c b/src/multi.c index 65ec38a8d0e..c8287645643 100644 --- a/src/multi.c +++ b/src/multi.c @@ -1,3 +1,32 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" /* ================================ MULTI/EXEC ============================== */ @@ -40,6 +69,20 @@ void queueMultiCommand(redisClient *c) { c->mstate.count++; } +void discardTransaction(redisClient *c) { + freeClientMultiState(c); + initClientMultiState(c); + c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS|REDIS_DIRTY_EXEC); + unwatchAllKeys(c); +} + +/* Flag the transacation as DIRTY_EXEC so that EXEC will fail. + * Should be called every time there is an error while queueing a command. */ +void flagTransaction(redisClient *c) { + if (c->flags & REDIS_MULTI) + c->flags |= REDIS_DIRTY_EXEC; +} + void multiCommand(redisClient *c) { if (c->flags & REDIS_MULTI) { addReplyError(c,"MULTI calls can not be nested"); @@ -54,23 +97,17 @@ void discardCommand(redisClient *c) { addReplyError(c,"DISCARD without MULTI"); return; } - - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS);; - unwatchAllKeys(c); + discardTransaction(c); addReply(c,shared.ok); } /* Send a MULTI command to all the slaves and AOF file. Check the execCommand - * implememntation for more information. */ -void execCommandReplicateMulti(redisClient *c) { + * implementation for more information. */ +void execCommandPropagateMulti(redisClient *c) { robj *multistring = createStringObject("MULTI",5); - if (server.aof_state != REDIS_AOF_OFF) - feedAppendOnlyFile(server.multiCommand,c->db->id,&multistring,1); - if (listLength(server.slaves)) - replicationFeedSlaves(server.slaves,c->db->id,&multistring,1); + propagate(server.multiCommand,c->db->id,&multistring,1, + REDIS_PROPAGATE_AOF|REDIS_PROPAGATE_REPL); decrRefCount(multistring); } @@ -79,29 +116,26 @@ void execCommand(redisClient *c) { robj **orig_argv; int orig_argc; struct redisCommand *orig_cmd; + int must_propagate = 0; /* Need to propagate MULTI/EXEC to AOF / slaves? */ if (!(c->flags & REDIS_MULTI)) { addReplyError(c,"EXEC without MULTI"); return; } - /* Check if we need to abort the EXEC if some WATCHed key was touched. - * A failed EXEC will return a multi bulk nil object. */ - if (c->flags & REDIS_DIRTY_CAS) { - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS); - unwatchAllKeys(c); - addReply(c,shared.nullmultibulk); - return; + /* Check if we need to abort the EXEC because: + * 1) Some WATCHed key was touched. + * 2) There was a previous error while queueing commands. + * A failed EXEC in the first case returns a multi bulk nil object + * (technically it is not an error but a special behavior), while + * in the second an EXECABORT error is returned. */ + if (c->flags & (REDIS_DIRTY_CAS|REDIS_DIRTY_EXEC)) { + addReply(c, c->flags & REDIS_DIRTY_EXEC ? shared.execaborterr : + shared.nullmultibulk); + discardTransaction(c); + goto handle_monitor; } - /* Replicate a MULTI request now that we are sure the block is executed. - * This way we'll deliver the MULTI/..../EXEC block as a whole and - * both the AOF and the replication link will have the same consistency - * and atomicity guarantees. */ - execCommandReplicateMulti(c); - /* Exec all the queued commands */ unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */ orig_argv = c->argv; @@ -112,6 +146,16 @@ void execCommand(redisClient *c) { c->argc = c->mstate.commands[j].argc; c->argv = c->mstate.commands[j].argv; c->cmd = c->mstate.commands[j].cmd; + + /* Propagate a MULTI request once we encounter the first write op. + * This way we'll deliver the MULTI/..../EXEC block as a whole and + * both the AOF and the replication link will have the same consistency + * and atomicity guarantees. */ + if (!must_propagate && !(c->cmd->flags & REDIS_CMD_READONLY)) { + execCommandPropagateMulti(c); + must_propagate = 1; + } + call(c,REDIS_CALL_FULL); /* Commands may alter argc/argv, restore mstate. */ @@ -122,13 +166,19 @@ void execCommand(redisClient *c) { c->argv = orig_argv; c->argc = orig_argc; c->cmd = orig_cmd; - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS); - /* Make sure the EXEC command is always replicated / AOF, since we - * always send the MULTI command (we can't know beforehand if the - * next operations will contain at least a modification to the DB). */ - server.dirty++; + discardTransaction(c); + /* Make sure the EXEC command will be propagated as well if MULTI + * was already propagated. */ + if (must_propagate) server.dirty++; + +handle_monitor: + /* Send EXEC to clients waiting data from MONITOR. We do it here + * since the natural order of commands execution is actually: + * MUTLI, EXEC, ... commands inside transaction ... + * Instead EXEC is flagged as REDIS_CMD_SKIP_MONITOR in the command + * table, and we do it here with correct ordering. */ + if (listLength(server.monitors) && !server.loading) + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); } /* ===================== WATCH (CAS alike for MULTI/EXEC) =================== @@ -164,13 +214,13 @@ void watchForKey(redisClient *c, robj *key) { } /* This key is not already watched in this DB. Let's add it */ clients = dictFetchValue(c->db->watched_keys,key); - if (!clients) { + if (!clients) { clients = listCreate(); dictAdd(c->db->watched_keys,key,clients); incrRefCount(key); } listAddNodeTail(clients,c); - /* Add the new key to the lits of keys watched by this client */ + /* Add the new key to the list of keys watched by this client */ wk = zmalloc(sizeof(*wk)); wk->key = key; wk->db = c->db; diff --git a/src/networking.c b/src/networking.c index 40aad83606a..9f258292fe2 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1,5 +1,35 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #include "redis.h" #include +#include static void setProtocolError(redisClient *c, int pos); @@ -22,7 +52,6 @@ int listMatchObjects(void *a, void *b) { redisClient *createClient(int fd) { redisClient *c = zmalloc(sizeof(redisClient)); - c->bufpos = 0; /* passing -1 as fd it is possible to create a non connected client. * This is useful since all the Redis commands needs to be executed @@ -30,7 +59,9 @@ redisClient *createClient(int fd) { * contexts (for instance a Lua script) we need a non connected client. */ if (fd != -1) { anetNonBlock(NULL,fd); - anetTcpNoDelay(NULL,fd); + anetEnableTcpNoDelay(NULL,fd); + if (server.tcpkeepalive) + anetKeepAlive(NULL,fd,server.tcpkeepalive); if (aeCreateFileEvent(server.el,fd,AE_READABLE, readQueryFromClient, c) == AE_ERR) { @@ -41,8 +72,12 @@ redisClient *createClient(int fd) { } selectDb(c,0); + c->id = server.next_client_id++; c->fd = fd; + c->name = NULL; + c->bufpos = 0; c->querybuf = sdsempty(); + c->querybuf_peak = 0; c->reqtype = 0; c->argc = 0; c->argv = NULL; @@ -51,24 +86,28 @@ redisClient *createClient(int fd) { c->bulklen = -1; c->sentlen = 0; c->flags = 0; - c->lastinteraction = time(NULL); + c->ctime = c->lastinteraction = server.unixtime; c->authenticated = 0; c->replstate = REDIS_REPL_NONE; + c->repl_put_online_on_ack = 0; + c->reploff = 0; + c->repl_ack_off = 0; + c->repl_ack_time = 0; + c->slave_listening_port = 0; + c->slave_capa = SLAVE_CAPA_NONE; c->reply = listCreate(); c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; - listSetFreeMethod(c->reply,decrRefCount); + listSetFreeMethod(c->reply,decrRefCountVoid); listSetDupMethod(c->reply,dupClientReplyValue); - c->bpop.keys = NULL; - c->bpop.count = 0; + c->bpop.keys = dictCreate(&setDictType,NULL); c->bpop.timeout = 0; c->bpop.target = NULL; - c->io_keys = listCreate(); c->watched_keys = listCreate(); - listSetFreeMethod(c->io_keys,decrRefCount); c->pubsub_channels = dictCreate(&setDictType,NULL); c->pubsub_patterns = listCreate(); - listSetFreeMethod(c->pubsub_patterns,decrRefCount); + c->peerid = NULL; + listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); if (fd != -1) listAddNodeTail(server.clients,c); initClientMultiState(c); @@ -82,19 +121,21 @@ redisClient *createClient(int fd) { * returns REDIS_OK, and make sure to install the write handler in our event * loop so that when the socket is writable new data gets written. * - * If the client should not receive new data, because it is a fake client - * or a slave, or because the setup of the write handler failed, the function - * returns REDIS_ERR. + * If the client should not receive new data, because it is a fake client, + * a master, a slave not yet online, or because the setup of the write handler + * failed, the function returns REDIS_ERR. * * Typically gets called every time a reply is built, before adding more * data to the clients output buffers. If the function returns REDIS_ERR no * data should be appended to the output buffers. */ int prepareClientToWrite(redisClient *c) { if (c->flags & REDIS_LUA_CLIENT) return REDIS_OK; + if ((c->flags & REDIS_MASTER) && + !(c->flags & REDIS_MASTER_FORCE_REPLY)) return REDIS_ERR; if (c->fd <= 0) return REDIS_ERR; /* Fake client */ if (c->bufpos == 0 && listLength(c->reply) == 0 && (c->replstate == REDIS_REPL_NONE || - c->replstate == REDIS_REPL_ONLINE) && + c->replstate == REDIS_REPL_ONLINE) && !c->repl_put_online_on_ack && aeCreateFileEvent(server.el, c->fd, AE_WRITABLE, sendReplyToClient, c) == AE_ERR) return REDIS_ERR; return REDIS_OK; @@ -347,7 +388,7 @@ void *addDeferredMultiBulkLength(redisClient *c) { return listLast(c->reply); } -/* Populate the length object and try glueing it to the next chunk. */ +/* Populate the length object and try gluing it to the next chunk. */ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { listNode *ln = (listNode*)node; robj *len, *next; @@ -363,20 +404,29 @@ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { /* Only glue when the next node is non-NULL (an sds in this case) */ if (next->ptr != NULL) { + c->reply_bytes -= zmalloc_size_sds(len->ptr); + c->reply_bytes -= zmalloc_size_sds(next->ptr); len->ptr = sdscatlen(len->ptr,next->ptr,sdslen(next->ptr)); + c->reply_bytes += zmalloc_size_sds(len->ptr); listDelNode(c->reply,ln->next); } } asyncCloseClientOnOutputBufferLimitReached(c); } -/* Add a duble as a bulk reply */ +/* Add a double as a bulk reply */ void addReplyDouble(redisClient *c, double d) { char dbuf[128], sbuf[128]; int dlen, slen; - dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); - slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); - addReplyString(c,sbuf,slen); + if (isinf(d)) { + /* Libc in odd systems (Hi Solaris!) will format infinite in a + * different way, so better to handle it in an explicit way. */ + addReplyBulkCString(c, d > 0 ? "inf" : "-inf"); + } else { + dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); + slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); + addReplyString(c,sbuf,slen); + } } /* Add a long long as integer reply or bulk len / multi bulk count. @@ -413,7 +463,10 @@ void addReplyLongLong(redisClient *c, long long ll) { } void addReplyMultiBulkLen(redisClient *c, long length) { - addReplyLongLongWithPrefix(c,length,'*'); + if (length < REDIS_SHARED_BULKHDR_LEN) + addReply(c,shared.mbulkhdr[length]); + else + addReplyLongLongWithPrefix(c,length,'*'); } /* Create the length prefix of a bulk reply, example: $2234 */ @@ -435,7 +488,11 @@ void addReplyBulkLen(redisClient *c, robj *obj) { len++; } } - addReplyLongLongWithPrefix(c,len,'$'); + + if (len < REDIS_SHARED_BULKHDR_LEN) + addReply(c,shared.bulkhdr[len]); + else + addReplyLongLongWithPrefix(c,len,'$'); } /* Add a Redis Object as a bulk reply */ @@ -481,16 +538,19 @@ void copyClientOutputBuffer(redisClient *dst, redisClient *src) { dst->reply_bytes = src->reply_bytes; } -static void acceptCommonHandler(int fd) { +#define MAX_ACCEPTS_PER_CALL 1000 +static void acceptCommonHandler(int fd, int flags) { redisClient *c; if ((c = createClient(fd)) == NULL) { - redisLog(REDIS_WARNING,"Error allocating resoures for the client"); - close(fd); /* May be already closed, just ingore errors */ + redisLog(REDIS_WARNING, + "Error registering fd event for the new client: %s (fd=%d)", + strerror(errno),fd); + close(fd); /* May be already closed, just ignore errors */ return; } /* If maxclient directive is set and this is one client more... close the * connection. Note that we create the client instead to check before - * for this condition, since now the socket is already set in nonblocking + * for this condition, since now the socket is already set in non-blocking * mode and we can send an error for free using the Kernel I/O */ if (listLength(server.clients) > server.maxclients) { char *err = "-ERR max number of clients reached\r\n"; @@ -504,40 +564,48 @@ static void acceptCommonHandler(int fd) { return; } server.stat_numconnections++; + c->flags |= flags; } void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - int cport, cfd; - char cip[128]; + int cport, cfd, max = MAX_ACCEPTS_PER_CALL; + char cip[REDIS_IP_STR_LEN]; REDIS_NOTUSED(el); REDIS_NOTUSED(mask); REDIS_NOTUSED(privdata); - cfd = anetTcpAccept(server.neterr, fd, cip, &cport); - if (cfd == AE_ERR) { - redisLog(REDIS_WARNING,"Accepting client connection: %s", server.neterr); - return; + while(max--) { + cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno != EWOULDBLOCK) + redisLog(REDIS_WARNING, + "Accepting client connection: %s", server.neterr); + return; + } + redisLog(REDIS_VERBOSE,"Accepted %s:%d", cip, cport); + acceptCommonHandler(cfd,0); } - redisLog(REDIS_VERBOSE,"Accepted %s:%d", cip, cport); - acceptCommonHandler(cfd); } void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - int cfd; + int cfd, max = MAX_ACCEPTS_PER_CALL; REDIS_NOTUSED(el); REDIS_NOTUSED(mask); REDIS_NOTUSED(privdata); - cfd = anetUnixAccept(server.neterr, fd); - if (cfd == AE_ERR) { - redisLog(REDIS_WARNING,"Accepting client connection: %s", server.neterr); - return; + while(max--) { + cfd = anetUnixAccept(server.neterr, fd); + if (cfd == ANET_ERR) { + if (errno != EWOULDBLOCK) + redisLog(REDIS_WARNING, + "Accepting client connection: %s", server.neterr); + return; + } + redisLog(REDIS_VERBOSE,"Accepted connection to %s", server.unixsocket); + acceptCommonHandler(cfd,REDIS_UNIX_SOCKET); } - redisLog(REDIS_VERBOSE,"Accepted connection to %s", server.unixsocket); - acceptCommonHandler(cfd); } - static void freeClientArgv(redisClient *c) { int j; for (j = 0; j < c->argc; j++) @@ -546,80 +614,111 @@ static void freeClientArgv(redisClient *c) { c->cmd = NULL; } +/* Close all the slaves connections. This is useful in chained replication + * when we resync with our own master and want to force all our slaves to + * resync with us as well. */ +void disconnectSlaves(void) { + while (listLength(server.slaves)) { + listNode *ln = listFirst(server.slaves); + freeClient((redisClient*)ln->value); + } +} + void freeClient(redisClient *c) { listNode *ln; /* If this is marked as current client unset it */ if (server.current_client == c) server.current_client = NULL; - /* Note that if the client we are freeing is blocked into a blocking - * call, we have to set querybuf to NULL *before* to call - * unblockClientWaitingData() to avoid processInputBuffer() will get - * called. Also it is important to remove the file events after - * this, because this call adds the READABLE event. */ + /* If it is our master that's beging disconnected we should make sure + * to cache the state to try a partial resynchronization later. + * + * Note that before doing this we make sure that the client is not in + * some unexpected state, by checking its flags. */ + if (server.master && c->flags & REDIS_MASTER) { + redisLog(REDIS_WARNING,"Connection with master lost."); + if (!(c->flags & (REDIS_CLOSE_AFTER_REPLY| + REDIS_CLOSE_ASAP| + REDIS_BLOCKED| + REDIS_UNBLOCKED))) + { + replicationCacheMaster(c); + return; + } + } + + /* Log link disconnection with slave */ + if ((c->flags & REDIS_SLAVE) && !(c->flags & REDIS_MONITOR)) { + redisLog(REDIS_WARNING,"Connection with slave %s lost.", + replicationGetSlaveName(c)); + } + + /* Free the query buffer */ sdsfree(c->querybuf); c->querybuf = NULL; + + /* Deallocate structures used to block on blocking ops. */ if (c->flags & REDIS_BLOCKED) unblockClientWaitingData(c); + dictRelease(c->bpop.keys); /* UNWATCH all the keys */ unwatchAllKeys(c); listRelease(c->watched_keys); + /* Unsubscribe from all the pubsub channels */ pubsubUnsubscribeAllChannels(c,0); pubsubUnsubscribeAllPatterns(c,0); dictRelease(c->pubsub_channels); listRelease(c->pubsub_patterns); - /* Obvious cleanup */ - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + + /* Close socket, unregister events, and remove list of replies and + * accumulated arguments. */ + if (c->fd != -1) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + close(c->fd); + } listRelease(c->reply); freeClientArgv(c); - close(c->fd); + /* Remove from the list of clients */ - ln = listSearchKey(server.clients,c); - redisAssert(ln != NULL); - listDelNode(server.clients,ln); + if (c->fd != -1) { + ln = listSearchKey(server.clients,c); + redisAssert(ln != NULL); + listDelNode(server.clients,ln); + } + /* When client was just unblocked because of a blocking operation, - * remove it from the list with unblocked clients. */ + * remove it from the list of unblocked clients. */ if (c->flags & REDIS_UNBLOCKED) { ln = listSearchKey(server.unblocked_clients,c); redisAssert(ln != NULL); listDelNode(server.unblocked_clients,ln); } - listRelease(c->io_keys); - /* Master/slave cleanup. - * Case 1: we lost the connection with a slave. */ + + /* Master/slave cleanup Case 1: + * we lost the connection with a slave. */ if (c->flags & REDIS_SLAVE) { - if (c->replstate == REDIS_REPL_SEND_BULK && c->repldbfd != -1) - close(c->repldbfd); + if (c->replstate == REDIS_REPL_SEND_BULK) { + if (c->repldbfd != -1) close(c->repldbfd); + if (c->replpreamble) sdsfree(c->replpreamble); + } list *l = (c->flags & REDIS_MONITOR) ? server.monitors : server.slaves; ln = listSearchKey(l,c); redisAssert(ln != NULL); listDelNode(l,ln); + /* We need to remember the time when we started to have zero + * attached slaves, as after some time we'll free the replication + * backlog. */ + if (c->flags & REDIS_SLAVE && listLength(server.slaves) == 0) + server.repl_no_slaves_since = server.unixtime; + refreshGoodSlavesCount(); } - /* Case 2: we lost the connection with the master. */ - if (c->flags & REDIS_MASTER) { - server.master = NULL; - server.repl_state = REDIS_REPL_CONNECT; - server.repl_down_since = time(NULL); - /* Since we lost the connection with the master, we should also - * close the connection with all our slaves if we have any, so - * when we'll resync with the master the other slaves will sync again - * with us as well. Note that also when the slave is not connected - * to the master it will keep refusing connections by other slaves. - * - * We do this only if server.masterhost != NULL. If it is NULL this - * means the user called SLAVEOF NO ONE and we are freeing our - * link with the master, so no need to close link with slaves. */ - if (server.masterhost != NULL) { - while (listLength(server.slaves)) { - ln = listFirst(server.slaves); - freeClient((redisClient*)ln->value); - } - } - } + /* Master/slave cleanup Case 2: + * we lost the connection with the master. */ + if (c->flags & REDIS_MASTER) replicationHandleMasterDisconnection(); /* If this client was scheduled for async freeing we need to remove it * from the queue. */ @@ -629,9 +728,12 @@ void freeClient(redisClient *c) { listDelNode(server.clients_to_close,ln); } - /* Release memory */ + /* Release other dynamically allocated client structure fields, + * and finally release the client structure itself. */ + if (c->name) decrRefCount(c->name); zfree(c->argv); freeClientMultiState(c); + sdsfree(c->peerid); zfree(c); } @@ -640,7 +742,7 @@ void freeClient(redisClient *c) { * a context where calling freeClient() is not possible, because the client * should be valid for the continuation of the flow of the program. */ void freeClientAsync(redisClient *c) { - if (c->flags & REDIS_CLOSE_ASAP) return; + if (c->flags & REDIS_CLOSE_ASAP || c->flags & REDIS_LUA_CLIENT) return; c->flags |= REDIS_CLOSE_ASAP; listAddNodeTail(server.clients_to_close,c); } @@ -666,13 +768,8 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { while(c->bufpos > 0 || listLength(c->reply)) { if (c->bufpos > 0) { - if (c->flags & REDIS_MASTER) { - /* Don't reply to a master */ - nwritten = c->bufpos - c->sentlen; - } else { - nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); - if (nwritten <= 0) break; - } + nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); + if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -692,13 +789,8 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { continue; } - if (c->flags & REDIS_MASTER) { - /* Don't reply to a master */ - nwritten = objlen - c->sentlen; - } else { - nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen); - if (nwritten <= 0) break; - } + nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen); + if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -717,6 +809,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { * * However if we are over the maxmemory limit we ignore that and * just deliver as much data as it is possible to deliver. */ + server.stat_net_output_bytes += totwritten; if (totwritten > REDIS_MAX_WRITE_PER_EVENT && (server.maxmemory == 0 || zmalloc_used_memory() < server.maxmemory)) break; @@ -731,7 +824,13 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { return; } } - if (totwritten > 0) c->lastinteraction = time(NULL); + if (totwritten > 0) { + /* For clients representing masters we don't count sending data + * as an interaction, since we always send REPLCONF ACK commands + * that take some time to just fill the socket output buffer. + * We just rely on data / pings received for timeout detection. */ + if (!(c->flags & REDIS_MASTER)) c->lastinteraction = server.unixtime; + } if (c->bufpos == 0 && listLength(c->reply) == 0) { c->sentlen = 0; aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); @@ -751,40 +850,15 @@ void resetClient(redisClient *c) { if (!(c->flags & REDIS_MULTI)) c->flags &= (~REDIS_ASKING); } -void closeTimedoutClients(void) { - redisClient *c; - listNode *ln; - time_t now = time(NULL); - listIter li; - - listRewind(server.clients,&li); - while ((ln = listNext(&li)) != NULL) { - c = listNodeValue(ln); - if (server.maxidletime && - !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ - !(c->flags & REDIS_MASTER) && /* no timeout for masters */ - !(c->flags & REDIS_BLOCKED) && /* no timeout for BLPOP */ - dictSize(c->pubsub_channels) == 0 && /* no timeout for pubsub */ - listLength(c->pubsub_patterns) == 0 && - (now - c->lastinteraction > server.maxidletime)) - { - redisLog(REDIS_VERBOSE,"Closing idle client"); - freeClient(c); - } else if (c->flags & REDIS_BLOCKED) { - if (c->bpop.timeout != 0 && c->bpop.timeout < now) { - addReply(c,shared.nullmultibulk); - unblockClientWaitingData(c); - } - } - } -} - int processInlineBuffer(redisClient *c) { - char *newline = strstr(c->querybuf,"\r\n"); + char *newline; int argc, j; - sds *argv; + sds *argv, aux; size_t querylen; + /* Search for end of line */ + newline = strchr(c->querybuf,'\n'); + /* Nothing to do without a \r\n */ if (newline == NULL) { if (sdslen(c->querybuf) > REDIS_INLINE_MAX_SIZE) { @@ -794,16 +868,35 @@ int processInlineBuffer(redisClient *c) { return REDIS_ERR; } + /* Handle the \r\n case. */ + if (newline && newline != c->querybuf && *(newline-1) == '\r') + newline--; + /* Split the input buffer up to the \r\n */ querylen = newline-(c->querybuf); - argv = sdssplitlen(c->querybuf,querylen," ",1,&argc); + aux = sdsnewlen(c->querybuf,querylen); + argv = sdssplitargs(aux,&argc); + sdsfree(aux); + if (argv == NULL) { + addReplyError(c,"Protocol error: unbalanced quotes in request"); + setProtocolError(c,0); + return REDIS_ERR; + } + + /* Newline from slaves can be used to refresh the last ACK time. + * This is useful for a slave to ping back while loading a big + * RDB file. */ + if (querylen == 0 && c->flags & REDIS_SLAVE) + c->repl_ack_time = server.unixtime; /* Leave data after the first line of the query in the buffer */ - c->querybuf = sdsrange(c->querybuf,querylen+2,-1); + sdsrange(c->querybuf,querylen+2,-1); /* Setup argv array on client structure */ - if (c->argv) zfree(c->argv); - c->argv = zmalloc(sizeof(robj*)*argc); + if (argc) { + if (c->argv) zfree(c->argv); + c->argv = zmalloc(sizeof(robj*)*argc); + } /* Create redis objects for all arguments. */ for (c->argc = 0, j = 0; j < argc; j++) { @@ -821,14 +914,14 @@ int processInlineBuffer(redisClient *c) { /* Helper function. Trims query buffer to make the function that processes * multi bulk requests idempotent. */ static void setProtocolError(redisClient *c, int pos) { - if (server.verbosity >= REDIS_VERBOSE) { - sds client = getClientInfoString(c); + if (server.verbosity <= REDIS_VERBOSE) { + sds client = catClientInfoString(sdsempty(),c); redisLog(REDIS_VERBOSE, "Protocol error from client: %s", client); sdsfree(client); } c->flags |= REDIS_CLOSE_AFTER_REPLY; - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); } int processMultibulkBuffer(redisClient *c) { @@ -866,7 +959,7 @@ int processMultibulkBuffer(redisClient *c) { pos = (newline-c->querybuf)+2; if (ll <= 0) { - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); return REDIS_OK; } @@ -884,8 +977,10 @@ int processMultibulkBuffer(redisClient *c) { newline = strchr(c->querybuf+pos,'\r'); if (newline == NULL) { if (sdslen(c->querybuf) > REDIS_INLINE_MAX_SIZE) { - addReplyError(c,"Protocol error: too big bulk count string"); + addReplyError(c, + "Protocol error: too big bulk count string"); setProtocolError(c,0); + return REDIS_ERR; } break; } @@ -911,15 +1006,19 @@ int processMultibulkBuffer(redisClient *c) { pos += newline-(c->querybuf+pos)+2; if (ll >= REDIS_MBULK_BIG_ARG) { + size_t qblen; + /* If we are going to read a large object from network * try to make it likely that it will start at c->querybuf - * boundary so that we can optimized object creation + * boundary so that we can optimize object creation * avoiding a large copy of data. */ - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); pos = 0; + qblen = sdslen(c->querybuf); /* Hint the sds library about the amount of bytes this string is * going to contain. */ - c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2); + if (qblen < (size_t)ll+2) + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-qblen); } c->bulklen = ll; } @@ -929,7 +1028,7 @@ int processMultibulkBuffer(redisClient *c) { /* Not enough data (+2 == trailing \r\n) */ break; } else { - /* Optimization: if the buffer contanins JUST our bulk element + /* Optimization: if the buffer contains JUST our bulk element * instead of creating a new object by *copying* the sds we * just use the current sds string. */ if (pos == 0 && @@ -954,7 +1053,7 @@ int processMultibulkBuffer(redisClient *c) { } /* Trim to pos */ - if (pos) c->querybuf = sdsrange(c->querybuf,pos,-1); + if (pos) sdsrange(c->querybuf,pos,-1); /* We're done when c->multibulk == 0 */ if (c->multibulklen == 0) return REDIS_OK; @@ -1012,9 +1111,9 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { server.current_client = c; readlen = REDIS_IOBUF_LEN; /* If this is a multi bulk request, and we are processing a bulk reply - * that is large enough, try to maximize the probabilty that the query - * buffer contains excatly the SDS string representing the object, even - * at the risk of requring more read(2) calls. This way the function + * that is large enough, try to maximize the probability that the query + * buffer contains exactly the SDS string representing the object, even + * at the risk of requiring more read(2) calls. This way the function * processMultiBulkBuffer() can avoid copying buffers to create the * Redis Object representing the argument. */ if (c->reqtype == REDIS_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1 @@ -1026,6 +1125,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { } qblen = sdslen(c->querybuf); + if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); nread = read(fd, c->querybuf+qblen, readlen); if (nread == -1) { @@ -1043,13 +1143,15 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { } if (nread) { sdsIncrLen(c->querybuf,nread); - c->lastinteraction = time(NULL); + c->lastinteraction = server.unixtime; + if (c->flags & REDIS_MASTER) c->reploff += nread; + server.stat_net_input_bytes += nread; } else { server.current_client = NULL; return; } if (sdslen(c->querybuf) > server.client_max_querybuf_len) { - sds ci = getClientInfoString(c), bytes = sdsempty(); + sds ci = catClientInfoString(sdsempty(),c), bytes = sdsempty(); bytes = sdscatrepr(bytes,c->querybuf,64); redisLog(REDIS_WARNING,"Closing client that reached max query buffer length: %s (qbuf initial bytes: %s)", ci, bytes); @@ -1080,14 +1182,66 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, *biggest_input_buffer = bib; } -/* Turn a Redis client into an sds string representing its state. */ -sds getClientInfoString(redisClient *client) { - char ip[32], flags[16], events[3], *p; +/* This is a helper function for genClientPeerId(). + * It writes the specified ip/port to "peerid" as a null termiated string + * in the form ip:port if ip does not contain ":" itself, otherwise + * [ip]:port format is used (for IPv6 addresses basically). */ +void formatPeerId(char *peerid, size_t peerid_len, char *ip, int port) { + if (strchr(ip,':')) + snprintf(peerid,peerid_len,"[%s]:%d",ip,port); + else + snprintf(peerid,peerid_len,"%s:%d",ip,port); +} + +/* A Redis "Peer ID" is a colon separated ip:port pair. + * For IPv4 it's in the form x.y.z.k:port, example: "127.0.0.1:1234". + * For IPv6 addresses we use [] around the IP part, like in "[::1]:1234". + * For Unix sockets we use path:0, like in "/tmp/redis:0". + * + * A Peer ID always fits inside a buffer of REDIS_PEER_ID_LEN bytes, including + * the null term. + * + * The function returns REDIS_OK on succcess, and REDIS_ERR on failure. + * + * On failure the function still populates 'peerid' with the "?:0" string + * in case you want to relax error checking or need to display something + * anyway (see anetPeerToString implementation for more info). */ +int genClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { + char ip[REDIS_IP_STR_LEN]; int port; - time_t now = time(NULL); + + if (client->flags & REDIS_UNIX_SOCKET) { + /* Unix socket client. */ + snprintf(peerid,peerid_len,"%s:0",server.unixsocket); + return REDIS_OK; + } else { + /* TCP client. */ + int retval = anetPeerToString(client->fd,ip,sizeof(ip),&port); + formatPeerId(peerid,peerid_len,ip,port); + return (retval == -1) ? REDIS_ERR : REDIS_OK; + } +} + +/* This function returns the client peer id, by creating and caching it + * if client->peerid is NULL, otherwise returning the cached value. + * The Peer ID never changes during the life of the client, however it + * is expensive to compute. */ +char *getClientPeerId(redisClient *c) { + char peerid[REDIS_PEER_ID_LEN]; + + if (c->peerid == NULL) { + genClientPeerId(c,peerid,sizeof(peerid)); + c->peerid = sdsnew(peerid); + } + return c->peerid; +} + +/* Concatenate a string representing the state of a client in an human + * readable format, into the sds string 's'. */ +sds catClientInfoString(sds s, redisClient *client) { + char flags[16], events[3], *p; int emask; - anetPeerToString(client->fd,ip,&port); p = flags; if (client->flags & REDIS_SLAVE) { if (client->flags & REDIS_MONITOR) @@ -1102,6 +1256,7 @@ sds getClientInfoString(redisClient *client) { if (client->flags & REDIS_CLOSE_AFTER_REPLY) *p++ = 'c'; if (client->flags & REDIS_UNBLOCKED) *p++ = 'u'; if (client->flags & REDIS_CLOSE_ASAP) *p++ = 'A'; + if (client->flags & REDIS_UNIX_SOCKET) *p++ = 'U'; if (p == flags) *p++ = 'N'; *p++ = '\0'; @@ -1110,18 +1265,24 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_READABLE) *p++ = 'r'; if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; - return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", - ip,port,client->fd, - (long)(now - client->lastinteraction), + return sdscatfmt(s, + "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + (unsigned long long) client->id, + getClientPeerId(client), + client->fd, + client->name ? (char*)client->name->ptr : "", + (long long)(server.unixtime - client->ctime), + (long long)(server.unixtime - client->lastinteraction), flags, client->db->id, (int) dictSize(client->pubsub_channels), (int) listLength(client->pubsub_patterns), - (unsigned long) sdslen(client->querybuf), - (unsigned long) client->bufpos, - (unsigned long) listLength(client->reply), - getClientOutputBufferMemoryUsage(client), + (client->flags & REDIS_MULTI) ? client->mstate.count : -1, + (unsigned long long) sdslen(client->querybuf), + (unsigned long long) sdsavail(client->querybuf), + (unsigned long long) client->bufpos, + (unsigned long long) listLength(client->reply), + (unsigned long long) getClientOutputBufferMemoryUsage(client), events, client->lastcmd ? client->lastcmd->name : "NULL"); } @@ -1132,14 +1293,11 @@ sds getAllClientsInfoString(void) { redisClient *client; sds o = sdsempty(); + o = sdsMakeRoomFor(o,200*listLength(server.clients)); listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - sds cs; - client = listNodeValue(ln); - cs = getClientInfoString(client); - o = sdscatsds(o,cs); - sdsfree(cs); + o = catClientInfoString(o,client); o = sdscatlen(o,"\n",1); } return o; @@ -1151,31 +1309,133 @@ void clientCommand(redisClient *c) { redisClient *client; if (!strcasecmp(c->argv[1]->ptr,"list") && c->argc == 2) { + /* CLIENT LIST */ sds o = getAllClientsInfoString(); addReplyBulkCBuffer(c,o,sdslen(o)); sdsfree(o); - } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { + } else if (!strcasecmp(c->argv[1]->ptr,"kill")) { + /* CLIENT KILL + * CLIENT KILL