Upstream 25-05-2016

update changelog

fix NODEJS_ENABLED flag to do the right thing

fix valgrind

changelog

Author: mikeburkat

ChangeLog

@@ -71,3 +71,17 @@
 	* rtbkit/fees: creatives can now include some fees that are applied to the
 				   final bid price
 	* rtbkit/openrtb: improved OpenRTB parsing by reducing memory allocations
+
+2016-05-30 Michael Burkat <michael@datacratic.com>
+
+	* jml-buikd: Node, Docker, tcmalloc, python related changes.
+	* jml: fix in least squares function.
+	* rtbkit/common/analytics: new plugin to abstract use of zmq or http for logging.
+	* rtbkit/core/router: use new analytics plugin.
+	* rtbkti/core/post_auction: use new analytics plugin.
+	* rtbkit/plugins/adserver: use new analytics plugin.
+	* soa/service: replace rtbkit-deps libcurl by the system provided curl.
+	* soa/service: fix bugs preventing compiling with gcc 4.8.
+	* soa: fix NODEJS_ENABLED flag to do the right thing.
+	* rtbkit: support for ubuntu 14.04
+	* rtbkit: Should update rtbkit-deps to latest with this update.

Makefile

@@ -48,12 +48,16 @@ CXX_VERSION?=$(shell g++ --version | head -n1 | sed 's/.* //g')
 
 CFLAGS += -fno-strict-overflow -msse4.2
 
+ifeq ($(NODEJS_ENABLED), 1)
 PKGINCLUDE_PACKAGES = sigc++-2.0 cairomm-1.0
+else
+PKGINCLUDE_PACKAGES = sigc++-2.0
+endif
 
 PKGCONFIG_INCLUDE:=$(shell pkg-config --cflags-only-I $(PKGINCLUDE_PACKAGES))
 
 
-CXXFLAGS += -Wno-deprecated -Winit-self -fno-omit-frame-pointer -std=c++0x -fno-deduce-init-list -I$(NODE_PREFIX)/include/node -msse3 -Ileveldb/include -Wno-unused-but-set-variable -I$(LOCAL_INCLUDE_DIR) -I$(GEN) $(PKGCONFIG_INCLUDE) -Wno-psabi -D__GXX_EXPERIMENTAL_CXX0X__=1
+CXXFLAGS += -Wno-deprecated -Winit-self -fno-omit-frame-pointer -std=c++0x -fno-deduce-init-list -I$(NODE_PREFIX)/include/node -msse3 -Ileveldb/include -Wno-unused-but-set-variable -I$(LOCAL_INCLUDE_DIR) -I$(GEN) $(PKGCONFIG_INCLUDE) -Wno-psabi -D__GXX_EXPERIMENTAL_CXX0X__=1 -Wno-unused-local-typedefs -Wno-unused-variable -DNODEJS_ENABLED=$(NODEJS_ENABLED)
 CXXLINKFLAGS += -Wl,--copy-dt-needed-entries -Wl,--no-as-needed -L/usr/local/lib
 CFLAGS +=  -Wno-unused-but-set-variable
 

jml-build/arch/x86_64.mk

@@ -3,7 +3,7 @@ CXXFLAGS ?= $(INCLUDE) -pipe -Wall -Werror -Wno-sign-compare -Woverloaded-virtua
 CXXLINKFLAGS = -rdynamic $(foreach DIR,$(PWD)/$(BIN) $(PWD)/$(LIB) $(LOCAL_LIB_DIR),-L$(DIR) -Wl,--rpath-link,$(DIR)) -Wl,--rpath,\$$ORIGIN/../bin -Wl,--rpath,\$$ORIGIN/../lib -Wl,--copy-dt-needed-entries -Wl,--no-as-needed
 CXXLIBRARYFLAGS = -shared $(CXXLINKFLAGS) -lpthread
 CXXEXEFLAGS =$(CXXLINKFLAGS) -lpthread
-CXXEXEPOSTFLAGS := $(if $(MEMORY_ALLOC_LIBRARY),-l$(MEMORY_ALLOC_LIBRARY))
+CXXEXEPOSTFLAGS = $(if $(MEMORY_ALLOC_LIBRARY),-l$(MEMORY_ALLOC_LIBRARY))
 CXXNODEBUGFLAGS := -O3 -DBOOST_DISABLE_ASSERTS -DNDEBUG 
 CXXDEBUGFLAGS := -O1 -g3 -UBOOST_DISABLE_ASSERTS -UNDEBUG
 

jml-build/docker.mk

@@ -56,7 +56,7 @@ DOCKER_GET_REVISION_SCRIPT?=$(JML_BUILD)/get_git_revision.sh
 
 # DOCKER_PUSH: if this is defined, then docker will be asked to push the
 # container to the repository after it has been successfully built.
-#
+# N.B. the tag "latest" will be applied to the image before pushing
 # This should be added to local.mk, not edited here.
 
 #DOCKER_PUSH:=1
@@ -70,6 +70,8 @@ DOCKER_GET_REVISION_SCRIPT?=$(JML_BUILD)/get_git_revision.sh
 # DOCKER_TAG: if this is defined, the given tag will also be applied to the
 # built image.  By default it's "latest" which is expected by most docker
 # tooling, but can be changed to something else or undefined if required.
+DOCKER_TAG:= $(shell whoami)_latest
+DOCKER_TAGS += $(DOCKER_TAG)
 
 # DOCKER_COMMIT_ARGS: Anything in this variable (which should be overridden
 # on a per-target basis) will be passed to docker commit as arguments.
@@ -79,13 +81,10 @@ DOCKER_GET_REVISION_SCRIPT?=$(JML_BUILD)/get_git_revision.sh
 # container after the container is created.  It can be used to modify the
 # container before it is committed.
 
-DOCKER_TAG:=latest
-
-
 
 # Docker target (generic).  If you make docker_target_name, it will make
 # target_name and install it inside a docker image.
-#
+# 
 # In order to determine the tag, by default this rule will call the
 # get_git_revision script that will return a revision ID from git to tag
 # the image with.  The script will also ensure that everything used in the
@@ -94,18 +93,67 @@ DOCKER_TAG:=latest
 
 #docker_%:	$(TMPBIN)/%.iid
 
-docker_%: % $(DOCKER_GLOBAL_DEPS) $(DOCKER_TARGET_DEPS)
-	@BUILD=$(BUILD) bash $(DOCKER_GET_REVISION_SCRIPT) $(<) > $(TMPBIN)/$(<).rid $(if $(DOCKER_ALLOW_DIRTY), || true,)
-	echo "revision" `cat $(TMPBIN)/$(<).rid`
-	@echo "Building $(<) for use within docker"
-	+make TMPBIN=$(TMPBIN) LIB=$(TMPBIN)/docker-$(<)/lib BIN=$(TMPBIN)/docker-$(<)/bin ETC=$(TMPBIN)/docker-$(<)/etc $(<)
+altroot_prep_%: % $(DOCKER_GLOBAL_DEPS) $(DOCKER_TARGET_DEPS)
+	@BUILD=$(BUILD) bash $(DOCKER_GET_REVISION_SCRIPT) $(*) > $(TMPBIN)/$(*).rid $(if $(DOCKER_ALLOW_DIRTY), || true,)
+	echo "revision" `cat $(TMPBIN)/$(*).rid`
+	@echo "Building $(*) for use within docker"
+	make TMPBIN=$(TMPBIN) LIB=$(TMPBIN)/docker-$(*)/opt/lib BIN=$(TMPBIN)/docker-$(*)/opt/bin ALTROOT=$(TMPBIN)/docker-$(*) $(*)
+
+docker_%: altroot_prep_%  $(DOCKER_GLOBAL_DEPS) $(DOCKER_TARGET_DEPS)
 	@echo "Creating container"
-	@rm -f $(TMPBIN)/$(<).cid
-	docker run -cidfile $(TMPBIN)/$(<).cid -v `pwd`:/tmp/build $(DOCKER_BASE_IMAGE) sh /tmp/build/$(JML_BUILD)/docker_install_inside_container.sh /tmp/build/$(TMPBIN)/docker-$(<) $(if $(DOCKER_POST_INSTALL_SCRIPT),/tmp/build/$(DOCKER_POST_INSTALL_SCRIPT))
-	cat $(TMPBIN)/$(<).cid
-	echo docker commit `cat $(TMPBIN)/$(<).cid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(<):`cat $(TMPBIN)/$(<).rid`
-	docker commit $(DOCKER_COMMIT_ARGS) `cat $(TMPBIN)/$(<).cid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(<):`cat $(TMPBIN)/$(<).rid` > $(TMPBIN)/$<.iid && cat $(TMPBIN)/$<.iid
-	$(if $(DOCKER_TAG),docker tag `cat $(TMPBIN)/$(<).iid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(<):$(DOCKER_TAG))
-	@docker rm `cat $(TMPBIN)/$(<).cid`
-	$(if $(DOCKER_PUSH),docker push $(DOCKER_REGISTRY)$(DOCKER_USER)$(<))
-	@echo $(COLOR_WHITE)Created $(if $(DOCKER_PUSH),and pushed )$(COLOR_BOLD)$(DOCKER_REGISTRY)$(DOCKER_USER)$(<):`cat $(TMPBIN)/$(<).rid`$(COLOR_RESET) as image $(COLOR_WHITE)$(COLOR_BOLD)`cat $(TMPBIN)/$<.iid`$(COLOR_RESET)
+	@rm -f $(TMPBIN)/$(*).cid
+	docker run --cidfile $(TMPBIN)/$(*).cid -v `pwd`:/tmp/build $(DOCKER_BASE_IMAGE) sh /tmp/build/$(JML_BUILD)/docker_install_inside_container.sh /tmp/build/$(TMPBIN)/docker-$(*) $(if $(DOCKER_POST_INSTALL_SCRIPT),/tmp/build/$(DOCKER_POST_INSTALL_SCRIPT))
+	cat $(TMPBIN)/$(*).cid
+	echo docker commit `cat $(TMPBIN)/$(*).cid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):`cat $(TMPBIN)/$(*).rid`
+	docker commit $(DOCKER_COMMIT_ARGS) `cat $(TMPBIN)/$(*).cid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):`cat $(TMPBIN)/$(*).rid` > $(TMPBIN)/$(*).iid && cat $(TMPBIN)/$(*).iid
+	$(foreach tag,$(DOCKER_TAGS),docker tag -f `cat $(TMPBIN)/$(*).iid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):$(tag);)
+	$(if $(DOCKER_PUSH),docker tag -f `cat $(TMPBIN)/$(*).iid` $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):latest)
+	@docker rm `cat $(TMPBIN)/$(*).cid`
+	#$(if $(DOCKER_PUSH),$(if $(DOCKER_TAG), docker push $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):$(DOCKER_TAG)))
+	$(if $(DOCKER_PUSH),docker push $(DOCKER_REGISTRY)$(DOCKER_USER)$(*):latest)
+	@echo $(COLOR_WHITE)Created $(if $(DOCKER_PUSH),and pushed )$(COLOR_BOLD)$(DOCKER_REGISTRY)$(DOCKER_USER)$(*):`cat $(TMPBIN)/$(*).rid`$(COLOR_RESET) as image $(COLOR_WHITE)$(COLOR_BOLD)`cat $(TMPBIN)/$(*).iid`$(COLOR_RESET)
+
+
+## Functions
+# install_templated_file
+#  1: source file
+#  2: destination file
+#  3: mode (444)
+#  4: make target to add the rule as a prerequisite
+#  5: variable definition file
+#  6: template filter  # must take a file as sole parameter and output
+                       # the result on stdout
+define install_templated_file
+$$(if $(trace),$$(warning called "$(0)" "$(1)" "$(2)" "$(3)" "$(4)" "$(5)" "$(6)))
+$(eval mode := $(if $(3),-m $(3),-m 0444))
+$(eval target_add_prereq := $(4))
+$(eval var_file := $(if $(5),$(5)))
+$(eval new_target := $(0)-$(1)-$(2))
+
+$(new_target): $(1) $(var_file)
+	install -d `dirname $(2)`
+	$(6) $(1) >$(2)~
+	install $(mode) $(2)~ $(2)
+	rm $(2)~
+
+$(target_add_prereq): $(new_target)
+.PHONY: $(new_target)
+endef
+
+# install_directory
+#  $(1): source dir
+#  $(2): destination dir
+#  $(3): make target to add the rule as a prerequisite
+define install_directory
+$$(if $(trace),$$(warning called "$(0)" "$(1)" "$(2)" "$(3)"))
+$(eval target_add_prereq := $(3))
+$(eval new_target := $(0)-$(1)-$(2))
+
+$(new_target):
+	mkdir -p $(2)
+	cp -vR $(1) $(2)
+
+$(target_add_prereq): $(new_target)
+.PHONY: $(new_target)
+endef
+

jml-build/docker_install_inside_container.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
 
+set -e
+set -x
+
 echo "installing from " $1
 
-mkdir -p /opt
-cp -r $1/* /opt
-cp -r $1/etc/* /etc
+cd $1
+cp -av * /
 
-$2
+shift
+$*

jml-build/functions.mk

@@ -306,12 +306,25 @@ $$(LIB_$(1)_SO):	$$(dir $$(LIB_$(1)_SO))/.dir_exists $$(OBJFILES_$(1)) $$(foreac
 	$$(if $(verbose_build),@echo $$(LINK_$(1)_COMMAND2),@echo $$(LIB_$(1)_BUILD_NAME) $$(LIB_$(1)_FILENAME))
 	@$$(LINK_$(1)_COMMAND2)
 
+$$(tmpLIBNAME): $(LIB)/$$(tmpLIBNAME)$$(so)
+.PHONY:	$$(tmpLIBNAME)
+
 LIB_$(1)_DEPS := $(LIB)/$$(tmpLIBNAME)$$(so)
 
 libraries: $(LIB)/$$(tmpLIBNAME)$$(so)
 endif
 endef
 
+# add a forward dependency to a library, ie it depends upon a library defined further forwards
+# in the makefile.
+# $(1): name of the library
+# $(2): name of all libraries it has a forward dependency on
+define library_forward_dependency
+ifneq ($(PREMAKE),1)
+$$(if $$(LIB_$(1)_SO),,$$(error unknown library $(1) for forward dep))
+$$(LIB_$(1)_SO): $$(foreach lib,$(2), $(LIB)/lib$$(lib)$$(so))
+endif
+endef
 
 # add a program
 # $(1): name of the program
@@ -334,7 +347,7 @@ $(1)_OBJFILES:=$$(foreach file,$$(addsuffix .lo,$$(basename $$($(1)_PROGFILES:%=
 #$$(warning $(1)_OBJFILES = $$($(1)_OBJFILES))
 #$$(warning $(1)_PROGFILES = "$$($(1)_PROGFILES)")
 
-LINK_$(1)_COMMAND:=$$(CXX) $$(CXXFLAGS) $$(CXXEXEFLAGS) $$(CXXNODEBUGFLAGS) -o $(BIN)/$(1) -lexception_hook $(MALLOC_LIBRARY) -L$(LIB) -ldl $$($(1)_OBJFILES) $$(foreach lib,$(2), -l$$(lib)) $$(CXXEXEPOSTFLAGS)
+LINK_$(1)_COMMAND:=$$(CXX) $$(CXXFLAGS) $$(CXXEXEFLAGS) $$(CXXNODEBUGFLAGS) -o $(BIN)/$(1) -lexception_hook -L$(LIB) -ldl $$($(1)_OBJFILES) $$(foreach lib,$(2), -l$$(lib)) $$(CXXEXEPOSTFLAGS)
 
 
 $(BIN)/$(1):	$(BIN)/.dir_exists $$($(1)_OBJFILES) $$(foreach lib,$(2),$$(LIB_$$(lib)_DEPS)) $$(if $$(HAS_EXCEPTION_HOOK),$$(LIB)/libexception_hook.so)
@@ -381,7 +394,7 @@ $$(eval $$(call add_sources,$(1).cc))
 
 $(1)_OBJFILES:=$$(BUILD_$(CWD)/$(1).lo_OBJ)
 
-LINK_$(1)_COMMAND:=$$(CXX) $$(CXXFLAGS) $$(CXXEXEFLAGS) $$(CXXNODEBUGFLAGS) -o $(TESTS)/$(1) -lexception_hook $(MALLOC_LIBRARY) -ldl  $$($(1)_OBJFILES) $$(foreach lib,$(2), -l$$(lib)) $(if $(findstring boost,$(3)), -lboost_unit_test_framework) $$(CXXEXEPOSTFLAGS)
+LINK_$(1)_COMMAND:=$$(CXX) $$(CXXFLAGS) $$(CXXEXEFLAGS) $$(CXXNODEBUGFLAGS) -o $(TESTS)/$(1) -lexception_hook -ldl  $$($(1)_OBJFILES) $$(foreach lib,$(2), -l$$(lib)) $(if $(findstring boost,$(3)), -lboost_unit_test_framework) $$(CXXEXEPOSTFLAGS)
 
 $(TESTS)/$(1):	$(TESTS)/.dir_exists $(TEST_TMP)/.dir_exists  $$($(1)_OBJFILES) $$(foreach lib,$(2),$$(LIB_$$(lib)_DEPS)) $$(if $$(HAS_EXCEPTION_HOOK),$$(LIB)/libexception_hook.so)
 	$$(if $(verbose_build),@echo $$(LINK_$(1)_COMMAND),@echo "           $(COLOR_BLUE)[BIN]$(COLOR_RESET) $(1)")

jml-build/get_python_requirements.sh

@@ -42,6 +42,9 @@ if [[ ! -f "$req_file" ]]; then
     exit 1
 fi
 
+# fail on first error
+set -e
+
 # Remove commented and empty lines & loop through the requirements
 grep -vE '^#|^$' $req_file | while read line
 do

jml-build/node.mk

@@ -25,7 +25,11 @@ nodejs_dependencies: package.json
 
 dependencies: nodejs_dependencies
 
-$(BIN)/node_runner: $(BIN)/.dir_exists
+environment_ld_preload_hash := $(shell echo $(NODE_PRELOAD) $(NODE_PATH) $(NODE) | md5sum | cut -f 1 -d ' ' )
+$(TMP)/environment_$(environment_ld_preload_hash): $(TMP)/.dir_exists
+	@touch $@
+
+$(BIN)/node_runner: $(BIN)/.dir_exists $(TMP)/environment_$(environment_ld_preload_hash)
 	@echo '#!/bin/bash' > $@~
 	@echo 'exec /usr/bin/env NODE_PATH=$(NODE_PATH) $(NODE_PRELOAD) $(NODE) "$$@"' >> $@~
 	@chmod +x $@~

jml-build/python.mk

@@ -7,6 +7,8 @@ PYTHON_INCLUDE_PATH ?= $(VIRTUALENV)/include/python$(PYTHON_VERSION)
 PYTHON ?= python$(PYTHON_VERSION)
 PIP ?= pip
 PYFLAKES ?= true
+# Override this to run a cmd before installing python_requirements.txt
+PYTHON_DEPENDENCIES_PRE_CMD ?= true  
 
 PYTHON_PURE_LIB_PATH ?= $(BIN)
 PYTHON_PLAT_LIB_PATH ?= $(BIN)
@@ -23,12 +25,13 @@ $(VIRTUALENV)/bin/activate:
 
 python_dependencies: $(VIRTUALENV)/bin/activate
 
-PYTHON_EXECUTABLE ?= $(VIRTUALENV)/bin/python
+PYTHON_EXECUTABLE ?= $(PYTHON) $(PYTHON_ARGS)
 
 endif
 
 python_dependencies:
-	@if [ -f python_requirements.txt ]; then \
+	if [ -f python_requirements.txt ]; then \
+		$(PYTHON_DEPENDENCIES_PRE_CMD); \
 		$(PIP) install -r python_requirements.txt; \
 	fi
 
@@ -83,7 +86,7 @@ define python_test
 ifneq ($(PREMAKE),1)
 $$(if $(trace),$$(warning called python_test "$(1)" "$(2)" "$(3)" "$(4)"))
 
-TEST_$(1)_COMMAND := rm -f $(TESTS)/$(1).{passed,failed} && $(PYFLAKES) $(CWD)/$(1).py && ((set -o pipefail && PYTHONPATH=$(RUN_PYTHONPATH) $(PYTHON) $(CWD)/$(1).py > $(TESTS)/$(1).running 2>&1 && mv $(TESTS)/$(1).running $(TESTS)/$(1).passed) || (mv $(TESTS)/$(1).running $(TESTS)/$(1).failed && echo "                 $(COLOR_RED)$(1) FAILED$(COLOR_RESET)" && cat $(TESTS)/$(1).failed && false))
+TEST_$(1)_COMMAND := rm -f $(TESTS)/$(1).{passed,failed} && $(PYFLAKES) $(CWD)/$(1).py && ((set -o pipefail && PYTHONPATH=$(RUN_PYTHONPATH) $(PYTHON) $(PYTHON_ARGS) $(CWD)/$(1).py > $(TESTS)/$(1).running 2>&1 && mv $(TESTS)/$(1).running $(TESTS)/$(1).passed) || (mv $(TESTS)/$(1).running $(TESTS)/$(1).failed && echo "                 $(COLOR_RED)$(1) FAILED$(COLOR_RESET)" && cat $(TESTS)/$(1).failed && false))
 
 $(TESTS)/$(1).passed:	$(TESTS)/.dir_exists $(CWD)/$(1).py $$(foreach lib,$(2),$$(PYTHON_$$(lib)_DEPS)) $$(foreach pymod,$(2),$(TMPBIN)/$$(pymod)_pymod)
 	$$(if $(verbose_build),@echo '$$(TEST_$(1)_COMMAND)',@echo "      $(COLOR_VIOLET)[TESTCASE]$(COLOR_RESET) $(1)")
@@ -92,7 +95,7 @@ $(TESTS)/$(1).passed:	$(TESTS)/.dir_exists $(CWD)/$(1).py $$(foreach lib,$(2),$$
 
 $(1):	$(CWD)/$(1).py $$(foreach lib,$(2),$$(PYTHON_$$(lib)_DEPS)) $$(foreach pymod,$(2),$(TMPBIN)/$$(pymod)_pymod)
 	@$(PYFLAKES) $(CWD)/$(1).py
-	PYTHONPATH=$(RUN_PYTHONPATH) $(PYTHON) $(CWD)/$(1).py $($(1)_ARGS)
+	PYTHONPATH=$(RUN_PYTHONPATH) $(PYTHON) $(PYTHON_ARGS) $(CWD)/$(1).py $($(1)_ARGS)
 
 .PHONY: $(1)
 
@@ -159,7 +162,7 @@ PYTHON_$(1)_DEPS := $(PYTHON_BIN_PATH)/$(1) $$(foreach pymod,$(3),$$(PYTHON_$$(p
 .PHONY: run_$(1)
 
 run_$(1):	$(PYTHON_BIN_PATH)/$(1)
-	$(PYTHON) $(PYTHON_BIN_PATH)/$(1)  $($(1)_ARGS)
+	$(PYTHON) $(PYTHON_ARGS) $(PYTHON_BIN_PATH)/$(1)  $($(1)_ARGS)
 
 $(PYTHON_BIN_PATH)/$(1): $(CWD)/$(2) $(PYTHON_BIN_PATH)/.dir_exists $$(foreach pymod,$(3),$(TMPBIN)/$$(pymod)_pymod) $$(foreach pymod,$(3),$$(PYTHON_$$(pymod)_DEPS))
 	@echo "$(COLOR_BLUE)[PYTHON_PROGRAM]$(COLOR_RESET) $(1)"

jml-build/tcmalloc.mk

@@ -1,6 +1,6 @@
 ifeq ($(TCMALLOC_ENABLED),1)
 
-MALLOC_LIBRARY?=-ltcmalloc
+MEMORY_ALLOC_LIBRARY?=tcmalloc
 CXXFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
 CFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free