Merge pull request #145 from basho/ku-node-package-rebased

Node-package integration to build debs and rpms

.gitignore

@@ -1,7 +1,10 @@
 ebin/
 deps/
 tests/
-basho_bench
+/basho_bench
+/rel/basho_bench
+package
+.rebar
 *~
 #*#
 

Makefile

@@ -1,8 +1,23 @@
-.PHONY: deps
+REPO            ?= basho_bench
+
+PKG_REVISION    ?= $(shell git describe --tags)
+PKG_VERSION     ?= $(shell git describe --tags | tr - .)
+PKG_ID           = basho-bench-$(PKG_VERSION)
+PKG_BUILD        = 1
+BASE_DIR         = $(shell pwd)
+ERLANG_BIN       = $(shell dirname $(shell which erl))
+REBAR           ?= $(BASE_DIR)/rebar
+OVERLAY_VARS    ?=
+
 
 all: deps compile
 	./rebar skip_deps=true escriptize
 
+.PHONY: deps compile rel
+
+rel: deps compile
+	cd rel && ../rebar generate skip_deps=true $(OVERLAY_VARS)
+
 deps:
 	./rebar get-deps
 
@@ -33,3 +48,31 @@ JOBS := $(addprefix job,${TARGETS})
 
 all_results: ${JOBS} ; echo "$@ successfully generated."
 ${JOBS}: job%: ; Rscript --vanilla priv/summary.r -i tests/$*
+
+##
+## Packaging targets
+##
+.PHONY: package
+export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE
+
+package.src: deps
+	mkdir -p package
+	rm -rf package/$(PKG_ID)
+	git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -)
+	${MAKE} -C package/$(PKG_ID) deps
+	for dep in package/$(PKG_ID)/deps/*; do \
+	    echo "Processing dep: $${dep}"; \
+	    mkdir -p $${dep}/priv; \
+	    git --git-dir=$${dep}/.git describe --always --tags >$${dep}/priv/vsn.git; \
+        done
+	find package/$(PKG_ID) -depth -name ".git" -exec rm -rf {} \;
+	tar -C package -czf package/$(PKG_ID).tar.gz $(PKG_ID)
+
+dist: package.src
+	cp package/$(PKG_ID).tar.gz .
+
+package: package.src
+	${MAKE} -C package -f $(PKG_ID)/deps/node_package/Makefile
+
+pkgclean: distclean
+	rm -rf package

pkg.vars.config

@@ -0,0 +1,24 @@
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+%%
+%% Packaging
+%%
+{package_name, "basho-bench"}.
+{package_install_name, "basho_bench"}.
+{package_install_user, "basho-bench"}.
+{package_install_group, "basho-bench"}.
+{package_install_user_desc, "Basho-bench user"}.
+{package_shortdesc, "Basho benchmarking tool"}.
+{package_desc, "Benchmarking tool"}.
+{package_commands, {list, [[{name, "basho_bench"}]]}}.
+{package_patch_dir, "basho-patches"}.
+{bin_or_sbin, "bin"}.
+{license_type, "OSS"}.
+{copyright, "2014 Basho Technologies, Inc"}.
+{vendor_name, "Basho Technologies, Inc"}.
+{vendor_url, "http://basho.com"}.
+{vendor_contact_name, "Basho Package Maintainer"}.
+{vendor_contact_email, "packaging@basho.com"}.
+{license_full_text, "This software is provided under license from Basho Technologies."}.
+{solaris_pkgname, "BASHObasho-bench"}.

rebar.config

@@ -1,3 +1,4 @@
+{require_otp_vsn, "R15|R16|17"}.
 
 {deps,
  [
@@ -8,32 +9,30 @@
   %%       increments.  If someone wants to take advantage of a
   %%       new folsom feature, that desire + float incr must be
   %%       weighed.
+  {node_package, "2.0.*", {git, "git://github.com/basho/node_package", {tag, "2.0.0"}}},
   {folsom, ".*", {git, "git://github.com/basho/folsom.git", {branch, "boundary-0.7.1+basho-bench-float"}}},
-
-  {lager, "2.*", {git, "git://github.com/basho/lager", {tag, "2.0.3"}}},
+  {lager, "2.*", {git, "git://github.com/basho/lager", {tag, "2.1.0"}}},
   {ibrowse, ".*",
    {git, "git://github.com/cmullaparthi/ibrowse.git", {tag, "v3.0.4"}}},
+  {riakc, ".*",
+   {git, "git://github.com/basho/riak-erlang-client", {branch, "develop"}}},
+  {mochiweb, "2.9.*",
+   {git, "git://github.com/basho/mochiweb", {tag, "v2.9.0"}}},
+  {getopt, ".*",
+   {git, "git://github.com/jcomellas/getopt", {tag, "v0.8.2"}}},
+
   {casbench, "0.1",
    {git, "git://github.com/basho/casbench",
     "95ed55b494551577870984aeb1e0f683631a326f"}},
   {erlcql, ".*",
    {git, "git://github.com/rpt/erlcql.git",
-   {branch, "master"}}},
-  {riakc, ".*",
-   {git, "git://github.com/basho/riak-erlang-client", {branch, "develop"}}},
-  {mochiweb, "1.5.1*",
-   {git, "git://github.com/basho/mochiweb", {tag, "1.5.1p6"}}},
-  {velvet, "1.*",
-   {git, "git://github.com/basho/velvet",
-    "4bb0fd664ff065c4082ca8dd2e0683e920537d15"}},
-  {getopt, ".*",
-   {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}
+   {branch, "master"}}}
  ]}.
 
 {erl_opts, [{src_dirs, [src]},
            {parse_transform, lager_transform}]}.
 
-{escript_incl_apps, [lager, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, velvet, goldrush]}.
+{escript_incl_apps, [node_package, lager, getopt, bear, folsom, ibrowse, riakc, mochiweb, protobuffs, goldrush]}.
 
 {escript_emu_args, "%%! +K true -rsh ssh\n"}.
 %% Use this for the Java client bench driver

rel/files/basho_bench

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Pull environment for this install
+. "{{runner_base_dir}}/lib/env.sh"
+
+# Make sure CWD is set to runner run dir
+cd $RUNNER_BASE_DIR/lib/basho_bench*/ebin
+
+ERL_LIBS=$RUNNER_BASE_DIR $ERTS_PATH/escript basho_bench.beam "$@"

rel/files/install_upgrade.escript

@@ -0,0 +1,44 @@
+#!/usr/bin/env escript
+%%! -noshell -noinput
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ft=erlang ts=4 sw=4 et
+
+-define(TIMEOUT, 60000).
+-define(INFO(Fmt,Args), io:format(Fmt,Args)).
+
+main([NodeName, Cookie, ReleasePackage]) ->
+    TargetNode = start_distribution(NodeName, Cookie),
+    {ok, Vsn} = rpc:call(TargetNode, release_handler, unpack_release,
+                         [ReleasePackage], ?TIMEOUT),
+    ?INFO("Unpacked Release ~p~n", [Vsn]),
+    {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler,
+                                    check_install_release, [Vsn], ?TIMEOUT),
+    {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler,
+                                    install_release, [Vsn], ?TIMEOUT),
+    ?INFO("Installed Release ~p~n", [Vsn]),
+    ok = rpc:call(TargetNode, release_handler, make_permanent, [Vsn], ?TIMEOUT),
+    ?INFO("Made Release ~p Permanent~n", [Vsn]);
+main(_) ->
+    init:stop(1).
+
+start_distribution(NodeName, Cookie) ->
+    MyNode = make_script_node(NodeName),
+    {ok, _Pid} = net_kernel:start([MyNode, shortnames]),
+    erlang:set_cookie(node(), list_to_atom(Cookie)),
+    TargetNode = make_target_node(NodeName),
+    case {net_kernel:hidden_connect_node(TargetNode),
+          net_adm:ping(TargetNode)} of
+        {true, pong} ->
+            ok;
+        {_, pang} ->
+            io:format("Node ~p not responding to pings.\n", [TargetNode]),
+            init:stop(1)
+    end,
+    TargetNode.
+
+make_target_node(Node) ->
+    [_, Host] = string:tokens(atom_to_list(node()), "@"),
+    list_to_atom(lists:concat([Node, "@", Host])).
+
+make_script_node(Node) ->
+    list_to_atom(lists:concat([Node, "_upgrader_", os:getpid()])).

rel/files/vm.args

@@ -0,0 +1,19 @@
+## Name of the node
+-name healthb@127.0.0.1
+
+## Cookie for distributed erlang
+-setcookie healthb
+
+## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
+## (Disabled by default..use with caution!)
+##-heart
+
+## Enable kernel poll and a few async threads
+##+K true
+##+A 5
+
+## Increase number of concurrent ports/sockets
+##-env ERL_MAX_PORTS 4096
+
+## Tweak GC to run more often
+##-env ERL_FULLSWEEP_AFTER 10

rel/reltool.config

@@ -0,0 +1,63 @@
+%% -*- mode: erlang -*-
+%% ex: ft=erlang
+{sys, [
+       {lib_dirs, ["../deps"]},
+       {erts, [{mod_cond, derived}, {app_file, strip}]},
+       {app_file, strip},
+       {rel, "basho_bench", "0.10.0",
+        [
+         kernel,
+         stdlib,
+	 bear,
+	 lager,
+	 folsom,
+	 goldrush,
+	 riakc,
+	 ibrowse,
+	 mochiweb
+        ]},
+       {rel, "start_clean", "",
+        [
+         kernel,
+         stdlib
+        ]},
+       {boot_rel, "basho_bench"},
+       {profile, embedded},
+       {incl_cond, derived},
+       {excl_archive_filters, [".*"]}, %% Do not archive built libs
+       {excl_sys_filters, ["^bin/(?!start_clean.boot)",
+                           "^erts.*/bin/(dialyzer|typer)",
+                           "^erts.*/(doc|info|include|lib|man|src)"]},
+       {excl_app_filters, ["\.gitignore"]},
+       {app, basho_bench, [{mod_cond, app}, {incl_cond, include}, {lib_dir, ".."}]},
+       {app, hipe, [{incl_cond, exclude}]}
+      ]}.
+
+{target_dir, "basho_bench"}.
+{overlay_vars, "vars.config"}.
+
+{overlay, [
+           {template, "../deps/node_package/priv/base/env.sh",
+                       "lib/env.sh"},
+           {mkdir, "data/b_b"},
+
+           %% Copy base files for starting and interacting w/ node
+           {copy, "../deps/node_package/priv/base/erl",
+                  "{{erts_vsn}}/bin/erl"},
+           {copy, "../deps/node_package/priv/base/nodetool",
+                  "{{erts_vsn}}/bin/nodetool"},
+           {template, "../deps/node_package/priv/base/env.sh",
+                  "lib/env.sh"},
+           {copy, "files/vm.args", "etc/vm.args"},
+
+           {template, "files/basho_bench", "bin/basho_bench"},
+
+	   {copy, "../examples/cs.config.sample", "etc/cs.config"},
+	   {copy, "../examples/riakc_pb.config",  "etc/riakc_pb.config"},
+	   {copy, "../examples/httpraw.config",   "etc/httpraw.config"},
+	   {copy, "../examples/http.config",      "etc/http.config"},
+	   {copy, "../examples/null_test.config", "etc/null_test.config"}
+
+           %%{copy, "files/install_upgrade.escript", "bin/install_upgrade.escript"},
+
+          ]}.

rel/vars.config

@@ -0,0 +1,20 @@
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+
+%% Platform-specific installation paths
+{platform_bin_dir,  "./bin"}.
+{platform_data_dir, "./data"}.
+{platform_etc_dir,  "./etc"}.
+{platform_lib_dir,  "./lib"}.
+{platform_log_dir,  "./log"}.
+
+%%
+{data_dir,    "{{target_dir}}/data"}.
+{runner_script_dir,  "\`cd \\`dirname $0\\` && /bin/pwd\`"}.
+{runner_base_dir,    "{{runner_script_dir}}/.."}.
+{runner_etc_dir,     "$RUNNER_BASE_DIR/etc"}.
+{runner_log_dir,     "$RUNNER_BASE_DIR/log"}.
+{runner_lib_dir,     "$RUNNER_BASE_DIR/lib"}.
+{runner_patch_dir,   "$RUNNER_BASE_DIR/lib/basho-patches"}.
+{pipe_dir,           "/tmp/$RUNNER_BASE_DIR/"}.
+{runner_user,        ""}.

src/basho_bench.app.src

@@ -1,6 +1,6 @@
 {application, basho_bench,
  [{description, "Riak Benchmarking Suite"},
-  {vsn, "0.9"},
+  {vsn, git},
   {modules, []},
   {registered, [ basho_bench_sup ]},
   {applications, [kernel,

src/basho_bench.erl

@@ -307,7 +307,7 @@ setup_distributed_work() ->
     [pool:attach(SlaveName) || SlaveName <- SlaveNames],
     CodePaths = code:get_path(),
     rpc:multicall(SlaveNames, code, set_path, [CodePaths]),
-    Apps = [lager, basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, velvet, goldrush],
+    Apps = [lager, basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, goldrush],
     [distribute_app(App) || App <- Apps].
 
 

src/basho_bench_driver_cs.erl

@@ -569,11 +569,15 @@ auth_sig(AccessKey, SecretKey, Method, ContentType, Date, Headers, Resource) ->
                     CanonizedAmzHeaders,
                     Resource
                    ],
-    Signature = base64:encode(stanchion_utils:sha_mac(SecretKey, StringToSign)),
+    Signature = base64:encode(sha_mac(SecretKey, StringToSign)),
     ["AWS ", AccessKey, $:, Signature].
 
 %% CS utilities
 
+%% @doc copied from stanchion_utils
+sha_mac(KeyData, STS) ->
+    crypto:hmac(sha, KeyData, STS).
+
 setup_user_and_bucket(State) ->
     case basho_bench_config:get(cs_access_key, undefined) of
         undefined ->