Issue151 benchmark random

Makefile

@@ -101,6 +101,7 @@ install: $(TARGET)
 	mkdir -m 0755 -p $(PREFIX)/share/flit/data/db
 	mkdir -m 0755 -p $(PREFIX)/share/flit/config
 	mkdir -m 0755 -p $(PREFIX)/share/flit/litmus-tests
+	mkdir -m 0755 -p $(PREFIX)/share/flit/benchmarks
 	mkdir -m 0755 -p $(PREFIX)/share/licenses/flit
 	ln -sf ../share/flit/scripts/flit.py $(PREFIX)/bin/flit
 	install -m 0755 $(TARGET) $(PREFIX)/lib/$(notdir $(TARGET))
@@ -120,6 +121,7 @@ install: $(TARGET)
 	install -m 0644 $(CONFIG_DIR)/flit-default.toml.in $(PREFIX)/share/flit/config/
 	install -m 0644 $(LITMUS_TESTS) $(PREFIX)/share/flit/litmus-tests/
 	install -m 0644 LICENSE $(PREFIX)/share/licenses/flit/
+	cp -r benchmarks/* $(PREFIX)/share/flit/benchmarks/
 	@echo "Generating $(INSTALL_FLIT_CONFIG)"
 	@# Make the flitconfig.py script specifying this installation information
 	@echo "'''"                                                                  > $(INSTALL_FLIT_CONFIG)

README.md

@@ -52,6 +52,7 @@ Contents:
   * [Available Compiler Flags](documentation/available-compiler-flags.md)
 * [Writing Test Cases](documentation/writing-test-cases.md)
 * [Test Executable](documentation/test-executable.md)
+* [Benchmarks](documentation/benchmarks.md)
 * [Database Structure](documentation/database-structure.md)
 * [Analyze Results](documentation/analyze-results.md)
 * **Extra Tools**

benchmarks/README.md

@@ -0,0 +1,14 @@
+# Benchmarks
+
+[Top-Level Documentation](../README.md)
+
+These benchmarks give insights into the usefulness of FLiT.  Each benchmark has
+its own goals, either to demonstrate how FLiT can be useful, or to give
+insights into specific tools and functionalities.
+
+These benchmarks are also part of the installation, and can be copied from
+there to exercise FLiT's capabilities.  They are installed in
+
+`$(PREFIX)/share/flit/benchmarks`
+
+[Top-Level Documentation](../README.md)

benchmarks/random/README.md

@@ -0,0 +1,11 @@
+# Random Benchmark
+
+This benchmark is to test that the C++ `<random>` header is portable accross
+compilers.  My guess is that they will be since I suspect they are implemented
+in the GLibC, but I'm not sure.  Hence the tests.  This can give us confidence
+that this is indeed safe to depend on in FLiT tests.
+
+We also test the `rand()` and `srand()` functions that are part of the standard
+`<cstdlib>` header file.  However, we do not test the non-portable `random()`,
+`srandom()`, `initstate()`, and `setstate()` functions that are part of the
+POSIX standard, but are not part of the C++ standard.

benchmarks/random/custom.mk

@@ -0,0 +1,27 @@
+# This file is included at the end of the copied Makefile.  If you have some
+# things you want to change about the Makefile, it is best to do it here.
+
+# additional source files to compile other than what is in '.' and 'tests/'
+# since those directories are added by a wildcard.
+SOURCE         +=
+
+# required compiler flags
+# for example, include directories
+#   CC_REQUIRED += -I<path>
+# or defines
+#   CC_REQUIRED += -DDEBUG_ENABLED=1
+CC_REQUIRED    +=
+
+# required linker flags
+# for example, link libraries
+#   LD_REQUIRED += -L<library-path> -l<library-name>
+# or rpath
+#   LD_REQUIRED += -Wl,-rpath=<abs-path-to-library-dir>
+LD_REQUIRED    +=
+
+# compiler and linker flags respectively - specifically for a dev build
+# - DEV_CFLAGS:   non-recorded compiler flags (such as includes)
+# - DEV_LDFLAGS:  linker flags (also not under test)
+DEV_CFLAGS     +=
+DEV_LDFLAGS    +=
+

benchmarks/random/flit-config.toml

@@ -0,0 +1,47 @@
+# Autogenerated by "flit init"
+#   flit version v2.0-alpha.3
+
+[database]
+
+# older versions of flit supported postgres.  that has been removed.  only
+# sqlite is supported at the moment.
+type = 'sqlite'
+
+# if relative path, it is relative to the directory containing this
+# configuration file.
+filepath = 'results.sqlite'
+
+# For now, only one host is supported, all others are ignored
+[[hosts]]
+
+name = 'yoga-manjaro'
+flit_path = '/home/bentley/git/FLiT/scripts/flitcli/flit.py'
+config_dir = '/home/bentley/git/FLiT/scripts/flitcli/config'
+
+# The settings for "make dev"
+[hosts.dev_build]
+# compiler_name must be found in [[hosts.compilers]] list under name attribute
+# but the optimization level and switches do not need to be in the compiler list
+compiler_name = 'g++'
+optimization_level = '-O2'
+switches = '-funsafe-math-optimizations'
+
+# The ground truth compilation to use in analysis, for "make gt"
+[hosts.ground_truth]
+# compiler_name must be found in [[hosts.compilers]] list under name attribute
+# but the optimization level and switches do not need to be in the compiler list
+compiler_name = 'g++'
+optimization_level = '-O0'
+switches = ''
+
+  # This host's list of compilers.
+  # For now, only used for hosts.ground_truth and hosts.dev_build.
+  # TODO: use this list to generate the Makefile
+  [[hosts.compilers]]
+
+  # binary can be an absolute path, relative path, or binary name (found in
+  # PATH).  If you want to specify a compiler in the same directory as this
+  # config file, prepend with a "./" (e.g. "./my-compiler")
+  binary = 'g++'
+  name = 'g++'
+

benchmarks/random/main.cpp

@@ -0,0 +1,5 @@
+#include "flit.h"
+
+int main(int argCount, char* argList[]) {
+  return flit::runFlitTests(argCount, argList);
+}

benchmarks/random/tests/Rand.cpp

@@ -0,0 +1,114 @@
+/* -- LICENSE BEGIN --
+ *
+ * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC.
+ *
+ * Produced at the Lawrence Livermore National Laboratory
+ *
+ * Written by
+ *   Michael Bentley (mikebentley15@gmail.com),
+ *   Geof Sawaya (fredricflinstone@gmail.com),
+ *   and Ian Briggs (ian.briggs@utah.edu)
+ * under the direction of
+ *   Ganesh Gopalakrishnan
+ *   and Dong H. Ahn.
+ *
+ * LLNL-CODE-743137
+ *
+ * All rights reserved.
+ *
+ * This file is part of FLiT. For details, see
+ *   https://pruners.github.io/flit
+ * Please also read
+ *   https://github.com/PRUNERS/FLiT/blob/master/LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the disclaimer below.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the disclaimer
+ *   (as noted below) in the documentation and/or other materials
+ *   provided with the distribution.
+ *
+ * - Neither the name of the LLNS/LLNL nor the names of its
+ *   contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL
+ * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Additional BSD Notice
+ *
+ * 1. This notice is required to be provided under our contract
+ *    with the U.S. Department of Energy (DOE). This work was
+ *    produced at Lawrence Livermore National Laboratory under
+ *    Contract No. DE-AC52-07NA27344 with the DOE.
+ *
+ * 2. Neither the United States Government nor Lawrence Livermore
+ *    National Security, LLC nor any of their employees, makes any
+ *    warranty, express or implied, or assumes any liability or
+ *    responsibility for the accuracy, completeness, or usefulness of
+ *    any information, apparatus, product, or process disclosed, or
+ *    represents that its use would not infringe privately-owned
+ *    rights.
+ *
+ * 3. Also, reference herein to any specific commercial products,
+ *    process, or services by trade name, trademark, manufacturer or
+ *    otherwise does not necessarily constitute or imply its
+ *    endorsement, recommendation, or favoring by the United States
+ *    Government or Lawrence Livermore National Security, LLC. The
+ *    views and opinions of authors expressed herein do not
+ *    necessarily state or reflect those of the United States
+ *    Government or Lawrence Livermore National Security, LLC, and
+ *    shall not be used for advertising or product endorsement
+ *    purposes.
+ *
+ * -- LICENSE END --
+ */
+
+#include <flit.h>
+
+#include <string>
+#include <vector>
+
+#include <cstdlib>
+
+template <typename T>
+class Rand : public flit::TestBase<T> {
+public:
+  Rand(std::string id) : flit::TestBase<T>(std::move(id)) {}
+  virtual size_t getInputsPerRun() override { return 1; }
+  virtual std::vector<T> getDefaultInput() override { return { 0, 42, 24, 12, 103 }; }
+protected:
+  virtual flit::Variant run_impl(const std::vector<T> &ti) override {
+    FLIT_UNUSED(ti);
+    return {};
+  }
+protected:
+  using flit::TestBase<T>::id;
+};
+
+// Only implement the test for double precision
+template <>
+flit::Variant Rand<double>::run_impl(const std::vector<double> &ti) {
+  size_t seed = ti[0];
+  srand(seed);
+  return rand();
+}
+
+REGISTER_TYPE(Rand);