Merge pull request #154 from PRUNERS/issue151-benchmark-random

Issue151 benchmark random

Fixes #151

Makefile

@@ -101,6 +101,7 @@ install: $(TARGET)
  mkdir -m 0755 -p $(PREFIX)/share/flit/data/db
  mkdir -m 0755 -p $(PREFIX)/share/flit/config
  mkdir -m 0755 -p $(PREFIX)/share/flit/litmus-tests
+ mkdir -m 0755 -p $(PREFIX)/share/flit/benchmarks
  mkdir -m 0755 -p $(PREFIX)/share/licenses/flit
  ln -sf ../share/flit/scripts/flit.py $(PREFIX)/bin/flit
  install -m 0755 $(TARGET) $(PREFIX)/lib/$(notdir $(TARGET))
@@ -120,6 +121,7 @@ install: $(TARGET)
  install -m 0644 $(CONFIG_DIR)/flit-default.toml.in $(PREFIX)/share/flit/config/
  install -m 0644 $(LITMUS_TESTS) $(PREFIX)/share/flit/litmus-tests/
  install -m 0644 LICENSE $(PREFIX)/share/licenses/flit/
+ cp -r benchmarks/* $(PREFIX)/share/flit/benchmarks/
  @echo "Generating $(INSTALL_FLIT_CONFIG)"
  @# Make the flitconfig.py script specifying this installation information
  @echo "'''" > $(INSTALL_FLIT_CONFIG)

README.md

@@ -58,6 +58,7 @@ Contents:
  * [Available Compiler Flags](documentation/available-compiler-flags.md)
 * [Writing Test Cases](documentation/writing-test-cases.md)
 * [Test Executable](documentation/test-executable.md)
+* [Benchmarks](documentation/benchmarks.md)
 * [Database Structure](documentation/database-structure.md)
 * [Analyze Results](documentation/analyze-results.md)
 * **Extra Tools**

benchmarks/README.md

@@ -0,0 +1,14 @@
+# Benchmarks
+
+[Top-Level Documentation](../README.md)
+
+These benchmarks give insights into the usefulness of FLiT. Each benchmark has
+its own goals, either to demonstrate how FLiT can be useful, or to give
+insights into specific tools and functionalities.
+
+These benchmarks are also part of the installation, and can be copied from
+there to exercise FLiT's capabilities. They are installed in
+
+`$(PREFIX)/share/flit/benchmarks`
+
+[Top-Level Documentation](../README.md)

benchmarks/random/README.md

@@ -0,0 +1,11 @@
+# Random Benchmark
+
+This benchmark is to test that the C++ `<random>` header is portable accross
+compilers. My guess is that they will be since I suspect they are implemented
+in the GLibC, but I'm not sure. Hence the tests. This can give us confidence
+that this is indeed safe to depend on in FLiT tests.
+
+We also test the `rand()` and `srand()` functions that are part of the standard
+`<cstdlib>` header file. However, we do not test the non-portable `random()`,
+`srandom()`, `initstate()`, and `setstate()` functions that are part of the
+POSIX standard, but are not part of the C++ standard.

benchmarks/random/custom.mk

@@ -0,0 +1,27 @@
+# This file is included at the end of the copied Makefile. If you have some
+# things you want to change about the Makefile, it is best to do it here.
+
+# additional source files to compile other than what is in '.' and 'tests/'
+# since those directories are added by a wildcard.
+SOURCE +=
+
+# required compiler flags
+# for example, include directories
+# CC_REQUIRED += -I<path>
+# or defines
+# CC_REQUIRED += -DDEBUG_ENABLED=1
+CC_REQUIRED +=
+
+# required linker flags
+# for example, link libraries
+# LD_REQUIRED += -L<library-path> -l<library-name>
+# or rpath
+# LD_REQUIRED += -Wl,-rpath=<abs-path-to-library-dir>
+LD_REQUIRED +=
+
+# compiler and linker flags respectively - specifically for a dev build
+# - DEV_CFLAGS: non-recorded compiler flags (such as includes)
+# - DEV_LDFLAGS: linker flags (also not under test)
+DEV_CFLAGS +=
+DEV_LDFLAGS +=
+

benchmarks/random/flit-config.toml

@@ -0,0 +1,47 @@
+# Autogenerated by "flit init"
+# flit version v2.0-alpha.3
+
+[database]
+
+# older versions of flit supported postgres. that has been removed. only
+# sqlite is supported at the moment.
+type = 'sqlite'
+
+# if relative path, it is relative to the directory containing this
+# configuration file.
+filepath = 'results.sqlite'
+
+# For now, only one host is supported, all others are ignored
+[[hosts]]
+
+name = 'yoga-manjaro'
+flit_path = '/home/bentley/git/FLiT/scripts/flitcli/flit.py'
+config_dir = '/home/bentley/git/FLiT/scripts/flitcli/config'
+
+# The settings for "make dev"
+[hosts.dev_build]
+# compiler_name must be found in [[hosts.compilers]] list under name attribute
+# but the optimization level and switches do not need to be in the compiler list
+compiler_name = 'g++'
+optimization_level = '-O2'
+switches = '-funsafe-math-optimizations'
+
+# The ground truth compilation to use in analysis, for "make gt"
+[hosts.ground_truth]
+# compiler_name must be found in [[hosts.compilers]] list under name attribute
+# but the optimization level and switches do not need to be in the compiler list
+compiler_name = 'g++'
+optimization_level = '-O0'
+switches = ''
+
+ # This host's list of compilers.
+ # For now, only used for hosts.ground_truth and hosts.dev_build.
+ # TODO: use this list to generate the Makefile
+ [[hosts.compilers]]
+
+ # binary can be an absolute path, relative path, or binary name (found in
+ # PATH). If you want to specify a compiler in the same directory as this
+ # config file, prepend with a "./" (e.g. "./my-compiler")
+ binary = 'g++'
+ name = 'g++'
+

benchmarks/random/main.cpp

@@ -0,0 +1,5 @@
+#include "flit.h"
+
+int main(int argCount, char* argList[]) {
+ return flit::runFlitTests(argCount, argList);
+}

benchmarks/random/tests/Rand.cpp

@@ -0,0 +1,114 @@
+/* -- LICENSE BEGIN --
+ *
+ * Copyright (c) 2015-2018, Lawrence Livermore National Security, LLC.
+ *
+ * Produced at the Lawrence Livermore National Laboratory
+ *
+ * Written by
+ * Michael Bentley (mikebentley15@gmail.com),
+ * Geof Sawaya (fredricflinstone@gmail.com),
+ * and Ian Briggs (ian.briggs@utah.edu)
+ * under the direction of
+ * Ganesh Gopalakrishnan
+ * and Dong H. Ahn.
+ *
+ * LLNL-CODE-743137
+ *
+ * All rights reserved.
+ *
+ * This file is part of FLiT. For details, see
+ * https://pruners.github.io/flit
+ * Please also read
+ * https://github.com/PRUNERS/FLiT/blob/master/LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the disclaimer below.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the disclaimer
+ * (as noted below) in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * - Neither the name of the LLNS/LLNL nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL
+ * SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Additional BSD Notice
+ *
+ * 1. This notice is required to be provided under our contract
+ * with the U.S. Department of Energy (DOE). This work was
+ * produced at Lawrence Livermore National Laboratory under
+ * Contract No. DE-AC52-07NA27344 with the DOE.
+ *
+ * 2. Neither the United States Government nor Lawrence Livermore
+ * National Security, LLC nor any of their employees, makes any
+ * warranty, express or implied, or assumes any liability or
+ * responsibility for the accuracy, completeness, or usefulness of
+ * any information, apparatus, product, or process disclosed, or
+ * represents that its use would not infringe privately-owned
+ * rights.
+ *
+ * 3. Also, reference herein to any specific commercial products,
+ * process, or services by trade name, trademark, manufacturer or
+ * otherwise does not necessarily constitute or imply its
+ * endorsement, recommendation, or favoring by the United States
+ * Government or Lawrence Livermore National Security, LLC. The
+ * views and opinions of authors expressed herein do not
+ * necessarily state or reflect those of the United States
+ * Government or Lawrence Livermore National Security, LLC, and
+ * shall not be used for advertising or product endorsement
+ * purposes.
+ *
+ * -- LICENSE END --
+ */
+
+#include <flit.h>
+
+#include <string>
+#include <vector>
+
+#include <cstdlib>
+
+template <typename T>
+class Rand : public flit::TestBase<T> {
+public:
+ Rand(std::string id) : flit::TestBase<T>(std::move(id)) {}
+ virtual size_t getInputsPerRun() override { return 1; }
+ virtual std::vector<T> getDefaultInput() override { return { 0, 42, 24, 12, 103 }; }
+protected:
+ virtual flit::Variant run_impl(const std::vector<T> &ti) override {
+ FLIT_UNUSED(ti);
+ return {};
+ }
+protected:
+ using flit::TestBase<T>::id;
+};
+
+// Only implement the test for double precision
+template <>
+flit::Variant Rand<double>::run_impl(const std::vector<double> &ti) {
+ size_t seed = ti[0];
+ srand(seed);
+ return rand();
+}
+
+REGISTER_TYPE(Rand);