Merge pull request p4lang#2 from p4lang/master

Update

.travis.yml

@@ -18,16 +18,27 @@ cache:
 
 env:
   - CTEST_PARALLEL_LEVEL=4
+    UNIFIED=ON
+    ENABLE_GMP=ON
+  - CTEST_PARALLEL_LEVEL=4
+    UNIFIED=OFF
+    ENABLE_GMP=ON
+  - CTEST_PARALLEL_LEVEL=4
+    UNIFIED=ON
+    ENABLE_GMP=OFF
 
 before_install:
   - tools/install_os_deps.sh
   - if [[ $TRAVIS_OS_NAME == 'osx' ]] ; then export PATH="/usr/local/opt/bison/bin:$PATH" ; fi
 
 install:
   - if [[ $TRAVIS_OS_NAME == 'linux' ]] ; then tools/start_ccache; fi
-  - if [[ $TRAVIS_OS_NAME == 'linux' ]] ; then docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=test . ; fi
-  - if [[ $TRAVIS_OS_NAME == 'osx' ]] ; then ./bootstrap.sh -DCMAKE_BUILD_TYPE=RELEASE && cd build && make -j 2 ; fi
+  # To flush out issues with unified vs. non-unified builds, do a non-unified
+  # build before continuing with the rest, which produces a unified build.
+  # This is done here on MacOS; for Linux, this is done in Dockerfile.
+  - if [[ $TRAVIS_OS_NAME == 'linux' ]] ; then docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=$UNIFIED --build-arg ENABLE_GMP=$ENABLE_GMP . ; fi
+  - if [[ $TRAVIS_OS_NAME == 'osx' ]] ; then ./bootstrap.sh -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_UNIFIED_COMPILATION=$UNIFIED -DENABLE_GMP=$ENABLE_GMP && cd build && make -j2; fi
 
 script:
-  - if [[ $TRAVIS_OS_NAME == 'linux' ]] ; then docker run -w /p4c/build -e CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random ; fi
-  - if [[ $TRAVIS_OS_NAME == 'osx' ]] ; then ctest --output-on-failure -j 2 --schedule-random -LE "ebpf$" ; fi
+  - if [[ $TRAVIS_OS_NAME == 'linux' && $UNIFIED == ON ]] ; then docker run -w /p4c/build -e CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random ; fi
+  - if [[ $TRAVIS_OS_NAME == 'osx' && $UNIFIED == ON ]] ; then ctest --output-on-failure -j 2 --schedule-random -LE "bpf$" ; fi

CMakeLists.txt

@@ -32,11 +32,17 @@ OPTION (ENABLE_DOCS "Build the documentation" OFF)
 OPTION (ENABLE_GTESTS "Enable building and running GTest unit tests" ON)
 OPTION (ENABLE_BMV2 "Build the BMV2 backend (required for the full test suite)" ON)
 OPTION (ENABLE_EBPF "Build the EBPF backend (required for the full test suite)" ON)
+OPTION (ENABLE_UBPF "Build the uBPF backend (required for the full test suite)" ON)
 OPTION (ENABLE_P4TEST "Build the P4Test backend (required for the full test suite)" ON)
 OPTION (ENABLE_P4C_GRAPHS "Build the p4c-graphs backend" ON)
 OPTION (ENABLE_PROTOBUF_STATIC "Link against Protobuf statically" ON)
 OPTION (ENABLE_GC "Use libgc" ON)
 OPTION (ENABLE_MULTITHREAD "Use multithreading" OFF)
+OPTION (ENABLE_GMP "Use GMP library" ON)
+
+set(MAX_LOGGING_LEVEL 10 CACHE STRING "Control the maximum logging level for -T logs")
+set_property(CACHE MAX_LOGGING_LEVEL PROPERTY STRINGS 0 1 2 3 4 5 6 7 8 9 10)
+add_definitions(-DMAX_LOGGING_LEVEL=${MAX_LOGGING_LEVEL})
 
 if (NOT CMAKE_BUILD_TYPE)
   set (CMAKE_BUILD_TYPE "RELEASE")
@@ -48,7 +54,7 @@ if (NOT $ENV{P4C_VERSION} STREQUAL "")
 else()
   # Semantic version numbering: <major>.<minor>.<patch>[-rcX]
   # Examples: 0.5.1, 1.0.0-rc1, 1.0.1-alpha
-  set (P4C_SEM_VERSION_STRING "1.1.0-rc1")
+  set (P4C_SEM_VERSION_STRING "1.2.0")
   string (REGEX MATCH "([0-9]+)\\.([0-9]+)\\.([0-9]+)([-0-9a-z\\.]*).*"
     __p4c_version ${P4C_SEM_VERSION_STRING})
   set (P4C_VERSION_MAJOR ${CMAKE_MATCH_1})
@@ -71,7 +77,7 @@ set (P4C_CXX_FLAGS "")
 set (P4C_LIB_DEPS)
 
 # required tools and libraries
-find_package (PythonInterp REQUIRED)
+find_package (PythonInterp 3 REQUIRED)
 find_package (FLEX REQUIRED)
 find_package (BISON 3.0.2 REQUIRED)
 if (ENABLE_PROTOBUF_STATIC)
@@ -98,19 +104,26 @@ if (ENABLE_GC)
 endif ()
 if (ENABLE_MULTITHREAD)
   add_definitions(-DMULTITHREAD)
-  set(THREADS_PREFER_PTHREAD_FLAG ON)
-  find_package(Threads REQUIRED)
-  set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${CMAKE_THREAD_LIBS_INIT}")
 endif()
-find_package (LibGmp REQUIRED)
+# we require -pthread to make std::call_once work, even if we're not using threads...
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${CMAKE_THREAD_LIBS_INIT}")
 include_directories(${Boost_INCLUDE_DIRS})
 include_directories(${PROTOBUF_INCLUDE_DIRS})
-include_directories(${LIBGMP_INCLUDE_DIR})
 include_directories(${LIBGC_INCLUDE_DIR})
 set (HAVE_LIBBOOST_IOSTREAMS 1)
-set (HAVE_LIBGMP 1)
-set (HAVE_LIBGMPXX 1)
-set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${Boost_LIBRARIES};${LIBGMP_LIBRARIES};${LIBGC_LIBRARIES}")
+set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${Boost_LIBRARIES}")
+if (ENABLE_GMP)
+  find_package (LibGmp REQUIRED)
+  include_directories(${LIBGMP_INCLUDE_DIR})
+  set (HAVE_LIBGMP 1)
+  set (HAVE_LIBGMPXX 1)
+  set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${LIBGMP_LIBRARIES}")
+endif ()
+if (ENABLE_GC)
+  set (P4C_LIB_DEPS "${P4C_LIB_DEPS};${LIBGC_LIBRARIES}")
+endif ()
 
 # other required libraries
 p4c_add_library (rt clock_gettime HAVE_CLOCK_GETTIME)
@@ -126,7 +139,9 @@ check_include_file_cxx (cxxabi.h HAVE_CXXABI_H)
 
 # set libraries to be used with check_function_exists
 set (CMAKE_REQUIRED_LIBRARIES_PRECHECK ${CMAKE_REQUIRED_LIBRARIES})
-set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LIBGC_LIBRARIES})
+if (ENABLE_GC)
+  set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LIBGC_LIBRARIES})
+endif ()
 
 include (CheckFunctionExists)
 check_function_exists (memchr HAVE_MEMCHR)
@@ -263,6 +278,9 @@ endif ()
 if (ENABLE_EBPF)
     add_subdirectory (backends/ebpf)
 endif ()
+if (ENABLE_UBPF)
+    add_subdirectory (backends/ubpf)
+endif ()
 if (ENABLE_P4TEST)
     add_subdirectory (backends/p4test)
 endif ()
@@ -309,10 +327,19 @@ add_custom_command (OUTPUT ${IR_GENERATED_SRCS}
 add_custom_target(genIR DEPENDS ${IR_GENERATED_SRCS})
 
 # Header files
+# p4test needs all the backend include files, whether the backend is enabled or not
+# Note that we only provide the headers for the build env, they are only installed by the
+# backend specific target.
+set (OTHER_HEADERS
+  backends/ebpf/p4include/ebpf_model.p4
+  )
 add_custom_target(update_includes ALL
-  #FIXME -- should only run this when headers change -- how to accomplish that?
-  COMMAND ${CMAKE_COMMAND} -E copy_directory ${P4C_SOURCE_DIR}/p4include ${P4C_BINARY_DIR}/p4include
-)
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${P4C_BINARY_DIR}/p4include
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${P4C_SOURCE_DIR}/p4include/*.p4 ${P4C_BINARY_DIR}/p4include
+  COMMAND for h in ${OTHER_HEADERS} \; do
+    ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/\$$h ${P4C_BINARY_DIR}/p4include \;
+  done
+  )
 
 # Installation
 # Targets install themselves. Here we install the core headers

Dockerfile

@@ -11,59 +11,10 @@ ARG MAKEFLAGS=-j2
 # removed from the image.
 ARG IMAGE_TYPE=build
 
-ENV P4C_DEPS bison \
-             build-essential \
-             cmake \
-             curl \
-             flex \
-             g++ \
-             libboost-dev \
-             libboost-graph-dev \
-             libboost-iostreams1.58-dev \
-             libfl-dev \
-             libgc-dev \
-             libgmp-dev \
-             pkg-config \
-             tcpdump
-ENV P4C_EBPF_DEPS libpcap-dev \
-             libelf-dev \
-             llvm \
-             clang \
-             iproute2 \
-             net-tools
-ENV P4C_RUNTIME_DEPS cpp \
-                     libboost-graph1.58.0 \
-                     libboost-iostreams1.58.0 \
-                     libgc1c2 \
-                     libgmp10 \
-                     libgmpxx4ldbl \
-                     python
-ENV P4C_PIP_PACKAGES ipaddr \
-                     pyroute2 \
-                     ply==3.8 \
-                     scapy==2.4.0
+# Whether to do a unified build.
+ARG ENABLE_UNIFIED_COMPILATION=ON
 
-# We install pip with get-pip.py (https://pip.pypa.io/en/stable/installing/)
-# since the Ubuntu package manager's version of pip seems to be broken on Ubuntu
-# 16.04.
+# Delegate the build to tools/travis-build.
 COPY . /p4c/
 WORKDIR /p4c/
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends $P4C_DEPS $P4C_EBPF_DEPS $P4C_RUNTIME_DEPS && \
-    mkdir /tmp/pip && cd /tmp/pip && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py && cd - && rm -rf /tmp/pip && \
-    pip install $P4C_PIP_PACKAGES && \
-    mkdir build && \
-    cd build && \
-    cmake .. '-DCMAKE_CXX_FLAGS:STRING=-O3' && \
-    make && \
-    make install && \
-    /usr/local/bin/ccache -p -s && \
-    ( \
-      (test "$IMAGE_TYPE" = "build" && \
-        apt-get purge -y $P4C_DEPS && \
-        apt-get autoremove --purge -y && \
-        rm -rf /p4c /var/cache/apt/* /var/lib/apt/lists/* && \
-        echo 'Build image ready') || \
-      (test "$IMAGE_TYPE" = "test" && \
-        echo 'Test image ready') \
-    )
+RUN chmod u+x tools/travis-build && tools/travis-build

README.md

@@ -2,10 +2,14 @@
 
 # p4c
 
-p4c is a new, alpha-quality reference compiler for the P4 programming language.
+p4c is a reference compiler for the P4 programming language.
 It supports both P4-14 and P4-16; you can find more information about P4
 [here](http://p4.org) and the specifications for both versions of the language
 [here](https://p4.org/specs).
+One fact attesting to the level of quality and completeness of p4c's
+code is that its front-end code, mid-end code, and p4c-graphs back end
+are used as the basis for at least one commercially supported P4
+compiler.
 
 p4c is modular; it provides a standard frontend and midend which can be combined
 with a target-specific backend to create a complete P4 compiler. The goal is to
@@ -104,7 +108,8 @@ dot -Tpdf ParserImpl.dot > ParserImpl.pdf
 
 2.  Install [dependencies](#dependencies). You can find specific instructions
     for Ubuntu 16.04 [here](#ubuntu-dependencies) and for macOS 10.12
-    [here](#macos-dependencies).
+    [here](#macos-dependencies).  You can also look at the
+    [travis installation script](tools/travis-build).
 
 3.  Build. Building should also take place in a subdirectory named `build`.
     ```
@@ -193,10 +198,14 @@ included with `p4c` are documented here:
 
 Most dependencies can be installed using `apt-get install`:
 
-`sudo apt-get install cmake g++ git automake libtool libgc-dev bison flex
+```bash
+$ sudo apt-get install cmake g++ git automake libtool libgc-dev bison flex
 libfl-dev libgmp-dev libboost-dev libboost-iostreams-dev
-libboost-graph-dev llvm pkg-config python python-scapy python-ipaddr python-ply
-tcpdump`
+libboost-graph-dev llvm pkg-config python python-scapy python-ipaddr python-ply python3-pip
+tcpdump
+
+$ pip3 install scapy ply
+```
 
 For documentation building:
 `sudo apt-get install -y doxygen graphviz texlive-full`
@@ -206,17 +215,17 @@ which is not available until Ubuntu 16.10. For earlier releases of Ubuntu,
 you'll need to install from source. You can find instructions
 [here](https://github.com/google/protobuf/blob/master/src/README.md). **We
 recommend that you use version
-[3.2.0](https://github.com/google/protobuf/releases/tag/v3.2.0)**. Earlier
+[3.6.1](https://github.com/google/protobuf/releases/tag/v3.6.1)**. Earlier
 versions in the 3 series may not be supported by other p4lang projects, such as
 [p4lang/PI](https://github.com/p4lang/PI). More recent versions may work as
-well, but all our CI testing is done with version 3.2.0. After cloning protobuf
-and before you build, check-out version 3.2.0:
+well, but all our CI testing is done with version 3.6.1. After cloning protobuf
+and before you build, check-out version 3.6.1:
 
-`git checkout v3.2.0`
+`git checkout v3.6.1`
 
 Please note that while all protobuf versions newer than 3.0 should work for
 `p4c` itself, you may run into trouble with some extensions and other p4lang
-projects unless you install version 3.2.0, so you may want to install from
+projects unless you install version 3.6.1, so you may want to install from
 source even on newer releases of Ubuntu.
 
 ## macOS dependencies
@@ -429,11 +438,11 @@ install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/driver/p4c.mybackend.cfg
 
 # Known issues
 
-The P4C compiler is in early development. Issues with the compiler are
-tracked on [GitHub](https://github.com/p4lang/p4c/issues). Before
-opening a new issue, please check whether a similar issue is already
-opened. Opening issues and submitting a pull request with fixes for
-those issues is much appreciated.
+Issues with the compiler are tracked on
+[GitHub](https://github.com/p4lang/p4c/issues). Before opening a new
+issue, please check whether a similar issue is already opened. Opening
+issues and submitting a pull request with fixes for those issues is
+much appreciated.
 
 In addition to the list of issues on Github, there are a number of
 currently unsupported features listed below:
@@ -448,7 +457,7 @@ access them from the IR
 
 * Nonstandard extension primitives from P4_14
   * Execute_meter extra arguments
-  * Recirculate variants
+  * Recirculate/clone/resubmit variants
   * Bypass_egress
   * Sample_ primitives
   * invalidate

backends/bmv2/CMakeLists.txt

@@ -88,13 +88,13 @@ build_unified(BMV2_BACKEND_COMMON_SRCS)
 add_library(bmv2backend ${BMV2_BACKEND_COMMON_SRCS})
 add_dependencies(bmv2backend genIR frontend)
 
-build_unified(BMV2_SIMPLE_SWiTCH_SRCS)
+build_unified(BMV2_SIMPLE_SWITCH_SRCS)
 add_executable(p4c-bm2-ss ${BMV2_SIMPLE_SWITCH_SRCS})
 target_link_libraries (p4c-bm2-ss bmv2backend ${P4C_LIBRARIES} ${P4C_LIB_DEPS})
 
 install(TARGETS p4c-bm2-ss RUNTIME DESTINATION ${P4C_RUNTIME_OUTPUT_DIRECTORY})
 
-build_unified(BMV2_PSA_SWiTCH_SRCS)
+build_unified(BMV2_PSA_SWITCH_SRCS)
 add_executable(p4c-bm2-psa ${BMV2_PSA_SWITCH_SRCS})
 target_link_libraries (p4c-bm2-psa bmv2backend ${P4C_LIBRARIES} ${P4C_LIB_DEPS})
 install(TARGETS p4c-bm2-psa RUNTIME DESTINATION ${P4C_RUNTIME_OUTPUT_DIRECTORY})
@@ -103,10 +103,10 @@ install(TARGETS p4c-bm2-psa RUNTIME DESTINATION ${P4C_RUNTIME_OUTPUT_DIRECTORY})
 # binary to be in the top level directory. This should go away when we
 # remove automake and fix the scripts.
 add_custom_target(linkbmv2
-  COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_BINARY_DIR}/p4c-bm2-ss ${P4C_BINARY_DIR}/p4c-bm2-ss
-  COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_BINARY_DIR}/p4c-bm2-psa ${P4C_BINARY_DIR}/p4c-bm2-psa
-  COMMAND ${CMAKE_COMMAND} -E create_symlink ${P4C_BINARY_DIR}/p4include ${CMAKE_CURRENT_BINARY_DIR}/p4include
-  COMMAND ${CMAKE_COMMAND} -E create_symlink ${P4C_BINARY_DIR}/p4_14include ${CMAKE_CURRENT_BINARY_DIR}/p4_14include
+  COMMAND ${CMAKE_COMMAND} -E create_symlink `realpath --relative-to=${P4C_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/p4c-bm2-ss` ${P4C_BINARY_DIR}/p4c-bm2-ss
+  COMMAND ${CMAKE_COMMAND} -E create_symlink `realpath --relative-to=${P4C_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/p4c-bm2-psa` ${P4C_BINARY_DIR}/p4c-bm2-psa
+  COMMAND ${CMAKE_COMMAND} -E create_symlink `realpath --relative-to=${CMAKE_CURRENT_BINARY_DIR} ${P4C_BINARY_DIR}/p4include` ${CMAKE_CURRENT_BINARY_DIR}/p4include
+  COMMAND ${CMAKE_COMMAND} -E create_symlink `realpath --relative-to=${CMAKE_CURRENT_BINARY_DIR} ${P4C_BINARY_DIR}/p4_14include` ${CMAKE_CURRENT_BINARY_DIR}/p4_14include
   )
 add_dependencies(p4c_driver linkbmv2)
 
@@ -184,8 +184,6 @@ set (XFAIL_TESTS
   # Next two use unknown externs
   testdata/p4_16_samples/issue1882-bmv2.p4
   testdata/p4_16_samples/issue1882-1-bmv2.p4
-  # Uses non-constant array index
-  testdata/p4_16_samples/issue1989-bmv2.p4
 )
 
 if (HAVE_SIMPLE_SWITCH)

backends/bmv2/README.md

@@ -14,9 +14,7 @@ To run and test this back-end you need some additional tools:
   https://github.com/p4lang/behavioral-model.git.  You may need to update your
   dynamic libraries after installing bmv2: `sudo ldconfig`
 
-- the Python scapy library for manipulating network packets `sudo pip install scapy`
-
-- the Python ipaddr library `sudo pip install ipaddr`
+- the Python scapy and ipaddr libraries `sudo pip3 install scapy ipaddr`
 
 # Unsupported P4_16 language features