modules/zstd: Add ZstdDecoder top level proc

.github/workflows/modules-zstd.yml

@@ -0,0 +1,73 @@
+name: Modules (ZSTD)
+on:
+  # Avoid triggering on pushes to /all/ open PR branches.
+  push:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  # This lets us trigger manually from the UI.
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Test ZSTD module (opt)
+    runs-on:
+      labels: ubuntu-22.04-64core
+    timeout-minutes: 600
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Restore Nightly Bazel Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: "~/.cache/bazel"
+          key: bazel-cache-nightly-${{ runner.os }}-${{ github.sha }}
+          restore-keys: bazel-cache-nightly-${{ runner.os }}-
+
+      - name: Install dependencies via apt
+        run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5  build-essential liblapack-dev libblas-dev gfortran
+
+      - name: Bazel Build Tools (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/dslx:interpreter_main //xls/dslx/ir_convert:ir_converter_main //xls/tools:opt_main //xls/tools:codegen_main
+
+      - name: Build ZSTD Module (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/modules/zstd:all
+
+      - name: Test ZSTD Module - DSLX Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_dslx_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Test ZSTD Module - CC Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_cc_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD verilog targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_verilog", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run ZSTD IR benchmark rules (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_ir_benchmark", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run synthesis benchmarks of the ZSTD module (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_benchmark_synth", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD place and route targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_place_and_route", kind(rule, //xls/modules/zstd/...))')

dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel

@@ -1,4 +1,4 @@
-# Copyright 2023 The XLS Authors
+# Copyright 2024 The XLS Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,38 +12,203 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-package(default_visibility = ["//visibility:public"])
+""" Builds zstd.
+"""
 
-licenses(["notice"])
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
 
-# Builds everything together similarly to the zstd cmake file:
-# https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt
-# but with legacy support and defines as in the zstd BUCK file:
-# https://github.com/facebook/zstd/blob/dev/lib/BUCK
+package(default_visibility = ["//visibility:public"])
 
-cc_library(
-    name = "zstd",
+filegroup(
+    name = "common_sources",
     srcs = glob([
-        "lib/common/*.h",
         "lib/common/*.c",
-        "lib/compress/*.h",
+        "lib/common/*.h",
+    ]),
+)
+
+filegroup(
+    name = "compress_sources",
+    srcs = glob([
         "lib/compress/*.c",
-        "lib/decompress/*.h",
+        "lib/compress/*.h",
+    ]),
+)
+
+filegroup(
+    name = "decompress_sources",
+    srcs = glob([
         "lib/decompress/*.c",
-        "lib/decompress/*.S",
-        "lib/deprecated/*.h",
-        "lib/deprecated/*.c",
-        "lib/dictBuilder/*.h",
+        "lib/decompress/*.h",
+    ]) + select({
+        "@platforms//os:windows": [],
+        "//conditions:default": glob(["lib/decompress/*.S"]),
+    }),
+)
+
+filegroup(
+    name = "dictbuilder_sources",
+    srcs = glob([
         "lib/dictBuilder/*.c",
-        "lib/legacy/*.h",
-        "lib/legacy/*.c",
+        "lib/dictBuilder/*.h",
     ]),
+)
+
+cc_library(
+    name = "zstd",
+    srcs = [
+        ":common_sources",
+        ":compress_sources",
+        ":decompress_sources",
+        ":dictbuilder_sources",
+    ],
     hdrs = [
+        "lib/zdict.h",
         "lib/zstd.h",
+        "lib/zstd_errors.h",
+    ],
+    includes = ["lib"],
+    linkopts = ["-pthread"],
+    linkstatic = True,
+    local_defines = [
+        "XXH_NAMESPACE=ZSTD_",
+        "ZSTD_MULTITHREAD",
+        "ZSTD_BUILD_SHARED=OFF",
+        "ZSTD_BUILD_STATIC=ON",
+    ] + select({
+        "@platforms//os:windows": ["ZSTD_DISABLE_ASM"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_binary(
+    name = "zstd_cli",
+    srcs = glob(
+        include = [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/datagen.c",
+            "programs/datagen.h",
+            "programs/platform.h",
+            "programs/util.h",
+        ],
+    ),
+    deps = [
+        ":datagen",
+        ":util",
+        ":zstd",
+    ],
+)
+
+cc_library(
+    name = "util",
+    srcs = [
+        "programs/platform.h",
+        "programs/util.c",
+    ],
+    hdrs = [
+        "lib/common/compiler.h",
+        "lib/common/debug.h",
+        "lib/common/mem.h",
+        "lib/common/portability_macros.h",
+        "lib/common/zstd_deps.h",
+        "programs/util.h",
+    ],
+)
+
+cc_library(
+    name = "datagen",
+    srcs = [
+        "programs/datagen.c",
+        "programs/platform.h",
+    ],
+    hdrs = ["programs/datagen.h"],
+    deps = [":util"],
+)
+
+cc_binary(
+    name = "datagen_cli",
+    srcs = [
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "tests/datagencli.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    includes = [
+        "programs",
+        "tests",
+    ],
+    deps = [":datagen"],
+)
+
+cc_test(
+    name = "fullbench",
+    srcs = [
+        "lib/decompress/zstd_decompress_internal.h",
+        "programs/benchfn.c",
+        "programs/benchfn.h",
+        "programs/benchzstd.c",
+        "programs/benchzstd.h",
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "programs/timefn.c",
+        "programs/timefn.h",
+        "tests/fullbench.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": ["-Wno-deprecated-declarations"],
+    }),
+    includes = [
+        "lib/common",
+        "programs",
+        "tests",
+    ],
+    deps = [
+        ":datagen",
+        ":zstd",
+    ],
+)
+
+# NOTE: Required because of direct zstd_compress.c include in decodecorpus sources
+cc_library(
+    name = "decodecorpus_includes",
+    hdrs = [
+        "lib/compress/zstd_compress.c",
+    ],
+)
+
+cc_binary(
+    name = "decodecorpus",
+    srcs = [
+        "tests/decodecorpus.c",
+    ] + glob(
+        [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/zstdcli.c",
+        ],
+    ),
+    deps = [
+        ":zstd",
+        ":decodecorpus_includes",
+    ],
+    includes = [
+        "lib/",
+        "lib/common/",
+        "lib/compress/",
+        "lib/dictBuilder/",
+        "lib/deprecated/",
+        "programs/",
     ],
-    strip_include_prefix = "lib",
     local_defines = [
-        "ZSTD_LEGACY_SUPPORT=4",
         "XXH_NAMESPACE=ZSTD_",
     ],
     visibility = ["//visibility:public"],

dependency_support/load_external.bzl

@@ -192,15 +192,6 @@ def load_external_repositories():
         url = "https://github.com/google/riegeli/archive/cb68d579f108c96831b6a7815da43ff24b4e5242.tar.gz",
     )
 
-    # Needed by fuzztest. Release 2024-03-30, current as of 2024-06-26
-    http_archive(
-        name = "net_zstd",
-        build_file = "@com_google_riegeli//third_party:net_zstd.BUILD",
-        sha256 = "30f35f71c1203369dc979ecde0400ffea93c27391bfd2ac5a9715d2173d92ff7",
-        strip_prefix = "zstd-1.5.6/lib",
-        urls = ["https://github.com/facebook/zstd/archive/v1.5.6.tar.gz"],
-    )
-
     # Needed by fuzztest. Release 2024-05-21, current as of 2024-06-26
     http_archive(
         name = "snappy",
@@ -331,3 +322,17 @@ def load_external_repositories():
         urls = ["https://github.com/bazelbuild/rules_pkg/releases/download/1.0.0/rules_pkg-1.0.0.tar.gz"],
         sha256 = "cad05f864a32799f6f9022891de91ac78f30e0fa07dc68abac92a628121b5b11",
     )
+
+    # Used in C++ tests of the ZSTD Module
+    # Transitive dependency of fuzztest (required by riegeli in fuzztest workspace)
+    # Version fdfb2aff released on 2024-07-31
+    # https://github.com/facebook/zstd/commit/fdfb2aff39dc498372d8c9e5f2330b692fea9794
+    # Updated 2024-08-08
+    http_archive(
+        name = "zstd",
+        sha256 = "9ace5a1b3c477048c6e034fe88d2abb5d1402ced199cae8e9eef32fdc32204df",
+        strip_prefix = "zstd-fdfb2aff39dc498372d8c9e5f2330b692fea9794",
+        urls = ["https://github.com/facebook/zstd/archive/fdfb2aff39dc498372d8c9e5f2330b692fea9794.zip"],
+        build_file = "//dependency_support/com_github_facebook_zstd:bundled.BUILD.bazel",
+    )
+

xls/examples/ram.x

@@ -55,7 +55,7 @@ pub fn ReadWordReq<NUM_PARTITIONS:u32, ADDR_WIDTH:u32>(addr:uN[ADDR_WIDTH]) ->
 }
 
 // Behavior of reads and writes to the same address in the same "tick".
-enum SimultaneousReadWriteBehavior : u2 {
+pub enum SimultaneousReadWriteBehavior : u2 {
   // The read shows the contents at the address before the write.
   READ_BEFORE_WRITE = 0,
   // The read shows the contents at the address after the write.
@@ -160,7 +160,7 @@ fn write_word_test() {
 
 // Function to compute num partitions (e.g. mask width) for a data_width-wide
 // word divided into word_partition_size-chunks.
-fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
+pub fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
   match word_partition_size {
     u32:0 => u32:0,
     _ => (word_partition_size + data_width - u32:1) / word_partition_size,
@@ -251,7 +251,7 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
     if read_req_valid && ASSERT_VALID_READ {
       let mem_initialized_as_bits =
         std::convert_to_bits_msb0(array_rev(mem_initialized[read_req.addr]));
-      assert_eq(read_req.mask & !mem_initialized_as_bits, uN[NUM_PARTITIONS]:0)
+      assert!(read_req.mask & !mem_initialized_as_bits == uN[NUM_PARTITIONS]:0, "memory_not_initialized")
     } else { () };
 
     let (value_to_write, written_mem_initialized) = write_word(
@@ -265,11 +265,8 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
       match SIMULTANEOUS_READ_WRITE_BEHAVIOR {
         SimultaneousReadWriteBehavior::READ_BEFORE_WRITE => mem[read_req.addr],
         SimultaneousReadWriteBehavior::WRITE_BEFORE_READ => value_to_write,
-        SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => {
-          // Assertion failure, we have a conflicting read and write.
-          assert_eq(true, false);
-          mem[read_req.addr]  // Need to return something.
-        },
+        SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => fail!("conflicting_read_and_write", mem[read_req.addr]),
+        _ => fail!("impossible_case", uN[DATA_WIDTH]:0),
       }
     } else { mem[read_req.addr] };
     let read_resp_value = ReadResp<DATA_WIDTH> {