Merge pull request #1375 from ucb-bar/use-fat-jar

Use fat jar's to remove SBT invocations

.github/workflows/chipyard-full-flow.yml

@@ -115,7 +115,7 @@ jobs:
 
           cd vlsi
 
-          # NOTE: most conda installs are in separate conda envs because they mess up 
+          # NOTE: most conda installs are in separate conda envs because they mess up
           #   each other's versions (for no apparent reason) and we need the latest versions
           conda config --add channels defaults
           conda config --add channels litex-hub
@@ -166,7 +166,7 @@ jobs:
     name: cleanup
     needs: [run-tutorial]
     runs-on: ferry
-    if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
+    if: ${{ always() }}
     steps:
       - name: Delete repo copy and conda env
         run: |

.gitignore

@@ -26,3 +26,4 @@ project/metals.sbt
 project/project/
 .ivy2
 .sbt
+.classpath_cache/

build.sbt

@@ -4,14 +4,34 @@ import Tests._
 // implicit one
 lazy val chipyardRoot = Project("chipyardRoot", file("."))
 
+// keep chisel/firrtl specific class files, rename other conflicts
+val chiselFirrtlMergeStrategy = CustomMergeStrategy.rename { dep =>
+  import sbtassembly.Assembly.{Project, Library}
+  val nm = dep match {
+    case p: Project => p.name
+    case l: Library => l.moduleCoord.name
+  }
+  if (Seq("firrtl", "chisel3").contains(nm.split("_")(0))) { // split by _ to avoid checking on major/minor version
+    dep.target
+  } else {
+    "renamed/" + dep.target
+  }
+}
+
 lazy val commonSettings = Seq(
   organization := "edu.berkeley.cs",
   version := "1.6",
   scalaVersion := "2.13.10",
   assembly / test := {},
-  assembly / assemblyMergeStrategy := { _ match {
-    case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard
-    case _ => MergeStrategy.first}},
+  assembly / assemblyMergeStrategy := {
+    case PathList("chisel3", "stage", xs @ _*) => chiselFirrtlMergeStrategy
+    case PathList("firrtl", "stage", xs @ _*) => chiselFirrtlMergeStrategy
+    // should be safe in JDK11: https://stackoverflow.com/questions/54834125/sbt-assembly-deduplicate-module-info-class
+    case x if x.endsWith("module-info.class") => MergeStrategy.discard
+    case x =>
+      val oldStrategy = (assembly / assemblyMergeStrategy).value
+      oldStrategy(x)
+  },
   scalacOptions ++= Seq(
     "-deprecation",
     "-unchecked",
@@ -86,8 +106,6 @@ lazy val hardfloat  = (project in rocketChipDir / "hardfloat")
   .settings(commonSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "org.scala-lang" % "scala-reflect" % scalaVersion.value,
-      "org.json4s" %% "json4s-jackson" % "3.6.6",
       "org.scalatest" %% "scalatest" % "3.2.0" % "test"
     )
   )
@@ -97,8 +115,6 @@ lazy val rocketMacros  = (project in rocketChipDir / "macros")
   .settings(
     libraryDependencies ++= Seq(
       "org.scala-lang" % "scala-reflect" % scalaVersion.value,
-      "org.json4s" %% "json4s-jackson" % "3.6.6",
-      "org.scalatest" %% "scalatest" % "3.2.0" % "test"
     )
   )
 
@@ -225,7 +241,6 @@ lazy val iocell = Project(id = "iocell", base = file("./tools/barstools/") / "sr
 lazy val tapeout = (project in file("./tools/barstools/"))
   .settings(chiselSettings)
   .settings(chiselTestSettings)
-  .enablePlugins(sbtassembly.AssemblyPlugin)
   .settings(commonSettings)
 
 lazy val dsptools = freshProject("dsptools", file("./tools/dsptools"))

common.mk

@@ -16,7 +16,6 @@ HELP_COMPILATION_VARIABLES += \
 "   EXTRA_SIM_LDFLAGS         = additional LDFLAGS for building simulators" \
 "   EXTRA_SIM_SOURCES         = additional simulation sources needed for simulator" \
 "   EXTRA_SIM_REQS            = additional make requirements to build the simulator" \
-"   ENABLE_SBT_THIN_CLIENT    = if set, use sbt's experimental thin client (works best when overridding SBT_BIN with the mainline sbt script)" \
 "   ENABLE_CUSTOM_FIRRTL_PASS = if set, enable custom firrtl passes (SFC lowers to LowFIRRTL & MFC converts to Verilog)" \
 "   ENABLE_YOSYS_FLOW         = if set, add compilation flags to enable the vlsi flow for yosys(tutorial flow)" \
 "   EXTRA_CHISEL_OPTIONS      = additional options to pass to the Chisel compiler" \
@@ -52,7 +51,6 @@ HELP_COMMANDS += \
 "   firrtl                      = generate intermediate firrtl files from chisel elaboration" \
 "   run-tests                   = run all assembly and benchmark tests" \
 "   launch-sbt                  = start sbt terminal" \
-"   {shutdown,start}-sbt-server = shutdown or start sbt server if using ENABLE_SBT_THIN_CLIENT" \
 "   find-config-fragments       = list all config. fragments"
 
 #########################################################################################
@@ -105,12 +103,24 @@ $(BOOTROM_TARGETS): $(build_dir)/bootrom.%.img: $(TESTCHIP_RSRCS_DIR)/testchipip
 	cp -f $< $@
 
 #########################################################################################
-# create firrtl file rule and variables
+# compile scala jars
+#########################################################################################
+$(CHIPYARD_CLASSPATH_TARGETS) &: $(SCALA_SOURCES) $(SCALA_BUILDTOOL_DEPS)
+	mkdir -p $(dir $@)
+	$(call run_sbt_assembly,$(SBT_PROJECT),$(CHIPYARD_CLASSPATH))
+
+# order only dependency between sbt runs needed to avoid concurrent sbt runs
+$(TAPEOUT_CLASSPATH_TARGETS) &: $(SCALA_SOURCES) $(SCALA_BUILDTOOL_DEPS) | $(CHIPYARD_CLASSPATH_TARGETS)
+	mkdir -p $(dir $@)
+	$(call run_sbt_assembly,tapeout,$(TAPEOUT_CLASSPATH))
+
+#########################################################################################
+# verilog generation pipeline
 #########################################################################################
 # AG: must re-elaborate if cva6 sources have changed... otherwise just run firrtl compile
-$(FIRRTL_FILE) $(ANNO_FILE) $(CHISEL_LOG_FILE) &: $(SCALA_SOURCES) $(SCALA_BUILDTOOL_DEPS) $(EXTRA_GENERATOR_REQS)
+$(FIRRTL_FILE) $(ANNO_FILE) $(CHISEL_LOG_FILE) &: $(CHIPYARD_CLASSPATH_TARGETS) $(EXTRA_GENERATOR_REQS)
 	mkdir -p $(build_dir)
-	(set -o pipefail && $(call run_scala_main,$(SBT_PROJECT),$(GENERATOR_PACKAGE).Generator,\
+	(set -o pipefail && $(call run_jar_scala_main,$(CHIPYARD_CLASSPATH),$(GENERATOR_PACKAGE).Generator,\
 		--target-dir $(build_dir) \
 		--name $(long_name) \
 		--top-module $(MODEL_PACKAGE).$(MODEL) \
@@ -195,9 +205,9 @@ endif
 	if [ $(SFC_LEVEL) = none ]; then cat $(EXTRA_ANNO_FILE) > $(FINAL_ANNO_FILE); fi
 
 $(SFC_MFC_TARGETS) &: private TMP_DIR := $(shell mktemp -d -t cy-XXXXXXXX)
-$(SFC_MFC_TARGETS) &: $(FIRRTL_FILE) $(FINAL_ANNO_FILE) $(SFC_LEVEL) $(EXTRA_FIRRTL_OPTIONS)
+$(SFC_MFC_TARGETS) &: $(TAPEOUT_CLASSPATH_TARGETS) $(FIRRTL_FILE) $(FINAL_ANNO_FILE) $(SFC_LEVEL) $(EXTRA_FIRRTL_OPTIONS)
 	rm -rf $(GEN_COLLATERAL_DIR)
-	$(call run_scala_main,tapeout,barstools.tapeout.transforms.GenerateModelStageMain,\
+	$(call run_jar_scala_main,$(TAPEOUT_CLASSPATH),barstools.tapeout.transforms.GenerateModelStageMain,\
 		--no-dedup \
 		--output-file $(SFC_FIRRTL_BASENAME) \
 		--output-annotation-file $(SFC_ANNO_FILE) \
@@ -263,12 +273,12 @@ $(TOP_SMEMS_CONF) $(MODEL_SMEMS_CONF) &:  $(MFC_SMEMS_CONF) $(MFC_MODEL_HRCHY_JS
 
 # This file is for simulation only. VLSI flows should replace this file with one containing hard SRAMs
 TOP_MACROCOMPILER_MODE ?= --mode synflops
-$(TOP_SMEMS_FILE) $(TOP_SMEMS_FIR) &: $(TOP_SMEMS_CONF)
-	$(call run_scala_main,tapeout,barstools.macros.MacroCompiler,-n $(TOP_SMEMS_CONF) -v $(TOP_SMEMS_FILE) -f $(TOP_SMEMS_FIR) $(TOP_MACROCOMPILER_MODE))
+$(TOP_SMEMS_FILE) $(TOP_SMEMS_FIR) &: $(TAPEOUT_CLASSPATH_TARGETS) $(TOP_SMEMS_CONF)
+	$(call run_jar_scala_main,$(TAPEOUT_CLASSPATH),barstools.macros.MacroCompiler,-n $(TOP_SMEMS_CONF) -v $(TOP_SMEMS_FILE) -f $(TOP_SMEMS_FIR) $(TOP_MACROCOMPILER_MODE))
 
 MODEL_MACROCOMPILER_MODE = --mode synflops
-$(MODEL_SMEMS_FILE) $(MODEL_SMEMS_FIR) &: $(MODEL_SMEMS_CONF) | $(TOP_SMEMS_FILE)
-	$(call run_scala_main,tapeout,barstools.macros.MacroCompiler, -n $(MODEL_SMEMS_CONF) -v $(MODEL_SMEMS_FILE) -f $(MODEL_SMEMS_FIR) $(MODEL_MACROCOMPILER_MODE))
+$(MODEL_SMEMS_FILE) $(MODEL_SMEMS_FIR) &: $(TAPEOUT_CLASSPATH_TARGETS) $(MODEL_SMEMS_CONF) | $(TOP_SMEMS_FILE)
+	$(call run_jar_scala_main,$(TAPEOUT_CLASSPATH),barstools.macros.MacroCompiler, -n $(MODEL_SMEMS_CONF) -v $(MODEL_SMEMS_FILE) -f $(MODEL_SMEMS_FIR) $(MODEL_MACROCOMPILER_MODE))
 
 ########################################################################################
 # remove duplicate files and headers in list of simulation file inputs
@@ -393,35 +403,19 @@ endif
 #######################################
 # Rules for building DRAMSim2 library
 #######################################
-
 dramsim_dir = $(base_dir)/tools/DRAMSim2
 dramsim_lib = $(dramsim_dir)/libdramsim.a
 
 $(dramsim_lib):
 	$(MAKE) -C $(dramsim_dir) $(notdir $@)
 
 ################################################
-# Helper to run SBT or manage the SBT server
+# Helper to run SBT
 ################################################
-
 SBT_COMMAND ?= shell
 .PHONY: launch-sbt
 launch-sbt:
-	cd $(base_dir) && $(SBT_NON_THIN) "$(SBT_COMMAND)"
-
-.PHONY: check-thin-client
-check-thin-client:
-ifeq (,$(ENABLE_SBT_THIN_CLIENT))
-	$(error ENABLE_SBT_THIN_CLIENT not set.)
-endif
-
-.PHONY: shutdown-sbt-server
-shutdown-sbt-server: check-thin-client
-	cd $(base_dir) && $(SBT) "shutdown"
-
-.PHONY: start-sbt-server
-start-sbt-server: check-thin-client
-	cd $(base_dir) && $(SBT) "exit"
+	cd $(base_dir) && $(SBT) "$(SBT_COMMAND)"
 
 #########################################################################################
 # print help text (and other help)

project/plugins.sbt

@@ -1,3 +1,3 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1")
 addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.4")
-addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.3")
+addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.6")

scripts/repo-clean.sh

@@ -26,7 +26,7 @@ rm -rf $RDIR/toolchains/esp-tools/riscv-tests/build.log
     popd
 )
 (
-    pushd $RDIR/generators/cva6/src/main/resources/vsrc
+    pushd $RDIR/generators/cva6/src/main/resources/cva6/vsrc
     if [ -d cva6 ]
     then
 	git submodule deinit -f cva6

sims/vcs/Makefile

@@ -93,7 +93,7 @@ $(output_dir)/%.fsdb: $(output_dir)/% $(sim_debug)
 #########################################################################################
 .PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) $(sim_prefix)-* ucli.key
+	rm -rf $(CLASSPATH_CACHE) $(gen_dir) $(sim_prefix)-* ucli.key
 
 clean-sim:
 	rm -rf $(model_dir) $(build_dir)/vc_hdrs.h $(sim) $(sim).daidir ucli.key

sims/verilator/Makefile

@@ -226,7 +226,7 @@ $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 #########################################################################################
 .PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) $(sim_prefix)-*
+	rm -rf $(CLASSPATH_CACHE) $(gen_dir) $(sim_prefix)-*
 
 clean-sim:
 	rm -rf $(model_dir) $(sim)

sims/xcelium/Makefile

@@ -128,7 +128,7 @@ $(output_dir)/%.vcd: $(output_dir)/% $(sim_debug)
 #########################################################################################
 .PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) $(sim_prefix)-*
+	rm -rf $(CLASSPATH_CACHE) $(gen_dir) $(sim_prefix)-*
 
 clean-sim:
 	rm -rf $(model_dir) $(sim) $(sim_workdir) $(sim_run_tcl) ucli.key bpad_*.err sigusrdump.out dramsim*.log

variables.mk

@@ -8,7 +8,7 @@ HELP_COMPILATION_VARIABLES = \
 "   JAVA_TOOL_OPTIONS = if overridden, set underlying java tool options (default sets misc. sizes and tmp dir)" \
 "   SBT_OPTS          = set additional sbt command line options (these take the form -Dsbt.<option>=<setting>) " \
 "                       See https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html\#Command+Line+Options" \
-"   SBT_BIN           = if overridden, used to invoke sbt (default is to invoke sbt by sbt-launch.jar)" \
+"   SBT               = if overridden, used to invoke sbt (default is to invoke sbt by sbt-launch.jar)" \
 "   FIRRTL_LOGLEVEL   = if overridden, set firrtl log level (default is error)"
 
 HELP_PROJECT_VARIABLES = \
@@ -146,6 +146,14 @@ ifeq ($(GENERATOR_PACKAGE),hwacha)
 	long_name=$(MODEL_PACKAGE).$(CONFIG)
 endif
 
+# classpaths
+CLASSPATH_CACHE ?= $(base_dir)/.classpath_cache
+CHIPYARD_CLASSPATH ?= $(CLASSPATH_CACHE)/chipyard.jar
+TAPEOUT_CLASSPATH ?= $(CLASSPATH_CACHE)/tapeout.jar
+# if *_CLASSPATH is a true java classpath, it can be colon-delimited list of paths (on *nix)
+CHIPYARD_CLASSPATH_TARGETS ?= $(subst :, ,$(CHIPYARD_CLASSPATH))
+TAPEOUT_CLASSPATH_TARGETS ?= $(subst :, ,$(TAPEOUT_CLASSPATH))
+
 # chisel generated outputs
 FIRRTL_FILE ?= $(build_dir)/$(long_name).fir
 ANNO_FILE   ?= $(build_dir)/$(long_name).anno.json
@@ -206,32 +214,37 @@ sim_common_files       ?= $(build_dir)/sim_files.common.f
 #########################################################################################
 JAVA_HEAP_SIZE ?= 8G
 JAVA_TMP_DIR ?= $(base_dir)/.java_tmp
-export JAVA_TOOL_OPTIONS ?= -Xmx$(JAVA_HEAP_SIZE) -Xss8M -Dsbt.supershell=false -Djava.io.tmpdir=$(JAVA_TMP_DIR)
+export JAVA_TOOL_OPTIONS ?= -Xmx$(JAVA_HEAP_SIZE) -Xss8M -Djava.io.tmpdir=$(JAVA_TMP_DIR)
 
 #########################################################################################
 # default sbt launch command
 #########################################################################################
 SCALA_BUILDTOOL_DEPS = $(SBT_SOURCES)
 
-SBT_THIN_CLIENT_TIMESTAMP = $(base_dir)/project/target/active.json
-
-ifdef ENABLE_SBT_THIN_CLIENT
-SCALA_BUILDTOOL_DEPS += $(SBT_THIN_CLIENT_TIMESTAMP)
-# enabling speeds up sbt loading
-# use with sbt script or sbtn to bypass error code issues
-SBT_CLIENT_FLAG = --client
-endif
-
 # passes $(JAVA_TOOL_OPTIONS) from env to java
-export SBT_OPTS ?= -Dsbt.ivy.home=$(base_dir)/.ivy2 -Dsbt.global.base=$(base_dir)/.sbt -Dsbt.boot.directory=$(base_dir)/.sbt/boot/ -Dsbt.color=always
-SBT_BIN ?= java -jar $(ROCKETCHIP_DIR)/sbt-launch.jar $(SBT_OPTS)
-SBT = $(SBT_BIN) $(SBT_CLIENT_FLAG)
-SBT_NON_THIN = $(subst $(SBT_CLIENT_FLAG),,$(SBT))
+export SBT_OPTS ?= -Dsbt.ivy.home=$(base_dir)/.ivy2 -Dsbt.global.base=$(base_dir)/.sbt -Dsbt.boot.directory=$(base_dir)/.sbt/boot/ -Dsbt.color=always -Dsbt.supershell=false -Dsbt.server.forcestart=true
+SBT ?= java -jar $(ROCKETCHIP_DIR)/sbt-launch.jar $(SBT_OPTS)
 
+# (1) - classpath of the fat jar
+# (2) - main class
+# (3) - main class arguments
+define run_jar_scala_main
+	cd $(base_dir) && java -cp $(1) $(2) $(3)
+endef
+
+# (1) - sbt project
+# (2) - main class
+# (3) - main class arguments
 define run_scala_main
 	cd $(base_dir) && $(SBT) ";project $(1); runMain $(2) $(3)"
 endef
 
+# (1) - sbt project to assemble
+# (2) - classpath file(s) to create
+define run_sbt_assembly
+	cd $(base_dir) && $(SBT) ";project $(1); set assembly / assemblyOutputPath := file(\"$(2)\"); assembly"
+endef
+
 FIRRTL_LOGLEVEL ?= error
 
 #########################################################################################

vlsi/Makefile

@@ -171,4 +171,4 @@ $(OBJ_DIR)/hammer.d: $(HAMMER_D_DEPS)
 #########################################################################################
 .PHONY: clean
 clean:
-	rm -rf $(VLSI_OBJ_DIR) hammer-vlsi*.log __pycache__ output.json $(GENERATED_CONFS) $(gen_dir) $(SIM_CONF) $(SIM_DEBUG_CONF) $(SIM_TIMING_CONF) $(POWER_CONF)
+	rm -rf $(VLSI_OBJ_DIR) hammer-vlsi*.log __pycache__ output.json $(GENERATED_CONFS) $(CLASSPATH_CACHE) $(gen_dir) $(SIM_CONF) $(SIM_DEBUG_CONF) $(SIM_TIMING_CONF) $(POWER_CONF)