Make TLBusWrappers' topology a configurable Field

src/main/scala/devices/tilelink/CanHaveBuiltInDevices.scala

@@ -17,6 +17,11 @@ sealed trait BuiltInDevices {
 }
 
 object BuiltInDevices {
+  def none = new BuiltInDevices {
+    val errorOpt = None
+    val zeroOpt = None
+  }
+
   def attach(
     params: HasBuiltInDeviceParams with HasTLBusParams,
     outwardNode: TLOutwardNode)(implicit p: Parameters) = new BuiltInDevices {

src/main/scala/groundtest/GroundTestSubsystem.scala

@@ -16,7 +16,6 @@ import scala.math.max
 case object TileId extends Field[Int]
 
 class GroundTestSubsystem(implicit p: Parameters) extends BaseSubsystem
-    with HasHierarchicalBusTopology
     with CanHaveMasterAXI4MemPort {
   val tileParams = p(GroundTestTilesKey)
   val tiles = tileParams.zipWithIndex.map { case(c, i) => LazyModule(c.build(i, p)) }

src/main/scala/subsystem/Attachable.scala

@@ -0,0 +1,60 @@
+// See LICENSE.SiFive for license details.
+
+package freechips.rocketchip.subsystem
+
+import scala.language.dynamics
+import freechips.rocketchip.config.Parameters
+import freechips.rocketchip.diplomacy.{LazyModule, LazyScope}
+import freechips.rocketchip.diplomaticobjectmodel.HasLogicalTreeNode
+import freechips.rocketchip.prci.ClockGroupEphemeralNode
+import freechips.rocketchip.tilelink.TLBusWrapper
+import freechips.rocketchip.util.{Location, LocationMap}
+
+/** These traits are intended to make it possible to configure to which
+  * buses optional devices are attached, even after a subsystem has been instantiated.
+  * Consider them experimental for now.
+  */
+
+/** More nodes can be added to subclasses of this after they have been instantiated,
+  * and the implicit context-dependent Parameters object is available.
+  * These qualities comprise the fundamental capabilities of dynamic configurability.
+  */
+trait LazyScopeWithParameters extends LazyScope { this: LazyModule =>
+  implicit val p: Parameters
+}
+
+/** Layers of hierarchy with this trait will be represented as objects in the Object Model */
+trait HasLogicalHierarchy extends LazyScopeWithParameters with HasLogicalTreeNode { this: LazyModule => }
+
+/** Layers of hierarchy with this trait contain attachment points for neworks of power, clock, reset, and interrupt resources */
+trait HasPRCILocations extends HasLogicalHierarchy { this: LazyModule =>
+  implicit val asyncClockGroupsNode: ClockGroupEphemeralNode
+  val ibus: InterruptBusWrapper
+  val anyLocationMap = LocationMap.empty[Any]
+}
+
+/** Layers of hierarchy with this trait contain attachment points for TileLink interfaces */
+trait HasTileLinkLocations extends HasPRCILocations { this: LazyModule =>
+  val tlBusWrapperLocationMap = LocationMap.empty[TLBusWrapper]
+  def locateTLBusWrapper(location: Location[TLBusWrapper]): TLBusWrapper = locateTLBusWrapper(location.name)
+  def locateTLBusWrapper(name: String): TLBusWrapper = tlBusWrapperLocationMap(Location[TLBusWrapper](name))
+}
+
+/** Subclasses of this trait have the ability to instantiate things inside a context that has TL attachement locations */
+trait CanInstantiateWithinContextThatHasTileLinkLocations {
+  def instantiate(context: HasTileLinkLocations)(implicit p: Parameters): Unit
+}
+
+/** Subclasses of this trait have the ability to connect things inside a context that has TL attachement locations */
+trait CanConnectWithinContextThatHasTileLinkLocations {
+  def connect(context: HasTileLinkLocations)(implicit p: Parameters): Unit
+}
+
+/** Attachable things provide a standard interface by which other things may attach themselves to this target.
+  * Right now the trait is mostly for backwards compatibility, and in case it eventually becomes valuable
+  * to be able to define additional resources available to agents trying to attach themselves, other than
+  * what is being made available via the LocationMaps in trait HasTileLinkLocations.
+  */
+trait Attachable extends HasTileLinkLocations { this: LazyModule =>
+  def locateTLBusWrapper(location: TLBusWrapperLocation): TLBusWrapper = locateTLBusWrapper(location.name)
+}

src/main/scala/subsystem/BankedL2Params.scala

@@ -0,0 +1,83 @@
+// See LICENSE.SiFive for license details.
+
+package freechips.rocketchip.subsystem
+
+import chisel3.util.isPow2
+import freechips.rocketchip.config._
+import freechips.rocketchip.devices.tilelink.BuiltInDevices
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.interrupts._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.util._
+import CoherenceManagerWrapper._
+
+/** Global cache coherence granularity, which applies to all caches, for now. */
+case object CacheBlockBytes extends Field[Int](64)
+
+/** L2 Broadcast Hub configuration */
+case object BroadcastKey extends Field(BroadcastParams())
+
+case class BroadcastParams(
+  nTrackers:  Int     = 4,
+  bufferless: Boolean = false)
+
+/** L2 memory subsystem configuration */
+case object BankedL2Key extends Field(BankedL2Params())
+
+case class BankedL2Params(
+  nBanks: Int = 1,
+  coherenceManager: CoherenceManagerInstantiationFn = broadcastManager
+) {
+  require (isPow2(nBanks) || nBanks == 0)
+}
+
+case class CoherenceManagerWrapperParams(
+    blockBytes: Int,
+    beatBytes: Int,
+    nBanks: Int,
+    name: String)
+  (val coherenceManager: CoherenceManagerInstantiationFn)
+  extends HasTLBusParams 
+  with TLBusWrapperInstantiationLike
+{
+  val dtsFrequency = None
+  def instantiate(context: HasTileLinkLocations, loc: Location[TLBusWrapper])(implicit p: Parameters): CoherenceManagerWrapper = {
+    val cmWrapper = LazyModule(new CoherenceManagerWrapper(this, context))
+    cmWrapper.suggestName(loc.name + "_wrapper")
+    cmWrapper.halt.foreach { context.anyLocationMap += loc.halt(_) }
+    context.tlBusWrapperLocationMap += (loc -> cmWrapper)
+    cmWrapper
+  }
+}
+
+class CoherenceManagerWrapper(params: CoherenceManagerWrapperParams, context: HasTileLinkLocations)(implicit p: Parameters) extends TLBusWrapper(params, params.name) {
+  val (tempIn, tempOut, halt) = params.coherenceManager(context)
+
+  // TODO could remove temp if we could get access to .edges from InwardNodeHandle
+  val viewNode = TLIdentityNode()
+  def busView: TLEdge = viewNode.edges.out.head
+  val inwardNode = tempIn :*= viewNode
+  val builtInDevices = BuiltInDevices.none
+
+  private def banked(node: TLOutwardNode): TLOutwardNode =
+    if (params.nBanks == 0) node else { TLTempNode() :=* BankBinder(params.nBanks, params.blockBytes) :*= node }
+  val outwardNode = banked(tempOut)
+}
+
+object CoherenceManagerWrapper {
+  type CoherenceManagerInstantiationFn = HasTileLinkLocations => (TLInwardNode, TLOutwardNode, Option[IntOutwardNode])
+
+  val broadcastManager: CoherenceManagerInstantiationFn = { context =>
+    implicit val p = context.p
+    val BroadcastParams(nTrackers, bufferless) = p(BroadcastKey)
+    val bh = LazyModule(new TLBroadcast(p(CacheBlockBytes), nTrackers, bufferless))
+    val ss = TLSourceShrinker(nTrackers)
+    ss :*= bh.node
+    (bh.node, ss, None)
+  }
+
+  val incoherentManager: CoherenceManagerInstantiationFn = { _ =>
+    val node = TLNameNode("no_coherence_manager")
+    (node, node, None)
+  }
+}

src/main/scala/subsystem/BaseSubsystem.scala

@@ -7,22 +7,14 @@ import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.diplomaticobjectmodel.HasLogicalTreeNode
 import freechips.rocketchip.diplomaticobjectmodel.logicaltree._
-import freechips.rocketchip.tilelink.TLBusWrapper
 import freechips.rocketchip.prci._
+import freechips.rocketchip.tilelink.TLBusWrapper
 import freechips.rocketchip.util._
 
-
-case object SystemBusKey extends Field[SystemBusParams]
-case object FrontBusKey extends Field[FrontBusParams]
-case object PeripheryBusKey extends Field[PeripheryBusParams]
-case object ControlBusKey extends Field[PeripheryBusParams]
-case object MemoryBusKey extends Field[MemoryBusParams]
-case object BankedL2Key extends Field(BankedL2Params())
-
-case object BuildSystemBus extends Field[Parameters => SystemBus](p => new SystemBus(p(SystemBusKey))(p))
-
 case object SubsystemDriveAsyncClockGroupsKey extends Field[Option[ClockGroupDriverParameters]](Some(ClockGroupDriverParameters(1)))
 case object AsyncClockGroupsKey extends Field[ClockGroupEphemeralNode](ClockGroupEphemeralNode()(ValName("async_clock_groups")))
+case class TLNetworkTopologyLocated(where: HierarchicalLocation) extends Field[Seq[CanInstantiateWithinContextThatHasTileLinkLocations with CanConnectWithinContextThatHasTileLinkLocations]]
+case class TLManagerViewpointLocated(where: HierarchicalLocation) extends Field[Location[TLBusWrapper]](SBUS)
 
 class HierarchicalLocation(override val name: String) extends Location[LazyScope](name)
 case object InTile extends HierarchicalLocation("InTile")
@@ -45,72 +37,59 @@ abstract class BareSubsystemModuleImp[+L <: BareSubsystem](_outer: L) extends La
   println(outer.dts)
 }
 
-/** These traits are intended to make it possible to configure to which
-  * buses optional devices are attached, even after a subsystem has been instantiated.
-  * Consider them experimental for now.
-  */
-
-trait Attachable extends LazyScope
-    with HasLogicalTreeNode
-    with HasBusLocationFunction { this: LazyModule =>
-  implicit val p: Parameters
-  implicit val asyncClockGroupsNode: ClockGroupEphemeralNode
-  val ibus: InterruptBusWrapper
-}
-
-trait HasBusLocationFunction {
-  type BusLocationFunction = PartialFunction[TLBusWrapperLocation, TLBusWrapper]
-  def locateTLBusWrapper: BusLocationFunction
-}
-
-/** This class the cases matched in baseBusLocateFunc below.
-  * Extend/override them to offer novel attachment locations.
-  */
-class TLBusWrapperLocation(name: String) extends Location[TLBusWrapper](name)
-case object SBUS extends TLBusWrapperLocation("subsystem_sbus")
-case object PBUS extends TLBusWrapperLocation("subsystem_pbus")
-case object FBUS extends TLBusWrapperLocation("subsystem_fbus")
-case object MBUS extends TLBusWrapperLocation("subsystem_mbus")
-case object CBUS extends TLBusWrapperLocation("subsystem_cbus")
-
 trait SubsystemResetScheme
 case object ResetSynchronous extends SubsystemResetScheme
 case object ResetAsynchronous extends SubsystemResetScheme
 case object ResetAsynchronousFull extends SubsystemResetScheme
 
 case object SubsystemResetSchemeKey extends Field[SubsystemResetScheme](ResetSynchronous)
 
-/** Base Subsystem class with no peripheral devices or ports added */
-abstract class BaseSubsystem(implicit p: Parameters) extends BareSubsystem
-    with Attachable {
-
-  override val module: BaseSubsystemModuleImp[BaseSubsystem]
-
-  // These are wrappers around the standard buses available in all subsytems, where
-  // peripherals, tiles, ports, and other masters and slaves can attach themselves.
+/** Concrete attachment points for PRCI-related signals.
+  * These aren't actually very configurable, yet.
+  */
+trait HasConfigurablePRCILocations { this: HasPRCILocations =>
   val ibus = new InterruptBusWrapper()
-  val sbus = LazyModule(p(BuildSystemBus)(p))
-  val pbus = LazyModule(new PeripheryBus(p(PeripheryBusKey), "subsystem_pbus"))
-  val fbus = LazyModule(new FrontBus(p(FrontBusKey)))
-  val mbus = LazyModule(new MemoryBus(p(MemoryBusKey)))
-  val cbus = LazyModule(new PeripheryBus(p(ControlBusKey), "subsystem_cbus"))
-
-  def locateTLBusWrapper: BusLocationFunction = {
-    case SBUS => sbus
-    case PBUS => pbus
-    case FBUS => fbus
-    case MBUS => mbus
-    case CBUS => cbus
-  }
-
   implicit val asyncClockGroupsNode = p(AsyncClockGroupsKey)
   val async_clock_groups =
     p(SubsystemDriveAsyncClockGroupsKey)
       .map(_.drive(asyncClockGroupsNode))
       .getOrElse(InModuleBody { HeterogeneousBag[ClockGroupBundle](Nil) })
+}
+
+/** Look up the topology configuration for the TL buses located within this layer of the hierarchy */
+trait HasConfigurableTLNetworkTopology { this: HasTileLinkLocations =>
+  val location: HierarchicalLocation
+
+  // Calling these functions populates tlBusWrapperLocationMap and connects the locations to each other.
+  val topology = p(TLNetworkTopologyLocated(location))
+  topology.map(_.instantiate(this))
+  topology.foreach(_.connect(this))
+
+  // This is used lazily at DTS binding time to get a view of the network
+  lazy val topManagers = tlBusWrapperLocationMap(p(TLManagerViewpointLocated(location))).unifyManagers
+}
+
+/** Base Subsystem class with no peripheral devices, ports or cores added yet */
+abstract class BaseSubsystem(val location: HierarchicalLocation = InSubsystem)
+                            (implicit p: Parameters)
+  extends BareSubsystem
+  with Attachable
+  with HasConfigurablePRCILocations
+  with HasConfigurableTLNetworkTopology
+{
+  override val module: BaseSubsystemModuleImp[BaseSubsystem]
+
+  // TODO must there really always be an "sbus"?
+  val sbus = tlBusWrapperLocationMap(SBUS)
+  tlBusWrapperLocationMap.lift(SBUS).map { _.clockGroupNode := asyncClockGroupsNode }
+
+  // TODO deprecate these public members to see where users are manually hardcoding a particular bus that might actually not exist in a certain dynamic topology
+  val pbus = tlBusWrapperLocationMap.lift(PBUS).getOrElse(sbus)
+  val fbus = tlBusWrapperLocationMap.lift(FBUS).getOrElse(sbus)
+  val mbus = tlBusWrapperLocationMap.lift(MBUS).getOrElse(sbus)
+  val cbus = tlBusWrapperLocationMap.lift(CBUS).getOrElse(sbus)
 
   // Collect information for use in DTS
-  lazy val topManagers = sbus.unifyManagers
   ResourceBinding {
     val managers = topManagers
     val max = managers.flatMap(_.address).map(_.max).max
@@ -136,15 +115,7 @@ abstract class BaseSubsystem(implicit p: Parameters) extends BareSubsystem
 
   lazy val logicalTreeNode = new SubsystemLogicalTreeNode()
 
-  private val buses = Seq(
-    sbus,
-    pbus,
-    fbus,
-    mbus,
-    cbus
-  )
-
-  buses.foreach { bus =>
+  tlBusWrapperLocationMap.values.foreach { bus =>
     val builtIn = bus.builtInDevices
     builtIn.errorOpt.foreach { error =>
       LogicalModuleTree.add(logicalTreeNode, error.logicalTreeNode)