File Serialization of Annotations

src/main/scala/firrtl/Emitter.scala

@@ -14,11 +14,14 @@ import firrtl.PrimOps._
 import firrtl.WrappedExpression._
 import Utils._
 import MemPortUtils.{memPortField, memType}
-import firrtl.options.{HasShellOptions, PhaseException, ShellOption, Unserializable}
-import firrtl.stage.{RunFirrtlTransformAnnotation, TransformManager}
+import firrtl.options.{HasShellOptions, CustomFileEmission, ShellOption, PhaseException}
+import firrtl.options.Viewer.view
+import firrtl.stage.{FirrtlFileAnnotation, FirrtlOptions, RunFirrtlTransformAnnotation, TransformManager}
 // Datastructures
 import scala.collection.mutable.ArrayBuffer
 
+import java.io.File
+
 case class EmitterException(message: String) extends PassException(message)
 
 // ***** Annotations for telling the Emitters what to emit *****
@@ -92,17 +95,35 @@ final case class EmittedFirrtlModule(name: String, value: String, outputSuffix:
 final case class EmittedVerilogModule(name: String, value: String, outputSuffix: String) extends EmittedModule
 
 /** Traits for Annotations containing emitted components */
-sealed trait EmittedAnnotation[T <: EmittedComponent] extends NoTargetAnnotation with Unserializable {
+sealed trait EmittedAnnotation[T <: EmittedComponent] extends NoTargetAnnotation with CustomFileEmission {
   val value: T
+
+  override protected def baseFileName(annotations: AnnotationSeq): String = {
+    view[FirrtlOptions](annotations).outputFileName.getOrElse(value.name)
+  }
+
+  override protected val suffix: Option[String] = Some(value.outputSuffix)
+
+}
+sealed trait EmittedCircuitAnnotation[T <: EmittedCircuit] extends EmittedAnnotation[T] {
+
+  override def getBytes = value.value.getBytes
+
+}
+sealed trait EmittedModuleAnnotation[T <: EmittedModule] extends EmittedAnnotation[T] {
+
+  override def getBytes = value.value.getBytes
+
 }
-sealed trait EmittedCircuitAnnotation[T <: EmittedCircuit] extends EmittedAnnotation[T]
-sealed trait EmittedModuleAnnotation[T <: EmittedModule] extends EmittedAnnotation[T]
 
 case class EmittedFirrtlCircuitAnnotation(value: EmittedFirrtlCircuit)
-    extends EmittedCircuitAnnotation[EmittedFirrtlCircuit]
+    extends EmittedCircuitAnnotation[EmittedFirrtlCircuit] {
+
+  override def replacements(file: File): AnnotationSeq = Seq(FirrtlFileAnnotation(file.toString))
+
+}
 case class EmittedVerilogCircuitAnnotation(value: EmittedVerilogCircuit)
     extends EmittedCircuitAnnotation[EmittedVerilogCircuit]
-
 case class EmittedFirrtlModuleAnnotation(value: EmittedFirrtlModule)
     extends EmittedModuleAnnotation[EmittedFirrtlModule]
 case class EmittedVerilogModuleAnnotation(value: EmittedVerilogModule)
@@ -465,10 +486,10 @@ class VerilogEmitter extends SeqTransform with Emitter {
     throw EmitterException("Cannot emit verification statements in Verilog" +
       "(2001). Use the SystemVerilog emitter instead.")
   }
-  
-  /** 
+
+  /**
     * Store Emission option per Target
-    * Guarantee only one emission option per Target 
+    * Guarantee only one emission option per Target
     */
   private[firrtl] class EmissionOptionMap[V <: EmissionOption](val df : V) {
     private val m = collection.mutable.HashMap[ReferenceTarget, V]().withDefaultValue(df)
@@ -480,9 +501,9 @@ class VerilogEmitter extends SeqTransform with Emitter {
     }
     def apply(key: ReferenceTarget): V = m.apply(key)
   }
-  
+
   /** Provide API to retrieve EmissionOptions based on the provided [[AnnotationSeq]]
-    * 
+    *
     * @param annotations : AnnotationSeq to be searched for EmissionOptions
     *
     */
@@ -500,16 +521,16 @@ class VerilogEmitter extends SeqTransform with Emitter {
 
     def getRegisterEmissionOption(target: ReferenceTarget): RegisterEmissionOption =
       registerEmissionOption(target)
-      
+
     def getWireEmissionOption(target: ReferenceTarget): WireEmissionOption =
       wireEmissionOption(target)
-      
+
     def getPortEmissionOption(target: ReferenceTarget): PortEmissionOption =
       portEmissionOption(target)
-      
+
     def getNodeEmissionOption(target: ReferenceTarget): NodeEmissionOption =
       nodeEmissionOption(target)
-      
+
     def getConnectEmissionOption(target: ReferenceTarget): ConnectEmissionOption =
       connectEmissionOption(target)
 

src/main/scala/firrtl/options/StageAnnotations.scala

@@ -4,6 +4,9 @@ package firrtl.options
 
 import firrtl.AnnotationSeq
 import firrtl.annotations.{Annotation, NoTargetAnnotation}
+import firrtl.options.Viewer.view
+
+import java.io.File
 
 import scopt.OptionParser
 
@@ -14,6 +17,63 @@ sealed trait StageOption { this: Annotation => }
   */
 trait Unserializable { this: Annotation => }
 
+/** Mix-in that lets an [[firrtl.annotations.Annotation Annotation]] serialize itself to a file separate from the output
+  * annotation file.
+  *
+  * This can be used as a mechanism to serialize an [[firrtl.options.Unserializable Unserializable]] annotation or to
+  * write ancillary collateral used by downstream tooling, e.g., a TCL script or an FPGA constraints file. Any
+  * annotations using this mix-in will be serialized by the [[firrtl.options.phases.WriteOutputAnnotations
+  * WriteOutputAnnotations]] phase. This is one of the last phases common to all [[firrtl.options.Stage Stages]] and
+  * should not have to be called/included manually.
+  *
+  * Note: from the perspective of transforms generating annotations that mix-in this trait, the serialized files are not
+  * expected to be available to downstream transforms. Communication of information between transforms must occur
+  * through the annotations that will eventually be serialized to files.
+  */
+trait CustomFileEmission { this: Annotation =>
+
+  /** Output filename where serialized content will be written
+    *
+    * The full annotation sequence is a parameter to allow for the location where this annotation will be serialized to
+    * be a function of other annotations, e.g., if the location where information is written is controlled by a separate
+    * file location annotation.
+    *
+    * @param annotations the annotation sequence at the time of emission
+    */
+  protected def baseFileName(annotations: AnnotationSeq): String
+
+  /** Optional suffix of the output file */
+  protected def suffix: Option[String]
+
+  /** A method that can convert this annotation to bytes that will be written to a file.
+    *
+    * If you only need to serialize a string, you can use the `getBytes` method:
+    * {{{
+    *  def getBytes: Iterable[Byte] = myString.getBytes
+    * }}}
+    */
+  def getBytes: Iterable[Byte]
+
+  /** Optionally, a sequence of annotations that will replace this annotation in the output annotation file.
+    *
+    * A non-empty implementation of this method is a mechanism for telling a downstream [[firrtl.options.Stage Stage]]
+    * how to deserialize the information that was serialized to a separate file. For example, if a FIRRTL circuit is
+    * serialized to a separate file, this method could include an input file annotation that a later stage can use to
+    * read the serialized FIRRTL circuit back in.
+    */
+  def replacements(file: File): AnnotationSeq = Seq.empty
+
+  /** Method that returns the filename where this annotation will be serialized.
+    *
+    * @param annotations the annotations at the time of serialization
+    */
+  final def filename(annotations: AnnotationSeq): File = {
+    val name = view[StageOptions](annotations).getBuildFileName(baseFileName(annotations), suffix)
+    new File(name)
+  }
+
+}
+
 /** Holds the name of the target directory
   *  - set with `-td/--target-dir`
   *  - if unset, a [[TargetDirAnnotation]] will be generated with the

src/main/scala/firrtl/options/StageOptions.scala

@@ -59,11 +59,13 @@ class StageOptions private [firrtl] (
       } else {
         new File(targetDir + "/" + f)
       }
-    }.getCanonicalFile
+    }.toPath.normalize.toFile
 
-    val parent = file.getParentFile
-
-    if (!parent.exists) { parent.mkdirs() }
+    file.getParentFile match {
+      case null =>
+      case parent if (!parent.exists) => parent.mkdirs()
+      case _ =>
+    }
 
     file.toString
   }

src/main/scala/firrtl/options/phases/WriteOutputAnnotations.scala

@@ -3,11 +3,12 @@
 package firrtl.options.phases
 
 import firrtl.AnnotationSeq
-import firrtl.annotations.{DeletedAnnotation, JsonProtocol}
-import firrtl.options.{Phase, StageOptions, Unserializable, Viewer}
-import firrtl.options.Dependency
+import firrtl.annotations.{Annotation, DeletedAnnotation, JsonProtocol}
+import firrtl.options.{CustomFileEmission, Dependency, Phase, PhaseException, StageOptions, Unserializable, Viewer}
 
-import java.io.PrintWriter
+import java.io.{BufferedWriter, File, FileWriter, PrintWriter}
+
+import scala.collection.mutable
 
 /** [[firrtl.options.Phase Phase]] that writes an [[AnnotationSeq]] to a file. A file is written if and only if a
   * [[StageOptions]] view has a non-empty [[StageOptions.annotationFileOut annotationFileOut]].
@@ -27,10 +28,34 @@ class WriteOutputAnnotations extends Phase {
   /** Write the input [[AnnotationSeq]] to a fie. */
   def transform(annotations: AnnotationSeq): AnnotationSeq = {
     val sopts = Viewer[StageOptions].view(annotations)
-    val serializable = annotations.filter{
-      case _: Unserializable    => false
-      case _: DeletedAnnotation => sopts.writeDeleted
-      case _                    => true
+    val filesWritten = mutable.HashMap.empty[String, Annotation]
+    val serializable: AnnotationSeq = annotations.toSeq.flatMap {
+      case _: Unserializable     => None
+      case a: DeletedAnnotation  => if (sopts.writeDeleted) { Some(a) } else { None }
+      case a: CustomFileEmission =>
+        val filename = a.filename(annotations)
+        val canonical = filename.getCanonicalPath()
+
+        filesWritten.get(canonical) match {
+          case None =>
+            val w = new BufferedWriter(new FileWriter(filename))
+            a.getBytes.foreach( w.write(_) )
+            w.close()
+            filesWritten(canonical) = a
+          case Some(first) =>
+            val msg =
+              s"""|Multiple CustomFileEmission annotations would be serialized to the same file, '$canonical'
+                  |  - first writer:
+                  |      class: ${first.getClass.getName}
+                  |      trimmed serialization: ${first.serialize.take(80)}
+                  |  - second writer:
+                  |      class: ${a.getClass.getName}
+                  |      trimmed serialization: ${a.serialize.take(80)}
+                  |""".stripMargin
+            throw new PhaseException(msg)
+        }
+        a.replacements(filename)
+      case a => Some(a)
     }
 
     sopts.annotationFileOut match {

src/main/scala/firrtl/stage/FirrtlStage.scala

@@ -8,8 +8,7 @@ import firrtl.options.phases.DeletedWrapper
 import firrtl.stage.phases.CatchExceptions
 
 class FirrtlPhase
-    extends PhaseManager(targets=Seq(Dependency[firrtl.stage.phases.Compiler],
-                                     Dependency[firrtl.stage.phases.WriteEmitted])) {
+    extends PhaseManager(targets=Seq(Dependency[firrtl.stage.phases.Compiler])) {
 
   override def invalidates(a: Phase) = false
 

src/main/scala/firrtl/stage/package.scala

@@ -39,11 +39,9 @@ package object stage {
 
   private [firrtl] implicit object FirrtlExecutionResultView extends OptionsView[FirrtlExecutionResult] with LazyLogging {
 
-    private lazy val dummyWriteEmitted = new WriteEmitted
-
     def view(options: AnnotationSeq): FirrtlExecutionResult = {
       val emittedRes = options
-        .collect{ case DeletedAnnotation(dummyWriteEmitted.name, a: EmittedAnnotation[_]) => a.value.value }
+        .collect{ case a: EmittedAnnotation[_] => a.value.value }
         .mkString("\n")
 
       val emitters = options.collect{ case RunFirrtlTransformAnnotation(e: Emitter) => e }

src/main/scala/firrtl/stage/phases/Compiler.scala

@@ -51,7 +51,7 @@ class Compiler extends Phase with Translator[AnnotationSeq, Seq[CompilerRun]] {
         Dependency[AddCircuit],
         Dependency[AddImplicitOutputFile])
 
-  override def optionalPrerequisiteOf = Seq(Dependency[WriteEmitted])
+  override def optionalPrerequisiteOf = Seq.empty
 
   override def invalidates(a: Phase) = false
 

src/main/scala/firrtl/stage/phases/WriteEmitted.scala

@@ -24,6 +24,8 @@ import java.io.PrintWriter
   *
   * Any annotations written to files will be deleted.
   */
+@deprecated("Annotations that mixin the CustomFileEmission trait are automatically serialized by stages." +
+              "This will be removed in FIRRTL 1.5", "FIRRTL 1.4.0")
 class WriteEmitted extends Phase {
 
   override def prerequisites = Seq.empty

src/test/scala/firrtl/stage/phases/tests/DriverCompatibilitySpec.scala

@@ -125,7 +125,7 @@ class DriverCompatibilitySpec extends AnyFlatSpec with Matchers with PrivateMeth
   new PhaseFixture(new AddImplicitFirrtlFile) {
     val annotations = Seq( TopNameAnnotation("foo") )
     val expected = annotations.toSet +
-      FirrtlFileAnnotation(new File("foo.fir").getCanonicalPath)
+      FirrtlFileAnnotation(new File("foo.fir").getPath())
 
     phase.transform(annotations).toSet should be (expected)
   }

src/test/scala/firrtl/testutils/FirrtlSpec.scala

@@ -104,13 +104,13 @@ trait FirrtlRunners extends BackendCompilationUtilities {
     val customName = s"${prefix}_custom"
     val customAnnos = getBaseAnnos(customName) ++: toAnnos((new GetNamespace) +: customTransforms) ++: customAnnotations
 
-    val customResult = (new firrtl.stage.FirrtlStage).run(customAnnos)
+    val customResult = (new firrtl.stage.FirrtlStage).execute(Array.empty, customAnnos)
     val nsAnno = customResult.collectFirst { case m: ModuleNamespaceAnnotation => m }.get
 
     val refSuggestedName = s"${prefix}_ref"
     val refAnnos = getBaseAnnos(refSuggestedName) ++: Seq(RunFirrtlTransformAnnotation(new RenameModules), nsAnno)
 
-    val refResult = (new firrtl.stage.FirrtlStage).run(refAnnos)
+    val refResult = (new firrtl.stage.FirrtlStage).execute(Array.empty, refAnnos)
     val refName = refResult.collectFirst({ case stage.FirrtlCircuitAnnotation(c) => c.main }).getOrElse(refSuggestedName)
 
     assert(BackendCompilationUtilities.yosysExpectSuccess(customName, refName, testDir, timesteps))
@@ -145,7 +145,7 @@ trait FirrtlRunners extends BackendCompilationUtilities {
       annotations ++:
       (customTransforms ++ extraCheckTransforms).map(RunFirrtlTransformAnnotation(_))
 
-    (new firrtl.stage.FirrtlStage).run(annos)
+    (new firrtl.stage.FirrtlStage).execute(Array.empty, annos)
 
     testDir
   }

src/test/scala/firrtlTests/execution/VerilogExecution.scala

@@ -21,7 +21,8 @@ trait VerilogExecution extends TestExecution {
     // Run FIRRTL, emit Verilog file
     val cAnno = FirrtlCircuitAnnotation(c)
     val tdAnno = TargetDirAnnotation(testDir.getAbsolutePath)
-    (new FirrtlStage).run(AnnotationSeq(Seq(cAnno, tdAnno) ++ customAnnotations))
+
+    (new FirrtlStage).execute(Array.empty, AnnotationSeq(Seq(cAnno, tdAnno)) ++ customAnnotations)
 
     // Copy harness resource to test directory
     val harness = new File(testDir, s"top.cpp")