[Flang][Driver] Add infrastructure for basic frontend actions and fil…

…e I/O

This patch introduces the dependencies required to read and manage input files
provided by the command line option. It also adds the infrastructure to create
and write to output files. The output is sent to either stdout or a file
(specified with the `-o` flag).

Separately, in order to be able to test the code for file I/O, it adds
infrastructure to create frontend actions. As a basic testable example, it adds
the `InputOutputTest` FrontendAction. The sole purpose of this action is to
read a file from the command line and print it either to stdout or the output
file.  This action is run by using the `-test-io` flag also introduced in this
patch (available for `flang-new` and `flang-new -fc1`). With this patch:
```
flang-new -test-io input-file.f90
```
will read input-file.f90 and print it in the output file.

The `InputOutputTest` frontend action has been introduced primarily to
facilitate testing. It is hidden from users (i.e. it's only displayed with
`--help-hidden`). Currently Clang doesn’t have an equivalent action.

`-test-io` is used to trigger the InputOutputTest action in the Flang frontend
driver. This patch makes sure that “flang-new” forwards it to “flang-new -fc1"
by creating a preprocessor job. However, in Flang.cpp, `-test-io` is passed to
“flang-new -fc1” without `-E`. This way we make sure that the preprocessor is
_not_ run in the frontend driver. This is the desired behaviour: `-test-io`
should only read the input file and print it to the output stream.

co-authored-by: Andrzej Warzynski <andrzej.warzynski@arm.com>

Differential Revision: https://reviews.llvm.org/D87989

clang/include/clang/Driver/Options.h

@@ -36,7 +36,8 @@ enum ClangFlags {
   LinkOption = (1 << 13),
   FlangOption = (1 << 14),
   FC1Option = (1 << 15),
-  Ignored = (1 << 16),
+  FlangOnlyOption = (1 << 16),
+  Ignored = (1 << 17),
 };
 
 enum ID {

clang/include/clang/Driver/Options.td

@@ -60,6 +60,10 @@ def LinkOption : OptionFlag;
 // flang mode.
 def FlangOption : OptionFlag;
 
+// FlangOnlyOption - This option should only be used by Flang (i.e. it is not
+// available for Clang)
+def FlangOnlyOption : OptionFlag;
+
 // FC1Option - This option should be accepted by flang -fc1.
 def FC1Option : OptionFlag;
 
@@ -2130,8 +2134,8 @@ def gno_embed_source : Flag<["-"], "gno-embed-source">, Group<g_flags_Group>,
     Flags<[DriverOption]>,
     HelpText<"Restore the default behavior of not embedding source text in DWARF debug sections">;
 def headerpad__max__install__names : Joined<["-"], "headerpad_max_install_names">;
-def help : Flag<["-", "--"], "help">, Flags<[CC1Option,CC1AsOption, FC1Option, FlangOption]>,
-  HelpText<"Display available options">;
+def help : Flag<["-", "--"], "help">, Flags<[CC1Option,CC1AsOption, FC1Option,
+    FlangOption]>, HelpText<"Display available options">;
 def ibuiltininc : Flag<["-"], "ibuiltininc">,
   HelpText<"Enable builtin #include directories even when -nostdinc is used "
            "before or after -ibuiltininc. "
@@ -2811,7 +2815,8 @@ def nostdincxx : Flag<["-"], "nostdinc++">, Flags<[CC1Option]>,
 def nostdlib : Flag<["-"], "nostdlib">, Group<Link_Group>;
 def nostdlibxx : Flag<["-"], "nostdlib++">;
 def object : Flag<["-"], "object">;
-def o : JoinedOrSeparate<["-"], "o">, Flags<[DriverOption, RenderAsInput, CC1Option, CC1AsOption]>,
+def o : JoinedOrSeparate<["-"], "o">, Flags<[DriverOption, RenderAsInput,
+  CC1Option, CC1AsOption, FC1Option, FlangOption]>,
   HelpText<"Write output to <file>">, MetaVarName<"<file>">;
 def pagezero__size : JoinedOrSeparate<["-"], "pagezero_size">;
 def pass_exit_codes : Flag<["-", "--"], "pass-exit-codes">, Flags<[Unsupported]>;
@@ -3546,6 +3551,12 @@ def fno_sycl : Flag<["-"], "fno-sycl">, Group<sycl_Group>, Flags<[CoreOption]>,
 def sycl_std_EQ : Joined<["-"], "sycl-std=">, Group<sycl_Group>, Flags<[CC1Option, NoArgumentUnused, CoreOption]>,
   HelpText<"SYCL language standard to compile for.">, Values<"2017, 121, 1.2.1, sycl-1.2.1">;
 
+//===----------------------------------------------------------------------===//
+// FlangOption and FC1 Options
+//===----------------------------------------------------------------------===//
+def test_io : Flag<["-"], "test-io">, Flags<[HelpHidden, FlangOption, FC1Option, FlangOnlyOption]>, Group<Action_Group>,
+  HelpText<"Run the InputOuputTest action. Use for development and testing only.">;
+
 //===----------------------------------------------------------------------===//
 // CC1 Options
 //===----------------------------------------------------------------------===//

clang/lib/Driver/Driver.cpp

@@ -211,6 +211,11 @@ InputArgList Driver::ParseArgStrings(ArrayRef<const char *> ArgStrings,
   std::tie(IncludedFlagsBitmask, ExcludedFlagsBitmask) =
       getIncludeExcludeOptionFlagMasks(IsClCompatMode);
 
+  // Make sure that Flang-only options don't pollute the Clang output
+  // TODO: Make sure that Clang-only options don't pollute Flang output
+  if (!IsFlangMode())
+    ExcludedFlagsBitmask |= options::FlangOnlyOption;
+
   unsigned MissingArgIndex, MissingArgCount;
   InputArgList Args =
       getOpts().ParseArgs(ArgStrings, MissingArgIndex, MissingArgCount,
@@ -1573,6 +1578,8 @@ void Driver::PrintHelp(bool ShowHidden) const {
 
   if (IsFlangMode())
     IncludedFlagsBitmask |= options::FlangOption;
+  else
+    ExcludedFlagsBitmask |= options::FlangOnlyOption;
 
   std::string Usage = llvm::formatv("{0} [options] file...", Name).str();
   getOpts().PrintHelp(llvm::outs(), Usage.c_str(), DriverTitle.c_str(),
@@ -1628,6 +1635,11 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
   unsigned int DisableFlags =
       options::NoDriverOption | options::Unsupported | options::Ignored;
 
+  // Make sure that Flang-only options don't pollute the Clang output
+  // TODO: Make sure that Clang-only options don't pollute Flang output
+  if (!IsFlangMode())
+    DisableFlags |= options::FlangOnlyOption;
+
   // Distinguish "--autocomplete=-someflag" and "--autocomplete=-someflag,"
   // because the latter indicates that the user put space before pushing tab
   // which should end up in a file completion.

clang/lib/Driver/ToolChains/Flang.cpp

@@ -30,12 +30,18 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
   CmdArgs.push_back("-fc1");
 
-  CmdArgs.push_back("-triple");
-  CmdArgs.push_back(Args.MakeArgString(TripleStr));
-
+  // TODO: Eventually all actions will require a triple (e.g. `-triple
+  // aarch64-unknown-linux-gnu`). However, `-triple` is currently not supported
+  // by `flang-new -fc1`, so we only add it selectively to actions that we
+  // don't support/execute just yet.
   if (isa<PreprocessJobAction>(JA)) {
-    CmdArgs.push_back("-E");
+    if (C.getArgs().hasArg(options::OPT_test_io))
+      CmdArgs.push_back("-test-io");
+    else
+      CmdArgs.push_back("-E");
   } else if (isa<CompileJobAction>(JA) || isa<BackendJobAction>(JA)) {
+    CmdArgs.push_back("-triple");
+    CmdArgs.push_back(Args.MakeArgString(TripleStr));
     if (JA.getType() == types::TY_Nothing) {
       CmdArgs.push_back("-fsyntax-only");
     } else if (JA.getType() == types::TY_AST) {
@@ -52,6 +58,8 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
       assert(false && "Unexpected output type!");
     }
   } else if (isa<AssembleJobAction>(JA)) {
+    CmdArgs.push_back("-triple");
+    CmdArgs.push_back(Args.MakeArgString(TripleStr));
     CmdArgs.push_back("-emit-obj");
   } else {
     assert(false && "Unexpected action class for Flang tool.");

clang/lib/Driver/Types.cpp

@@ -325,10 +325,12 @@ types::getCompilationPhases(const clang::driver::Driver &Driver,
   // Filter to compiler mode. When the compiler is run as a preprocessor then
   // compilation is not an option.
   // -S runs the compiler in Assembly listing mode.
+  // -test-io is used by Flang to run InputOutputTest action
   if (Driver.CCCIsCPP() || DAL.getLastArg(options::OPT_E) ||
       DAL.getLastArg(options::OPT__SLASH_EP) ||
       DAL.getLastArg(options::OPT_M, options::OPT_MM) ||
-      DAL.getLastArg(options::OPT__SLASH_P))
+      DAL.getLastArg(options::OPT__SLASH_P) ||
+      DAL.getLastArg(options::OPT_test_io))
     LastPhase = phases::Preprocess;
 
   // --precompile only runs up to precompilation.

clang/test/Driver/immediate-options.c

@@ -3,8 +3,12 @@
 // HELP-NOT: ast-dump
 // HELP-NOT: driver-mode
 
+// Make sure that Flang-only options are not available in Clang
+// HELP-NOT: test-io
+
 // RUN: %clang --help-hidden | FileCheck %s -check-prefix=HELP-HIDDEN
 // HELP-HIDDEN: driver-mode
+// HELP-HIDDEN-NOT: test-io
 
 // RUN: %clang -dumpversion | FileCheck %s -check-prefix=DUMPVERSION
 // DUMPVERSION: {{[0-9]+\.[0-9.]+}}

flang/include/flang/Frontend/CompilerInstance.h

@@ -9,6 +9,9 @@
 #define LLVM_FLANG_FRONTEND_COMPILERINSTANCE_H
 
 #include "flang/Frontend/CompilerInvocation.h"
+#include "flang/Frontend/FrontendAction.h"
+#include "flang/Parser/provenance.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include <cassert>
 #include <memory>
@@ -20,18 +23,68 @@ class CompilerInstance {
   /// The options used in this compiler instance.
   std::shared_ptr<CompilerInvocation> invocation_;
 
+  /// Flang file  manager.
+  std::shared_ptr<Fortran::parser::AllSources> allSources_;
+
   /// The diagnostics engine instance.
   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_;
 
+  /// Holds information about the output file.
+  struct OutputFile {
+    std::string filename_;
+    OutputFile(std::string inputFilename)
+        : filename_(std::move(inputFilename)) {}
+  };
+
+  /// Output stream that doesn't support seeking (e.g. terminal, pipe).
+  /// This stream is normally wrapped in buffer_ostream before being passed
+  /// to users (e.g. via CreateOutputFile).
+  std::unique_ptr<llvm::raw_fd_ostream> nonSeekStream_;
+
+  /// The list of active output files.
+  std::list<OutputFile> outputFiles_;
+
+  /// Holds the output stream provided by the user. Normally, users of
+  /// CompilerInstance will call CreateOutputFile to obtain/create an output
+  /// stream. If they want to provide their own output stream, this field will
+  /// facilitate this. It is optional and will normally be just a nullptr.
+  std::unique_ptr<llvm::raw_pwrite_stream> outputStream_;
+
 public:
   explicit CompilerInstance();
 
   ~CompilerInstance();
+
+  /// @name Compiler Invocation
+  /// {
+
   CompilerInvocation &GetInvocation() {
     assert(invocation_ && "Compiler instance has no invocation!");
     return *invocation_;
   };
 
+  /// Replace the current invocation.
+  void SetInvocation(std::shared_ptr<CompilerInvocation> value);
+
+  /// }
+  /// @name File manager
+  /// {
+
+  /// Return the current allSources.
+  Fortran::parser::AllSources &GetAllSources() const { return *allSources_; }
+
+  bool HasAllSources() const { return allSources_ != nullptr; }
+
+  /// }
+  /// @name High-Level Operations
+  /// {
+
+  /// Execute the provided action against the compiler's
+  /// CompilerInvocation object.
+  /// \param act - The action to execute.
+  /// \return - True on success.
+  bool ExecuteAction(FrontendAction &act);
+
   /// }
   /// @name Forwarding Methods
   /// {
@@ -60,7 +113,7 @@ class CompilerInstance {
     return *diagnostics_;
   }
 
-  /// SetDiagnostics - Replace the current diagnostics engine.
+  /// Replace the current diagnostics engine.
   void SetDiagnostics(clang::DiagnosticsEngine *value);
 
   clang::DiagnosticConsumer &GetDiagnosticClient() const {
@@ -75,30 +128,80 @@ class CompilerInstance {
     return *diagnostics_;
   }
 
+  /// {
+  /// @name Output Files
+  /// {
+
+  /// Add an output file onto the list of tracked output files.
+  ///
+  /// \param outFile - The output file info.
+  void AddOutputFile(OutputFile &&outFile);
+
+  /// Clear the output file list.
+  void ClearOutputFiles(bool eraseFiles);
+
+  /// Create the default output file (based on the invocation's options) and
+  /// add it to the list of tracked output files. If the name of the output
+  /// file is not provided, it is derived from the input file.
+  ///
+  /// \param binary     The mode to open the file in.
+  /// \param baseInput  If the invocation contains no output file name (i.e.
+  ///                   outputFile_ in FrontendOptions is empty), the input path
+  ///                   name to use for deriving the output path.
+  /// \param extension  The extension to use for output names derived from
+  ///                   \p baseInput.
+  /// \return           ostream for the output file or nullptr on error.
+  std::unique_ptr<llvm::raw_pwrite_stream> CreateDefaultOutputFile(
+      bool binary = true, llvm::StringRef baseInput = "",
+      llvm::StringRef extension = "");
+
+  /// Create a new output file
+  ///
+  /// \param outputPath   The path to the output file.
+  /// \param error [out]  On failure, the error.
+  /// \param binary       The mode to open the file in.
+  /// \return             ostream for the output file or nullptr on error.
+  std::unique_ptr<llvm::raw_pwrite_stream> CreateOutputFile(
+      llvm::StringRef outputPath, std::error_code &error, bool binary);
+
   /// }
   /// @name Construction Utility Methods
   /// {
 
-  /// Create a DiagnosticsEngine object with a the TextDiagnosticPrinter.
+  /// Create a DiagnosticsEngine object
   ///
-  /// If no diagnostic client is provided, this creates a
-  /// DiagnosticConsumer that is owned by the returned diagnostic
-  /// object, if using directly the caller is responsible for
-  /// releasing the returned DiagnosticsEngine's client eventually.
+  /// If no diagnostic client is provided, this method creates a
+  /// DiagnosticConsumer that is owned by the returned diagnostic object. If
+  /// using directly the caller is responsible for releasing the returned
+  /// DiagnosticsEngine's client eventually.
   ///
   /// \param opts - The diagnostic options; note that the created text
   /// diagnostic object contains a reference to these options.
   ///
-  /// \param client If non-NULL, a diagnostic client that will be
-  /// attached to (and, then, owned by) the returned DiagnosticsEngine
-  /// object.
+  /// \param client - If non-NULL, a diagnostic client that will be attached to
+  /// (and optionally, depending on /p shouldOwnClient, owned by) the returned
+  /// DiagnosticsEngine object.
   ///
   /// \return The new object on success, or null on failure.
   static clang::IntrusiveRefCntPtr<clang::DiagnosticsEngine> CreateDiagnostics(
       clang::DiagnosticOptions *opts,
       clang::DiagnosticConsumer *client = nullptr, bool shouldOwnClient = true);
   void CreateDiagnostics(
       clang::DiagnosticConsumer *client = nullptr, bool shouldOwnClient = true);
+
+  /// }
+  /// @name Output Stream Methods
+  /// {
+  void SetOutputStream(std::unique_ptr<llvm::raw_pwrite_stream> outStream) {
+    outputStream_ = std::move(outStream);
+  }
+
+  bool IsOutputStreamNull() { return (outputStream_ == nullptr); }
+
+  // Allow the frontend compiler to write in the output stream.
+  void WriteOutputStream(const std::string &message) {
+    *outputStream_ << message;
+  }
 };
 
 } // end namespace Fortran::frontend

flang/include/flang/Frontend/CompilerInvocation.h

@@ -24,7 +24,7 @@ bool ParseDiagnosticArgs(clang::DiagnosticOptions &opts,
 
 class CompilerInvocationBase {
 public:
-  /// Options controlling the diagnostic engine.$
+  /// Options controlling the diagnostic engine.
   llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> diagnosticOpts_;
 
   CompilerInvocationBase();