Handle includes in pre- and post-deployment scripts (fixes #23) (#61)

* Another attempt at #23

* Generate pre- and post-deploy scripts using SqlTools BatchParser (#23)

* Use reflection to accurately set _currentFilePath (#23)

* Check included files for incremental builds (#23)

* Checkout submodules in the CI pipeline

* Remove ugly hack

* Remove _currentFilePath entirely

* Test post-deployment script includes

* Resolve comments from PR review

* Resolve comments from PR review

* Compile individual files from ManagedBatchParser

* Refactor ModelValidationError and add an overload constructor

* Fix LogicalName for ManagedBatchParser localization resources

* Resolve comments from PR review

Co-authored-by: Jonathan Mezach <jonathan.mezach@rr-wfm.com>
Co-authored-by: Rosenberg, Jeff <jeff.rosenberg@icfnext.com>

.github/workflows/main.yml

@@ -23,6 +23,7 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0 # avoid shallow clone so nbgv can do its work.
+        submodules: 'true' # need to check out the SqlTools submodule to build successfully.
 
     # Install .NET Core SDK
     # TOOO: Remove this workaround once https://github.com/actions/setup-dotnet/issues/25 gets fixed

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "sqltoolsservice"]
+	path = sqltoolsservice
+	url = https://github.com/microsoft/sqltoolsservice.git

src/DacpacTool/BaseOptions.cs

@@ -0,0 +1,10 @@
+using System.IO;
+using Microsoft.SqlServer.Dac.Model;
+
+namespace MSBuild.Sdk.SqlProj.DacpacTool
+{
+    public abstract class BaseOptions
+    {
+        public bool Debug { get; set; }
+    }
+}

src/DacpacTool/BuildOptions.cs

@@ -3,7 +3,7 @@
 
 namespace MSBuild.Sdk.SqlProj.DacpacTool
 {
-    public class BuildOptions
+    public class BuildOptions : BaseOptions
     {
         public string Name { get; set; }
         public string Version { get; set; }
@@ -16,6 +16,5 @@ public class BuildOptions
         public FileInfo PreDeploy { get; set; }
         public FileInfo PostDeploy { get; set; }
         public FileInfo RefactorLog { get; set; }
-        public bool Debug { get; set; }
     }
 }

src/DacpacTool/DacpacTool.csproj

@@ -6,10 +6,29 @@
     <LangVersion>8.0</LangVersion>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <SqlToolsPath>../../sqltoolsservice/src</SqlToolsPath>
+    <ManagedBatchParserPath>$(SqlToolsPath)/Microsoft.SqlTools.ManagedBatchParser</ManagedBatchParserPath>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.SqlServer.DACFX" Version="150.4826.1" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20427.1" />
     <PackageReference Include="System.Security.Permissions" Version="4.7.0" />
   </ItemGroup>
 
+  <!-- References to Microsoft.SqlTools.ManagedBatchParser -->
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="161.41011.9" />
+    <ProjectReference Include="$(SqlToolsPath)/Microsoft.SqlTools.Hosting/Microsoft.SqlTools.Hosting.csproj" />
+    <Compile Include="$(ManagedBatchParserPath)/Localization/*.cs" />
+    <EmbeddedResource Include="$(ManagedBatchParserPath)/Localization/sr.resx" LogicalName="Microsoft.SqlTools.ManagedBatchParser.Localization.SR.resources" />
+    <None Include="$(ManagedBatchParserPath)/Localization/sr.strings" />
+    <Compile Include="$(ManagedBatchParserPath)/BatchParser/**/*.cs" />
+    <Compile Remove="$(ManagedBatchParserPath)/BatchParser/ExecutionEngineCode/ExecutionEngine.cs" />
+    <Compile Remove="$(ManagedBatchParserPath)/BatchParser/ExecutionEngineCode/Batch.cs" />
+    <Compile Remove="$(ManagedBatchParserPath)/BatchParser/ExecutionEngineCode/BatchParser*Args.cs" />
+    <Compile Remove="$(ManagedBatchParserPath)/BatchParser/BatchParserWrapper.cs" />
+  </ItemGroup>
+
 </Project>

src/DacpacTool/DeployOptions.cs

@@ -2,7 +2,7 @@
 
 namespace MSBuild.Sdk.SqlProj.DacpacTool
 {
-    public class DeployOptions
+    public class DeployOptions : BaseOptions
     {
         public FileInfo Input { get; set; }
         public string TargetServerName { get; set; }
@@ -11,6 +11,5 @@ public class DeployOptions
         public string TargetPassword { get; set; }
         public string[] Property { get; set; }
         public string[] SqlCmdVar { get; set; }
-        public bool Debug { get; set; }
     }
 }

src/DacpacTool/IncludeVariableResolver.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.SqlTools.ServiceLayer.BatchParser;
+using Microsoft.SqlTools.ServiceLayer.BatchParser.ExecutionEngineCode;
+
+namespace MSBuild.Sdk.SqlProj.DacpacTool
+{
+    // This was just copied from Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser.TestVariableResolver
+    // No special functionality needed for our project
+    internal sealed class IncludeVariableResolver : IVariableResolver
+    {
+        private StringBuilder outputString;
+        private BatchParserSqlCmd batchParserSqlCmd;
+
+        public IncludeVariableResolver()
+        {
+            this.outputString = new StringBuilder();
+            batchParserSqlCmd = new BatchParserSqlCmd();
+        }
+
+        public string GetVariable(PositionStruct pos, string name)
+        {
+            return batchParserSqlCmd.GetVariable(pos, name);
+        }
+
+        public void SetVariable(PositionStruct pos, string name, string value)
+        {
+            outputString.AppendFormat("Setting variable {0} to [{1}]\n", name, value);
+            batchParserSqlCmd.SetVariable(pos, name, value);
+        }
+    }
+}

src/DacpacTool/InspectOptions.cs

@@ -0,0 +1,10 @@
+using System.IO;
+
+namespace MSBuild.Sdk.SqlProj.DacpacTool
+{
+    public class InspectOptions : BaseOptions
+    {
+        public FileInfo PreDeploy { get; set; }
+        public FileInfo PostDeploy { get; set; }
+    }
+}

src/DacpacTool/ModelValidationError.cs

@@ -9,34 +9,62 @@ namespace MSBuild.Sdk.SqlProj.DacpacTool
     /// </summary>
     public class ModelValidationError
     {
-        private readonly DacModelError _modelError;
         private readonly string _sourceName;
+        private readonly int _line;
+        private readonly int _column;
+        private readonly ModelErrorType _errorType;
+        private readonly ModelErrorSeverity _severity;
+        private readonly string _prefix;
+        private readonly int _errorCode;
+        private readonly string _message;
 
         public ModelValidationError(DacModelError modelError, string sourceName)
         {
-            _modelError = modelError ?? throw new ArgumentNullException(nameof(modelError));
-            _sourceName = sourceName.Replace("MSSQL::", string.Empty);
+            modelError = modelError ?? throw new ArgumentNullException(nameof(modelError));
+            _sourceName = String.IsNullOrEmpty(sourceName) ? 
+                            modelError.SourceName : 
+                            sourceName.Replace("MSSQL::", string.Empty);
+            _line = modelError.Line;
+            _column = modelError.Column;
+            _errorType = modelError.ErrorType;
+            _severity = modelError.Severity;
+            _prefix = modelError.Prefix;
+            _errorCode = modelError.ErrorCode;
+            _message = modelError.Message;
         }
 
-        public ModelErrorSeverity Severity { get => _modelError.Severity; }
+        public ModelValidationError(string sourceName, int line, int column, ModelErrorType errorType,
+            ModelErrorSeverity severity, string prefix, int errorCode, string message)
+        {
+            _sourceName = sourceName;
+            _line = line;
+            _column = column;
+            _errorType = errorType;
+            _severity = severity;
+            _prefix = prefix;
+            _errorCode = errorCode;
+            _message = message;
+        }
+
+        public ModelErrorSeverity Severity { get => _severity; }
 
         public override string ToString()
         {
             var stringBuilder = new StringBuilder();
-            stringBuilder.Append(_sourceName ?? _modelError.SourceName);
+            stringBuilder.Append(_sourceName);
             stringBuilder.Append('(');
-            stringBuilder.Append(_modelError.Line);
+            stringBuilder.Append(_line);
             stringBuilder.Append(',');
-            stringBuilder.Append(_modelError.Column);
+            stringBuilder.Append(_column);
             stringBuilder.Append("):");
-            stringBuilder.Append(_modelError.ErrorType);
+            stringBuilder.Append(_errorType);
             stringBuilder.Append(' ');
-            stringBuilder.Append(_modelError.Severity);
+            stringBuilder.Append(_severity);
             stringBuilder.Append(' ');
-            stringBuilder.Append(_modelError.Prefix);
-            stringBuilder.Append(_modelError.ErrorCode);
+            stringBuilder.Append(_prefix);
+            stringBuilder.Append(_errorCode);
             stringBuilder.Append(": ");
-            stringBuilder.Append(_modelError.Message);
+            stringBuilder.Append(_message);
 
             return stringBuilder.ToString();
         }

src/DacpacTool/PackageBuilder.cs

@@ -6,6 +6,7 @@
 using Microsoft.Data.Tools.Schema.Sql.Packaging;
 using Microsoft.SqlServer.Dac;
 using Microsoft.SqlServer.Dac.Model;
+using Microsoft.SqlTools.ServiceLayer.BatchParser;
 
 namespace MSBuild.Sdk.SqlProj.DacpacTool
 {
@@ -305,7 +306,8 @@ private void WritePart(FileInfo file, Package package, string path)
 
             using (var stream = part.GetStream())
             {
-                var buffer = Encoding.UTF8.GetBytes(File.ReadAllText(file.FullName, Encoding.UTF8) + Environment.NewLine + "GO" + Environment.NewLine);
+                var parser = new ScriptParser(file.FullName, new IncludeVariableResolver());
+                var buffer = Encoding.UTF8.GetBytes(parser.GenerateScript());
                 stream.Write(buffer, 0, buffer.Length);
             }
         }

src/DacpacTool/Program.cs

@@ -33,6 +33,16 @@ static async Task<int> Main(string[] args)
             };
             buildCommand.Handler = CommandHandler.Create<BuildOptions>(BuildDacpac);
 
+            var collectIncludesCommand = new Command("collect-includes")
+            {
+                new Option<FileInfo>(new string[] { "--predeploy" }, "Filename of optional pre-deployment script"),
+                new Option<FileInfo>(new string[] { "--postdeploy" }, "Filename of optional post-deployment script"),
+#if DEBUG
+                new Option<bool>(new string[] { "--debug" }, "Waits for a debugger to attach")
+#endif
+            };
+            collectIncludesCommand.Handler = CommandHandler.Create<InspectOptions>(InspectIncludes);
+
             var deployCommand = new Command("deploy")
             {
                 new Option<FileInfo>(new string[] { "--input", "-i" }, "Path to the .dacpac package to deploy"),
@@ -48,7 +58,7 @@ static async Task<int> Main(string[] args)
             };
             deployCommand.Handler = CommandHandler.Create<DeployOptions>(DeployDacpac);
 
-            var rootCommand = new RootCommand { buildCommand, deployCommand };
+            var rootCommand = new RootCommand { buildCommand, collectIncludesCommand, deployCommand };
             rootCommand.Description = "Command line tool for generating a SQL Server Data-Tier Application Framework package (dacpac)";
 
             return await rootCommand.InvokeAsync(args);
@@ -57,7 +67,7 @@ static async Task<int> Main(string[] args)
         private static int BuildDacpac(BuildOptions options)
         {
             // Wait for a debugger to attach
-            WaitForDebuggerToAttach(options.Debug);
+            WaitForDebuggerToAttach(options);
 
             // Set metadata for the package
             using var packageBuilder = new PackageBuilder();
@@ -132,10 +142,37 @@ private static int BuildDacpac(BuildOptions options)
             return 0;
         }
 
+        private static int InspectIncludes(InspectOptions options)
+        {
+            // Wait for a debugger to attach
+            WaitForDebuggerToAttach(options);
+
+            var scriptInspector = new ScriptInspector();
+
+            // Add predeployment and postdeployment scripts
+            if (options.PreDeploy != null) 
+            {
+                scriptInspector.AddPreDeploymentScript(options.PreDeploy);
+            }
+            if (options.PostDeploy != null)
+            {
+                scriptInspector.AddPostDeploymentScript(options.PostDeploy);
+            }
+
+            // Write all included files to stdout
+            var includedFiles = scriptInspector.IncludedFiles;
+            foreach (var file in includedFiles)
+            {
+                Console.Out.WriteLine(file);
+            }
+
+            return 0;
+        }
+
         private static int DeployDacpac(DeployOptions options)
         {
             // Wait for a debugger to attach
-            WaitForDebuggerToAttach(options.Debug);
+            WaitForDebuggerToAttach(options);
 
             try
             {
@@ -187,9 +224,9 @@ private static int DeployDacpac(DeployOptions options)
         }
 
         [Conditional("DEBUG")]
-        private static void WaitForDebuggerToAttach(bool waitForDebuggerToAttach)
+        private static void WaitForDebuggerToAttach(BaseOptions options)
         {
-            if (waitForDebuggerToAttach)
+            if (options.Debug)
             {
                 Console.WriteLine("Waiting for debugger to attach");
                 while (!Debugger.IsAttached)

src/DacpacTool/ScriptInspector.cs

@@ -0,0 +1,31 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+
+namespace MSBuild.Sdk.SqlProj.DacpacTool
+{
+    public sealed class ScriptInspector
+    {
+        private readonly List<string> _includedFiles = new List<string>();
+        public IEnumerable<string> IncludedFiles
+        {
+            get => _includedFiles;
+        }
+
+        public void AddPreDeploymentScript(FileInfo script)
+        {
+            AddIncludedFiles(script);
+        }
+
+        public void AddPostDeploymentScript(FileInfo script)
+        {
+            AddIncludedFiles(script);
+        }
+
+        private void AddIncludedFiles(FileInfo file)
+        {
+            var parser = new ScriptParser(file.FullName, new IncludeVariableResolver());
+            _includedFiles.AddRange(parser.CollectFileNames());
+        }
+    }
+}