Fix batch parser test (#239)

* fixed batch parser test

* fix batch parser test

* fix batch parser test

* fixed baseline tests for parser and fixed trace output logic

* Update RunEnvironmentInfo.cs

* checking error logs on AppVeyor

* checking error logs on app veyor

* changed file reading encoding

* adding logs to app veyor build

* changed encoding of baseline files

* added error logs for app veyor

* changed error logs for app veyor

* changed how file stream works in batch parser tests

* changed baseline and testscript encodings

* cleaned code for necessary batch parser tests

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/BatchParser/BatchParserTests.cs

@@ -8,10 +8,10 @@ using System.Globalization;
 using System.IO;
 using System.Text;
 using Microsoft.SqlTools.ServiceLayer.BatchParser;
-using Microsoft.SqlTools.ServiceLayer.QueryExecution;
 using Microsoft.SqlTools.ServiceLayer.Test.Common;
 using Microsoft.SqlTools.ServiceLayer.Test.Common.Baselined;
 using Xunit;
+using Microsoft.SqlTools.ServiceLayer.QueryExecution;
 
 namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
 {
@@ -27,7 +27,7 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
         public void InitializeTest()
         {
             CategoryName = "BatchParser";
-            this.TraceOutputDirectory = RunEnvironmentInfo.GetTestDataLocation();
+            this.TraceOutputDirectory = RunEnvironmentInfo.GetTraceOutputLocation();
             TestInitialize();
         }
 
@@ -39,22 +39,36 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
 
             TestCommandHandler handler = new TestCommandHandler(output);
             IVariableResolver resolver = new TestVariableResolver(new StringBuilder());
-            Parser p = new Parser(
+            using (Parser p = new Parser(
                 handler,
                 resolver,
                 new StringReader(script),
-                "test");
+                "test"))
+            {
                 p.ThrowOnUnresolvedVariable = true;
-
                 handler.SetParser(p);
 
                 Assert.Throws<BatchParserException>(() => p.Parse());
             }
+        }
+
+        private static Stream GenerateStreamFromString(string s)
+        {
+            MemoryStream stream = new MemoryStream();
+            StreamWriter writer = new StreamWriter(stream);
+            writer.Write(s);
+            writer.Flush();
+            stream.Position = 0;
+            return stream;
+        }
 
         public void TokenizeWithLexer(string filename, StringBuilder output)
         {
-            
-            using (Lexer lexer = new Lexer(new StreamReader(File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)), filename))
+            // Create a new file by changing CRLFs to LFs and generate a new steam
+            // or the tokens generated by the lexer will always have off by one errors
+            string input = File.ReadAllText(filename).Replace("\r\n", "\n");
+            var inputStream = GenerateStreamFromString(input);
+            using (Lexer lexer = new Lexer(new StreamReader(inputStream), filename))
             {
                 
                 string inputText = File.ReadAllText(filename);
@@ -71,9 +85,9 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
                     {
                         lexer.ConsumeToken();
                         token = lexer.CurrentToken;
-                        roundtripTextBuilder.Append(token.Text);
+                        roundtripTextBuilder.Append(token.Text.Replace("\r\n", "\n"));
                         outputBuilder.AppendLine(GetTokenString(token));
-                        tokenizedInput.Append('[').Append(GetTokenCode(token.TokenType)).Append(':').Append(token.Text).Append(']');
+                        tokenizedInput.Append('[').Append(GetTokenCode(token.TokenType)).Append(':').Append(token.Text.Replace("\r\n", "\n")).Append(']');
                     } while (token.TokenType != LexerTokenType.Eof);
                 }
                 catch (BatchParserException ex)
@@ -91,7 +105,7 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
                 if (lexerError == false)
                 {
                     // Verify that all text from tokens can be recombined into original string
-                    Assert.Equal<string>(inputText, roundtripTextBuilder.ToString().Replace("\r\n", "\n"));
+                    Assert.Equal<string>(inputText, roundtripTextBuilder.ToString());
                 }
             }
         }
@@ -113,9 +127,18 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
             }
         }
 
+        private static void CopyToOutput(string sourceDirectory, string filename)
+        {
+            File.Copy(Path.Combine(sourceDirectory, filename), filename, true);
+            FileUtilities.SetFileReadWrite(filename);
+        }
+
         [Fact]
         public void BatchParserTest()
         {
+            CopyToOutput(FilesLocation, "TS-err-cycle1.txt");
+            CopyToOutput(FilesLocation, "cycle2.txt");
+
             Start("err-blockComment");
             Start("err-blockComment2");
             Start("err-varDefinition");
@@ -151,18 +174,23 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
         {
             try
             {
+                // Create a new file by changing CRLFs to LFs and generate a new steam
+                // or the tokens generated by the lexer will always have off by one errors
                 TestCommandHandler commandHandler = new TestCommandHandler(output);
+                string input = File.ReadAllText(filename).Replace("\r\n", "\n");
+                var inputStream = GenerateStreamFromString(input);
+                StreamReader streamReader = new StreamReader(inputStream);
 
-                Parser parser = new Parser(
+                using (Parser parser = new Parser(
                     commandHandler,
                     new TestVariableResolver(output),
-                    new StreamReader(File.Open(filename, FileMode.Open)),
-                    filename);
-
+                    streamReader,
+                    filename))
+                {
                     commandHandler.SetParser(parser);
-
                     parser.Parse();
                 }
+            }
             catch (BatchParserException ex)
             {
                 output.AppendLine(string.Format(CultureInfo.CurrentCulture, "[PARSER ERROR: code {0} at {1} - {2} in {3}, token text: {4}, message: {5}]", ex.ErrorCode, GetPositionString(ex.Begin), GetPositionString(ex.End), GetFilenameOnly(ex.Begin.Filename), ex.Text, ex.Message));
@@ -185,7 +213,7 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
                 string tokenText = token.Text;
                 if (tokenText != null)
                 {
-                    tokenText = tokenText.Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t");
+                    tokenText = tokenText.Replace("\r\n", "\\n").Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t");
                 }
                 string tokenFilename = token.Filename;
                 tokenFilename = GetFilenameOnly(tokenFilename);
@@ -216,14 +244,14 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
 
             try
             {
-                baseline = GetFileContent(baselineFilename);
+                baseline = GetFileContent(baselineFilename).Replace("\r\n", "\n");
             }
             catch (FileNotFoundException)
             {
                 baseline = string.Empty;
             }
 
-            string outputString = output.ToString();
+            string outputString = output.ToString().Replace("\r\n", "\n");
 
             Console.WriteLine(baselineFilename);
 

test/Microsoft.SqlTools.ServiceLayer.Test.Common/Baselined/BaselinedTest.cs

@@ -360,7 +360,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common.Baselined
         {
             Trace.WriteLine(string.Format("GetFileContent for [{0}]", Path.GetFullPath(path)));
 
-            using (StreamReader sr = new StreamReader(File.Open(path, FileMode.Open), Encoding.Unicode))
+            using (StreamReader sr = new StreamReader(File.Open(path, FileMode.Open), Encoding.UTF8))
             {
                 return sr.ReadToEnd();
             }
@@ -406,13 +406,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common.Baselined
         /// <param name="text">The content for the trace file</param>
         public void WriteTraceFile(string traceFile, string text)
         {
-            Stream traceStream = GetStreamFromString(traceFile);
-            using (StreamWriter sw = new StreamWriter(traceStream, Encoding.Unicode))
-            {
-                sw.Write(text);
-                sw.Flush();
-                sw.Dispose();
-            }
+            File.WriteAllText(traceFile, text);
         }
 
         /// <summary>

test/Microsoft.SqlTools.ServiceLayer.Test.Common/RunEnvironmentInfo.cs

@@ -18,6 +18,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
     public class RunEnvironmentInfo
     {
         private static string cachedTestFolderPath;
+        private static string cachedTraceFolderPath;
 
         public static bool IsLabMode()
         {
@@ -64,10 +65,48 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             }
             return testFolderPath;
         }
+
+        /// <summary>
+        /// Location of all trace data (expected output)
+        /// </summary>
+        /// <returns>The full path to the trace data directory</returns>
+        public static string GetTraceOutputLocation()
+        {
+            string traceFolderPath;
+            string testPath = @"test\Microsoft.SqlTools.ServiceLayer.Test.Common\Trace";
+            string projectPath = Environment.GetEnvironmentVariable(Constants.ProjectPath);
+
+            if (projectPath != null)
+            {
+                traceFolderPath = Path.Combine(projectPath, testPath);
+            }
+            else
+            {
+                if (cachedTraceFolderPath != null)
+                {
+                    traceFolderPath = cachedTraceFolderPath;
+                }
+                else
+                {
+                    // We are running tests locally, which means we expect to be running inside the bin\debug\netcoreapp directory
+                    // Test Files should be found at the root of the project so go back the necessary number of directories for this 
+                    // to be found. We are manually specifying the testFolderPath here for clarity on where to expect this
+
+                    string assemblyDir = Path.GetDirectoryName(typeof(Scripts).GetTypeInfo().Assembly.Location);
+                    string defaultPath = Path.Combine(assemblyDir, GoUpNDirectories(4));
+                    traceFolderPath = Path.Combine(defaultPath, "Microsoft.SqlTools.ServiceLayer.Test.Common", "TestData");
+
+                    cachedTraceFolderPath = traceFolderPath;
+                }
+            }
+            return traceFolderPath;
+        }
+
         private static string GoUpNDirectories(int n)
         {
             string up = ".." + (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "\\" : "/");
             return string.Concat(Enumerable.Repeat(up, n));
+
         }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/BatchParser/TestScripts/TS-input.txt

@@ -1,4 +1,4 @@
-GO 2
+GO 2
 BEGIN
 :r input-2.txt
 :r "input-2.txt"

test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceProvider.cs

@@ -135,7 +135,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
                 }
                 TestServiceProvider.hasInitServices = true;
 
-                const string hostName = "SQ Tools Test Service Host";
+                const string hostName = "SQL Tools Test Service Host";
                 const string hostProfileId = "SQLToolsTestService";
                 Version hostVersion = new Version(1, 0);