Invoke-SqlCmd handles ":!!if" in inconsistent way (PS5 vs PS6) (#806)

* Invoke-SqlCmd handles ":!!if" in inconsistent way (PS5 vs PS6)

Fix: Set wordBounday to false for "!!" in Lexer.cs

* Issue : Invoke-SqlCmd handles ":!!if" in inconsistent way (PS5 vs PS6)
TFS: http://sqlbuvsts01:8080/Main/SQL%20Server/_workitems#id=12817630&triage=true&_a=edit
Fix:
1) Wrote two test cases in BatchParserTests.cs

src/Microsoft.SqlTools.ManagedBatchParser/BatchParser/Lexer.cs

@@ -43,8 +43,9 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
         /// </summary>
         public Token CurrentToken
         {
-            get {
+            get
-                return currentToken; 
+            {
+                return currentToken;
             }
         }
 
@@ -83,7 +84,7 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
                 if (CurrentTokenType == LexerTokenType.Eof)
                 {
                     popInputAtNextConsume = true;
-                    if(inputStack.Count > 0)
+                    if (inputStack.Count > 0)
                     {
                         // report as empty NewLine token
                         currentToken = new Token(
@@ -473,7 +474,7 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
                 SetToken(LexerTokenType.Eof);
                 return true;
             }
-            
+
             if (ch.HasValue)
             {
                 if (IsNewLineChar(ch.Value))
@@ -600,10 +601,10 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
             string text = currentInput.FlushBufferedText();
 
             currentToken = new Token(
-                lexerTokenType, 
+                lexerTokenType,
-                tokenBeginPosition, 
+                tokenBeginPosition,
                 new PositionStruct(currentInput.CurrentLine, currentInput.CurrentColumn, currentInput.CurrentOffset, currentInput.Filename),
-                text, 
+                text,
                 currentInput.Filename);
         }
 
@@ -625,7 +626,7 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
             if (wordBoundary)
             {
                 char? ch = Lookahead(text.Length);
-                if (ch != null && IsWhitespaceChar(ch.Value) == false && IsNewLineChar(ch.Value) == false 
+                if (ch != null && IsWhitespaceChar(ch.Value) == false && IsNewLineChar(ch.Value) == false
                     && ch != '$' && ch != '/' && ch != '-' && ch != '\'' && ch != '"' && ch != '(' && ch != '[' && ch != '!')
                 {
                     return false;
@@ -645,91 +646,108 @@ namespace Microsoft.SqlTools.ServiceLayer.BatchParser
         {
             Consume(); // colon
 
-            if (TryAccept("reset", true))
+            /*
-            {
+              TryAccept(text: "!!", wordBoundary: false)
-                SetToken(LexerTokenType.Reset);
+              ----------------------
-                return true;
+              Reason: 
-            }
+                * The behavior is differ from SqlCmd and current implementation. 
-            else if (TryAccept("ed", true))
+                * Example: 
-            {
+                    * !!Dir (without whitespace)
-                SetToken(LexerTokenType.Ed);
+                        *  Command: Invoke-Sqlcmd -Query ':!!if exist foo.txt del foo.txt' 
-                return true;
+                        *  Result : Incorrect syntax near ':'.  (InCorrect)
-            }
+                        
-            else if (TryAccept("!!", true))
+                    * !! DIR (with whitespace)
+                        *  Command: Invoke-Sqlcmd -Query ':!! if exist foo.txt del foo.txt' 
+                        *  Result :  Command Execute is not supported. (Correct)
+                        
+                * TryAccept return false , if we pass true to parameter wordBoundary and LexerTokenType will set to Text instead of Execute.
+                * So by passing wordBoundary as false, I will get TryAccept as true and  LexerTokenType will set to Execute.
+             */
+            if (TryAccept(text: "!!", wordBoundary: false))
             {
                 SetToken(LexerTokenType.Execute);
                 return true;
             }
-            else if (TryAccept("quit", true))
+            else if (TryAccept(text: "connect", wordBoundary: true))
-            {
-                SetToken(LexerTokenType.Quit);
-                return true;
-            }
-            else if (TryAccept("exit", true))
-            {
-                SetToken(LexerTokenType.Exit);
-                return true;
-            }
-            else if (TryAccept("r", true))
-            {
-                SetToken(LexerTokenType.Include);
-                return true;
-            }
-            else if (TryAccept("serverlist", true))
-            {
-                SetToken(LexerTokenType.Serverlist);
-                return true;
-            }
-            else if (TryAccept("setvar", true))
-            {
-                SetToken(LexerTokenType.Setvar);
-                return true;
-            }
-            else if (TryAccept("list", true))
-            {
-                SetToken(LexerTokenType.List);
-                return true;
-            }
-            else if (TryAccept("error", true))
-            {
-                SetToken(LexerTokenType.ErrorCommand);
-                return true;
-            }
-            else if (TryAccept("out", true))
-            {
-                SetToken(LexerTokenType.Out);
-                return true;
-            }
-            else if (TryAccept("perftrace", true))
-            {
-                SetToken(LexerTokenType.Perftrace);
-                return true;
-            }
-            else if (TryAccept("connect", true))
             {
                 SetToken(LexerTokenType.Connect);
                 return true;
             }
-            else if (TryAccept("on error", true))
+            else if (TryAccept(text: "ed", wordBoundary: true))
             {
-                SetToken(LexerTokenType.OnError);
+                SetToken(LexerTokenType.Ed);
                 return true;
             }
-            else if (TryAccept("help", true))
+            else if (TryAccept(text: "error", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.ErrorCommand);
+                return true;
+            }
+            else if (TryAccept(text: "exit", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Exit);
+                return true;
+            }
+            else if (TryAccept(text: "help", wordBoundary: true))
             {
                 SetToken(LexerTokenType.Help);
                 return true;
             }
-            else if (TryAccept("xml", true))
+            else if (TryAccept(text: "list", wordBoundary: true))
             {
-                SetToken(LexerTokenType.Xml);
+                SetToken(LexerTokenType.List);
                 return true;
             }
-            else if (TryAccept("listvar", true))
+            else if (TryAccept(text: "listvar", wordBoundary: true))
             {
                 SetToken(LexerTokenType.ListVar);
                 return true;
             }
+            else if (TryAccept(text: "on error", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.OnError);
+                return true;
+            }
+            else if (TryAccept(text: "out", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Out);
+                return true;
+            }
+            else if (TryAccept(text: "perftrace", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Perftrace);
+                return true;
+            }
+            else if (TryAccept(text: "quit", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Quit);
+                return true;
+            }
+            else if (TryAccept(text: "r", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Include);
+                return true;
+            }
+            else if (TryAccept(text: "reset", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Reset);
+                return true;
+            }
+            else if (TryAccept(text: "serverlist", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Serverlist);
+                return true;
+            }
+            else if (TryAccept(text: "setvar", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Setvar);
+                return true;
+            }
+            else if (TryAccept(text: "xml", wordBoundary: true))
+            {
+                SetToken(LexerTokenType.Xml);
+                return true;
+            }
 
             return false;
         }

test/Microsoft.SqlTools.ManagedBatchParser.IntegrationTests/BatchParser/BatchParserTests.cs

@@ -3,17 +3,17 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
-using System;
-using System.Data.SqlClient;
-using System.Globalization;
-using System.IO;
-using System.Text;
 using Microsoft.SqlTools.ManagedBatchParser.IntegrationTests.TSQLExecutionEngine;
 using Microsoft.SqlTools.ManagedBatchParser.IntegrationTests.Utility;
 using Microsoft.SqlTools.ServiceLayer.BatchParser;
 using Microsoft.SqlTools.ServiceLayer.BatchParser.ExecutionEngineCode;
 using Microsoft.SqlTools.ServiceLayer.Test.Common;
 using Microsoft.SqlTools.ServiceLayer.Test.Common.Baselined;
+using System;
+using System.Data.SqlClient;
+using System.Globalization;
+using System.IO;
+using System.Text;
 using Xunit;
 
 namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
@@ -122,7 +122,6 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
             }
         }
 
-
         // Verify the execution by passing long value , Except a exception.
         [Fact]
         public void VerifyInvalidNumber()
@@ -140,8 +139,8 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
             {
                 p.ThrowOnUnresolvedVariable = true;
                 handler.SetParser(p);
-                // This test will fail because we are passing invalid number. 
+                // This test will fail because we are passing invalid number.
-                // Exception will be raised from  ParseGo() 
+                // Exception will be raised from  ParseGo()
                 Assert.Throws<BatchParserException>(() => p.Parse());
             }
         }
@@ -223,7 +222,6 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
         {
             try
             {
-
                 string query = ":SETVAR    a 10";
                 var inputStream = GenerateStreamFromString(query);
                 using (Lexer lexer = new Lexer(new StreamReader(inputStream), "Test.sql"))
@@ -232,15 +230,52 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
                 }
                 executionResult = ScriptExecutionResult.Success;
             }
-            catch (Exception ex)
+            catch (Exception)
             {
-
                 executionResult = ScriptExecutionResult.Failure;
-
             }
             //  we doesn't expect any exception or testCase failures
             Assert.Equal<ScriptExecutionResult>(ScriptExecutionResult.Success, executionResult);
-             
+        }
+
+        // This test case is to verify that, Powershell's Invoke-SqlCmd handles ":!!if" in an inconsistent way
+        // Inconsistent way means, instead of throwing an exception as "Command Execute is not supported." it was throwing "Incorrect syntax near ':'."
+        [Fact]
+        public void VerifySqlCmdExecute()
+        {
+            string query = ":!!if exist foo.txt del foo.txt";
+            var inputStream = GenerateStreamFromString(query);
+            TestCommandHandler handler = new TestCommandHandler(new StringBuilder());
+            IVariableResolver resolver = new TestVariableResolver(new StringBuilder());
+            using (Parser p = new Parser(
+                handler,
+                resolver,
+                new StringReader(query),
+                "test"))
+            {
+                p.ThrowOnUnresolvedVariable = true;
+                handler.SetParser(p);
+
+                var exception = Assert.Throws<BatchParserException>(() => p.Parse());
+                // Verify the message should be "Command Execute is not supported."
+                Assert.Equal("Command Execute is not supported.", exception.Message);
+            }
+        }
+
+        // This test case is to verify that, Lexer type for :!!If was set to "Text" instead of "Execute"
+        [Fact]
+        public void VerifyLexerTypeOfSqlCmdIFisExecute()
+        {
+            string query = ":!!if exist foo.txt del foo.txt";
+            var inputStream = GenerateStreamFromString(query);
+            LexerTokenType type = LexerTokenType.None;
+            using (Lexer lexer = new Lexer(new StreamReader(inputStream), "Test.sql"))
+            {
+                lexer.ConsumeToken();
+                type = lexer.CurrentTokenType;
+            }
+            // we are expecting the lexer type should to be Execute.
+            Assert.Equal("Execute", type.ToString());
         }
 
         // Verify the custom exception functionality by raising user defined error.
@@ -271,7 +306,7 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
         {
             using (ExecutionEngine executionEngine = new ExecutionEngine())
             {
-                string query = @"SELECT 1+2 
+                string query = @"SELECT 1+2
                                 Go 2";
                 using (SqlConnection con = new SqlConnection(CONNECTION_STRING))
                 {
@@ -287,7 +322,7 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
         }
 
         // Verify whether the batchParser execute SqlCmd.
-        //[Fact]   //  This Testcase should execute and pass, But it is failing now. 
+        //[Fact]   //  This Testcase should execute and pass, But it is failing now.
         public void VerifyIsSqlCmd()
         {
             using (ExecutionEngine executionEngine = new ExecutionEngine())
@@ -299,11 +334,10 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
                     TestExecutor testExecutor = new TestExecutor(query, con, new ExecutionEngineConditions());
                     testExecutor.Run();
                     Assert.True(testExecutor.ResultCountQueue.Count >= 1);
-
                 }
             }
         }
-     
+
         // Verify whether the executionEngine execute Batch
         [Fact]
         public void VerifyExecuteBatch()
@@ -318,11 +352,11 @@ namespace Microsoft.SqlTools.ManagedBatchParser.UnitTests.BatchParser
                     executionEngine.BatchParserExecutionFinished += OnBatchParserExecutionFinished;
                     executionEngine.ExecuteBatch(new ScriptExecutionArgs(query, con, 15, new ExecutionEngineConditions(), new BatchParserMockEventHandler()));
                     Assert.Equal(ScriptExecutionResult.Success, executionResult);
-
                 }
             }
         }
-        // Capture the event once batch finish execution. 
+
+        // Capture the event once batch finish execution.
         private void OnBatchParserExecutionFinished(object sender, BatchParserExecutionFinishedEventArgs e)
         {
             executionResult = e.ExecutionResult;