port batch parser wrapper (#232)

* Initial commit for GitHub IO pages

* Add initial doxfx content

* Update manifest.json

* Update manifest.json

* Set theme jekyll-theme-cayman

* Set theme jekyll-theme-slate

* Set theme jekyll-theme-minimal

* Set theme jekyll-theme-tactile

* Clear out theme setting

* Remove API docs

* Revert "Adding Milliseconds to DateTime fields (#173)" (#197)

This reverts commit 431dfa4156.

* ported new BatchParser

* added BatchParser tests

* fixing merge conflicts

* fix build issues

* cleaned code and addressed comments from code review

* addressed code review and made BatchParser logic more efficient

* fixed batch parser tests

* changed class name to fix build issues

* fixed merge conflicts

* added path for lab mode baseline tests

* changed env path for lab mode

* added env variable to appveyor

* testing env variable for appveyor

* fixed lab build

* debug appveyor build

* testing changes for appveyor

* changed trace env path

* debugging appveyor build

* changed baseline env path

* debugging

* debugging

* debugging

* switched on trace flag

* debugging

* debugging

* changed build config

* changed baseline files

* checking baseline output

* changed baseline files

* debug baseline tests

* debugging baseline

* debugging

* debugging

* debug

* debugging

* testing baseline format

* debug

* debug

* debug

* debug

* debug

* newline debug

* changed baseline file

* debug

* test

* try new way to read

* added execution engine tests

* change test

* testing file encoding

* moved execution engine tests to integration

* try compare without spaces

* removed no op test

* added env variable for travis

* put batch parser tests to integration too

* put batch parser in integration

* try new baseline string match

* compare baseline test logic changed

* changed baseline logic as well as cleaned tests

* fix build for travis CI

* fix travis CI issues

* fixed highlighting bugs on vscode

* code review changes

* ported new BatchParser

* added BatchParser tests

* Initial commit for GitHub IO pages

* Add initial doxfx content

* Update manifest.json

* Update manifest.json

* Set theme jekyll-theme-cayman

* Set theme jekyll-theme-slate

* Set theme jekyll-theme-minimal

* Set theme jekyll-theme-tactile

* Clear out theme setting

* Remove API docs

* Revert "Adding Milliseconds to DateTime fields (#173)" (#197)

This reverts commit 431dfa4156.

* fixing merge conflicts

* fix build issues

* cleaned code and addressed comments from code review

* addressed code review and made BatchParser logic more efficient

* fixed batch parser tests

* changed class name to fix build issues

* fixed merge conflicts

* added path for lab mode baseline tests

changed env path for lab mode

added env variable to appveyor

testing env variable for appveyor

fixed lab build

debug appveyor build

testing changes for appveyor

changed trace env path

debugging appveyor build

changed baseline env path

debugging

debugging

debugging

switched on trace flag

debugging

debugging

changed build config

changed baseline files

checking baseline output

changed baseline files

debug baseline tests

debugging baseline

debugging

debugging

debug

debugging

testing baseline format

debug

debug

debug

debug

debug

newline debug

changed baseline file

debug

test

try new way to read

added execution engine tests

change test

testing file encoding

moved execution engine tests to integration

try compare without spaces

removed no op test

added env variable for travis

* put batch parser tests to integration too

* put batch parser in integration

try new baseline string match

* compare baseline test logic changed

* changed baseline logic as well as cleaned tests

* fix build for travis CI

* fix travis CI issues

* fixed highlighting bugs on vscode

* code review changes

* fixed filestream writer test

* added localization string

* added localization string

* generated new string files again

* code review changes
This commit is contained in:
Aditya Bist
2017-02-10 16:51:26 -08:00
committed by GitHub
parent 1a5685c387
commit eb4f2f2b91
412 changed files with 82047 additions and 704 deletions

View File

@@ -0,0 +1,52 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Data.SqlClient;
using System.Diagnostics;
using Microsoft.SqlTools.ServiceLayer.BatchParser.ExecutionEngineCode;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
internal class BatchParserMockEventHandler : IBatchEventsHandler
{
public SqlError Error { get; private set; }
/// <summary>
/// fired when there is an error message from the server
/// </summary>
public void OnBatchError(object sender, BatchErrorEventArgs args)
{
Debug.WriteLine("{0}", args.Message);
Error = args.Error;
}
/// <summary>
/// fired when there is a message from the server
/// </summary>
public void OnBatchMessage(object sender, BatchMessageEventArgs args)
{
Debug.WriteLine("{0}", args.Message);
}
/// <summary>
/// fired when there is a new result set available. It is guarnteed
/// to be fired from the same thread that called Execute method
/// </summary>
public void OnBatchResultSetProcessing(object sender, BatchResultSetEventArgs args) { }
/// <summary>
/// fired when we've done absolutely all actions for the current result set
/// </summary>
public void OnBatchResultSetFinished(object sender, EventArgs args) { }
/// <summary>
/// fired when the batch recieved cancel request BEFORE it
/// initiates cancel operation. Note that it is fired from a
/// different thread then the one used to kick off execution
/// </summary>
public void OnBatchCancelling(object sender, EventArgs args) { }
}
}

View File

@@ -0,0 +1,65 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using Microsoft.SqlTools.ServiceLayer.BatchParser;
using Microsoft.SqlTools.ServiceLayer.BatchParser.ExecutionEngineCode;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
public class BatchParserSqlCmdTests : IDisposable
{
private BatchParserSqlCmd bpcmd;
private PositionStruct testPOS;
public BatchParserSqlCmdTests()
{
bpcmd = new BatchParserSqlCmd();
testPOS = new PositionStruct();
bpcmd.SetVariable(testPOS, "variable1", "test1");
bpcmd.SetVariable(testPOS, "variable2", "test2");
bpcmd.SetVariable(testPOS, "variable3", "test3");
}
public void Dispose()
{
Dispose(true);
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (bpcmd != null)
{
bpcmd = null;
}
}
}
[Fact]
public void CheckSetVariable()
{
Assert.Equal(bpcmd.InternalVariables.Count, 3);
bpcmd.SetVariable(testPOS, "variable4", "test4");
bpcmd.SetVariable(testPOS, "variable5", "test5");
bpcmd.SetVariable(testPOS, "variable6", "test6");
Assert.Equal(bpcmd.InternalVariables.Count, 6);
}
[Fact]
public void CheckGetVariable()
{
string value = bpcmd.GetVariable(testPOS, "variable1");
Assert.Equal("test1", value);
value = bpcmd.GetVariable(testPOS, "variable2");
Assert.Equal("test2", value);
value = bpcmd.GetVariable(testPOS, "variable3");
Assert.Equal("test3", value);
}
}
}

View File

@@ -0,0 +1,253 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Globalization;
using System.IO;
using System.Text;
using Microsoft.SqlTools.ServiceLayer.BatchParser;
using Microsoft.SqlTools.ServiceLayer.QueryExecution;
using Microsoft.SqlTools.ServiceLayer.Test.Common;
using Microsoft.SqlTools.ServiceLayer.Test.Common.Baselined;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
public class BatchParserTests : BaselinedTest
{
private bool testFailed = false;
public BatchParserTests()
{
InitializeTest();
}
public void InitializeTest()
{
CategoryName = "BatchParser";
this.TraceOutputDirectory = RunEnvironmentInfo.GetTestDataLocation();
TestInitialize();
}
[Fact]
public void VerifyThrowOnUnresolvedVariable()
{
string script = "print '$(NotDefined)'";
StringBuilder output = new StringBuilder();
TestCommandHandler handler = new TestCommandHandler(output);
IVariableResolver resolver = new TestVariableResolver(new StringBuilder());
Parser p = new Parser(
handler,
resolver,
new StringReader(script),
"test");
p.ThrowOnUnresolvedVariable = true;
handler.SetParser(p);
Assert.Throws<BatchParserException>(() => p.Parse());
}
public void TokenizeWithLexer(string filename, StringBuilder output)
{
using (Lexer lexer = new Lexer(new StreamReader(File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)), filename))
{
string inputText = File.ReadAllText(filename);
inputText = inputText.Replace("\r\n", "\n");
StringBuilder roundtripTextBuilder = new StringBuilder();
StringBuilder outputBuilder = new StringBuilder();
StringBuilder tokenizedInput = new StringBuilder();
bool lexerError = false;
Token token = null;
try
{
do
{
lexer.ConsumeToken();
token = lexer.CurrentToken;
roundtripTextBuilder.Append(token.Text);
outputBuilder.AppendLine(GetTokenString(token));
tokenizedInput.Append('[').Append(GetTokenCode(token.TokenType)).Append(':').Append(token.Text).Append(']');
} while (token.TokenType != LexerTokenType.Eof);
}
catch (BatchParserException ex)
{
lexerError = true;
outputBuilder.AppendLine(string.Format(CultureInfo.CurrentCulture, "[ERROR: code {0} at {1} - {2} in {3}, message: {4}]", ex.ErrorCode, GetPositionString(ex.Begin), GetPositionString(ex.End), GetFilenameOnly(ex.Begin.Filename), ex.Message));
}
output.AppendLine("Lexer tokenized input:");
output.AppendLine("======================");
output.AppendLine(tokenizedInput.ToString());
output.AppendLine("Tokens:");
output.AppendLine("=======");
output.AppendLine(outputBuilder.ToString());
if (lexerError == false)
{
// Verify that all text from tokens can be recombined into original string
Assert.Equal<string>(inputText, roundtripTextBuilder.ToString().Replace("\r\n", "\n"));
}
}
}
private string GetTokenCode(LexerTokenType lexerTokenType)
{
switch (lexerTokenType)
{
case LexerTokenType.Text:
return "T";
case LexerTokenType.Whitespace:
return "WS";
case LexerTokenType.NewLine:
return "NL";
case LexerTokenType.Comment:
return "C";
default:
return lexerTokenType.ToString();
}
}
static void CopyToOutput(string sourceDirectory, string filename)
{
File.Copy(Path.Combine(sourceDirectory, filename), filename, true);
FileUtilities.SetFileReadWrite(filename);
}
[Fact]
public void BatchParserTest()
{
CopyToOutput(FilesLocation, "TS-err-cycle1.txt");
CopyToOutput(FilesLocation, "cycle2.txt");
Start("err-blockComment");
Start("err-blockComment2");
Start("err-varDefinition");
Start("err-varDefinition2");
Start("err-varDefinition3");
Start("err-varDefinition4");
Start("err-varDefinition5");
Start("err-varDefinition6");
Start("err-varDefinition7");
Start("err-varDefinition8");
Start("err-varDefinition9");
Start("err-variableRef");
Start("err-variableRef2");
Start("err-variableRef3");
Start("err-variableRef4");
Start("err-cycle1");
Start("input");
Start("input2");
Start("pass-blockComment");
Start("pass-lineComment");
Start("pass-lineComment2");
Start("pass-noBlockComments");
Start("pass-noLineComments");
Start("pass-varDefinition");
Start("pass-varDefinition2");
Start("pass-varDefinition3");
Start("pass-varDefinition4");
Start("pass-command-and-comment");
Assert.False(testFailed, "At least one of test cases failed. Check output for details.");
}
public void TestParser(string filename, StringBuilder output)
{
try
{
TestCommandHandler commandHandler = new TestCommandHandler(output);
Parser parser = new Parser(
commandHandler,
new TestVariableResolver(output),
new StreamReader(File.Open(filename, FileMode.Open)),
filename);
commandHandler.SetParser(parser);
parser.Parse();
}
catch (BatchParserException ex)
{
output.AppendLine(string.Format(CultureInfo.CurrentCulture, "[PARSER ERROR: code {0} at {1} - {2} in {3}, token text: {4}, message: {5}]", ex.ErrorCode, GetPositionString(ex.Begin), GetPositionString(ex.End), GetFilenameOnly(ex.Begin.Filename), ex.Text, ex.Message));
}
}
private string GetPositionString(PositionStruct pos)
{
return string.Format(CultureInfo.InvariantCulture, "{0}:{1} [{2}]", pos.Line, pos.Column, pos.Offset);
}
private string GetTokenString(Token token)
{
if (token == null)
{
return "(null)";
}
else
{
string tokenText = token.Text;
if (tokenText != null)
{
tokenText = tokenText.Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t");
}
string tokenFilename = token.Filename;
tokenFilename = GetFilenameOnly(tokenFilename);
return string.Format(CultureInfo.CurrentCulture, "[Token {0} at {1}({2}:{3} [{4}] - {5}:{6} [{7}]): '{8}']",
token.TokenType,
tokenFilename,
token.Begin.Line, token.Begin.Column, token.Begin.Offset,
token.End.Line, token.End.Column, token.End.Offset,
tokenText);
}
}
internal static string GetFilenameOnly(string fullPath)
{
return fullPath != null ? Path.GetFileName(fullPath) : null;
}
public override void Run()
{
string inputFilename = GetTestscriptFilePath(CurrentTestName);
StringBuilder output = new StringBuilder();
TokenizeWithLexer(inputFilename, output);
TestParser(inputFilename, output);
string baselineFilename = GetBaselineFilePath(CurrentTestName);
string baseline;
try
{
baseline = GetFileContent(baselineFilename);
}
catch (FileNotFoundException)
{
baseline = string.Empty;
}
string outputString = output.ToString();
Console.WriteLine(baselineFilename);
if (string.Compare(baseline, outputString, StringComparison.Ordinal) != 0)
{
DumpToTrace(CurrentTestName, outputString);
string outputFilename = Path.Combine(TraceFilePath, GetBaselineFileName(CurrentTestName));
Console.WriteLine(":: Output does not match the baseline!");
Console.WriteLine("code --diff \"" + baselineFilename + "\" \"" + outputFilename + "\"");
Console.WriteLine();
Console.WriteLine(":: To update the baseline:");
Console.WriteLine("copy \"" + outputFilename + "\" \"" + baselineFilename + "\"");
Console.WriteLine();
testFailed = true;
}
}
}
}

View File

@@ -0,0 +1,51 @@
using Microsoft.SqlTools.ServiceLayer.BatchParser;
using Microsoft.SqlTools.ServiceLayer.BatchParser.ExecutionEngineCode;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
public class BatchParserWrapperTests
{
private BatchParserWrapper parserWrapper;
public BatchParserWrapperTests()
{
parserWrapper = new BatchParserWrapper();
}
[Fact]
public void CheckSimpleSingleSQLBatchStatement()
{
string sqlScript = "select * from sys.objects";
var batches = parserWrapper.GetBatches(sqlScript);
Assert.Equal(1, batches.Count);
BatchDefinition batch = batches[0];
Assert.Equal(sqlScript, batch.BatchText);
Assert.Equal(1, batch.StartLine);
Assert.Equal(1, batch.StartColumn);
Assert.Equal(2, batch.EndLine);
Assert.Equal(sqlScript.Length, batch.EndColumn);
}
[Fact]
public void CheckComment()
{
string sqlScript = "-- this is a comment --";
var batches = parserWrapper.GetBatches(sqlScript);
Assert.Equal(1, batches.Count);
BatchDefinition batch = batches[0];
Assert.Equal(sqlScript, batch.BatchText);
Assert.Equal(1, batch.StartLine);
Assert.Equal(1, batch.StartColumn);
Assert.Equal(2, batch.EndLine);
Assert.Equal(sqlScript.Length, batch.EndColumn);
}
[Fact]
public void CheckNoOps()
{
string sqlScript = "GO";
var batches = parserWrapper.GetBatches(sqlScript);
Assert.Equal(0, batches.Count);
}
}
}

View File

@@ -0,0 +1,92 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.IO;
using System.Text;
using System.Globalization;
using Microsoft.SqlTools.ServiceLayer.BatchParser;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
internal class TestCommandHandler : ICommandHandler
{
private Parser parser;
private StringBuilder outputString;
public TestCommandHandler(StringBuilder outputString)
{
this.outputString = outputString;
}
public void SetParser(Parser parser)
{
this.parser = parser;
}
public BatchParserAction Go(TextBlock batch, int repeatCount)
{
string textWithVariablesResolved;
string textWithVariablesUnresolved;
LineInfo lineInfoVarsResolved;
LineInfo lineInfoVarsUnresolved;
batch.GetText(true, out textWithVariablesResolved, out lineInfoVarsResolved);
batch.GetText(false, out textWithVariablesUnresolved, out lineInfoVarsUnresolved);
outputString.AppendFormat(CultureInfo.InvariantCulture, "*** Execute batch ({0})\n", repeatCount);
if (string.Compare(textWithVariablesUnresolved, textWithVariablesResolved, StringComparison.Ordinal) != 0)
{
outputString.AppendLine("Text with variables not resolved:");
outputString.AppendLine(textWithVariablesResolved);
outputString.AppendLine("Text with variables not resolved:");
outputString.AppendLine(textWithVariablesUnresolved);
int i = 0;
outputString.AppendLine("Mapping from resolved string to unresolved:");
while (i <= textWithVariablesResolved.Length)
{
PositionStruct pos = lineInfoVarsResolved.GetStreamPositionForOffset(i);
string character = i < textWithVariablesResolved.Length ? ("" + textWithVariablesResolved[i]).Replace("\n", @"\n").Replace("\r", @"\r") : "EOF";
outputString.AppendFormat(CultureInfo.InvariantCulture, "Pos [{0}] {1}:{2} \"{3}\"",
i,
BatchParserTests.GetFilenameOnly(pos.Filename),
pos.Offset,
character);
outputString.AppendLine();
i++;
}
}
else
{
outputString.AppendLine("Batch text:");
outputString.AppendLine(textWithVariablesUnresolved);
}
outputString.AppendLine();
return BatchParserAction.Continue;
}
public BatchParserAction OnError(Token token, OnErrorAction action)
{
outputString.AppendFormat(CultureInfo.InvariantCulture, "*** PARSER: On error: {0}", action.ToString());
outputString.AppendLine();
return BatchParserAction.Continue;
}
public BatchParserAction Include(TextBlock filename, out TextReader stream, out string newFilename)
{
string resolvedFilename;
LineInfo lineInfo;
filename.GetText(true, out resolvedFilename, out lineInfo);
outputString.AppendFormat(CultureInfo.InvariantCulture, "*** PARSER: Include file \"{0}\"\n", resolvedFilename);
outputString.AppendLine();
string currentFilename = lineInfo.GetStreamPositionForOffset(0).Filename;
string currentFilePath = Path.Combine(Path.GetDirectoryName(currentFilename), resolvedFilename);
stream = new StreamReader(File.Open(currentFilePath, FileMode.Open));
newFilename = resolvedFilename;
return BatchParserAction.Continue;
}
}
}

View File

@@ -0,0 +1,48 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.SqlTools.ServiceLayer.BatchParser;
namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.BatchParser
{
internal sealed class TestVariableResolver : IVariableResolver
{
Dictionary<string, string> variables = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
private StringBuilder outputString;
public TestVariableResolver(StringBuilder outputString)
{
this.outputString = outputString;
}
public string GetVariable(PositionStruct pos, string name)
{
if (variables.ContainsKey(name))
{
return variables[name];
}
else
{
return null;
}
}
public void SetVariable(PositionStruct pos, string name, string value)
{
outputString.AppendFormat("Setting variable {0} to [{1}]\n", name, value);
if (value == null)
{
variables.Remove(name);
}
else
{
variables[name] = value;
}
}
}
}