From ccd2c9caa9c53239e8d3b4991f07af6ed9d507ed Mon Sep 17 00:00:00 2001 From: Kevin Cunnane Date: Tue, 21 Feb 2017 18:21:59 -0800 Subject: [PATCH] Update Keywords list to include most tokens (#244) - Fixes https://github.com/Microsoft/vscode-mssql/issues/705 - Updated the keyword list to include most common keyword ranges and added test to cover top 10 missing keywords - Fixed a bug in TestUtilities.cs where text range was not defined correctly. --- .../Formatter/Impl/FormatContext.cs | 51 +++++++++---------- .../Formatter/Impl/FormatterTokens.cs | 8 +++ .../KeywordCaseConversion_Lowercase.sql | 50 ++++++++++++++++++ .../KeywordCaseConversion_Uppercase.sql | 50 ++++++++++++++++++ .../TestFiles/KeywordCaseConversion.sql | 41 +++++++++++++++ .../TestUtilities.cs | 5 +- .../Formatter/GeneralFormatterTests.cs | 32 ++++++++++++ 7 files changed, 208 insertions(+), 29 deletions(-) create mode 100644 test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Lowercase.sql create mode 100644 test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Uppercase.sql create mode 100644 test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/TestFiles/KeywordCaseConversion.sql create mode 100644 test/Microsoft.SqlTools.ServiceLayer.Test/Formatter/GeneralFormatterTests.cs diff --git a/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatContext.cs b/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatContext.cs index 61837b89..a45366ee 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatContext.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatContext.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text; using Babel.ParserGenerator; +using Microsoft.SqlServer.Management.SqlParser.Parser; using Microsoft.SqlServer.Management.SqlParser.SqlCodeDom; namespace Microsoft.SqlTools.ServiceLayer.Formatter @@ -32,32 +33,30 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter private void LoadKeywordIdentifiers() { KeywordIdentifiers = new HashSet(); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_FROM); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_SELECT); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_TABLE); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_CREATE); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_USEDB); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_NOT); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_NULL); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_IDENTITY); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_ORDER); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_BY); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_DESC); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_ASC); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_GROUP); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_WHERE); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_JOIN); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_ON); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_UNION); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_ALL); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_EXCEPT); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_INTERSECT); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_INTO); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_DEFAULT); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_WITH); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_AS); + + List keywordRanges = new List() + { + new [] { Tokens.TOKEN_OR.ToInt(), Tokens.TOKEN_NOT.ToInt() }, + new [] { Tokens.TOKEN_ELSE.ToInt(), Tokens.TOKEN_UNPIVOT.ToInt() }, + new [] { Tokens.TOKEN_ALL.ToInt(), Tokens.TOKEN_PERCENT.ToInt() }, + // TODO is TOKEN_PROC_SEMI a keyword? + new [] { Tokens.TOKEN_OPENQUERY.ToInt(), Tokens.TOKEN_FREETEXT.ToInt() }, + new [] { Tokens.TOKEN_c_MOVE.ToInt(), Tokens.TOKEN_c_ROLLUP.ToInt() }, + new [] { Tokens.TOKEN_REVERT.ToInt(), Tokens.TOKEN_c_OPENJSON.ToInt() }, + new [] { Tokens.TOKEN_s_CDA_TYPE.ToInt(), Tokens.TOKEN_s_CDA_POLICY.ToInt() }, + new [] { Tokens.TOKEN_BEGIN_CS.ToInt(), Tokens.TOKEN_END_CS.ToInt() } + // Note: after this it becomes hard to interpret actual keywords in the Tokens enum. + // Should review and re-assess later + }; + + foreach(int[] range in keywordRanges) + { + for(int i = range[0]; i <= range[1]; i++) + { + KeywordIdentifiers.Add(i); + } + } KeywordIdentifiers.Add(FormatterTokens.LEX_BATCH_SEPERATOR); - KeywordIdentifiers.Add(FormatterTokens.TOKEN_IS); } public string FormattedSql @@ -168,13 +167,11 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter { TokenData tok = Script.TokenManager.TokenList[i]; Replacements.Add(new Replacement(tok.StartIndex, sql, sql.ToUpperInvariant())); - sql = sql.ToUpperInvariant(); } else if (FormatOptions.LowercaseKeywords) { TokenData tok = Script.TokenManager.TokenList[i]; Replacements.Add(new Replacement(tok.StartIndex, sql, sql.ToLowerInvariant())); - sql = sql.ToLowerInvariant(); } } } diff --git a/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatterTokens.cs b/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatterTokens.cs index 38b1e008..f8272a5a 100644 --- a/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatterTokens.cs +++ b/src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatterTokens.cs @@ -57,4 +57,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter public static readonly int LAST_TOKEN = ResolveTokenId("LAST_TOKEN"); } + + static class TokenConverter + { + public static int ToInt(this Tokens token) + { + return (int)token; + } + } } diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Lowercase.sql b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Lowercase.sql new file mode 100644 index 00000000..c58bcdec --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Lowercase.sql @@ -0,0 +1,50 @@ +-- Case: common type keywords + +create table [SimpleTable] +( + + -- this is a comment before document + [DocumentID] INT identity (1, 1) not null, + + -- this is a comment before Title + [Title] NVARCHAR (50) not null, + + + -- this is a comment before FileName + [FileName] NVARCHAR (400) not null, + + -- this is a comment before FileExtension + [FileExtension] nvarchar(8) +); +go +create view v1 +as + select * + from [SimpleTable] +go +create procedure p1 +as +begin + select * + from [SimpleTable] +end +go +insert into t +default values +go + +go +insert openquery (OracleSvr, 'SELECT name FROM joe.titles') +values + ('NewTitle'); +go +insert into myTable + (FileName, FileType, Document) +select 'Text1.txt' as FileName, + '.txt' as FileType, + * +from openrowset(bulk N'C:\Text1.txt', siNGLE_BLOB) as Document; +go +select * +from openxml (@idoc, '/ROOT/Customers') +exec sp_xml_removedocument @idoc; \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Uppercase.sql b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Uppercase.sql new file mode 100644 index 00000000..8a78f018 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Uppercase.sql @@ -0,0 +1,50 @@ +-- Case: common type keywords + +CREATE TABLE [SimpleTable] +( + + -- this is a comment before document + [DocumentID] INT IDENTITY (1, 1) NOT NULL, + + -- this is a comment before Title + [Title] NVARCHAR (50) NOT NULL, + + + -- this is a comment before FileName + [FileName] NVARCHAR (400) NOT NULL, + + -- this is a comment before FileExtension + [FileExtension] nvarchar(8) +); +GO +CREATE VIEW v1 +AS + SELECT * + FROM [SimpleTable] +GO +CREATE PROCEDURE p1 +AS +BEGIN + SELECT * + FROM [SimpleTable] +END +GO +INSERT INTO t +DEFAULT VALUES +GO + +GO +INSERT OPENQUERY (OracleSvr, 'SELECT name FROM joe.titles') +VALUES + ('NewTitle'); +GO +INSERT INTO myTable + (FileName, FileType, Document) +SELECT 'Text1.txt' AS FileName, + '.txt' AS FileType, + * +FROM OPENROWSET(BULK N'C:\Text1.txt', siNGLE_BLOB) AS Document; +GO +SELECT * +FROM OPENXML (@idoc, '/ROOT/Customers') +EXEC sp_xml_removedocument @idoc; \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/TestFiles/KeywordCaseConversion.sql b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/TestFiles/KeywordCaseConversion.sql new file mode 100644 index 00000000..a1bdfc40 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/TestFiles/KeywordCaseConversion.sql @@ -0,0 +1,41 @@ +-- Case: common type keywords + +cReaTe tAbLe [SimpleTable] +( + + -- this is a comment before document + [DocumentID] INT iDentIty (1, 1) NOT NULL, + + -- this is a comment before Title + [Title] NVARCHAR (50) NOT NULL, + + + -- this is a comment before FileName + [FileName] NVARCHAR (400) NOT NULL, + + -- this is a comment before FileExtension + [FileExtension] nvarchar(8) +); +GO +CREATE vIEw v1 aS select * from [SimpleTable] +GO +CREATE pRoceduRe p1 aS +bEgIn +select * from [SimpleTable] +eNd +GO +iNsert iNto t dEfault vAlues +GO + +GO +iNsert oPenQUERY (OracleSvr, 'SELECT name FROM joe.titles') +VALUES ('NewTitle'); +GO +INSERT INTO myTable(FileName, FileType, Document) + SELECT 'Text1.txt' AS FileName, + '.txt' AS FileType, + * FROM openROWSET(bUlK N'C:\Text1.txt', siNGLE_BLOB) AS Document; +GO +sELECT * +FROM openXML (@idoc, '/ROOT/Customers') +exEC sp_xml_removedocument @idoc; \ No newline at end of file diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestUtilities.cs b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestUtilities.cs index 695be588..733918cb 100644 --- a/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestUtilities.cs +++ b/test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestUtilities.cs @@ -54,9 +54,10 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common int startRange = Math.Max(firstDiffIndex - 50, 0); int endRange = Math.Min(firstDiffIndex + 50, minEnd); + int length = endRange - startRange; - string baselineDiff = ShowWhitespace(baseline.Substring(startRange, endRange)); - string actualDiff = ShowWhitespace(actual.Substring(startRange, endRange)); + string baselineDiff = ShowWhitespace(baseline.Substring(startRange, length)); + string actualDiff = ShowWhitespace(actual.Substring(startRange, length)); return "\r\nFirst Diff:\r\n===== Baseline =====\r\n" + baselineDiff + "\r\n===== Actual =====\r\n" diff --git a/test/Microsoft.SqlTools.ServiceLayer.Test/Formatter/GeneralFormatterTests.cs b/test/Microsoft.SqlTools.ServiceLayer.Test/Formatter/GeneralFormatterTests.cs new file mode 100644 index 00000000..a310a2b9 --- /dev/null +++ b/test/Microsoft.SqlTools.ServiceLayer.Test/Formatter/GeneralFormatterTests.cs @@ -0,0 +1,32 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.SqlTools.ServiceLayer.Formatter; +using Xunit; + +namespace Microsoft.SqlTools.ServiceLayer.Test.Formatter +{ + public class GeneralFormatterTests : FormatterUnitTestsBase + { + [Fact] + public void KeywordCaseConversionUppercase() + { + LoadAndFormatAndCompare("KeywordCaseConversion", + GetInputFile("KeywordCaseConversion.sql"), + GetBaselineFile("KeywordCaseConversion_Uppercase.sql"), + new FormatOptions() { KeywordCasing = CasingOptions.Uppercase }, + verifyFormat: true); + } + [Fact] + public void KeywordCaseConversionLowercase() + { + LoadAndFormatAndCompare("KeywordCaseConversion", + GetInputFile("KeywordCaseConversion.sql"), + GetBaselineFile("KeywordCaseConversion_Lowercase.sql"), + new FormatOptions() { KeywordCasing = CasingOptions.Lowercase }, + verifyFormat: true); + } + } +}