ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-21 01:25:42 -05:00
Code Issues Packages Projects Releases Wiki Activity

Update Keywords list to include most tokens (#244)

Browse Source 
- Fixes https://github.com/Microsoft/vscode-mssql/issues/705
- Updated the keyword list to include most common keyword ranges and added test to cover top 10 missing keywords
- Fixed a bug in TestUtilities.cs where text range was not defined correctly.
This commit is contained in:
Kevin Cunnane
2017-02-21 18:21:59 -08:00
committed by GitHub
parent ef012a4f98
commit ccd2c9caa9
7 changed files with 208 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatContext.cs
View File

@@ -8,6 +8,7 @@ using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Text;
using Babel.ParserGenerator;
using Microsoft.SqlServer.Management.SqlParser.Parser;
using Microsoft.SqlServer.Management.SqlParser.SqlCodeDom;
namespace Microsoft.SqlTools.ServiceLayer.Formatter
@@ -32,32 +33,30 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter
private void LoadKeywordIdentifiers()
{
KeywordIdentifiers = new HashSet<int>();
KeywordIdentifiers.Add(FormatterTokens.TOKEN_FROM);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_SELECT);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_TABLE);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_CREATE);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_USEDB);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_NOT);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_NULL);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_IDENTITY);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_ORDER);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_BY);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_DESC);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_ASC);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_GROUP);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_WHERE);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_JOIN);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_ON);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_UNION);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_ALL);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_EXCEPT);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_INTERSECT);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_INTO);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_DEFAULT);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_WITH);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_AS);
List<int[]> keywordRanges = new List<int[]>()
{
new [] { Tokens.TOKEN_OR.ToInt(), Tokens.TOKEN_NOT.ToInt() },
new [] { Tokens.TOKEN_ELSE.ToInt(), Tokens.TOKEN_UNPIVOT.ToInt() },
new [] { Tokens.TOKEN_ALL.ToInt(), Tokens.TOKEN_PERCENT.ToInt() },
// TODO is TOKEN_PROC_SEMI a keyword?
new [] { Tokens.TOKEN_OPENQUERY.ToInt(), Tokens.TOKEN_FREETEXT.ToInt() },
new [] { Tokens.TOKEN_c_MOVE.ToInt(), Tokens.TOKEN_c_ROLLUP.ToInt() },
new [] { Tokens.TOKEN_REVERT.ToInt(), Tokens.TOKEN_c_OPENJSON.ToInt() },
new [] { Tokens.TOKEN_s_CDA_TYPE.ToInt(), Tokens.TOKEN_s_CDA_POLICY.ToInt() },
new [] { Tokens.TOKEN_BEGIN_CS.ToInt(), Tokens.TOKEN_END_CS.ToInt() }
// Note: after this it becomes hard to interpret actual keywords in the Tokens enum.
// Should review and re-assess later
};
foreach(int[] range in keywordRanges)
{
for(int i = range[0]; i <= range[1]; i++)
{
KeywordIdentifiers.Add(i);
}
}
KeywordIdentifiers.Add(FormatterTokens.LEX_BATCH_SEPERATOR);
KeywordIdentifiers.Add(FormatterTokens.TOKEN_IS);
}
public string FormattedSql
@@ -168,13 +167,11 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter
{
TokenData tok = Script.TokenManager.TokenList[i];
Replacements.Add(new Replacement(tok.StartIndex, sql, sql.ToUpperInvariant()));
sql = sql.ToUpperInvariant();
}
else if (FormatOptions.LowercaseKeywords)
{
TokenData tok = Script.TokenManager.TokenList[i];
Replacements.Add(new Replacement(tok.StartIndex, sql, sql.ToLowerInvariant()));
sql = sql.ToLowerInvariant();
}
}
}

8
src/Microsoft.SqlTools.ServiceLayer/Formatter/Impl/FormatterTokens.cs
View File

@@ -57,4 +57,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Formatter
public static readonly int LAST_TOKEN = ResolveTokenId("LAST_TOKEN");
}
static class TokenConverter
{
public static int ToInt(this Tokens token)
{
return (int)token;
}
}
}

50
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Lowercase.sql Normal file
View File

@@ -0,0 +1,50 @@
-- Case: common type keywords
create table [SimpleTable]
(
-- this is a comment before document
[DocumentID] INT identity (1, 1) not null,
-- this is a comment before Title
[Title] NVARCHAR (50) not null,
-- this is a comment before FileName
[FileName] NVARCHAR (400) not null,
-- this is a comment before FileExtension
[FileExtension] nvarchar(8)
);
go
create view v1
as
select *
from [SimpleTable]
go
create procedure p1
as
begin
select *
from [SimpleTable]
end
go
insert into t
default values
go
go
insert openquery (OracleSvr, 'SELECT name FROM joe.titles')
values
('NewTitle');
go
insert into myTable
(FileName, FileType, Document)
select 'Text1.txt' as FileName,
'.txt' as FileType,
*
from openrowset(bulk N'C:\Text1.txt', siNGLE_BLOB) as Document;
go
select *
from openxml (@idoc, '/ROOT/Customers')
exec sp_xml_removedocument @idoc;

50
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/BaselineFiles/KeywordCaseConversion_Uppercase.sql Normal file
View File

@@ -0,0 +1,50 @@
-- Case: common type keywords
CREATE TABLE [SimpleTable]
(
-- this is a comment before document
[DocumentID] INT IDENTITY (1, 1) NOT NULL,
-- this is a comment before Title
[Title] NVARCHAR (50) NOT NULL,
-- this is a comment before FileName
[FileName] NVARCHAR (400) NOT NULL,
-- this is a comment before FileExtension
[FileExtension] nvarchar(8)
);
GO
CREATE VIEW v1
AS
SELECT *
FROM [SimpleTable]
GO
CREATE PROCEDURE p1
AS
BEGIN
SELECT *
FROM [SimpleTable]
END
GO
INSERT INTO t
DEFAULT VALUES
GO
GO
INSERT OPENQUERY (OracleSvr, 'SELECT name FROM joe.titles')
VALUES
('NewTitle');
GO
INSERT INTO myTable
(FileName, FileType, Document)
SELECT 'Text1.txt' AS FileName,
'.txt' AS FileType,
*
FROM OPENROWSET(BULK N'C:\Text1.txt', siNGLE_BLOB) AS Document;
GO
SELECT *
FROM OPENXML (@idoc, '/ROOT/Customers')
EXEC sp_xml_removedocument @idoc;

41
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestData/TSqlFormatter/TestFiles/KeywordCaseConversion.sql Normal file
View File

@@ -0,0 +1,41 @@
-- Case: common type keywords
cReaTe tAbLe [SimpleTable]
(
-- this is a comment before document
[DocumentID] INT iDentIty (1, 1) NOT NULL,
-- this is a comment before Title
[Title] NVARCHAR (50) NOT NULL,
-- this is a comment before FileName
[FileName] NVARCHAR (400) NOT NULL,
-- this is a comment before FileExtension
[FileExtension] nvarchar(8)
);
GO
CREATE vIEw v1 aS select * from [SimpleTable]
GO
CREATE pRoceduRe p1 aS
bEgIn
select * from [SimpleTable]
eNd
GO
iNsert iNto t dEfault vAlues
GO
GO
iNsert oPenQUERY (OracleSvr, 'SELECT name FROM joe.titles')
VALUES ('NewTitle');
GO
INSERT INTO myTable(FileName, FileType, Document)
SELECT 'Text1.txt' AS FileName,
'.txt' AS FileType,
* FROM openROWSET(bUlK N'C:\Text1.txt', siNGLE_BLOB) AS Document;
GO
sELECT *
FROM openXML (@idoc, '/ROOT/Customers')
exEC sp_xml_removedocument @idoc;

5
test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestUtilities.cs
View File

@@ -54,9 +54,10 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
int startRange = Math.Max(firstDiffIndex - 50, 0);
int endRange = Math.Min(firstDiffIndex + 50, minEnd);
int length = endRange - startRange;
string baselineDiff = ShowWhitespace(baseline.Substring(startRange, endRange));
string actualDiff = ShowWhitespace(actual.Substring(startRange, endRange));
string baselineDiff = ShowWhitespace(baseline.Substring(startRange, length));
string actualDiff = ShowWhitespace(actual.Substring(startRange, length));
return "\r\nFirst Diff:\r\n===== Baseline =====\r\n"
+ baselineDiff
+ "\r\n===== Actual =====\r\n"

32
test/Microsoft.SqlTools.ServiceLayer.Test/Formatter/GeneralFormatterTests.cs Normal file
View File

@@ -0,0 +1,32 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using Microsoft.SqlTools.ServiceLayer.Formatter;
using Xunit;
namespace Microsoft.SqlTools.ServiceLayer.Test.Formatter
{
public class GeneralFormatterTests : FormatterUnitTestsBase
{
[Fact]
public void KeywordCaseConversionUppercase()
{
LoadAndFormatAndCompare("KeywordCaseConversion",
GetInputFile("KeywordCaseConversion.sql"),
GetBaselineFile("KeywordCaseConversion_Uppercase.sql"),
new FormatOptions() { KeywordCasing = CasingOptions.Uppercase },
verifyFormat: true);
}
[Fact]
public void KeywordCaseConversionLowercase()
{
LoadAndFormatAndCompare("KeywordCaseConversion",
GetInputFile("KeywordCaseConversion.sql"),
GetBaselineFile("KeywordCaseConversion_Lowercase.sql"),
new FormatOptions() { KeywordCasing = CasingOptions.Lowercase },
verifyFormat: true);
}
}
}