Add serialization service support (#840)

Added a method that handles serialization requests
Support splitting save over multiple requests to reduce overall message size
Added unit tests
String changes used a new version of the string tool for generation.
Will publish PR separately for the changes to build & localization
so this can run on Mac without .Net Core 1.0

src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/DbColumnWrapper.cs

@@ -3,6 +3,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Data;
 using System.Data.Common;
@@ -93,104 +94,8 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
             DataType = column.DataType;
             DataTypeName = column.DataTypeName.ToLowerInvariant();
 
-            // Determine the SqlDbType
-            SqlDbType type;
-            if (Enum.TryParse(DataTypeName, true, out type))
-            {
-                SqlDbType = type;
-            }
-            else
-            {
-                switch (DataTypeName)
-                {
-                    case "numeric":
-                        SqlDbType = SqlDbType.Decimal;
-                        break;
-                    case "sql_variant":
-                        SqlDbType = SqlDbType.Variant;
-                        break;
-                    case "timestamp":
-                        SqlDbType = SqlDbType.VarBinary;
-                        break;
-                    case "sysname":
-                        SqlDbType = SqlDbType.NVarChar;
-                        break;
-                    default:
-                        SqlDbType = DataTypeName.EndsWith(".sys.hierarchyid") ? SqlDbType.NVarChar : SqlDbType.Udt;
-                        break;
-                }
-            }
-
-            // We want the display name for the column to always exist
-            ColumnName = string.IsNullOrEmpty(column.ColumnName)
-                ? SR.QueryServiceColumnNull
-                : column.ColumnName;
-
-            switch (DataTypeName)
-            {
-                case "varchar":
-                case "nvarchar":
-                    IsChars = true;
-
-                    Debug.Assert(ColumnSize.HasValue);
-                    if (ColumnSize.Value == int.MaxValue)
-                    {
-                        //For Yukon, special case nvarchar(max) with column name == "Microsoft SQL Server 2005 XML Showplan" -
-                        //assume it is an XML showplan.
-                        //Please note this field must be in sync with a similar field defined in QESQLBatch.cs.
-                        //This is not the best fix that we could do but we are trying to minimize code impact
-                        //at this point. Post Yukon we should review this code again and avoid
-                        //hard-coding special column name in multiple places.
-                        const string yukonXmlShowPlanColumn = "Microsoft SQL Server 2005 XML Showplan";
-                        if (column.ColumnName == yukonXmlShowPlanColumn)
-                        {
-                            // Indicate that this is xml to apply the right size limit
-                            // Note we leave chars type as well to use the right retrieval mechanism.
-                            IsXml = true;
-                        }
-                        IsLong = true;
-                    }
-                    break;
-                case "text":
-                case "ntext":
-                    IsChars = true;
-                    IsLong = true;
-                    break;
-                case "xml":
-                    IsXml = true;
-                    IsLong = true;
-                    break;
-                case "binary":
-                case "image":
-                    IsBytes = true;
-                    IsLong = true;
-                    break;
-                case "varbinary":
-                case "rowversion":
-                    IsBytes = true;
-
-                    Debug.Assert(ColumnSize.HasValue);
-                    if (ColumnSize.Value == int.MaxValue)
-                    {
-                        IsLong = true;
-                    }
-                    break;
-                case "sql_variant":
-                    IsSqlVariant = true;
-                    break;
-                default:
-                    if (!AllServerDataTypes.Contains(DataTypeName))
-                    {
-                        // treat all UDT's as long/bytes data types to prevent the CLR from attempting
-                        // to load the UDT assembly into our process to call ToString() on the object.
-
-                        IsUdt = true;
-                        IsBytes = true;
-                        IsLong = true;
-                    }
-                    break;
-            }
-
+            DetermineSqlDbType();
+            AddNameAndDataFields(column.ColumnName);
 
             if (IsUdt)
             {
@@ -215,6 +120,19 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
             }
         }
 
+        public DbColumnWrapper(ColumnInfo columnInfo)
+        {
+            DataTypeName = columnInfo.DataTypeName.ToLowerInvariant();
+            DetermineSqlDbType();
+            DataType = TypeConvertor.ToNetType(this.SqlDbType);
+            if (DataType == typeof(String))
+            {
+                this.ColumnSize = int.MaxValue;
+            }
+            AddNameAndDataFields(columnInfo.Name);
+        }
+
+
         /// <summary>
         /// Default constructor, used for deserializing JSON RPC only
         /// </summary>
@@ -299,5 +217,176 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
 
         #endregion
 
+
+        private void DetermineSqlDbType()
+        {
+            // Determine the SqlDbType
+            SqlDbType type;
+            if (Enum.TryParse(DataTypeName, true, out type))
+            {
+                SqlDbType = type;
+            }
+            else
+            {
+                switch (DataTypeName)
+                {
+                    case "numeric":
+                        SqlDbType = SqlDbType.Decimal;
+                        break;
+                    case "sql_variant":
+                        SqlDbType = SqlDbType.Variant;
+                        break;
+                    case "timestamp":
+                        SqlDbType = SqlDbType.VarBinary;
+                        break;
+                    case "sysname":
+                        SqlDbType = SqlDbType.NVarChar;
+                        break;
+                    default:
+                        SqlDbType = DataTypeName.EndsWith(".sys.hierarchyid") ? SqlDbType.NVarChar : SqlDbType.Udt;
+                        break;
+                }
+            }
+        }
+
+        private void AddNameAndDataFields(string columnName)
+        {
+            // We want the display name for the column to always exist
+            ColumnName = string.IsNullOrEmpty(columnName)
+                ? SR.QueryServiceColumnNull
+                : columnName;
+
+            switch (DataTypeName)
+            {
+                case "varchar":
+                case "nvarchar":
+                    IsChars = true;
+
+                    Debug.Assert(ColumnSize.HasValue);
+                    if (ColumnSize.Value == int.MaxValue)
+                    {
+                        //For Yukon, special case nvarchar(max) with column name == "Microsoft SQL Server 2005 XML Showplan" -
+                        //assume it is an XML showplan.
+                        //Please note this field must be in sync with a similar field defined in QESQLBatch.cs.
+                        //This is not the best fix that we could do but we are trying to minimize code impact
+                        //at this point. Post Yukon we should review this code again and avoid
+                        //hard-coding special column name in multiple places.
+                        const string yukonXmlShowPlanColumn = "Microsoft SQL Server 2005 XML Showplan";
+                        if (columnName == yukonXmlShowPlanColumn)
+                        {
+                            // Indicate that this is xml to apply the right size limit
+                            // Note we leave chars type as well to use the right retrieval mechanism.
+                            IsXml = true;
+                        }
+                        IsLong = true;
+                    }
+                    break;
+                case "text":
+                case "ntext":
+                    IsChars = true;
+                    IsLong = true;
+                    break;
+                case "xml":
+                    IsXml = true;
+                    IsLong = true;
+                    break;
+                case "binary":
+                case "image":
+                    IsBytes = true;
+                    IsLong = true;
+                    break;
+                case "varbinary":
+                case "rowversion":
+                    IsBytes = true;
+
+                    Debug.Assert(ColumnSize.HasValue);
+                    if (ColumnSize.Value == int.MaxValue)
+                    {
+                        IsLong = true;
+                    }
+                    break;
+                case "sql_variant":
+                    IsSqlVariant = true;
+                    break;
+                default:
+                    if (!AllServerDataTypes.Contains(DataTypeName))
+                    {
+                        // treat all UDT's as long/bytes data types to prevent the CLR from attempting
+                        // to load the UDT assembly into our process to call ToString() on the object.
+
+                        IsUdt = true;
+                        IsBytes = true;
+                        IsLong = true;
+                    }
+                    break;
+            }
+        }
+    }
+
+
+
+    /// <summary>
+    /// Convert a base data type to another base data type
+    /// </summary>
+    public sealed class TypeConvertor
+    {
+        private static Dictionary<SqlDbType,Type> _typeMap = new Dictionary<SqlDbType,Type>();
+
+        static TypeConvertor()
+        {
+            _typeMap[SqlDbType.BigInt] = typeof(Int64);
+            _typeMap[SqlDbType.Binary] = typeof(Byte);
+            _typeMap[SqlDbType.Bit] = typeof(Boolean);
+            _typeMap[SqlDbType.Char] = typeof(String);
+            _typeMap[SqlDbType.DateTime] = typeof(DateTime);
+            _typeMap[SqlDbType.Decimal] = typeof(Decimal);
+            _typeMap[SqlDbType.Float] = typeof(Double);
+            _typeMap[SqlDbType.Image] = typeof(Byte[]);
+            _typeMap[SqlDbType.Int] = typeof(Int32);
+            _typeMap[SqlDbType.Money] = typeof(Decimal);
+            _typeMap[SqlDbType.NChar] = typeof(String);
+            _typeMap[SqlDbType.NChar] = typeof(String);
+            _typeMap[SqlDbType.NChar] = typeof(String);
+            _typeMap[SqlDbType.NText] = typeof(String);
+            _typeMap[SqlDbType.NVarChar] = typeof(String);
+            _typeMap[SqlDbType.Real] = typeof(Single);
+            _typeMap[SqlDbType.UniqueIdentifier] = typeof(Guid);
+            _typeMap[SqlDbType.SmallDateTime] = typeof(DateTime);
+            _typeMap[SqlDbType.SmallInt] = typeof(Int16);
+            _typeMap[SqlDbType.SmallMoney] = typeof(Decimal);
+            _typeMap[SqlDbType.Text] = typeof(String);
+            _typeMap[SqlDbType.Timestamp] = typeof(Byte[]);
+            _typeMap[SqlDbType.TinyInt] = typeof(Byte);
+            _typeMap[SqlDbType.VarBinary] = typeof(Byte[]);
+            _typeMap[SqlDbType.VarChar] = typeof(String);
+            _typeMap[SqlDbType.Variant] = typeof(Object);
+            // Note: treating as string
+            _typeMap[SqlDbType.Xml] = typeof(String);
+            _typeMap[SqlDbType.TinyInt] = typeof(Byte);
+            _typeMap[SqlDbType.TinyInt] = typeof(Byte);
+            _typeMap[SqlDbType.TinyInt] = typeof(Byte);
+            _typeMap[SqlDbType.TinyInt] = typeof(Byte);
+        }
+
+        private TypeConvertor()
+        {
+
+        }
+
+
+        /// <summary>
+        /// Convert TSQL type to .Net data type
+        /// </summary>
+        /// <param name="sqlDbType"></param>
+        /// <returns></returns>
+        public static Type ToNetType(SqlDbType sqlDbType)
+        {
+            Type netType;
+            if (!_typeMap.TryGetValue(sqlDbType, out netType))
+            {
+                netType = typeof(String);
+            }
+            return netType;
+        }
     }
 }

src/Microsoft.SqlTools.ServiceLayer/QueryExecution/Contracts/Serialization.cs

@@ -3,14 +3,77 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
+using System.Data;
 using Microsoft.SqlTools.Hosting.Protocol.Contracts;
+using Microsoft.SqlTools.Utility;
 
 namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
 {
+
+
+    public class ColumnInfo
+    {
+        /// <summary>
+        /// Name of this column
+        /// </summary>
+        public string Name { get; set; }
+
+        public string DataTypeName { get; set; } 
+
+        public ColumnInfo()
+        {
+        }
+
+        public ColumnInfo(string name, string dataTypeName)
+        {
+            this.Name = name;
+            this.DataTypeName = dataTypeName;
+        }
+    }
+
+    public interface ISerializationParams
+    {
+        
+        /// <summary>
+        /// Path to file that the serialized results will be stored in
+        /// </summary>
+        string FilePath { get; set; }
+
+        /// <summary>
+        /// Results that are to be serialized into 'SaveFormat' format
+        /// </summary>
+        DbCellValue[][] Rows { get; set; }
+
+        /// <summary>
+        /// Whether the current set of Rows passed in is the last for this file
+        // </summary>
+        bool IsLastBatch { get; set; }
+    }
     /// <summary>
     /// Class used for storing results and how the results are to be serialized
     /// </summary>
-    public class SaveResultsInfo
+    public class SerializeDataContinueRequestParams : ISerializationParams
+    {
+        /// <summary>
+        /// Path to file that the serialized results will be stored in
+        /// </summary>
+        public string FilePath { get; set; }
+
+        /// <summary>
+        /// Results that are to be serialized into 'SaveFormat' format
+        /// </summary>
+        public DbCellValue[][] Rows { get; set; }
+
+        /// <summary>
+        /// Whether the current set of Rows passed in is the last for this file
+        // </summary>
+        public bool IsLastBatch { get; set; }
+
+    }
+    /// <summary>
+    /// Class used for storing results and how the results are to be serialized
+    /// </summary>
+    public class SerializeDataStartRequestParams : GeneralRequestDetails, ISerializationParams
     {
         /// <summary>
         /// String representation of the type that service is supposed to serialize to
@@ -21,37 +84,97 @@ namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts
         /// <summary>
         /// Path to file that the serialized results will be stored in
         /// </summary>
-        public string SavePath { get; set; }
+        public string FilePath { get; set; }
 
         /// <summary>
         /// Results that are to be serialized into 'SaveFormat' format
         /// </summary>
         public DbCellValue[][] Rows { get; set; }
 
-        /// <summary>
-        /// Whether the current set of Rows passed in is the last for this file
-        // </summary>
-        public bool IsLast { get; set; }
+        public ColumnInfo[] Columns { get; set; }
 
+        /// <summary>
+        /// Whether this is the only request expected for this file.
+        // </summary>
+        public bool IsLastBatch { get; set; }
+
+        public SerializeDataStartRequestParams()
+        {
+        }
         /// <summary>
         /// Constructor
         /// </summary>
-        public SaveResultsInfo(string saveFormat, 
+        public SerializeDataStartRequestParams(string saveFormat, 
             string savePath, 
             DbCellValue[][] rows, 
             bool isLast)
         {
             this.SaveFormat = saveFormat;
-            this.SavePath = savePath;
+            this.FilePath = savePath;
             this.Rows = Rows;
-            this.IsLast = isLast;
+            this.IsLastBatch = isLast;
+        }
+
+        internal bool IncludeHeaders
+        {
+            get { return this.GetOptionValue<bool>(SerializationOptionsHelper.IncludeHeaders); }
+            set { this.SetOptionValue<bool>(SerializationOptionsHelper.IncludeHeaders, value); }
+        }
+
+        internal string Delimiter
+        {
+            get { return this.GetOptionValue<string>(SerializationOptionsHelper.Delimiter); }
+            set { this.SetOptionValue<string>(SerializationOptionsHelper.Delimiter, value); }
+        }
+
+        internal string LineSeparator
+        {
+            get { return this.GetOptionValue<string>(SerializationOptionsHelper.LineSeparator); }
+            set { this.SetOptionValue<string>(SerializationOptionsHelper.LineSeparator, value); }
+        }
+
+        internal string TextIdentifier
+        {
+            get { return this.GetOptionValue<string>(SerializationOptionsHelper.TextIdentifier); }
+            set { this.SetOptionValue<string>(SerializationOptionsHelper.TextIdentifier, value); }
+        }
+
+        internal string Encoding
+        {
+            get { return this.GetOptionValue<string>(SerializationOptionsHelper.Encoding); }
+            set { this.SetOptionValue<string>(SerializationOptionsHelper.Encoding, value); }
+        }
+
+        internal bool Formatted
+        {
+            get { return this.GetOptionValue<bool>(SerializationOptionsHelper.Formatted); }
+            set { this.SetOptionValue<bool>(SerializationOptionsHelper.Formatted, value); }
         }
     }
 
-    public class SaveAsRequest
+    public class SerializeDataResult
     {
-        public static readonly
-            RequestType<SaveResultsInfo, SaveResultRequestResult> Type =
-            RequestType<SaveResultsInfo, SaveResultRequestResult>.Create("query/saveAs");
+        public string Messages { get; set; }
+
+        public bool Succeeded { get; set; }
+    }
+
+    public class SerializeStartRequest
+    {
+        public static readonly RequestType<SerializeDataStartRequestParams, SerializeDataResult> Type = RequestType<SerializeDataStartRequestParams, SerializeDataResult>.Create("serialize/start");
+    }
+    public class SerializeContinueRequest
+    {
+        public static readonly RequestType<SerializeDataContinueRequestParams, SerializeDataResult> Type = RequestType<SerializeDataContinueRequestParams, SerializeDataResult>.Create("serialize/continue");
+    }
+
+    class SerializationOptionsHelper
+    {
+        internal const string IncludeHeaders = "includeHeaders";
+        internal const string Delimiter = "delimiter";
+        internal const string LineSeparator = "lineSeparator";
+        internal const string TextIdentifier = "textIdentifier";
+        internal const string Encoding = "encoding";
+        internal const string Formatted = "formatted";
     }
 }