Feature/schemacompare Exclude-Include and Options enhancement (#799)

* Initial code for Including/Excluding individual changes (no tests added yet)

* Adding Exclude include tests. Default options call and additional options tests.

* Taking PR comments

* Retry in test for reliability

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/SchemaCompare/SchemaCompareServiceOptionsTests.cs

@@ -3,9 +3,6 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 using Microsoft.SqlTools.Hosting.Protocol;
-using Microsoft.SqlTools.ServiceLayer.DacFx;
-using Microsoft.SqlTools.ServiceLayer.DacFx.Contracts;
-using Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility;
 using Microsoft.SqlTools.ServiceLayer.SchemaCompare;
 using Microsoft.SqlTools.ServiceLayer.SchemaCompare.Contracts;
 using Microsoft.SqlTools.ServiceLayer.TaskServices;
@@ -16,6 +13,7 @@ using System;
 using System.IO;
 using System.Threading.Tasks;
 using Xunit;
+using Microsoft.SqlTools.ServiceLayer.SchemaCopmare;
 
 namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.SchemaCompare
 {
@@ -111,7 +109,6 @@ END
 
         private async Task<Mock<RequestContext<SchemaCompareResult>>> SendAndValidateSchemaCompareRequestDacpacToDacpacWithOptions(string sourceScript, string targetScript, DeploymentOptions nodiffOption, DeploymentOptions shouldDiffOption)
         {
-
             var result = SchemaCompareTestUtils.GetLiveAutoCompleteTestObjects();
             var schemaCompareRequestContext = new Mock<RequestContext<SchemaCompareResult>>();
             schemaCompareRequestContext.Setup(x => x.SendResult(It.IsAny<SchemaCompareResult>())).Returns(Task.FromResult(new object()));
@@ -373,17 +370,21 @@ END
         }
 
         /// <summary>
-        /// Verify the schema compare script generation comparing dacpac and db with and excluding table valued function
+        /// Verify the schema compare default creation test
         /// </summary>
         [Fact]
-        public void ValidateSchemaCompareOptionsDefault()
+        public void ValidateSchemaCompareOptionsDefaultAgainstDacFx()
         {
             DeploymentOptions deployOptions = new DeploymentOptions();
             DacDeployOptions dacOptions = new DacDeployOptions();
 
             System.Reflection.PropertyInfo[] deploymentOptionsProperties = deployOptions.GetType().GetProperties();
             System.Reflection.PropertyInfo[] ddProperties = dacOptions.GetType().GetProperties();
-            
+
+            // Note that DatabaseSpecification and sql cmd variables list is not present in Sqltools service - its not settable and is not used by ADS options.
+            // TODO : update this test if the above options are added later
+            Assert.True(deploymentOptionsProperties.Length == ddProperties.Length - 2 , $"Number of properties is not same Deployment options : {deploymentOptionsProperties.Length} DacFx options : {ddProperties.Length}");
+
             foreach (var deployOptionsProp in deploymentOptionsProperties)
             {
                 var dacProp = dacOptions.GetType().GetProperty(deployOptionsProp.Name);
@@ -391,10 +392,49 @@ END
 
                 var deployOptionsValue = deployOptionsProp.GetValue(deployOptions);
                 var dacValue = dacProp.GetValue(dacOptions);
-                
-                Assert.True((deployOptionsValue == null && dacValue == null) || deployOptionsValue.Equals(dacValue), $"DacFx DacDeploy property not equal to Tools Service DeploymentOptions for { deployOptionsProp.Name}, SchemaCompareOptions value: {deployOptionsValue} and DacDeployOptions value: {dacValue} ");
+
+                if (deployOptionsProp.Name != "ExcludeObjectTypes") // do not compare for ExcludeObjectTypes because it will be different
+                {
+                    Assert.True((deployOptionsValue == null && dacValue == null) || deployOptionsValue.Equals(dacValue), $"DacFx DacDeploy property not equal to Tools Service DeploymentOptions for { deployOptionsProp.Name}, SchemaCompareOptions value: {deployOptionsValue} and DacDeployOptions value: {dacValue} ");
+                }
             }
         }
 
+        /// <summary>
+        /// Verify the schema compare default creation test
+        /// </summary>
+        [Fact]
+        public async void ValidateSchemaCompareGetDefaultOptionsCallFromService()
+        {
+            DeploymentOptions deployOptions = new DeploymentOptions();
+            var schemaCompareRequestContext = new Mock<RequestContext<SchemaCompareOptionsResult>>();
+            schemaCompareRequestContext.Setup(x => x.SendResult(It.IsAny<SchemaCompareOptionsResult>())).Returns(Task.FromResult(new object()));
+            schemaCompareRequestContext.Setup((RequestContext<SchemaCompareOptionsResult> x) => x.SendResult(It.Is<SchemaCompareOptionsResult>((options) => this.OptionsEqualsDefault(options) == true))).Returns(Task.FromResult(new object()));
+            
+            SchemaCompareGetOptionsParams p = new SchemaCompareGetOptionsParams();
+            await SchemaCompareService.Instance.HandleSchemaCompareGetDefaultOptionsRequest(p, schemaCompareRequestContext.Object);
+        }
+
+        private bool OptionsEqualsDefault(SchemaCompareOptionsResult options)
+        {
+            DeploymentOptions defaultOpt = new DeploymentOptions();
+            DeploymentOptions actualOpt = options.DefaultDeploymentOptions;
+
+            System.Reflection.PropertyInfo[] deploymentOptionsProperties = defaultOpt.GetType().GetProperties();
+            foreach(var v in deploymentOptionsProperties)
+            {
+                var defaultP = v.GetValue(defaultOpt);
+                var actualP = v.GetValue(actualOpt);
+                if (v.Name == "ExcludeObjectTypes")
+                {
+                    Assert.True((defaultP as ObjectType[]).Length == (actualP as ObjectType[]).Length, $"Number of excluded objects is different; expected: {(defaultP as ObjectType[]).Length} actual: {(actualP as ObjectType[]).Length}");
+                }
+                else
+                {
+                    Assert.True((defaultP == null && actualP == null) || defaultP.Equals(actualP), $"Actual Property from Service is not equal to default property for { v.Name}, Actual value: {actualP} and Default value: {defaultP}");
+                }
+            }
+            return true;
+        }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/SchemaCompare/SchemaCompareServiceTests.cs

@@ -3,9 +3,6 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 using Microsoft.SqlTools.Hosting.Protocol;
-using Microsoft.SqlTools.ServiceLayer.DacFx;
-using Microsoft.SqlTools.ServiceLayer.DacFx.Contracts;
-using Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility;
 using Microsoft.SqlTools.ServiceLayer.SchemaCompare;
 using Microsoft.SqlTools.ServiceLayer.SchemaCompare.Contracts;
 using Microsoft.SqlTools.ServiceLayer.TaskServices;
@@ -13,6 +10,7 @@ using Microsoft.SqlTools.ServiceLayer.Test.Common;
 using Moq;
 using System;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
 
@@ -73,11 +71,7 @@ CREATE TABLE [dbo].[table3]
                 };
 
                 SchemaCompareOperation schemaCompareOperation = new SchemaCompareOperation(schemaCompareParams, null, null);
-                schemaCompareOperation.Execute(TaskExecutionMode.Execute);
-
-                Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
-                Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
-                Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
+                ValidateSchemaCompareWithExcludeIncludeResults(schemaCompareOperation);
 
                 // cleanup
                 SchemaCompareTestUtils.VerifyAndCleanup(sourceDacpacFilePath);
@@ -120,11 +114,7 @@ CREATE TABLE [dbo].[table3]
                 };
 
                 SchemaCompareOperation schemaCompareOperation = new SchemaCompareOperation(schemaCompareParams, result.ConnectionInfo, result.ConnectionInfo);
-                schemaCompareOperation.Execute(TaskExecutionMode.Execute);
-
-                Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
-                Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
-                Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
+                ValidateSchemaCompareWithExcludeIncludeResults(schemaCompareOperation);
             }
             finally
             {
@@ -163,11 +153,7 @@ CREATE TABLE [dbo].[table3]
                 };
 
                 SchemaCompareOperation schemaCompareOperation = new SchemaCompareOperation(schemaCompareParams, result.ConnectionInfo, null);
-                schemaCompareOperation.Execute(TaskExecutionMode.Execute);
-
-                Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
-                Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
-                Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
+                ValidateSchemaCompareWithExcludeIncludeResults(schemaCompareOperation);
 
                 // cleanup
                 SchemaCompareTestUtils.VerifyAndCleanup(targetDacpacFilePath);
@@ -208,13 +194,8 @@ CREATE TABLE [dbo].[table3]
                 };
 
                 SchemaCompareOperation schemaCompareOperation = new SchemaCompareOperation(schemaCompareParams, result.ConnectionInfo, result.ConnectionInfo);
-                schemaCompareOperation.Execute(TaskExecutionMode.Execute);
-
-                Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
-                Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
-                Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
-
-                // generate script
+                
+                // generate script params
                 var generateScriptParams = new SchemaCompareGenerateScriptParams
                 {
                     TargetDatabaseName = targetDb.DatabaseName,
@@ -222,8 +203,7 @@ CREATE TABLE [dbo].[table3]
                     ScriptFilePath = Path.Combine(folderPath, string.Concat(sourceDb.DatabaseName, "_", "Update.publish.sql"))
                 };
 
-                SchemaCompareGenerateScriptOperation generateScriptOperation = new SchemaCompareGenerateScriptOperation(generateScriptParams, schemaCompareOperation.ComparisonResult);
-                generateScriptOperation.Execute(TaskExecutionMode.Execute);
+                ValidateSchemaCompareScriptGenerationWithExcludeIncludeResults(schemaCompareOperation, generateScriptParams);
 
                 // cleanup
                 SchemaCompareTestUtils.VerifyAndCleanup(generateScriptParams.ScriptFilePath);
@@ -266,11 +246,6 @@ CREATE TABLE [dbo].[table3]
                 };
 
                 SchemaCompareOperation schemaCompareOperation = new SchemaCompareOperation(schemaCompareParams, result.ConnectionInfo, result.ConnectionInfo);
-                schemaCompareOperation.Execute(TaskExecutionMode.Execute);
-
-                Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
-                Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
-                Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
 
                 // generate script
                 var generateScriptParams = new SchemaCompareGenerateScriptParams
@@ -280,9 +255,8 @@ CREATE TABLE [dbo].[table3]
                     ScriptFilePath = Path.Combine(folderPath, string.Concat(sourceDb.DatabaseName, "_", "Update.publish.sql"))
                 };
 
-                SchemaCompareGenerateScriptOperation generateScriptOperation = new SchemaCompareGenerateScriptOperation(generateScriptParams, schemaCompareOperation.ComparisonResult);
-                generateScriptOperation.Execute(TaskExecutionMode.Execute);
-
+                ValidateSchemaCompareScriptGenerationWithExcludeIncludeResults(schemaCompareOperation, generateScriptParams);
+                
                 // cleanup
                 SchemaCompareTestUtils.VerifyAndCleanup(generateScriptParams.ScriptFilePath);
                 SchemaCompareTestUtils.VerifyAndCleanup(sourceDacpacFilePath);
@@ -354,7 +328,7 @@ CREATE TABLE [dbo].[table3]
                 Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
                 Assert.True(schemaCompareOperation.ComparisonResult.IsEqual);
                 Assert.Empty(schemaCompareOperation.ComparisonResult.Differences);
-
+                
                 // cleanup
                 SchemaCompareTestUtils.VerifyAndCleanup(sourceDacpacFilePath);
             }
@@ -429,7 +403,108 @@ CREATE TABLE [dbo].[table3]
             }
             return schemaCompareRequestContext;
         }
+        
+        private void ValidateSchemaCompareWithExcludeIncludeResults(SchemaCompareOperation schemaCompareOperation)
+        {
+            schemaCompareOperation.Execute(TaskExecutionMode.Execute);
 
+            Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
+            Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
+            Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
+
+            // create Diff Entry from Difference
+            
+            DiffEntry diff = SchemaCompareOperation.CreateDiffEntry(schemaCompareOperation.ComparisonResult.Differences.First(), null);
+
+            int initial = schemaCompareOperation.ComparisonResult.Differences.Count();
+            SchemaCompareNodeParams schemaCompareExcludeNodeParams = new SchemaCompareNodeParams()
+            {
+                OperationId = schemaCompareOperation.OperationId,
+                DiffEntry = diff,
+                IncludeRequest = false,
+                TaskExecutionMode = TaskExecutionMode.Execute
+            };
+            SchemaCompareIncludeExcludeNodeOperation nodeExcludeOperation = new SchemaCompareIncludeExcludeNodeOperation(schemaCompareExcludeNodeParams, schemaCompareOperation.ComparisonResult);
+            nodeExcludeOperation.Execute(TaskExecutionMode.Execute);
+
+            int afterExclude = schemaCompareOperation.ComparisonResult.Differences.Count();
+
+            Assert.True(initial == afterExclude, $"Changes should be same again after excluding/including, before {initial}, now {afterExclude}");
+            
+            SchemaCompareNodeParams schemaCompareincludeNodeParams = new SchemaCompareNodeParams()
+            {
+                OperationId = schemaCompareOperation.OperationId,
+                DiffEntry = diff,
+                IncludeRequest = true,
+                TaskExecutionMode = TaskExecutionMode.Execute
+            };
+
+            SchemaCompareIncludeExcludeNodeOperation nodeIncludeOperation = new SchemaCompareIncludeExcludeNodeOperation(schemaCompareincludeNodeParams, schemaCompareOperation.ComparisonResult);
+            nodeIncludeOperation.Execute(TaskExecutionMode.Execute);
+            int afterInclude = schemaCompareOperation.ComparisonResult.Differences.Count();
+
+
+            Assert.True(initial == afterInclude, $"Changes should be same again after excluding/including, before:{initial}, now {afterInclude}");
+        }
+        
+        private void ValidateSchemaCompareScriptGenerationWithExcludeIncludeResults(SchemaCompareOperation schemaCompareOperation, SchemaCompareGenerateScriptParams generateScriptParams)
+        {
+            schemaCompareOperation.Execute(TaskExecutionMode.Execute);
+
+            Assert.True(schemaCompareOperation.ComparisonResult.IsValid);
+            Assert.False(schemaCompareOperation.ComparisonResult.IsEqual);
+            Assert.NotNull(schemaCompareOperation.ComparisonResult.Differences);
+
+            SchemaCompareGenerateScriptOperation generateScriptOperation = new SchemaCompareGenerateScriptOperation(generateScriptParams, schemaCompareOperation.ComparisonResult);
+            generateScriptOperation.Execute(TaskExecutionMode.Execute);
+
+            string initialScript = File.ReadAllText(generateScriptParams.ScriptFilePath);
+
+            // create Diff Entry from on Difference
+            DiffEntry diff = SchemaCompareOperation.CreateDiffEntry(schemaCompareOperation.ComparisonResult.Differences.First(), null);
+
+            int initial = schemaCompareOperation.ComparisonResult.Differences.Count();
+            SchemaCompareNodeParams schemaCompareExcludeNodeParams = new SchemaCompareNodeParams()
+            {
+                OperationId = schemaCompareOperation.OperationId,
+                DiffEntry = diff,
+                IncludeRequest = false,
+                TaskExecutionMode = TaskExecutionMode.Execute
+            };
+            SchemaCompareIncludeExcludeNodeOperation nodeExcludeOperation = new SchemaCompareIncludeExcludeNodeOperation(schemaCompareExcludeNodeParams, schemaCompareOperation.ComparisonResult);
+            nodeExcludeOperation.Execute(TaskExecutionMode.Execute);
+
+            int afterExclude = schemaCompareOperation.ComparisonResult.Differences.Count();
+
+            Assert.True(initial == afterExclude, $"Changes should be same again after excluding/including, before {initial}, now {afterExclude}");
+
+            generateScriptOperation = new SchemaCompareGenerateScriptOperation(generateScriptParams, schemaCompareOperation.ComparisonResult);
+            generateScriptOperation.Execute(TaskExecutionMode.Execute);
+
+            string afterExcludeScript = File.ReadAllText(generateScriptParams.ScriptFilePath);
+            Assert.True(initialScript.Length > afterExcludeScript.Length, $"Script should be affected (less statements) exclude operation, before {initialScript}, now {afterExcludeScript}");
+
+            SchemaCompareNodeParams schemaCompareincludeNodeParams = new SchemaCompareNodeParams()
+            {
+                OperationId = schemaCompareOperation.OperationId,
+                DiffEntry = diff,
+                IncludeRequest = true,
+                TaskExecutionMode = TaskExecutionMode.Execute
+            };
+
+            SchemaCompareIncludeExcludeNodeOperation nodeIncludeOperation = new SchemaCompareIncludeExcludeNodeOperation(schemaCompareincludeNodeParams, schemaCompareOperation.ComparisonResult);
+            nodeIncludeOperation.Execute(TaskExecutionMode.Execute);
+            int afterInclude = schemaCompareOperation.ComparisonResult.Differences.Count();
+            
+            Assert.True(initial == afterInclude, $"Changes should be same again after excluding/including:{initial}, now {afterInclude}");
+
+            generateScriptOperation = new SchemaCompareGenerateScriptOperation(generateScriptParams, schemaCompareOperation.ComparisonResult);
+            generateScriptOperation.Execute(TaskExecutionMode.Execute);
+
+            string afterIncludeScript = File.ReadAllText(generateScriptParams.ScriptFilePath);
+            Assert.True(initialScript.Length == afterIncludeScript.Length, $"Changes should be same as inital since we included what we excluded, before {initialScript}, now {afterIncludeScript}");
+        }
+        
         /// <summary>
         /// Verify the schema compare request comparing two dacpacs
         /// </summary>

test/Microsoft.SqlTools.ServiceLayer.IntegrationTests/SchemaCompare/SchemaCompareTestUtils.cs

@@ -10,6 +10,7 @@ using Microsoft.SqlTools.ServiceLayer.Test.Common;
 using NUnit.Framework;
 using System;
 using System.IO;
+using static Microsoft.SqlTools.ServiceLayer.IntegrationTests.Utility.LiveConnectionHelper;
 
 namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.SchemaCompare
 {
@@ -50,7 +51,21 @@ namespace Microsoft.SqlTools.ServiceLayer.IntegrationTests.SchemaCompare
 
         internal static LiveConnectionHelper.TestConnectionResult GetLiveAutoCompleteTestObjects()
         {
-            var result = LiveConnectionHelper.InitLiveConnectionInfo();
+            // Adding retry for reliability - otherwise it caused test to fail in lab
+            TestConnectionResult result = null;
+            int retry = 3;
+
+            while (retry > 0)
+            {
+                result = LiveConnectionHelper.InitLiveConnectionInfo();
+                if (result != null && result.ConnectionInfo != null)
+                {
+                    return result;
+                }
+                System.Threading.Thread.Sleep(1000);
+                retry--;
+            }
+
             return result;
         }
     }