Dropped Ledger Columns folders for ledger primary tables (#1688)

* dropped ledger columns strings, nodetypes, definition

* work in progress

* WIP- dropped columns folder showing for all tables

* Dropped Ledger Column folder only for primary ledger tables

* cleaning up

* undoing changes to kusto layer

* formatting

* undoing formatting

* undoing formatting

* removing IsLedger property and instead filtering on NodeTypeId

* updating column path test

src/Microsoft.SqlTools.ServiceLayer/Localization/sr.cs

@@ -2045,6 +2045,14 @@ namespace Microsoft.SqlTools.ServiceLayer
             }
         }
 
+        public static string SchemaHierarchy_DroppedLedgerColumns
+        {
+            get
+            {
+                return Keys.GetString(Keys.SchemaHierarchy_DroppedLedgerColumns);
+            }
+        }
+
         public static string SchemaHierarchy_DroppedLedgerTables
         {
             get
@@ -10877,6 +10885,9 @@ namespace Microsoft.SqlTools.ServiceLayer
             public const string SchemaHierarchy_ExternalTables = "SchemaHierarchy_ExternalTables";
 
 
+            public const string SchemaHierarchy_DroppedLedgerColumns = "SchemaHierarchy_DroppedLedgerColumns";
+
+
             public const string SchemaHierarchy_DroppedLedgerTables = "SchemaHierarchy_DroppedLedgerTables";
 
 

src/Microsoft.SqlTools.ServiceLayer/Localization/sr.resx

@@ -1280,6 +1280,10 @@
     <value>External Tables</value>
     <comment></comment>
   </data>
+  <data name="SchemaHierarchy_DroppedLedgerColumns" xml:space="preserve">
+    <value>Dropped Ledger Columns</value>
+    <comment></comment>
+  </data>
   <data name="SchemaHierarchy_DroppedLedgerTables" xml:space="preserve">
     <value>Dropped Ledger Tables</value>
     <comment></comment>

src/Microsoft.SqlTools.ServiceLayer/Localization/sr.strings

@@ -630,6 +630,8 @@ SchemaHierarchy_ExternalResources = External Resources
 
 SchemaHierarchy_ExternalTables = External Tables
 
+SchemaHierarchy_DroppedLedgerColumns = Dropped Ledger Columns
+
 SchemaHierarchy_DroppedLedgerTables = Dropped Ledger Tables
 
 SchemaHierarchy_DroppedLedgerViews = Dropped Ledger Views

src/Microsoft.SqlTools.ServiceLayer/Localization/sr.xlf

@@ -6331,6 +6331,11 @@ The Query Processor estimates that implementing the following index could improv
         <target state="new">Dropped Ledger Views</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="SchemaHierarchy_DroppedLedgerColumns">
+        <source>Dropped Ledger Columns</source>
+        <target state="new">Dropped Ledger Columns</target>
+        <note></note>
+      </trans-unit>
       <trans-unit id="SqlProjectModelNotFound">
         <source>Could not find SQL model from project: {0}.</source>
         <target state="new">Could not find SQL model from project: {0}.</target>

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/Nodes/NodeObservableCollection.cs

@@ -38,7 +38,8 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Nodes
                 bool anyDroppedFolders = this.Any(
                         node => node is FolderNode &&
                         (node.NodeTypeId == NodeTypes.DroppedLedgerTables ||
-                         node.NodeTypeId == NodeTypes.DroppedLedgerViews));
+                         node.NodeTypeId == NodeTypes.DroppedLedgerViews ||
+                         node.NodeTypeId == NodeTypes.DroppedLedgerColumns));
                 return this.FirstOrDefault() is SmoTreeNode && !anyDroppedFolders;
             }
         }

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/Nodes/NodeTypes.cs

@@ -142,5 +142,6 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Nodes
         ColumnEncryptionKeys,
         DroppedLedgerTables,
         DroppedLedgerViews,
+        DroppedLedgerColumns,
     }
 }

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/SmoModel/SmoChildFactoryBase.cs

@@ -53,6 +53,7 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.SmoModel
             {
                 allChildren.RemoveAll(x => x.IsSystemObject);
             }
+
             if (context != null && context.ValidFor != 0 && context.ValidFor != ValidForFlag.All)
             {
                 allChildren.RemoveAll(x =>
@@ -64,6 +65,22 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.SmoModel
                     }
                     return false;
                 });
+
+                // Remove the Dropped Ledger Columns folder if this isn't under a ledger table
+                allChildren.RemoveAll(x =>
+                {
+                    if (x.NodeTypeId == NodeTypes.DroppedLedgerColumns)
+                    {
+                        Table? parentTable = context.Parent as Table;
+                        if (parentTable == null ||
+                            !(parentTable.LedgerType == LedgerTableType.UpdatableLedgerTable ||
+                            parentTable.LedgerType == LedgerTableType.AppendOnlyLedgerTable))
+                        {
+                            return true;
+                        }
+                    }
+                    return false;
+                });
             }
         }
 

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/SmoModel/SmoColumnCustomNode.cs

@@ -89,6 +89,11 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.SmoModel
             {
                 Name = "XmlDocumentConstraint",
                 ValidFor = ValidForFlag.NotSqlDw
+            },
+            new NodeSmoProperty
+            {
+                Name = "IsDroppedLedgerColumn",
+                ValidFor = ValidForFlag.Sql2022|ValidForFlag.AzureV12
             }
         });
 

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/SmoModel/SmoTreeNodes.cs

@@ -1644,6 +1644,34 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.SmoModel
     {
         public override IEnumerable<string> ApplicableParents() { return new[] { "Columns" }; }
 
+        public override IEnumerable<INodeFilter> Filters
+        {
+            get
+            {
+                var filters = new List<INodeFilter>();
+                filters.Add(new NodePropertyFilter
+                {
+                    Property = "IsDroppedLedgerColumn",
+                    Type = typeof(bool),
+                    ValidFor = ValidForFlag.Sql2022|ValidForFlag.AzureV12,
+                    Values = new List<object> { 0 },
+                });
+                return filters;
+            }
+        }
+
+        protected override void OnExpandPopulateFolders(IList<TreeNode> currentChildren, TreeNode parent)
+        {
+            currentChildren.Add(new FolderNode {
+                NodeValue = SR.SchemaHierarchy_DroppedLedgerColumns,
+                NodeType = "Folder",
+                NodeTypeId = NodeTypes.DroppedLedgerColumns,
+                IsSystemObject = false,
+                ValidFor = ValidForFlag.Sql2022|ValidForFlag.AzureV12,
+                SortPriority = Int32.MaxValue,
+            });
+        }
+
         internal override Type[] ChildQuerierTypes
         {
             get
@@ -1663,6 +1691,46 @@ namespace Microsoft.SqlTools.ServiceLayer.ObjectExplorer.SmoModel
         }
     }
 
+    [Export(typeof(ChildFactory))]
+    [Shared]
+    internal partial class DroppedLedgerColumnsChildFactory : SmoChildFactoryBase
+    {
+        public override IEnumerable<string> ApplicableParents() { return new[] { "DroppedLedgerColumns" }; }
+
+        public override IEnumerable<INodeFilter> Filters
+        {
+            get
+            {
+                var filters = new List<INodeFilter>();
+                filters.Add(new NodePropertyFilter
+                {
+                    Property = "IsDroppedLedgerColumn",
+                    Type = typeof(bool),
+                    ValidFor = ValidForFlag.Sql2022|ValidForFlag.AzureV12,
+                    Values = new List<object> { 1 },
+                });
+                return filters;
+            }
+        }
+
+        internal override Type[] ChildQuerierTypes
+        {
+            get
+            {
+                return new [] { typeof(SqlColumnQuerier), };
+            }
+        }
+
+        public override TreeNode CreateChild(TreeNode parent, object context)
+        {
+            var child = new SmoTreeNode();
+            child.IsAlwaysLeaf = true;
+            child.NodeType = "Column";
+            InitializeChild(parent, child, context);
+            return child;
+        }
+    }
+
     [Export(typeof(ChildFactory))]
     [Shared]
     internal partial class KeysChildFactory : SmoChildFactoryBase

src/Microsoft.SqlTools.ServiceLayer/ObjectExplorer/SmoModel/SmoTreeNodesDefinition.xml

@@ -224,7 +224,19 @@
     <Child Name="Statistics"/>
   </Node>
 
-  <Node Name="Columns" LocLabel="SR.SchemaHierarchy_Columns" BaseClass="ModelBased" Strategy="PopulateParentDetails" NodeType="Column" ChildQuerierTypes="SqlColumn" DisableSort=""/>
+  <Node Name="Columns" LocLabel="SR.SchemaHierarchy_Columns" BaseClass="ModelBased" Strategy="PopulateParentDetails" NodeType="Column" ChildQuerierTypes="SqlColumn" DisableSort="">
+    <Filters>
+      <Filter Property="IsDroppedLedgerColumn" Value="0" Type="bool" ValidFor="Sql2022|AzureV12" />
+    </Filters>
+    <Child Name="DroppedLedgerColumns"/>
+  </Node>
+
+  <Node Name="DroppedLedgerColumns" LocLabel="SR.SchemaHierarchy_DroppedLedgerColumns" BaseClass="ModelBased" Strategy="PopulateParentDetails" NodeType="Column" ChildQuerierTypes="SqlColumn" ValidFor="Sql2022|AzureV12" SortPriority="Int32.MaxValue">
+    <Filters>
+      <Filter Property="IsDroppedLedgerColumn" Value="1" Type="bool" ValidFor="Sql2022|AzureV12" />
+    </Filters>
+  </Node>
+
   <Node Name="Keys" LocLabel="SR.SchemaHierarchy_Keys" BaseClass="ModelBased" Strategy="ElementsInRelationship" NodeType="Key" ChildQuerierTypes="SqlIndex;SqlForeignKeyConstraint" ValidFor="NotSqlDw">
     <Filters>
       <Filter TypeToReverse="SqlIndex" Property="IndexKeyType" Type="Enum" ValidFor="AllOnPrem|AzureV12">

src/Microsoft.SqlTools.ServiceLayer/Scripting/ScripterCore.cs

@@ -700,6 +700,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Scripting
                 filterExpressions.Add("@GeneratedAlwaysType=0");
             }
 
+            // Check if we're called for EDIT for SQL2022+/Sterling+.
+            // We need to omit dropped ledger columns if such are present
             if (server.Version.Major >= 16 || (DatabaseEngineType.SqlAzureDatabase == server.DatabaseEngineType && server.Version.Major >= 12))
             {
                 filterExpressions.Add("@IsDroppedLedgerColumn=0");

test/Microsoft.SqlTools.ServiceLayer.UnitTests/ObjectExplorer/NodePathGeneratorTests.cs

@@ -85,17 +85,29 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.ObjectExplorer
             var expectedPaths = new List<string>
             {
                 "testServer/Databases/testDatabase/Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/testDatabase/Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/System Databases/testDatabase/Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/testDatabase/Tables/System Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/testDatabase/Tables/System Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/System Databases/testDatabase/Tables/System Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Tables/System Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/System Databases/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/testDatabase/Views/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/testDatabase/Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/System Databases/testDatabase/Views/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/testDatabase/Views/System Views/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/testDatabase/Views/System Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/System Databases/testDatabase/Views/System Views/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Views/System Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/Databases/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/testColumn",
-                "testServer/Databases/System Databases/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/testColumn"
+                "testServer/Databases/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/testColumn",
+                "testServer/Databases/System Databases/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn"
             };
 
             Assert.AreEqual(expectedPaths.Count, paths.Count);
@@ -112,11 +124,17 @@ namespace Microsoft.SqlTools.ServiceLayer.UnitTests.ObjectExplorer
             var expectedPaths = new List<string>
             {
                 "testServer/testDatabase/Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/testDatabase/Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/testDatabase/Tables/System Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/testDatabase/Tables/System Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/testColumn",
+                "testServer/testDatabase/Tables/Dropped Ledger Tables/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/testDatabase/Views/testSchema.testTable/Columns/testColumn",
+                "testServer/testDatabase/Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
                 "testServer/testDatabase/Views/System Views/testSchema.testTable/Columns/testColumn",
-                "testServer/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/testColumn"
+                "testServer/testDatabase/Views/System Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn",
+                "testServer/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/testColumn",
+                "testServer/testDatabase/Views/Dropped Ledger Views/testSchema.testTable/Columns/Dropped Ledger Columns/testColumn"
             };
 
             Assert.AreEqual(expectedPaths.Count, paths.Count);