Package published projects (#1291)

* Package published project

* sign and publish nuget packages

* Publish logs

* Sign before packaging

* Add res of packages

* Publish to specific folders

* add comment

* Remove unused package projects

.gitignore

@@ -40,7 +40,6 @@ msbuild.wrn
 # code coverage artifacts
 coverage.xml
 node_modules
-packages
 reports
 opencovertests.xml
 outputCobertura.xml
@@ -173,7 +172,6 @@ publish/
 *.nuget.props
 *.nuget.targets
 src/**/*.nupkg
-**/packages/*
 
 # Windows Azure Build Output
 csx/

azure-pipelines/build.yml

@@ -126,6 +126,41 @@ steps:
     MaxConcurrency: 5
   condition: and(succeeded(), eq(variables['signed'], true))
 
+- task: BatchScript@1
+  displayName: 'Package published projects'
+  inputs:
+    filename: build.cmd
+    arguments: '-target=dotnetpackpublished -mono'
+  continueOnError: true
+
+- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+  displayName: 'ESRP Code Signing - Nuget Package'
+  inputs:
+    ConnectedServiceName: 'Code Signing'
+    FolderPath: '$(Build.SourcesDirectory)/artifacts/nugetPackages'
+    Pattern: '*.nupkg'
+    signConfigType: 'inlineSignParams'
+    inlineOperation: |
+      [
+          {
+              "keyCode": "CP-401405",
+              "operationSetCode": "NuGetSign",
+              "parameters": [ ],
+              "toolName": "sign",
+              "toolVersion": "1.0"
+          },
+          {
+              "keyCode": "CP-401405",
+              "operationSetCode": "NuGetVerify",
+              "parameters": [ ],
+              "toolName": "sign",
+              "toolVersion": "1.0"
+          }
+      ]
+    SessionTimeout: '60'
+    MaxConcurrency: '50'
+    MaxRetryAttempts: '5'
+
 - task: ArchiveFiles@1
   displayName: 'Archive osx build'
   inputs:
@@ -171,10 +206,22 @@ steps:
 #     archiveFile: '$(Build.SourcesDirectory)/artifacts/package/Microsoft.SqlTools.ServiceLayer-win10-arm64-net5.0.zip'
 
 - task: PublishBuildArtifacts@1
-  displayName: 'Publish Artifact: drop'
+  displayName: 'Publish Artifact: build archives'
   inputs:
     PathtoPublish: '$(Build.SourcesDirectory)/artifacts/package'
 
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: nuget packages'
+  inputs:
+    PathtoPublish: '$(Build.SourcesDirectory)/artifacts/nugetPackages'
+    ArtifactName: 'packages'
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: logs'
+  inputs:
+    PathtoPublish: '$(Build.SourcesDirectory)/artifacts/logs'
+    ArtifactName: 'logs'
+
 - task: NuGetCommand@2
   displayName: 'NuGet push'
   condition: eq(variables['PUSH_SYMBOLS'], 'true')

build.cake

@@ -48,7 +48,10 @@ public class BuildPlan
     public string[] Frameworks { get; set; }
     public string[] Rids { get; set; }
     public string[] MainProjects { get; set; }
+    // The set of projects that we want to call dotnet pack on directly
     public string[] PackageProjects { get; set; }
+    // The set of projects that we want to call dotnet pack on which require publishing being done first
+    public string[] PackagePublishedProjects { get; set; }
 }
 
 var buildPlan = JsonConvert.DeserializeObject<BuildPlan>(
@@ -61,6 +64,7 @@ var toolsFolder = System.IO.Path.Combine(workingDirectory, buildPlan.BuildToolsF
 
 var sourceFolder = System.IO.Path.Combine(workingDirectory, "src");
 var testFolder = System.IO.Path.Combine(workingDirectory, "test");
+var packagesFolder = System.IO.Path.Combine(workingDirectory, "packages");
 
 var artifactFolder = System.IO.Path.Combine(workingDirectory, buildPlan.ArtifactsFolder);
 var publishFolder = System.IO.Path.Combine(artifactFolder, "publish");
@@ -227,8 +231,9 @@ Task("BuildTest")
         }
     }
 });
+
 /// <summary>
-///  Build Test projects.
+///  Packages projects specified in PackageProjects
 /// </summary>
 Task("DotnetPack")
     .IsDependentOn("Cleanup")
@@ -241,16 +246,29 @@ Task("DotnetPack")
         // For now, putting all nugets in the 1 directory
         var outputFolder = System.IO.Path.Combine(nugetPackageFolder);
         var projectFolder = System.IO.Path.Combine(sourceFolder, project);
-        var runLog = new List<string>();
-        Run(dotnetcli, $"pack --configuration {configuration} --output {outputFolder} \"{projectFolder}\"",
-            new RunOptions
-            {
-                StandardOutputListing = runLog
-            })
-        .ExceptionOnError($"Packaging {project} failed.");
-        System.IO.File.WriteAllLines(System.IO.Path.Combine(logFolder, $"{project}-pack.log"), runLog.ToArray());
+        DotnetPack(outputFolder, projectFolder, project);
     }
 });
+
+/// <summary>
+///  Packages projects specified in PackagePublishedProjects, these projects require that publishing be done first. Note that we
+///  don't do the publishing here because we need the binaries to be signed before being packaged up and that is done by the pipeline
+///  currently.
+/// </summary>
+Task("DotnetPackPublished")
+    .Does(() =>
+{
+    foreach (var project in buildPlan.PackagePublishedProjects)
+    {
+        // For now, putting all nugets in the 1 directory
+        var outputFolder = System.IO.Path.Combine(nugetPackageFolder);
+        var projectFolder = System.IO.Path.Combine(packagesFolder, project);
+        DotnetPack(outputFolder, projectFolder, project);
+    }
+});
+
+
+
 /// <summary>
 ///  Run all tests for .NET Desktop and .NET Core
 /// </summary>

build.json

@@ -26,5 +26,18 @@
   ],
   "PackageProjects": [
     "Microsoft.SqlTools.Hosting"
+  ],
+  "PackagePublishedProjects": [
+    "Microsoft.SqlToolsService",
+    "runtime.centos.7-x64.native.Microsoft.SqlToolsService",
+    "runtime.debian.8-x64.native.Microsoft.SqlToolsService",
+    "runtime.fedora.23-x64.native.Microsoft.SqlToolsService",
+    "runtime.linux-x64.native.Microsoft.SqlToolsService",
+    "runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService",
+    "runtime.osx-x64.native.Microsoft.SqlToolsService",
+    "runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService",
+    "runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService",
+    "runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService",
+    "runtime.win7-x64.native.Microsoft.SqlToolsService"
   ]
 }

packages/Microsoft.SqlToolsService/Microsoft.SqlToolsService.csproj

@@ -12,29 +12,23 @@
     <ProjectReference Include="../runtime.centos.7-x64.native.Microsoft.SqlToolsService/runtime.centos.7-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.debian.8-x64.native.Microsoft.SqlToolsService/runtime.debian.8-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.fedora.23-x64.native.Microsoft.SqlToolsService/runtime.fedora.23-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
-    <ProjectReference Include="../runtime.linux-64-x64.native.Microsoft.SqlToolsService/runtime.linux-64-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
+    <ProjectReference Include="../runtime.linux-x64.native.Microsoft.SqlToolsService/runtime.linux-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService/runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
-    <ProjectReference Include="../runtime.osx.10.11-x64.native.Microsoft.SqlToolsService/runtime.osx.10.11-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
+    <ProjectReference Include="../runtime.osx-x64.native.Microsoft.SqlToolsService/runtime.osx-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService/runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
-    <ProjectReference Include="../runtime.win10-arm.native.Microsoft.SqlToolsService/runtime.win10-arm.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
-    <ProjectReference Include="../runtime.win10-arm64.native.Microsoft.SqlToolsService/runtime.win10-arm64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
     <ProjectReference Include="../runtime.win7-x64.native.Microsoft.SqlToolsService/runtime.win7-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
-    <ProjectReference Include="../runtime.win7-x86.native.Microsoft.SqlToolsService/runtime.win7-x86.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="runtime.linux-64-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.debian.8-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.osx.10.11-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.win10-arm.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.win10-arm64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.win7-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
-    <PackageReference Include="runtime.win7-x86.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
+    <PackageReference Include="runtime.linux-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.debian.8-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.osx-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
+    <PackageReference Include="runtime.win7-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
   </ItemGroup>
 </Project>

packages/README.md

@@ -0,0 +1,3 @@
+This directory contains projects that only handle packaging up the published artifacts for SQL Tools Service.
+
+There is one root project - `Microsoft.SqlToolsService`, and then a sub-project for each runtime that we currently support. The root project has a package dependency on each of the sub projects so that consumers only need to take a reference to the root project to be able to get all the necessary files for each platform.

packages/runtime.centos.7-x64.native.Microsoft.SqlToolsService/runtime.centos.7-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/centos.7-x64/**" Pack="true" PackagePath="runtimes/centos.7-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/centos.7-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/centos.7-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.debian.8-x64.native.Microsoft.SqlToolsService/runtime.debian.8-x64.native.Microsoft.SqlToolsService.csproj

@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.Build.NoTargets">
   <PropertyGroup>
-    <PackageDescription>SQL Tools Service runtime components for win10-arm</PackageDescription>
+    <PackageDescription>SQL Tools Service runtime components for debian.8-x64</PackageDescription>
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win10-arm/**" Pack="true" PackagePath="runtimes/win10-arm/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/debian.8-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/debian.8-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.fedora.23-x64.native.Microsoft.SqlToolsService/runtime.fedora.23-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/fedora.23-x64/**" Pack="true" PackagePath="runtimes/fedora.23-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/fedora.23-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/fedora.23-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.linux-x64.native.Microsoft.SqlToolsService/runtime.linux-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/linux-64/**" Pack="true" PackagePath="runtimes/linux-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/linux-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/linux-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService/runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer-opensuse.13.2-x64-net5.0/**" Pack="true" PackagePath="runtimes/opensuse.13.2-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer-opensuse.13.2-x64-net5.0/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/opensuse.13.2-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.osx-x64.native.Microsoft.SqlToolsService/runtime.osx-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/osx-x64/**" Pack="true" PackagePath="runtimes/osx-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/osx-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/osx-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService/runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/rhel.7.2-x64/**" Pack="true" PackagePath="runtimes/rhel.7.2-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/rhel.7.2-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/rhel.7.2-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.14.04/**" Pack="true" PackagePath="runtimes/ubuntu.14.04-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.14.04/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/ubuntu.14.04-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.16.04/**" Pack="true" PackagePath="runtimes/ubuntu.16.04-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.16.04/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/ubuntu.16.04-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.win10-arm64.native.Microsoft.SqlToolsService/runtime.win10-arm64.native.Microsoft.SqlToolsService.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.Build.NoTargets">
-  <PropertyGroup>
-    <PackageDescription>SQL Tools Service runtime components for win10-arm64</PackageDescription>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win10-arm64/**" Pack="true" PackagePath="runtimes/win10-arm64/native" />
-  </ItemGroup>
-</Project>

packages/runtime.win7-x64.native.Microsoft.SqlToolsService/runtime.win7-x64.native.Microsoft.SqlToolsService.csproj

@@ -4,6 +4,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x64/**" Pack="true" PackagePath="runtimes/win7-x64/native" />
+    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/win7-x64/native" />
   </ItemGroup>
 </Project>

packages/runtime.win7-x86.native.Microsoft.SqlToolsService/runtime.win7-x86.native.Microsoft.SqlToolsService.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.Build.NoTargets">
-  <PropertyGroup>
-    <PackageDescription>SQL Tools Service runtime components for win7-x86</PackageDescription>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x86/**" Pack="true" PackagePath="runtimes/win7-x86/native" />
-  </ItemGroup>
-</Project>

scripts/runhelpers.cake

@@ -15,6 +15,10 @@ public class RunOptions
     /// </summary>
     public IList<string> StandardOutputListing { get; set; }
     /// <summary>
+    ///  Container logging the Error content.
+    /// </summary>
+    public IList<string> StandardErrorListing { get; set; }
+    /// <summary>
     ///  Desired maximum time-out for the process
     /// </summary>
     public int TimeOut { get; set; }
@@ -117,7 +121,8 @@ ExitStatus Run(string exec, string args, RunOptions runOptions)
             {
                 WorkingDirectory = workingDirectory,
                 UseShellExecute = false,
-                RedirectStandardOutput = runOptions.StandardOutputListing != null
+                RedirectStandardOutput = runOptions.StandardOutputListing != null,
+                RedirectStandardError = runOptions.StandardErrorListing != null,
             });
     if (runOptions.StandardOutputListing != null)
     {
@@ -130,6 +135,17 @@ ExitStatus Run(string exec, string args, RunOptions runOptions)
         };
         process.BeginOutputReadLine();
     }
+    if (runOptions.StandardErrorListing != null)
+    {
+        process.ErrorDataReceived += (s, e) =>
+        {
+            if (e.Data != null)
+            {
+                runOptions.StandardErrorListing.Add(e.Data);
+            }
+        };
+        process.BeginErrorReadLine();
+    }
     if (runOptions.TimeOut == 0)
     {
         process.WaitForExit();
@@ -202,3 +218,23 @@ public void KillProcessTree(Process process)
         process.Kill();
     }
 }
+
+public void DotnetPack(string outputFolder, string projectFolder, string project) {
+    var runLog = new List<string>();
+    var errorLog = new List<string>();
+    var logPath = System.IO.Path.Combine(logFolder, $"{project}-pack.log");
+    Information($"Packaging {projectFolder}");
+    try {
+        Run(dotnetcli, $"pack --configuration {configuration} --output {outputFolder} \"{projectFolder}\"",
+            new RunOptions
+            {
+                StandardOutputListing = runLog,
+                StandardErrorListing = errorLog
+            })
+        .ExceptionOnError($"Packaging {project} failed. See {logPath} for details.");
+    } catch(Exception) {
+        System.IO.File.WriteAllLines(logPath, runLog.ToArray());
+        throw;
+    }
+    System.IO.File.WriteAllLines(logPath, runLog.ToArray());
+}