Package published projects (#1291)

* Package published project

* sign and publish nuget packages

* Publish logs

* Sign before packaging

* Add res of packages

* Publish to specific folders

* add comment

* Remove unused package projects
This commit is contained in:
Charles Gagnon
2021-11-10 12:32:15 -08:00
committed by GitHub
parent 61401b61b0
commit c81a400752
19 changed files with 161 additions and 70 deletions

2
.gitignore vendored
View File

@@ -40,7 +40,6 @@ msbuild.wrn
# code coverage artifacts
coverage.xml
node_modules
packages
reports
opencovertests.xml
outputCobertura.xml
@@ -173,7 +172,6 @@ publish/
*.nuget.props
*.nuget.targets
src/**/*.nupkg
**/packages/*
# Windows Azure Build Output
csx/

View File

@@ -126,6 +126,41 @@ steps:
MaxConcurrency: 5
condition: and(succeeded(), eq(variables['signed'], true))
- task: BatchScript@1
displayName: 'Package published projects'
inputs:
filename: build.cmd
arguments: '-target=dotnetpackpublished -mono'
continueOnError: true
- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
displayName: 'ESRP Code Signing - Nuget Package'
inputs:
ConnectedServiceName: 'Code Signing'
FolderPath: '$(Build.SourcesDirectory)/artifacts/nugetPackages'
Pattern: '*.nupkg'
signConfigType: 'inlineSignParams'
inlineOperation: |
[
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetSign",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "1.0"
},
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetVerify",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "1.0"
}
]
SessionTimeout: '60'
MaxConcurrency: '50'
MaxRetryAttempts: '5'
- task: ArchiveFiles@1
displayName: 'Archive osx build'
inputs:
@@ -171,10 +206,22 @@ steps:
# archiveFile: '$(Build.SourcesDirectory)/artifacts/package/Microsoft.SqlTools.ServiceLayer-win10-arm64-net5.0.zip'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: drop'
displayName: 'Publish Artifact: build archives'
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/package'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: nuget packages'
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/nugetPackages'
ArtifactName: 'packages'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: logs'
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/logs'
ArtifactName: 'logs'
- task: NuGetCommand@2
displayName: 'NuGet push'
condition: eq(variables['PUSH_SYMBOLS'], 'true')

View File

@@ -48,7 +48,10 @@ public class BuildPlan
public string[] Frameworks { get; set; }
public string[] Rids { get; set; }
public string[] MainProjects { get; set; }
// The set of projects that we want to call dotnet pack on directly
public string[] PackageProjects { get; set; }
// The set of projects that we want to call dotnet pack on which require publishing being done first
public string[] PackagePublishedProjects { get; set; }
}
var buildPlan = JsonConvert.DeserializeObject<BuildPlan>(
@@ -61,6 +64,7 @@ var toolsFolder = System.IO.Path.Combine(workingDirectory, buildPlan.BuildToolsF
var sourceFolder = System.IO.Path.Combine(workingDirectory, "src");
var testFolder = System.IO.Path.Combine(workingDirectory, "test");
var packagesFolder = System.IO.Path.Combine(workingDirectory, "packages");
var artifactFolder = System.IO.Path.Combine(workingDirectory, buildPlan.ArtifactsFolder);
var publishFolder = System.IO.Path.Combine(artifactFolder, "publish");
@@ -227,8 +231,9 @@ Task("BuildTest")
}
}
});
/// <summary>
/// Build Test projects.
/// Packages projects specified in PackageProjects
/// </summary>
Task("DotnetPack")
.IsDependentOn("Cleanup")
@@ -241,16 +246,29 @@ Task("DotnetPack")
// For now, putting all nugets in the 1 directory
var outputFolder = System.IO.Path.Combine(nugetPackageFolder);
var projectFolder = System.IO.Path.Combine(sourceFolder, project);
var runLog = new List<string>();
Run(dotnetcli, $"pack --configuration {configuration} --output {outputFolder} \"{projectFolder}\"",
new RunOptions
{
StandardOutputListing = runLog
})
.ExceptionOnError($"Packaging {project} failed.");
System.IO.File.WriteAllLines(System.IO.Path.Combine(logFolder, $"{project}-pack.log"), runLog.ToArray());
DotnetPack(outputFolder, projectFolder, project);
}
});
/// <summary>
/// Packages projects specified in PackagePublishedProjects, these projects require that publishing be done first. Note that we
/// don't do the publishing here because we need the binaries to be signed before being packaged up and that is done by the pipeline
/// currently.
/// </summary>
Task("DotnetPackPublished")
.Does(() =>
{
foreach (var project in buildPlan.PackagePublishedProjects)
{
// For now, putting all nugets in the 1 directory
var outputFolder = System.IO.Path.Combine(nugetPackageFolder);
var projectFolder = System.IO.Path.Combine(packagesFolder, project);
DotnetPack(outputFolder, projectFolder, project);
}
});
/// <summary>
/// Run all tests for .NET Desktop and .NET Core
/// </summary>
@@ -338,7 +356,7 @@ Task("OnlyPublish")
.IsDependentOn("SRGen")
.IsDependentOn("CodeGen")
.Does(() =>
{
{
var packageName = buildPlan.PackageName;
foreach (var project in buildPlan.MainProjects)
{
@@ -361,7 +379,7 @@ Task("OnlyPublish")
//Only required for mac. We're assuming the openssl is installed in /usr/local/opt/openssl
//If that's not the case user has to run the command manually
if (!IsRunningOnWindows() && !IsRunningOnUnix() && runtime.Contains("osx"))
{
{
Run("install_name_tool", "-add_rpath /usr/local/opt/openssl/lib " + outputFolder + "/System.Security.Cryptography.Native.dylib");
}
}
@@ -379,7 +397,7 @@ Task("OnlyPublish")
}
}
CreateRunScript(System.IO.Path.Combine(publishFolder, project, "default"), scriptFolder);
}
}
});
/// <summary>
@@ -558,17 +576,17 @@ Task("SRGen")
try
{
var projects = System.IO.Directory.GetFiles(sourceFolder, "*.csproj", SearchOption.AllDirectories).ToList();
var locTemplateDir = System.IO.Path.Combine(sourceFolder, "../localization");
var locTemplateDir = System.IO.Path.Combine(sourceFolder, "../localization");
foreach(var project in projects) {
var projectDir = System.IO.Path.GetDirectoryName(project);
// set current directory to this project so relative paths can be reliably
// used. This is to address an issue with quoting differences between Windows
// and MacOS. On MacOS the dotnet command ends up calling SRGen with quotations around
// and MacOS. On MacOS the dotnet command ends up calling SRGen with quotations around
// the arguments stripped off. This causes SRGen to interpret the arg values as options
System.IO.Directory.SetCurrentDirectory(projectDir);
// build remaining paths relative to the project directory
var localizationDir = System.IO.Path.Combine(".", "Localization");
var projectName = (new System.IO.DirectoryInfo(projectDir)).Name;
@@ -597,7 +615,7 @@ Task("SRGen")
System.IO.File.Delete(outputCs);
}
if (!System.IO.Directory.Exists(inputXliff))
if (!System.IO.Directory.Exists(inputXliff))
{
System.IO.Directory.CreateDirectory(inputXliff);
}
@@ -628,23 +646,23 @@ Task("SRGen")
var xlfDoc = new XliffParser.XlfDocument(docName);
var xlfFile = xlfDoc.Files.Single();
// load a language template
// load a language template
var templateFileLocation = System.IO.Path.Combine(locTemplateDir, System.IO.Path.GetFileName(docName) + ".template");
var templateDoc = new XliffParser.XlfDocument(templateFileLocation);
var templateFile = templateDoc.Files.Single();
var templateFile = templateDoc.Files.Single();
// iterate through our tranlation units and prune invalid units
foreach (var unit in xlfFile.TransUnits)
{
// if a unit does not have a target it is invalid
// if a unit does not have a target it is invalid
if (unit.Target != null) {
templateFile.AddTransUnit(unit.Id, unit.Source, unit.Target, 0, 0);
}
}
}
// export modified template to RESX
var newPath = System.IO.Path.Combine(localizationDir, System.IO.Path.GetFileName(docName));
templateDoc.SaveAsResX(newPath.Replace("xlf","resx"));
templateDoc.SaveAsResX(newPath.Replace("xlf","resx"));
}
}
}

View File

@@ -26,5 +26,18 @@
],
"PackageProjects": [
"Microsoft.SqlTools.Hosting"
],
"PackagePublishedProjects": [
"Microsoft.SqlToolsService",
"runtime.centos.7-x64.native.Microsoft.SqlToolsService",
"runtime.debian.8-x64.native.Microsoft.SqlToolsService",
"runtime.fedora.23-x64.native.Microsoft.SqlToolsService",
"runtime.linux-x64.native.Microsoft.SqlToolsService",
"runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService",
"runtime.osx-x64.native.Microsoft.SqlToolsService",
"runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService",
"runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService",
"runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService",
"runtime.win7-x64.native.Microsoft.SqlToolsService"
]
}

View File

@@ -12,29 +12,23 @@
<ProjectReference Include="../runtime.centos.7-x64.native.Microsoft.SqlToolsService/runtime.centos.7-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.debian.8-x64.native.Microsoft.SqlToolsService/runtime.debian.8-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.fedora.23-x64.native.Microsoft.SqlToolsService/runtime.fedora.23-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.linux-64-x64.native.Microsoft.SqlToolsService/runtime.linux-64-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.linux-x64.native.Microsoft.SqlToolsService/runtime.linux-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService/runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.osx.10.11-x64.native.Microsoft.SqlToolsService/runtime.osx.10.11-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.osx-x64.native.Microsoft.SqlToolsService/runtime.osx-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService/runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService/runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.win10-arm.native.Microsoft.SqlToolsService/runtime.win10-arm.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.win10-arm64.native.Microsoft.SqlToolsService/runtime.win10-arm64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.win7-x64.native.Microsoft.SqlToolsService/runtime.win7-x64.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
<ProjectReference Include="../runtime.win7-x86.native.Microsoft.SqlToolsService/runtime.win7-x86.native.Microsoft.SqlToolsService.csproj" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="runtime.linux-64-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.debian.8-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.osx.10.11-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.win10-arm.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.win10-arm64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.win7-x64.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.win7-x86.native.Microsoft.SqlToolsService" Version="$(PackageVersion)" />
<PackageReference Include="runtime.linux-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.debian.8-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.opensuse.13.2-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.osx-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.rhel.7.2-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.ubuntu.14.04-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.ubuntu.16.04-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
<PackageReference Include="runtime.win7-x64.native.Microsoft.SqlToolsService" VersionOverride="$(PackageVersion)" />
</ItemGroup>
</Project>

3
packages/README.md Normal file
View File

@@ -0,0 +1,3 @@
This directory contains projects that only handle packaging up the published artifacts for SQL Tools Service.
There is one root project - `Microsoft.SqlToolsService`, and then a sub-project for each runtime that we currently support. The root project has a package dependency on each of the sub projects so that consumers only need to take a reference to the root project to be able to get all the necessary files for each platform.

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/centos.7-x64/**" Pack="true" PackagePath="runtimes/centos.7-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/centos.7-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/centos.7-x64/native" />
</ItemGroup>
</Project>

View File

@@ -1,9 +1,9 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<PackageDescription>SQL Tools Service runtime components for win10-arm</PackageDescription>
<PackageDescription>SQL Tools Service runtime components for debian.8-x64</PackageDescription>
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win10-arm/**" Pack="true" PackagePath="runtimes/win10-arm/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/debian.8-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/debian.8-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/fedora.23-x64/**" Pack="true" PackagePath="runtimes/fedora.23-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/fedora.23-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/fedora.23-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/linux-64/**" Pack="true" PackagePath="runtimes/linux-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/linux-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/linux-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer-opensuse.13.2-x64-net5.0/**" Pack="true" PackagePath="runtimes/opensuse.13.2-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer-opensuse.13.2-x64-net5.0/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/opensuse.13.2-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/osx-x64/**" Pack="true" PackagePath="runtimes/osx-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/osx-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/osx-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/rhel.7.2-x64/**" Pack="true" PackagePath="runtimes/rhel.7.2-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/rhel.7.2-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/rhel.7.2-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.14.04/**" Pack="true" PackagePath="runtimes/ubuntu.14.04-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.14.04/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/ubuntu.14.04-x64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.16.04/**" Pack="true" PackagePath="runtimes/ubuntu.16.04-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/ubuntu.16.04/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/ubuntu.16.04-x64/native" />
</ItemGroup>
</Project>

View File

@@ -1,9 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<PackageDescription>SQL Tools Service runtime components for win10-arm64</PackageDescription>
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win10-arm64/**" Pack="true" PackagePath="runtimes/win10-arm64/native" />
</ItemGroup>
</Project>

View File

@@ -4,6 +4,6 @@
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x64/**" Pack="true" PackagePath="runtimes/win7-x64/native" />
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x64/$(TargetFramework)/**" Pack="true" PackagePath="runtimes/win7-x64/native" />
</ItemGroup>
</Project>

View File

@@ -1,9 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<PackageDescription>SQL Tools Service runtime components for win7-x86</PackageDescription>
</PropertyGroup>
<ItemGroup>
<None Include="../../artifacts/publish/Microsoft.SqlTools.ServiceLayer/win7-x86/**" Pack="true" PackagePath="runtimes/win7-x86/native" />
</ItemGroup>
</Project>

View File

@@ -15,6 +15,10 @@ public class RunOptions
/// </summary>
public IList<string> StandardOutputListing { get; set; }
/// <summary>
/// Container logging the Error content.
/// </summary>
public IList<string> StandardErrorListing { get; set; }
/// <summary>
/// Desired maximum time-out for the process
/// </summary>
public int TimeOut { get; set; }
@@ -117,7 +121,8 @@ ExitStatus Run(string exec, string args, RunOptions runOptions)
{
WorkingDirectory = workingDirectory,
UseShellExecute = false,
RedirectStandardOutput = runOptions.StandardOutputListing != null
RedirectStandardOutput = runOptions.StandardOutputListing != null,
RedirectStandardError = runOptions.StandardErrorListing != null,
});
if (runOptions.StandardOutputListing != null)
{
@@ -130,6 +135,17 @@ ExitStatus Run(string exec, string args, RunOptions runOptions)
};
process.BeginOutputReadLine();
}
if (runOptions.StandardErrorListing != null)
{
process.ErrorDataReceived += (s, e) =>
{
if (e.Data != null)
{
runOptions.StandardErrorListing.Add(e.Data);
}
};
process.BeginErrorReadLine();
}
if (runOptions.TimeOut == 0)
{
process.WaitForExit();
@@ -202,3 +218,23 @@ public void KillProcessTree(Process process)
process.Kill();
}
}
public void DotnetPack(string outputFolder, string projectFolder, string project) {
var runLog = new List<string>();
var errorLog = new List<string>();
var logPath = System.IO.Path.Combine(logFolder, $"{project}-pack.log");
Information($"Packaging {projectFolder}");
try {
Run(dotnetcli, $"pack --configuration {configuration} --output {outputFolder} \"{projectFolder}\"",
new RunOptions
{
StandardOutputListing = runLog,
StandardErrorListing = errorLog
})
.ExceptionOnError($"Packaging {project} failed. See {logPath} for details.");
} catch(Exception) {
System.IO.File.WriteAllLines(logPath, runLog.ToArray());
throw;
}
System.IO.File.WriteAllLines(logPath, runLog.ToArray());
}