Azure SQL Hybrid Cloud Toolkit Notebooks Extension Command (#13286)

* added extension folder incomplete

* WIP extension progress

* notebook finally opens in side panel

* notebook now opens via notebook extension

* html file spaces restored

* package json fixed

* fixed vscode import issue

* more cleanup

* remove git stuff

* placeholder icon logos added

* fixed gulpfile

* cleanup changes

* vscode import fixed

* fixed main and yarn.lock

* added provided notebooks view

* formatting for package.json

* removed first command as its not necessary

* fixed notebook typo

* readded spaces

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/ActionType.cs

@@ -0,0 +1,12 @@
+namespace BatchWrapper
+{
+    /// <summary>
+    /// The type of sqlpackage action to perform.
+    /// </summary>
+    public enum ActionType
+    {
+        DefaultInvalid = -1,
+        Export,
+        Import
+    }
+}

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/App.config

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <startup> 
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
+    </startup>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Azure.Batch" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Azure.KeyVault.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-3.0.5.0" newVersion="3.0.5.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/BatchWrapper.csproj

@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{E64CD92E-2D2C-48F1-B6B1-A7AF819B06F1}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>BatchWrapper</RootNamespace>
+    <AssemblyName>BatchWrapper</AssemblyName>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <LangVersion>8.0</LangVersion>
+    <FileAlignment>512</FileAlignment>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <Deterministic>true</Deterministic>
+    <IsWebBootstrapper>false</IsWebBootstrapper>
+    <PublishUrl>publish\</PublishUrl>
+    <Install>true</Install>
+    <InstallFrom>Disk</InstallFrom>
+    <UpdateEnabled>false</UpdateEnabled>
+    <UpdateMode>Foreground</UpdateMode>
+    <UpdateInterval>7</UpdateInterval>
+    <UpdateIntervalUnits>Days</UpdateIntervalUnits>
+    <UpdatePeriodically>false</UpdatePeriodically>
+    <UpdateRequired>false</UpdateRequired>
+    <MapFileExtensions>true</MapFileExtensions>
+    <ApplicationRevision>0</ApplicationRevision>
+    <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
+    <UseApplicationTrust>false</UseApplicationTrust>
+    <BootstrapperEnabled>true</BootstrapperEnabled>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.Azure.Batch.Conventions.Files, Version=3.5.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Azure.Batch.Conventions.Files.3.5.1\lib\net461\Microsoft.Azure.Batch.Conventions.Files.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\WindowsAzure.Storage.9.3.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Net" />
+    <Reference Include="System.Net.Http.WebRequest" />
+    <Reference Include="System.Runtime" />
+    <Reference Include="System.Runtime.Serialization" />
+    <Reference Include="System.Web" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ActionType.cs" />
+    <Compile Include="Constants.cs" />
+    <Compile Include="Payload.cs" />
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <BootstrapperPackage Include=".NETFramework,Version=v4.7.2">
+      <Visible>False</Visible>
+      <ProductName>Microsoft .NET Framework 4.7.2 %28x86 and x64%29</ProductName>
+      <Install>true</Install>
+    </BootstrapperPackage>
+    <BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
+      <Visible>False</Visible>
+      <ProductName>.NET Framework 3.5 SP1</ProductName>
+      <Install>false</Install>
+    </BootstrapperPackage>
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+</Project>

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Constants.cs

@@ -0,0 +1,34 @@
+namespace BatchWrapper
+{
+    /// <summary>
+    /// Constants for the batch wrapper.
+    /// </summary>
+    public static class Constants
+    {
+        /// <summary>
+        /// Environment variable names present or needed during the batch task execution.
+        /// </summary>
+        public static class EnvironmentVariableNames
+        {
+            /// <summary>
+            /// Path to the directory containing the sqlpackage exe.
+            /// </summary>
+            internal const string AppPackagePrefix = "AZ_BATCH_APP_PACKAGE";
+
+            /// <summary>
+            /// Path to the working directory assigned to the batch task.
+            /// </summary>
+            internal const string TaskWorkingDir = "AZ_BATCH_TASK_WORKING_DIR";
+
+            /// <summary>
+            /// Path to the working directory assigned to the batch task.
+            /// </summary>
+            internal const string AzBatchTaskId = "AZ_BATCH_TASK_ID";
+
+            /// <summary>
+            /// Path to the working directory assigned to the batch task.
+            /// </summary>
+            internal const string JobContainerUrl = "JOB_CONTAINER_URL";
+        }
+    }
+}

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Payload.cs

@@ -0,0 +1,38 @@
+namespace BatchWrapper
+{
+    /// <summary>
+    /// The top-level object stored in the key vault for an import/export operation.
+    /// </summary>
+    public sealed class Payload
+    {
+        /// <summary>
+        /// The Name of sqlpackage to use for performing the import/export operation.
+        /// </summary>
+        public string ApplicatonPackageName{ get; set; }
+
+        /// <summary>
+        /// The Version of sqlpackage to use for performing the import/export operation.
+        /// </summary>
+        public string ApplicatonPackageVersion { get; set; }
+
+        /// <summary>
+        /// The type of sqlpackage action to perform.
+        /// </summary>
+        public ActionType Action { get; set; }
+
+        /// <summary>
+        /// The logical server name to export from or import to.
+        /// </summary>
+        public string LogicalServerName { get; set; }
+
+        /// <summary>
+        /// The database name to export from or import to.
+        /// </summary>
+        public string DatabaseName { get; set; }
+
+        /// <summary>
+        /// The server admin username.
+        /// </summary>
+        public string AccessToken { get; set; }
+    }
+}

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/Program.cs

@@ -0,0 +1,178 @@
+using Microsoft.Azure.Batch.Conventions.Files;
+using Microsoft.WindowsAzure.Storage.Blob;
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BatchWrapper
+{
+    public static class Program
+    {
+        private static string dataDirectory = "F:\\data";
+        private static string tempDirectory = "F:\\temp";
+        private static string[] directories = { dataDirectory, tempDirectory };
+
+        private static readonly TimeSpan stdoutFlushDelay = TimeSpan.FromSeconds(3);
+
+        private static void WriteLine(string message) => WriteLineInternal(Console.Out, message);
+        private static void WriteErrorLine(string message) => WriteLineInternal(Console.Error, message);
+        private static void WriteLineInternal(TextWriter writer, string message)
+        {
+            var lines = message?.Split('\n') ?? new string[0];
+            foreach (var line in lines)
+            {
+                writer.WriteLine($"[{DateTime.UtcNow:u}] {line?.TrimEnd()}");
+            }
+        }
+
+        public static async Task<int> Main(string[] args)
+        {
+            var assembly = typeof(Program).Assembly;
+            WriteLine($"{assembly.ManifestModule.Name} v{assembly.GetName().Version.ToString(3)}");
+
+            // Get the command payload 
+            var payload = new Payload();
+
+            if (args.Length > 0)
+            {
+                payload.Action = (ActionType)Enum.Parse(typeof(ActionType), args[0]);
+                payload.LogicalServerName = args[1] + ".database.windows.net";
+                payload.DatabaseName = args[2];
+                payload.AccessToken = args[3];
+                payload.ApplicatonPackageName = args[4];
+                payload.ApplicatonPackageVersion = args[5];
+            }
+
+            // Cleanup folders
+            foreach (string dir in directories)
+            {
+                if (Directory.Exists(dir))
+                {
+                    Directory.Delete(dir, true);
+                }
+
+                Directory.CreateDirectory(dir);
+            }
+
+            string sqlPackageBacpacFile = Path.Combine(dataDirectory, payload.DatabaseName + ".bacpac");
+            string sqlPackageLogPath = payload.DatabaseName + ".log";
+
+            var targetDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AppPackagePrefix + "_" + payload.ApplicatonPackageName + "#" + payload.ApplicatonPackageVersion);
+            var workingDir = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.TaskWorkingDir);
+
+            string taskId = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.AzBatchTaskId);
+            string jobContainerUrl = Environment.GetEnvironmentVariable(Constants.EnvironmentVariableNames.JobContainerUrl);
+
+            // Build the import/export command
+            var cmdBuilder = new StringBuilder();
+            cmdBuilder.Append($"/Action:{payload.Action}");
+            cmdBuilder.Append(" /MaxParallelism:16");
+            cmdBuilder.Append(String.Format(" /DiagnosticsFile:{0}", sqlPackageLogPath));
+            cmdBuilder.Append(" /p:CommandTimeout=604800");
+
+            switch (payload.Action)
+            {
+                case ActionType.Export:
+                    cmdBuilder.Append($" /SourceServerName:{payload.LogicalServerName}");
+                    cmdBuilder.Append($" /SourceDatabaseName:{payload.DatabaseName}");
+                    cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
+                    cmdBuilder.Append($" /TargetFile:{sqlPackageBacpacFile}");
+                    cmdBuilder.Append($" /SourceTimeout:30");
+                    cmdBuilder.Append(String.Format(" /p:TempDirectoryForTableData=\"{0}\"", tempDirectory));
+                    cmdBuilder.Append(" /p:VerifyFullTextDocumentTypesSupported=false");
+                    break;
+
+                case ActionType.Import:
+                    cmdBuilder.Append($" /TargetServerName:{payload.LogicalServerName}");
+                    cmdBuilder.Append($" /TargetDatabaseName:{payload.DatabaseName}");
+                    cmdBuilder.Append($" /AccessToken:{payload.AccessToken}");
+                    cmdBuilder.Append($" /TargetTimeout:30");
+                    cmdBuilder.Append($" /SourceFile:{sqlPackageBacpacFile}");
+                    break;
+
+                default:
+                    throw new ArgumentException($"Invalid action type: {payload.Action}");
+            }
+
+            if (payload.Action == ActionType.Import)
+            {
+                WriteLine(string.Format("Downloading {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
+                CloudBlobContainer container = new CloudBlobContainer(new Uri(jobContainerUrl));
+                CloudBlockBlob blob = container.GetBlockBlobReference(String.Format("$JobOutput/{0}.bacpac", payload.DatabaseName));
+                blob.DownloadToFile(sqlPackageBacpacFile, FileMode.CreateNew);
+
+                if (File.Exists(sqlPackageBacpacFile))
+                {
+                    WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
+                }
+                else
+                {
+                    throw new Exception(string.Format("{0} didn't download", sqlPackageBacpacFile));
+                }
+            }
+
+            // Perform the import/export process
+            var startTime = DateTimeOffset.UtcNow;
+            var process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    WorkingDirectory = workingDir,
+                    FileName = Path.Combine(targetDir, "sqlpackage.exe"),
+                    Arguments = cmdBuilder.ToString(),
+                    CreateNoWindow = true,
+                    UseShellExecute = false,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true
+                }
+            };
+
+            process.OutputDataReceived += (s, e) => WriteLine(e.Data);
+            process.ErrorDataReceived += (s, e) => WriteErrorLine(e.Data);
+            process.Start();
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+            process.WaitForExit();
+
+            WriteLine(String.Format("SqlPackage.exe exited with code: {0}", process.ExitCode));
+
+            if (payload.Action == ActionType.Export)
+            {
+                if (File.Exists(sqlPackageBacpacFile))
+                {
+                    WriteLine(string.Format("Downloaded {0} bacpac file to {1}", payload.DatabaseName, sqlPackageBacpacFile));
+                }
+                else
+                {
+                    throw new Exception(string.Format("{0} didn't downloaded", sqlPackageBacpacFile));
+                }
+
+                // Persist the Job Output 
+                JobOutputStorage jobOutputStorage = new JobOutputStorage(new Uri(jobContainerUrl));
+
+                await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageLogPath);
+                WriteLine(String.Format("Uploaded {0} to job account", sqlPackageLogPath));
+
+                await jobOutputStorage.SaveAsync(JobOutputKind.JobOutput, sqlPackageBacpacFile, payload.DatabaseName + ".bacpac");
+                WriteLine(String.Format("Uploaded {0} to job account", sqlPackageBacpacFile));
+            }
+
+            // We are tracking the disk file to save our standard output, but the node agent may take
+            // up to 3 seconds to flush the stdout stream to disk. So give the file a moment to catch up.
+            await Task.Delay(stdoutFlushDelay);
+
+            // Cleanup folders
+            foreach (string dir in directories)
+            {
+                if (Directory.Exists(dir))
+                {
+                    Directory.Delete(dir, true);
+                }
+            }
+
+            return process.ExitCode;
+        }
+    }
+}

extensions/azurehybridtoolkit/notebooks/hybridbook/Components/ADP/BatchWrapper/packages.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Microsoft.Azure.Batch" version="13.0.0" targetFramework="net472" />
+  <package id="Microsoft.Azure.Batch.Conventions.Files" version="3.5.1" targetFramework="net472" />
+  <package id="Microsoft.Azure.KeyVault.Core" version="3.0.5" targetFramework="net472" />
+  <package id="Microsoft.Rest.ClientRuntime" version="2.3.21" targetFramework="net472" />
+  <package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.19" targetFramework="net472" />
+  <package id="Newtonsoft.Json" version="12.0.3" targetFramework="net472" />
+  <package id="WindowsAzure.Storage" version="9.3.3" targetFramework="net472" />
+</packages>