ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-14 01:25:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Added delete materialized run endpoints (#858)

Browse Source 
* Added endpoint for fetching all notebook jobs

* Refractored NotebookJobInfo to AgentNotebookInfo to make it more consistent with the rest of the codebase

* Added Notebook History endpoint in contracts.

* Added Create, Update, Delete notebook endpoints. Also added separate fetch template, materialized notebook endpoints. This will make the Notebook Request and Notebook History responses lighter.

* AgentNotebookInfo is now derived from AgentJobInfo

* added fetch noteook history endpoint

* Added fetching materialized notebook endpoint

* Added code for cleaning up the directory

* Added create notebook api

* Added Update and delete notebook job

* Fixed notebook history API

* Added last run info to the script and template folder

* Added execute database feature for notebook Jobs

* SQL commands are now using sqlparameters to prevent
any injection attacks

* Changed rundate and runtime to string to preserve
leading zeros

* integration test for agentnotebooks api

* Made some changes mentioned in PR

* Refactored the code, removed enpoint logic from the notebook handler and
wrote test cases

* changes select statements, fixed a bug in the test job cleanup
and fixed other stuff mentioned in the PR.

* added notebook_error column in notebook history select statement

* Added get template notebook endpoint

* Added renaming and pinning notebook runs

* made tables names caps to handle servers with case sensitive queries

* made some changes to the enpoint points paths

* Added delete materialized notebooks endpoints

* Fixed a bug in delete, rename and pin apis where
requests from multiple clients with materializedID = 0
could have resulted in multiple rows getting created
on notebooks.nb_materialized table.

* fixed a merge conflict

* Made some changes mentioned in the PR
This commit is contained in:
Aasim Khan
2019-09-04 10:34:28 -07:00
committed by GitHub
parent 89623c1ae8
commit 8a12802fb9
4 changed files with 181 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/Microsoft.SqlTools.ServiceLayer/Agent/AgentService.cs
View File

@@ -131,7 +131,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
this.ServiceHost.SetRequestHandler(UpdateAgentNotebookRequest.Type, HandleUpdateAgentNotebookRequest);
this.ServiceHost.SetRequestHandler(UpdateAgentNotebookRunPinRequest.Type, HandleUpdateAgentNotebookRunPinRequest);
this.ServiceHost.SetRequestHandler(UpdateAgentNotebookRunNameRequest.Type, HandleUpdateAgentNotebookRunNameRequest);
this.ServiceHost.SetRequestHandler(DeleteNotebookMaterializedRequest.Type, HandleDeleteNotebookMaterializedRequest);
serviceHost.RegisterShutdownTask(async (shutdownParams, shutdownRequestContext) =>
{
@@ -1265,7 +1265,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
parameters.OwnerUri,
out connInfo);
result = GetAgentNotebookHistories(
result = await GetAgentNotebookHistories(
connInfo,
parameters.JobId,
parameters.JobName,
@@ -1320,7 +1320,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
ConnectionServiceInstance.TryFindConnection(
parameters.OwnerUri,
out connInfo);
result.NotebookTemplate = AgentNotebookHelper.GetTemplateNotebook(connInfo, parameters.JobId, parameters.TargetDatabase).Result;
result.NotebookTemplate = await AgentNotebookHelper.GetTemplateNotebook(connInfo, parameters.JobId, parameters.TargetDatabase);
result.Success = true;
}
catch (Exception e)
@@ -1426,7 +1426,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
// Calling update helper function
await AgentNotebookHelper.UpdateMaterializedNotebookName(
connInfo,
parameters.MaterializedId,
parameters.agentNotebookHistory,
parameters.TargetDatabase,
parameters.MaterializedNotebookName);
result.Success = true;
@@ -1455,7 +1455,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
// Calling update helper function
await AgentNotebookHelper.UpdateMaterializedNotebookPin(
connInfo,
parameters.MaterializedId,
parameters.agentNotebookHistory,
parameters.TargetDatabase,
parameters.MaterializedNotebookPin);
result.Success = true;
@@ -1470,7 +1470,36 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
});
}
public AgentNotebookHistoryResult GetAgentNotebookHistories(
internal async Task HandleDeleteNotebookMaterializedRequest(DeleteNotebookMaterializedParams parameters, RequestContext<ResultStatus> requestContext)
{
await Task.Run(async () =>
{
var result = new ResultStatus();
try
{
ConnectionInfo connInfo;
ConnectionServiceInstance.TryFindConnection(
parameters.OwnerUri,
out connInfo);
// Calling update helper function
await AgentNotebookHelper.DeleteMaterializedNotebook(
connInfo,
parameters.agentNotebookHistory,
parameters.TargetDatabase);
result.Success = true;
}
catch (Exception e)
{
result.Success = false;
result.ErrorMessage = e.ToString();
}
await requestContext.SendResult(result);
});
}
public async Task<AgentNotebookHistoryResult> GetAgentNotebookHistories
(
ConnectionInfo connInfo,
string jobId,
string jobName,
@@ -1519,7 +1548,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
var jobHistories = AgentUtilities.ConvertToAgentNotebookHistoryInfo(logEntries, job, steps);
// fetching notebook part of histories
Dictionary<string, DataRow> notebookHistoriesDict = new Dictionary<string, DataRow>();
DataTable materializedNotebookTable = AgentNotebookHelper.GetAgentNotebookHistories(connInfo, jobId, targetDatabase).Result;
DataTable materializedNotebookTable = await AgentNotebookHelper.GetAgentNotebookHistories(connInfo, jobId, targetDatabase);
foreach (DataRow materializedNotebookRow in materializedNotebookTable.Rows)
{
string materializedRunDateTime = materializedNotebookRow["run_date"].ToString() + materializedNotebookRow["run_time"].ToString();
@@ -1537,8 +1566,14 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
notebookHistory.MaterializedNotebookErrorInfo = notebookHistoriesDict[jobRuntime]["notebook_error"] as string;
notebookHistory.MaterializedNotebookName = notebookHistoriesDict[jobRuntime]["notebook_name"] as string;
notebookHistory.MaterializedNotebookPin = (bool)notebookHistoriesDict[jobRuntime]["pin"];
notebookHistory.MaterializedNotebookDeleted = (bool)notebookHistoriesDict[jobRuntime]["is_deleted"];
}
if (notebookHistory.MaterializedNotebookDeleted)
{
continue;
}
notebookHistories.Add(notebookHistory);
}
result.Histories = notebookHistories.ToArray();
tlog.CloseReader();

1
src/Microsoft.SqlTools.ServiceLayer/Agent/Contracts/AgentJobHistoryInfo.cs
View File

@@ -69,5 +69,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent.Contracts
public string MaterializedNotebookName { get; set; }
public int MaterializedNotebookErrorFlag { get; set; }
public string MaterializedNotebookErrorInfo { get; set; }
public bool MaterializedNotebookDeleted { get; set; }
}
}

32
src/Microsoft.SqlTools.ServiceLayer/Agent/Contracts/AgentNotebookRequest.cs
View File

@@ -219,7 +219,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent.Contracts
public class UpdateAgentNotebookRunNameParams : TaskRequestDetails
{
public string OwnerUri { get; set; }
public int MaterializedId { get; set; }
public AgentNotebookHistoryInfo agentNotebookHistory { get; set; }
public string MaterializedNotebookName { get; set; }
public string TargetDatabase { get; set; }
@@ -241,7 +241,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent.Contracts
public class UpdateAgentNotebookRunPinParams : TaskRequestDetails
{
public string OwnerUri { get; set; }
public string MaterializedId { get; set; }
public AgentNotebookHistoryInfo agentNotebookHistory{ get; set; }
public bool MaterializedNotebookPin { get; set; }
public string TargetDatabase { get; set; }
@@ -256,4 +256,32 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent.Contracts
RequestType<UpdateAgentNotebookRunPinParams, ResultStatus> Type =
RequestType<UpdateAgentNotebookRunPinParams, ResultStatus>.Create("agent/updatenotebookpin");
}
/// <summary>
/// SQL Agent Notebook materialized params
/// </summary>
public class DeleteNotebookMaterializedParams : TaskRequestDetails
{
public string OwnerUri { get; set; }
public string TargetDatabase { get; set; }
public AgentNotebookHistoryInfo agentNotebookHistory { get; set; }
}
/// <summary>
/// SQL Agent Notebook materialized result
/// </summary>
public class DeleteNotebookMaterializedResult : ResultStatus
{
public string NotebookMaterialized { get; set; }
}
/// <summary>
/// SQL Agent Notebook materialized request type
/// </summary>
public class DeleteNotebookMaterializedRequest
{
public static readonly
RequestType<DeleteNotebookMaterializedParams, ResultStatus> Type =
RequestType<DeleteNotebookMaterializedParams, ResultStatus>.Create("agent/deletenotebookmaterialized");
}
}

163
src/Microsoft.SqlTools.ServiceLayer/Agent/Jobs/AgentNotebookHelper.cs
View File

@@ -190,7 +190,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
ConfigAction.Drop,
runType);
if(!deleteJobResult.Item1)
if (!deleteJobResult.Item1)
{
throw new Exception(deleteJobResult.Item2);
}
@@ -200,7 +200,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
notebook.JobId,
notebook.TargetDatabase);
}
internal static async Task UpdateNotebook(
@@ -228,8 +228,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
notebook,
ConfigAction.Update,
runType);
if(!updateJobResult.Item1)
if (!updateJobResult.Item1)
{
throw new Exception(updateJobResult.Item2);
}
@@ -265,7 +265,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
run_date,
notebook_error,
pin,
notebook_name
notebook_name,
is_deleted
FROM
notebooks.nb_materialized
WHERE JOB_ID = @jobId";
@@ -281,13 +282,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
return result;
}
/// <summary>
///
/// </summary>
/// <param name="connInfo"></param>
/// <param name="materializedId"></param>
/// <param name="targetDatabase"></param>
/// <returns></returns>
public static async Task<string> GetMaterializedNotebook(
ConnectionInfo connInfo,
int materializedId,
@@ -340,12 +334,6 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
return templateNotebookRows["notebook"] as string;
}
/// <summary>
///
/// </summary>
/// <param name="notebookName"></param>
/// <param name="storageDatabase"></param>
/// <returns></returns>
public static AgentJobStepInfo[] CreateNotebookPowerShellStep(
string notebookName,
string storageDatabase)
@@ -438,6 +426,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
notebook NVARCHAR(MAX),
notebook_error NVARCHAR(MAX),
pin BIT NOT NULL DEFAULT 0,
is_deleted BIT NOT NULL DEFAULT 0,
notebook_name NVARCHAR(MAX) NOT NULL default('')
)
END
@@ -567,50 +556,45 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
targetDatabase);
}
public static async Task<string> GetTemplateFile(
ConnectionInfo connInfo,
string job_id,
string targetDatabase,
string templateFileContents)
{
string getNotebookTemplateQuery =
@"
SELECT notebook
from
notebooks.nb_template
where
job_id = @jobId;
";
List<SqlParameter> getNotebookTemplateQueryParams = new List<SqlParameter>();
getNotebookTemplateQueryParams.Add(new SqlParameter("job_id", getNotebookTemplateQueryParams));
DataSet templateDataSet = await AgentNotebookHelper.ExecuteSqlQueries(
connInfo,
getNotebookTemplateQuery,
getNotebookTemplateQueryParams,
targetDatabase);
DataTable templateDataTable = templateDataSet.Tables[0];
DataRow templateDataRow = templateDataTable.Rows[0];
return templateDataRow["notebook"] as string;
}
/// <summary>
/// Changing the name of materialized notebook runs. Special case is handled where new row is
/// added for failed jobs which do not have an entry into the materialized table
/// </summary>
/// <param name="connInfo">connectionInfo generated from OwnerUri</param>
/// <param name="agentNotebookHistory">actual history item to be pinned</param>
/// <param name="targetDatabase">database on which the notebook history is stored</param>
/// <param name="name">name for the materialized history</param>
/// <returns></returns>
public static async Task UpdateMaterializedNotebookName(
ConnectionInfo connInfo,
int materializedId,
AgentNotebookHistoryInfo agentNotebookHistory,
string targetDatabase,
string name)
{
string updateMaterializedNotebookNameQuery =
@"
@"
IF EXISTS
(SELECT * FROM notebooks.nb_materialized
WHERE job_id = @jobId AND run_time = @startTime AND run_date = @startDate)
BEGIN
UPDATE notebooks.nb_materialized
SET
notebook_name = @notebookName
notebook_name = @notebookName
WHERE
materialized_id = @materializedId
job_id = @jobId AND run_time = @startTime AND run_date = @startDate
END
ELSE
BEGIN
INSERT INTO notebooks.nb_materialized (job_id, run_time, run_date, notebook, notebook_error, notebook_name)
VALUES
(@jobID, @startTime, @startDate, '', '', @notebookName)
END
";
List<SqlParameter> updateMaterializedNotebookNameParams = new List<SqlParameter>();
updateMaterializedNotebookNameParams.Add(new SqlParameter("jobID", agentNotebookHistory.JobId));
updateMaterializedNotebookNameParams.Add(new SqlParameter("startTime", agentNotebookHistory.RunDate.ToString("HHmmss")));
updateMaterializedNotebookNameParams.Add(new SqlParameter("startDate", agentNotebookHistory.RunDate.ToString("yyyyMMdd")));
updateMaterializedNotebookNameParams.Add(new SqlParameter("notebookName", name));
updateMaterializedNotebookNameParams.Add(new SqlParameter("materializedId", materializedId));
await AgentNotebookHelper.ExecuteSqlQueries(
connInfo,
updateMaterializedNotebookNameQuery,
@@ -618,28 +602,97 @@ namespace Microsoft.SqlTools.ServiceLayer.Agent
targetDatabase);
}
/// <summary>
/// Changing the pin state of materialized notebook runs. Special case is handled where new row is
/// added for failed jobs which do not have an entry into the materialized table
/// </summary>
/// <param name="connInfo">connectionInfo generated from OwnerUri</param>
/// <param name="agentNotebookHistory">actual history item to be pinned</param>
/// <param name="targetDatabase">database on which the notebook history is stored</param>
/// <param name="pin">pin state for the history</param>
/// <returns></returns>
public static async Task UpdateMaterializedNotebookPin(
ConnectionInfo connInfo,
string materializedId,
AgentNotebookHistoryInfo agentNotebookHistory,
string targetDatabase,
bool pin)
{
string updateMaterializedNotebookPinQuery =
@"
@"
IF EXISTS
(SELECT * FROM notebooks.nb_materialized
WHERE job_id = @jobId AND run_time = @startTime AND run_date = @startDate)
BEGIN
UPDATE notebooks.nb_materialized
SET
pin = @notebookPin
pin = @notebookPin
WHERE
materialized_id = @materializedId
job_id = @jobId AND run_time = @startTime AND run_date = @startDate
END
ELSE
BEGIN
INSERT INTO notebooks.nb_materialized (job_id, run_time, run_date, notebook, notebook_error, pin)
VALUES
(@jobID, @startTime, @startDate, '', '', @notebookPin)
END
";
List<SqlParameter> updateMaterializedNotebookPinParams = new List<SqlParameter>();
updateMaterializedNotebookPinParams.Add(new SqlParameter("jobID", agentNotebookHistory.JobId));
updateMaterializedNotebookPinParams.Add(new SqlParameter("startTime", agentNotebookHistory.RunDate.ToString("HHmmss")));
updateMaterializedNotebookPinParams.Add(new SqlParameter("startDate", agentNotebookHistory.RunDate.ToString("yyyyMMdd")));
updateMaterializedNotebookPinParams.Add(new SqlParameter("notebookPin", pin));
updateMaterializedNotebookPinParams.Add(new SqlParameter("materializedId", materializedId));
await AgentNotebookHelper.ExecuteSqlQueries(
connInfo,
updateMaterializedNotebookPinQuery,
updateMaterializedNotebookPinParams,
targetDatabase);
}
/// <summary>
/// Delete a particular run of the job. Deletion mainly including clearing out the notebook,
/// and notebook-error. The API doesn't delete the row because some notebook runs that have job
/// error in them don't have an entry in the materialized table. For keeping track of those notebook
/// runs the entry is added into the table with is_delete set to 1.
/// </summary>
/// <param name="connInfo">connectionInfo generated from OwnerUri</param>
/// <param name="agentNotebookHistory">Actual history item to be deleted</param>
/// <param name="targetDatabase">database on which the notebook history is stored</param>
/// <returns></returns>
public static async Task DeleteMaterializedNotebook(
ConnectionInfo connInfo,
AgentNotebookHistoryInfo agentNotebookHistory,
string targetDatabase)
{
string deleteMaterializedNotebookQuery =
@"
IF EXISTS
(SELECT * FROM notebooks.nb_materialized
WHERE job_id = @jobId AND run_time = @startTime AND run_date = @startDate)
BEGIN
UPDATE notebooks.nb_materialized
SET is_deleted = 1,
notebook = '',
notebook_error = '',
WHERE
job_id = @jobId AND run_time = @startTime AND run_date = @startDate
END
ELSE
BEGIN
INSERT INTO notebooks.nb_materialized (job_id, run_time, run_date, notebook, notebook_error, is_deleted)
VALUES
(@jobID, @startTime, @startDate, '', '', 1)
END
";
List<SqlParameter> deleteMaterializedNotebookParams = new List<SqlParameter>();
deleteMaterializedNotebookParams.Add(new SqlParameter("jobID", agentNotebookHistory.JobId));
deleteMaterializedNotebookParams.Add(new SqlParameter("startTime", agentNotebookHistory.RunDate.ToString("HHmmss")));
deleteMaterializedNotebookParams.Add(new SqlParameter("startDate", agentNotebookHistory.RunDate.ToString("yyyyMMdd")));
deleteMaterializedNotebookParams.Add(new SqlParameter("materializedId", agentNotebookHistory.MaterializedNotebookId));
await AgentNotebookHelper.ExecuteSqlQueries(
connInfo,
deleteMaterializedNotebookQuery,
deleteMaterializedNotebookParams,
targetDatabase);
}
}
}