ckaczor/sqltoolsservice
1
0
Fork 0
mirror of https://github.com/ckaczor/sqltoolsservice.git synced 2026-01-15 01:25:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Writing Execute Results to Temp File (#35)

Browse Source 
* WIP for buffering in temporary file

* Adding support for writing to disk for buffering

* WIP - Adding file reader, factory for reader/writer

* Making long list use generics and implement IEnumerable

* Reading/Writing from file is working

* Removing unused 'skipValue' logic

* More tweaks to file buffer

Adding logic for cleaning up the temp files
Adding fix for empty/null column names

* Adding comments and cleanup

* Unit tests for FileStreamWrapper

* WIP adding more unit tests, and finishing up wiring up the output writers

* Finishing up initial unit tests

* Fixing bugs with long fields

* Squashed commit of the following:

commit df0ffc12a46cb286d801d08689964eac08ad71dd
Author: Benjamin Russell <beruss@microsoft.com>
Date:   Wed Sep 7 14:45:39 2016 -0700

    Removing last bit of async for file writing.

    We're seeing a 8x improvement of file write speeds!

commit 08a4b9f32e825512ca24d5dc03ef5acbf7cc6d94
Author: Benjamin Russell <beruss@microsoft.com>
Date:   Wed Sep 7 11:23:06 2016 -0700

    Removing async wrappers

* Rolling back test code for Program.cs

* Changes as per code review

* Fixing broken unit tests

* More fixes for codereview
This commit is contained in:
Benjamin Russell
2016-09-08 17:55:11 -07:00
committed by GitHub
parent 903eab61d1
commit 8aa3d524fc
24 changed files with 4050 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
src/Microsoft.SqlTools.ServiceLayer/QueryExecution/DataStorage/FileStreamWrapper.cs Normal file
View File

@@ -0,0 +1,282 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//
using System;
using System.Diagnostics;
using System.IO;
namespace Microsoft.SqlTools.ServiceLayer.QueryExecution.DataStorage
{
/// <summary>
/// Wrapper for a file stream, providing simplified creation, deletion, read, and write
/// functionality.
/// </summary>
public class FileStreamWrapper : IFileStreamWrapper
{
#region Member Variables
private byte[] buffer;
private int bufferDataSize;
private FileStream fileStream;
private long startOffset;
private long currentOffset;
#endregion
/// <summary>
/// Constructs a new FileStreamWrapper and initializes its state.
/// </summary>
public FileStreamWrapper()
{
// Initialize the internal state
bufferDataSize = 0;
startOffset = 0;
currentOffset = 0;
}
#region IFileStreamWrapper Implementation
/// <summary>
/// Initializes the wrapper by creating the internal buffer and opening the requested file.
/// If the file does not already exist, it will be created.
/// </summary>
/// <param name="fileName">Name of the file to open/create</param>
/// <param name="bufferLength">The length of the internal buffer</param>
/// <param name="accessMethod">
/// Whether or not the wrapper will be used for reading. If <c>true</c>, any calls to a
/// method that writes will cause an InvalidOperationException
/// </param>
public void Init(string fileName, int bufferLength, FileAccess accessMethod)
{
// Sanity check for valid buffer length, fileName, and accessMethod
if (bufferLength <= 0)
{
throw new ArgumentOutOfRangeException(nameof(bufferLength), "Buffer length must be a positive value");
}
if (string.IsNullOrWhiteSpace(fileName))
{
throw new ArgumentNullException(nameof(fileName), "File name cannot be null or whitespace");
}
if (accessMethod == FileAccess.Write)
{
throw new ArgumentException("Access method cannot be write-only", nameof(fileName));
}
// Setup the buffer
buffer = new byte[bufferLength];
// Open the requested file for reading/writing, creating one if it doesn't exist
fileStream = new FileStream(fileName, FileMode.OpenOrCreate, accessMethod, FileShare.ReadWrite,
bufferLength, false /*don't use asyncio*/);
// make file hidden
FileInfo fileInfo = new FileInfo(fileName);
fileInfo.Attributes |= FileAttributes.Hidden;
}
/// <summary>
/// Reads data into a buffer from the current offset into the file
/// </summary>
/// <param name="buf">The buffer to output the read data to</param>
/// <param name="bytes">The number of bytes to read into the buffer</param>
/// <returns>The number of bytes read</returns>
public int ReadData(byte[] buf, int bytes)
{
return ReadData(buf, bytes, currentOffset);
}
/// <summary>
/// Reads data into a buffer from the specified offset into the file
/// </summary>
/// <param name="buf">The buffer to output the read data to</param>
/// <param name="bytes">The number of bytes to read into the buffer</param>
/// <param name="offset">The offset into the file to start reading bytes from</param>
/// <returns>The number of bytes read</returns>
public int ReadData(byte[] buf, int bytes, long offset)
{
// Make sure that we're initialized before performing operations
if (buffer == null)
{
throw new InvalidOperationException("FileStreamWrapper must be initialized before performing operations");
}
MoveTo(offset);
int bytesCopied = 0;
while (bytesCopied < bytes)
{
int bufferOffset, bytesToCopy;
GetByteCounts(bytes, bytesCopied, out bufferOffset, out bytesToCopy);
Buffer.BlockCopy(buffer, bufferOffset, buf, bytesCopied, bytesToCopy);
bytesCopied += bytesToCopy;
if (bytesCopied < bytes && // did not get all the bytes yet
bufferDataSize == buffer.Length) // since current data buffer is full we should continue reading the file
{
// move forward one full length of the buffer
MoveTo(startOffset + buffer.Length);
}
else
{
// copied all the bytes requested or possible, adjust the current buffer pointer
currentOffset += bytesToCopy;
break;
}
}
return bytesCopied;
}
/// <summary>
/// Writes data to the underlying filestream, with buffering.
/// </summary>
/// <param name="buf">The buffer of bytes to write to the filestream</param>
/// <param name="bytes">The number of bytes to write</param>
/// <returns>The number of bytes written</returns>
public int WriteData(byte[] buf, int bytes)
{
// Make sure that we're initialized before performing operations
if (buffer == null)
{
throw new InvalidOperationException("FileStreamWrapper must be initialized before performing operations");
}
if (!fileStream.CanWrite)
{
throw new InvalidOperationException("This FileStreamWrapper canot be used for writing");
}
int bytesCopied = 0;
while (bytesCopied < bytes)
{
int bufferOffset, bytesToCopy;
GetByteCounts(bytes, bytesCopied, out bufferOffset, out bytesToCopy);
Buffer.BlockCopy(buf, bytesCopied, buffer, bufferOffset, bytesToCopy);
bytesCopied += bytesToCopy;
// adjust the current buffer pointer
currentOffset += bytesToCopy;
if (bytesCopied < bytes) // did not get all the bytes yet
{
Debug.Assert((int)(currentOffset - startOffset) == buffer.Length);
// flush buffer
Flush();
}
}
Debug.Assert(bytesCopied == bytes);
return bytesCopied;
}
/// <summary>
/// Flushes the internal buffer to the filestream
/// </summary>
public void Flush()
{
// Make sure that we're initialized before performing operations
if (buffer == null)
{
throw new InvalidOperationException("FileStreamWrapper must be initialized before performing operations");
}
if (!fileStream.CanWrite)
{
throw new InvalidOperationException("This FileStreamWrapper cannot be used for writing");
}
// Make sure we are at the right place in the file
Debug.Assert(fileStream.Position == startOffset);
int bytesToWrite = (int)(currentOffset - startOffset);
fileStream.Write(buffer, 0, bytesToWrite);
startOffset += bytesToWrite;
fileStream.Flush();
Debug.Assert(startOffset == currentOffset);
}
/// <summary>
/// Deletes the given file (ideally, created with this wrapper) from the filesystem
/// </summary>
/// <param name="fileName">The path to the file to delete</param>
public static void DeleteFile(string fileName)
{
File.Delete(fileName);
}
#endregion
/// <summary>
/// Perform calculations to determine how many bytes to copy and what the new buffer offset
/// will be for copying.
/// </summary>
/// <param name="bytes">Number of bytes requested to copy</param>
/// <param name="bytesCopied">Number of bytes copied so far</param>
/// <param name="bufferOffset">New offset to start copying from/to</param>
/// <param name="bytesToCopy">Number of bytes to copy in this iteration</param>
private void GetByteCounts(int bytes, int bytesCopied, out int bufferOffset, out int bytesToCopy)
{
bufferOffset = (int) (currentOffset - startOffset);
bytesToCopy = bytes - bytesCopied;
if (bytesToCopy > buffer.Length - bufferOffset)
{
bytesToCopy = buffer.Length - bufferOffset;
}
}
/// <summary>
/// Moves the internal buffer to the specified offset into the file
/// </summary>
/// <param name="offset">Offset into the file to move to</param>
private void MoveTo(long offset)
{
if (buffer.Length > bufferDataSize || // buffer is not completely filled
offset < startOffset || // before current buffer start
offset >= (startOffset + buffer.Length)) // beyond current buffer end
{
// init the offset
startOffset = offset;
// position file pointer
fileStream.Seek(startOffset, SeekOrigin.Begin);
// fill in the buffer
bufferDataSize = fileStream.Read(buffer, 0, buffer.Length);
}
// make sure to record where we are
currentOffset = offset;
}
#region IDisposable Implementation
private bool disposed;
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposed)
{
return;
}
if (disposing && fileStream != null)
{
if(fileStream.CanWrite) { Flush(); }
fileStream.Dispose();
}
disposed = true;
}
~FileStreamWrapper()
{
Dispose(false);
}
#endregion
}
}