mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-02-01 09:35:41 -05:00
Merge from vscode 05fc61ffb1aee9fd19173c32113daed079f9b7bd (#5074)
* Merge from vscode 05fc61ffb1aee9fd19173c32113daed079f9b7bd * fix tests
This commit is contained in:
Anthony Dresser
@@ -3,11 +3,11 @@
|
||||
* Licensed under the Source EULA. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as stream from 'vs/base/node/stream';
|
||||
import * as iconv from 'iconv-lite';
|
||||
import { isLinux, isMacintosh } from 'vs/base/common/platform';
|
||||
import { exec } from 'child_process';
|
||||
import { Readable, Writable } from 'stream';
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
|
||||
export const UTF8 = 'utf8';
|
||||
export const UTF8_with_bom = 'utf8bom';
|
||||
@@ -24,7 +24,12 @@ export interface IDecodeStreamOptions {
|
||||
overwriteEncoding?(detectedEncoding: string | null): string;
|
||||
}
|
||||
|
||||
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }> {
|
||||
export interface IDecodeStreamResult {
|
||||
detected: IDetectedEncodingResult;
|
||||
stream: NodeJS.ReadableStream;
|
||||
}
|
||||
|
||||
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
|
||||
if (!options.minBytesRequiredForDetection) {
|
||||
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_GUESS_BUFFER_MAX_LEN : NO_GUESS_BUFFER_MAX_LEN;
|
||||
}
|
||||
@@ -33,90 +38,88 @@ export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions
|
||||
options.overwriteEncoding = detected => detected || UTF8;
|
||||
}
|
||||
|
||||
return new Promise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }>((resolve, reject) => {
|
||||
|
||||
readable.on('error', reject);
|
||||
|
||||
readable.pipe(new class extends Writable {
|
||||
|
||||
private _decodeStream: NodeJS.ReadWriteStream;
|
||||
private _decodeStreamConstruction: Promise<void>;
|
||||
private _buffer: Buffer[] = [];
|
||||
private _bytesBuffered = 0;
|
||||
return new Promise<IDecodeStreamResult>((resolve, reject) => {
|
||||
const writer = new class extends Writable {
|
||||
private decodeStream: NodeJS.ReadWriteStream;
|
||||
private decodeStreamConstruction: Promise<void>;
|
||||
private buffer: Buffer[] = [];
|
||||
private bytesBuffered = 0;
|
||||
|
||||
_write(chunk: any, encoding: string, callback: Function): void {
|
||||
if (!Buffer.isBuffer(chunk)) {
|
||||
callback(new Error('data must be a buffer'));
|
||||
}
|
||||
|
||||
if (this._decodeStream) {
|
||||
// just a forwarder now
|
||||
this._decodeStream.write(chunk, callback);
|
||||
if (this.decodeStream) {
|
||||
this.decodeStream.write(chunk, callback); // just a forwarder now
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this._buffer.push(chunk);
|
||||
this._bytesBuffered += chunk.length;
|
||||
this.buffer.push(chunk);
|
||||
this.bytesBuffered += chunk.length;
|
||||
|
||||
if (this._decodeStreamConstruction) {
|
||||
// waiting for the decoder to be ready
|
||||
this._decodeStreamConstruction.then(_ => callback(), err => callback(err));
|
||||
// waiting for the decoder to be ready
|
||||
if (this.decodeStreamConstruction) {
|
||||
this.decodeStreamConstruction.then(() => callback(), err => callback(err));
|
||||
}
|
||||
|
||||
} else if (typeof options.minBytesRequiredForDetection === 'number' && this._bytesBuffered >= options.minBytesRequiredForDetection) {
|
||||
// buffered enough data, create stream and forward data
|
||||
// buffered enough data, create stream and forward data
|
||||
else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
|
||||
this._startDecodeStream(callback);
|
||||
}
|
||||
|
||||
} else {
|
||||
// only buffering
|
||||
// only buffering
|
||||
else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
|
||||
_startDecodeStream(callback: Function): void {
|
||||
|
||||
this._decodeStreamConstruction = Promise.resolve(detectEncodingFromBuffer({
|
||||
buffer: Buffer.concat(this._buffer), bytesRead: this._bytesBuffered
|
||||
this.decodeStreamConstruction = Promise.resolve(detectEncodingFromBuffer({
|
||||
buffer: Buffer.concat(this.buffer),
|
||||
bytesRead: this.bytesBuffered
|
||||
}, options.guessEncoding)).then(detected => {
|
||||
if (options.overwriteEncoding) {
|
||||
detected.encoding = options.overwriteEncoding(detected.encoding);
|
||||
}
|
||||
this._decodeStream = decodeStream(detected.encoding);
|
||||
for (const buffer of this._buffer) {
|
||||
this._decodeStream.write(buffer);
|
||||
}
|
||||
callback();
|
||||
resolve({ detected, stream: this._decodeStream });
|
||||
|
||||
this.decodeStream = decodeStream(detected.encoding);
|
||||
|
||||
for (const buffer of this.buffer) {
|
||||
this.decodeStream.write(buffer);
|
||||
}
|
||||
|
||||
callback();
|
||||
resolve({ detected, stream: this.decodeStream });
|
||||
}, err => {
|
||||
this.emit('error', err);
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
_final(callback: (err?: any) => any) {
|
||||
if (this._decodeStream) {
|
||||
// normal finish
|
||||
this._decodeStream.end(callback);
|
||||
} else {
|
||||
// we were still waiting for data...
|
||||
this._startDecodeStream(() => this._decodeStream.end(callback));
|
||||
|
||||
// normal finish
|
||||
if (this.decodeStream) {
|
||||
this.decodeStream.end(callback);
|
||||
}
|
||||
|
||||
// we were still waiting for data...
|
||||
else {
|
||||
this._startDecodeStream(() => this.decodeStream.end(callback));
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// errors
|
||||
readable.on('error', reject);
|
||||
|
||||
// pipe through
|
||||
readable.pipe(writer);
|
||||
});
|
||||
}
|
||||
|
||||
export function bomLength(encoding: string): number {
|
||||
switch (encoding) {
|
||||
case UTF8:
|
||||
return 3;
|
||||
case UTF16be:
|
||||
case UTF16le:
|
||||
return 2;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function decode(buffer: Buffer, encoding: string): string {
|
||||
return iconv.decode(buffer, toNodeEncoding(encoding));
|
||||
}
|
||||
@@ -129,7 +132,7 @@ export function encodingExists(encoding: string): boolean {
|
||||
return iconv.encodingExists(toNodeEncoding(encoding));
|
||||
}
|
||||
|
||||
export function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
|
||||
function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
|
||||
return iconv.decodeStream(toNodeEncoding(encoding));
|
||||
}
|
||||
|
||||
@@ -145,7 +148,7 @@ function toNodeEncoding(enc: string | null): string {
|
||||
return enc;
|
||||
}
|
||||
|
||||
export function detectEncodingByBOMFromBuffer(buffer: Buffer | null, bytesRead: number): string | null {
|
||||
export function detectEncodingByBOMFromBuffer(buffer: Buffer | VSBuffer | null, bytesRead: number): string | null {
|
||||
if (!buffer || bytesRead < 2) {
|
||||
return null;
|
||||
}
|
||||
@@ -177,39 +180,31 @@ export function detectEncodingByBOMFromBuffer(buffer: Buffer | null, bytesRead:
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects the Byte Order Mark in a given file.
|
||||
* If no BOM is detected, null will be passed to callback.
|
||||
*/
|
||||
export function detectEncodingByBOM(file: string): Promise<string | null> {
|
||||
return stream.readExactlyByFile(file, 3).then(({ buffer, bytesRead }) => detectEncodingByBOMFromBuffer(buffer, bytesRead), error => null);
|
||||
}
|
||||
|
||||
const MINIMUM_THRESHOLD = 0.2;
|
||||
const IGNORE_ENCODINGS = ['ascii', 'utf-8', 'utf-16', 'utf-32'];
|
||||
|
||||
/**
|
||||
* Guesses the encoding from buffer.
|
||||
*/
|
||||
export function guessEncodingByBuffer(buffer: Buffer): Promise<string | null> {
|
||||
return import('jschardet').then(jschardet => {
|
||||
jschardet.Constants.MINIMUM_THRESHOLD = MINIMUM_THRESHOLD;
|
||||
async function guessEncodingByBuffer(buffer: Buffer): Promise<string | null> {
|
||||
const jschardet = await import('jschardet');
|
||||
|
||||
const guessed = jschardet.detect(buffer);
|
||||
if (!guessed || !guessed.encoding) {
|
||||
return null;
|
||||
}
|
||||
jschardet.Constants.MINIMUM_THRESHOLD = MINIMUM_THRESHOLD;
|
||||
|
||||
const enc = guessed.encoding.toLowerCase();
|
||||
const guessed = jschardet.detect(buffer);
|
||||
if (!guessed || !guessed.encoding) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Ignore encodings that cannot guess correctly
|
||||
// (http://chardet.readthedocs.io/en/latest/supported-encodings.html)
|
||||
if (0 <= IGNORE_ENCODINGS.indexOf(enc)) {
|
||||
return null;
|
||||
}
|
||||
const enc = guessed.encoding.toLowerCase();
|
||||
|
||||
return toIconvLiteEncoding(guessed.encoding);
|
||||
});
|
||||
// Ignore encodings that cannot guess correctly
|
||||
// (http://chardet.readthedocs.io/en/latest/supported-encodings.html)
|
||||
if (0 <= IGNORE_ENCODINGS.indexOf(enc)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return toIconvLiteEncoding(guessed.encoding);
|
||||
}
|
||||
|
||||
const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = {
|
||||
@@ -270,9 +265,14 @@ export interface IDetectedEncodingResult {
|
||||
seemsBinary: boolean;
|
||||
}
|
||||
|
||||
export function detectEncodingFromBuffer(readResult: stream.ReadResult, autoGuessEncoding?: false): IDetectedEncodingResult;
|
||||
export function detectEncodingFromBuffer(readResult: stream.ReadResult, autoGuessEncoding?: boolean): Promise<IDetectedEncodingResult>;
|
||||
export function detectEncodingFromBuffer({ buffer, bytesRead }: stream.ReadResult, autoGuessEncoding?: boolean): Promise<IDetectedEncodingResult> | IDetectedEncodingResult {
|
||||
export interface IReadResult {
|
||||
buffer: Buffer | null;
|
||||
bytesRead: number;
|
||||
}
|
||||
|
||||
export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: false): IDetectedEncodingResult;
|
||||
export function detectEncodingFromBuffer(readResult: IReadResult, autoGuessEncoding?: boolean): Promise<IDetectedEncodingResult>;
|
||||
export function detectEncodingFromBuffer({ buffer, bytesRead }: IReadResult, autoGuessEncoding?: boolean): Promise<IDetectedEncodingResult> | IDetectedEncodingResult {
|
||||
|
||||
// Always first check for BOM to find out about encoding
|
||||
let encoding = detectEncodingByBOMFromBuffer(buffer, bytesRead);
|
||||
@@ -357,7 +357,7 @@ const windowsTerminalEncodings = {
|
||||
'1252': 'cp1252' // West European Latin
|
||||
};
|
||||
|
||||
export function resolveTerminalEncoding(verbose?: boolean): Promise<string> {
|
||||
export async function resolveTerminalEncoding(verbose?: boolean): Promise<string> {
|
||||
let rawEncodingPromise: Promise<string>;
|
||||
|
||||
// Support a global environment variable to win over other mechanics
|
||||
@@ -403,24 +403,23 @@ export function resolveTerminalEncoding(verbose?: boolean): Promise<string> {
|
||||
});
|
||||
}
|
||||
|
||||
return rawEncodingPromise.then(rawEncoding => {
|
||||
if (verbose) {
|
||||
console.log(`Detected raw terminal encoding: ${rawEncoding}`);
|
||||
}
|
||||
|
||||
if (!rawEncoding || rawEncoding.toLowerCase() === 'utf-8' || rawEncoding.toLowerCase() === UTF8) {
|
||||
return UTF8;
|
||||
}
|
||||
|
||||
const iconvEncoding = toIconvLiteEncoding(rawEncoding);
|
||||
if (iconv.encodingExists(iconvEncoding)) {
|
||||
return iconvEncoding;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
console.log('Unsupported terminal encoding, falling back to UTF-8.');
|
||||
}
|
||||
const rawEncoding = await rawEncodingPromise;
|
||||
if (verbose) {
|
||||
console.log(`Detected raw terminal encoding: ${rawEncoding}`);
|
||||
}
|
||||
|
||||
if (!rawEncoding || rawEncoding.toLowerCase() === 'utf-8' || rawEncoding.toLowerCase() === UTF8) {
|
||||
return UTF8;
|
||||
});
|
||||
}
|
||||
|
||||
const iconvEncoding = toIconvLiteEncoding(rawEncoding);
|
||||
if (iconv.encodingExists(iconvEncoding)) {
|
||||
return iconvEncoding;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
console.log('Unsupported terminal encoding, falling back to UTF-8.');
|
||||
}
|
||||
|
||||
return UTF8;
|
||||
}
|
||||
|
||||
@@ -5,63 +5,6 @@
|
||||
|
||||
import * as fs from 'fs';
|
||||
|
||||
export interface ReadResult {
|
||||
buffer: Buffer | null;
|
||||
bytesRead: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads totalBytes from the provided file.
|
||||
*/
|
||||
export function readExactlyByFile(file: string, totalBytes: number): Promise<ReadResult> {
|
||||
return new Promise<ReadResult>((resolve, reject) => {
|
||||
fs.open(file, 'r', null, (err, fd) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
function end(err: Error | null, resultBuffer: Buffer | null, bytesRead: number): void {
|
||||
fs.close(fd, closeError => {
|
||||
if (closeError) {
|
||||
return reject(closeError);
|
||||
}
|
||||
|
||||
if (err && (<any>err).code === 'EISDIR') {
|
||||
return reject(err); // we want to bubble this error up (file is actually a folder)
|
||||
}
|
||||
|
||||
return resolve({ buffer: resultBuffer, bytesRead });
|
||||
});
|
||||
}
|
||||
|
||||
const buffer = Buffer.allocUnsafe(totalBytes);
|
||||
let offset = 0;
|
||||
|
||||
function readChunk(): void {
|
||||
fs.read(fd, buffer, offset, totalBytes - offset, null, (err, bytesRead) => {
|
||||
if (err) {
|
||||
return end(err, null, 0);
|
||||
}
|
||||
|
||||
if (bytesRead === 0) {
|
||||
return end(null, buffer, offset);
|
||||
}
|
||||
|
||||
offset += bytesRead;
|
||||
|
||||
if (offset === totalBytes) {
|
||||
return end(null, buffer, offset);
|
||||
}
|
||||
|
||||
return readChunk();
|
||||
});
|
||||
}
|
||||
|
||||
readChunk();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a file until a matching string is found.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user