Remove all Big Data Cluster features (#21369)

This commit is contained in:
Cory Rivera
2022-12-07 12:28:17 -08:00
committed by GitHub
parent bb1f5bfffe
commit e2327c393a
213 changed files with 346 additions and 46800 deletions

View File

@@ -97,7 +97,6 @@ export const databaseDashboardSettingSchema: IJSONSchema = {
widget: {
'tasks-widget': [
'newQuery',
'mssqlCluster.task.newNotebook',
{ name: 'backup', when: 'connectionProvider == \'MSSQL\' && !mssql:iscloud && mssql:engineedition != 11 || connectionProvider == \'PGSQL\'' },
{ name: 'restore', when: 'connectionProvider == \'MSSQL\' && !mssql:iscloud && mssql:engineedition != 11 || connectionProvider == \'PGSQL\'' }
]

View File

@@ -79,7 +79,6 @@ const defaultVal = [
widget: {
'tasks-widget': [
'newQuery',
'mssqlCluster.task.newNotebook',
{ name: 'restore', when: 'connectionProvider == \'MSSQL\' && !mssql:iscloud && mssql:engineedition != 11 || connectionProvider == \'PGSQL\'' }]
},
gridItemConfig: {

View File

@@ -464,7 +464,7 @@ export class RunParametersAction extends TooltipFromLabelAction {
return;
}
const editor = this._notebookService.findNotebookEditor(context);
// Only run action for kernels that are supported (Python, PySpark, PowerShell)
// Only run action for kernels that are supported (Python, PowerShell)
let supportedKernels: string[] = [KernelsLanguage.Python, KernelsLanguage.PowerShell];
if (!supportedKernels.includes(editor.model.languageInfo.name)) {
// If the kernel is not supported indicate to user to use supported kernels

View File

@@ -1,89 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { localize } from 'vs/nls';
import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
export namespace constants {
export const userPropName = 'user';
export const knoxPortPropName = 'knoxport';
export const clusterPropName = 'clustername';
export const passwordPropName = 'password';
export const defaultKnoxPort = '30443';
}
/**
* This is a temporary connection definition, with known properties for Knox gateway connections.
* Long term this should be refactored to an extension contribution
*
* @export
*/
export class NotebookConnection {
private _host: string;
private _knoxPort: string;
constructor(private _connectionProfile: IConnectionProfile) {
if (!this._connectionProfile) {
throw new Error(localize('connectionInfoMissing', "connectionInfo is required"));
}
}
public get connectionProfile(): IConnectionProfile {
return this._connectionProfile;
}
public get host(): string {
if (!this._host) {
this.ensureHostAndPort();
}
return this._host;
}
/**
* Sets host and port values, using any ',' or ':' delimited port in the hostname in
* preference to the built in port.
*/
private ensureHostAndPort(): void {
this._host = this.connectionProfile.serverName;
this._knoxPort = NotebookConnection.getKnoxPortOrDefault(this.connectionProfile);
// determine whether the host has either a ',' or ':' in it
this.setHostAndPort(',');
this.setHostAndPort(':');
}
// set port and host correctly after we've identified that a delimiter exists in the host name
private setHostAndPort(delimeter: string): void {
let originalHost = this._host;
let index = originalHost.indexOf(delimeter);
if (index > -1) {
this._host = originalHost.slice(0, index);
this._knoxPort = originalHost.slice(index + 1);
}
}
public get user(): string {
return this._connectionProfile.options[constants.userPropName];
}
public get password(): string {
return this._connectionProfile.options[constants.passwordPropName];
}
public get knoxport(): string {
if (!this._knoxPort) {
this.ensureHostAndPort();
}
return this._knoxPort;
}
private static getKnoxPortOrDefault(connectionProfile: IConnectionProfile): string {
let port = connectionProfile.options[constants.knoxPortPropName];
if (!port) {
port = constants.defaultKnoxPort;
}
return port;
}
}

View File

@@ -472,7 +472,7 @@ suite('Notebook Actions', function (): void {
});
test('Should inform user kernel is not supported if Run with Parameters Action is run with unsupported kernels', async function (): Promise<void> {
// Kernels that are supported (Python, PySpark, PowerShell)
// Kernels that are supported (Python, PowerShell)
const testContents: azdata.nb.INotebookContents = {
cells: [{
@@ -517,7 +517,7 @@ suite('Notebook Actions', function (): void {
});
test('Should inform user that run with parameters is not supported for untitled notebooks', async function (): Promise<void> {
// Kernels that are supported (Python, PySpark, PowerShell)
// Kernels that are supported (Python, PowerShell)
const untitledUri = URI.parse('untitled:Notebook-0');
const testContents: azdata.nb.INotebookContents = {
cells: [{

View File

@@ -95,23 +95,6 @@ suite('Cell Model', function (): void {
assert.strictEqual(cell.language, 'python');
});
test('Should set cell language to python if defined as pyspark in languageInfo', async function (): Promise<void> {
let cellData: nb.ICellContents = {
cell_type: CellTypes.Code,
source: 'print(\'1\')',
metadata: { language: 'python' },
execution_count: 1
};
let notebookModel = new NotebookModelStub({
name: 'pyspark',
version: '',
mimetype: ''
});
let cell = factory.createCell(cellData, { notebook: notebookModel, isTrusted: false });
assert.strictEqual(cell.language, 'python');
});
test('Should keep cell language as python if cell has language override', async function (): Promise<void> {
let cellData: nb.ICellContents = {
cell_type: CellTypes.Code,
@@ -121,7 +104,7 @@ suite('Cell Model', function (): void {
};
let notebookModel = new NotebookModelStub({
name: 'scala',
name: 'powershell',
version: '',
mimetype: ''
});
@@ -560,7 +543,7 @@ suite('Cell Model', function (): void {
notebook: new NotebookModelStub({
name: '',
version: '',
mimetype: 'x-scala'
mimetype: ''
}),
isTrusted: false
});

View File

@@ -6,8 +6,8 @@
import * as assert from 'assert';
import * as TypeMoq from 'typemoq';
import { nb, ServerInfo } from 'azdata';
import { getHostAndPortFromEndpoint, isStream, getProvidersForFileName, asyncForEach, clusterEndpointsProperty, getClusterEndpoints, RawEndpoint, IEndpoint, getStandardKernelsForProvider, IStandardKernelWithProvider, rewriteUrlUsingRegex } from 'sql/workbench/services/notebook/browser/models/notebookUtils';
import { nb } from 'azdata';
import { isStream, getProvidersForFileName, asyncForEach, getStandardKernelsForProvider, IStandardKernelWithProvider } from 'sql/workbench/services/notebook/browser/models/notebookUtils';
import { INotebookService, DEFAULT_NOTEBOOK_PROVIDER, SQL_NOTEBOOK_PROVIDER } from 'sql/workbench/services/notebook/browser/notebookService';
import { NotebookServiceStub } from 'sql/workbench/contrib/notebook/test/stubs';
import { tryMatchCellMagic, extractCellMagicCommandPlusArgs } from 'sql/workbench/services/notebook/browser/utils';
@@ -209,80 +209,6 @@ suite('notebookUtils', function (): void {
await asyncForEach([1, 2, 3, 4], undefined);
});
test('getClusterEndpoints Test', async function (): Promise<void> {
let serverInfo = <ServerInfo>{
options: {}
};
serverInfo.options[clusterEndpointsProperty] = undefined;
let result = getClusterEndpoints(serverInfo);
assert.deepStrictEqual(result, []);
serverInfo.options[clusterEndpointsProperty] = [];
result = getClusterEndpoints(serverInfo);
assert.deepStrictEqual(result, []);
let testEndpoint = <RawEndpoint>{
serviceName: 'testName',
description: 'testDescription',
endpoint: 'testEndpoint',
protocol: 'testProtocol',
ipAddress: 'testIpAddress',
port: 1433
};
serverInfo.options[clusterEndpointsProperty] = [testEndpoint];
result = getClusterEndpoints(serverInfo);
assert.deepStrictEqual(result, [<IEndpoint>{
serviceName: testEndpoint.serviceName,
description: testEndpoint.description,
endpoint: testEndpoint.endpoint,
protocol: testEndpoint.protocol
}]);
testEndpoint.endpoint = undefined;
result = getClusterEndpoints(serverInfo);
assert.deepStrictEqual(result, [<IEndpoint>{
serviceName: testEndpoint.serviceName,
description: testEndpoint.description,
endpoint: 'https://testIpAddress:1433',
protocol: testEndpoint.protocol
}]);
});
test('getHostAndPortFromEndpoint Test', async function (): Promise<void> {
let result = getHostAndPortFromEndpoint('https://localhost:1433');
assert.strictEqual(result.host, 'localhost');
assert.strictEqual(result.port, '1433');
result = getHostAndPortFromEndpoint('tcp://localhost,12345');
assert.strictEqual(result.host, 'localhost');
assert.strictEqual(result.port, '12345');
result = getHostAndPortFromEndpoint('tcp://localhost');
assert.strictEqual(result.host, 'localhost');
assert.strictEqual(result.port, undefined);
result = getHostAndPortFromEndpoint('localhost');
assert.strictEqual(result.host, '');
assert.strictEqual(result.port, undefined);
result = getHostAndPortFromEndpoint('localhost:1433');
assert.strictEqual(result.host, '');
assert.strictEqual(result.port, undefined);
});
test('rewriteUrlUsingRegex Test', async function (): Promise<void> {
// Give a URL that should be rewritten
let html = '<a target="_blank" href="https://sparkhead-0.sparkhead-svc:8090/proxy/application_1/“>Link</a>';
let result = rewriteUrlUsingRegex(/(https?:\/\/sparkhead.*\/proxy)(.*)/g, html, '1.1.1.1', ':999', '/gateway/default/yarn/proxy');
assert.strictEqual(result, '<a target="_blank" href="https://1.1.1.1:999/gateway/default/yarn/proxy/application_1/“>Link</a>', 'Target URL does not match after substitution');
// Give a URL that should not be rewritten
html = '<a target="_blank" href="https://storage-0-0.storage-0-svc.mssql-cluster.svc.cluster.local:8044/node/containerlogs/container_7/root“>Link</a>';
result = rewriteUrlUsingRegex(/(https?:\/\/sparkhead.*\/proxy)(.*)/g, html, '1.1.1.1', ':999', '/gateway/default/yarn/proxy');
assert.strictEqual(result, '<a target="_blank" href="https://storage-0-0.storage-0-svc.mssql-cluster.svc.cluster.local:8044/node/containerlogs/container_7/root“>Link</a>', 'Target URL should not have been edited');
});
test('EditStack test', async function (): Promise<void> {
let maxStackSize = 200;
let stack = new RichTextEditStack(maxStackSize);

View File

@@ -12,13 +12,11 @@ export class ServerInfoContextKey implements IContextKey<ServerInfo> {
static ServerInfo = new RawContextKey<ServerInfo>('serverInfo', undefined);
static ServerMajorVersion = new RawContextKey<string>('serverMajorVersion', undefined);
static IsCloud = new RawContextKey<boolean>('isCloud', undefined);
static IsBigDataCluster = new RawContextKey<boolean>('isBigDataCluster', undefined);
static EngineEdition = new RawContextKey<number>('engineEdition', undefined);
private _serverInfo: IContextKey<ServerInfo>;
private _serverMajorVersion: IContextKey<string>;
private _isCloud: IContextKey<boolean>;
private _isBigDataCluster: IContextKey<boolean>;
private _engineEdition: IContextKey<number>;
constructor(
@@ -27,7 +25,6 @@ export class ServerInfoContextKey implements IContextKey<ServerInfo> {
this._serverInfo = ServerInfoContextKey.ServerInfo.bindTo(contextKeyService);
this._serverMajorVersion = ServerInfoContextKey.ServerMajorVersion.bindTo(contextKeyService);
this._isCloud = ServerInfoContextKey.IsCloud.bindTo(contextKeyService);
this._isBigDataCluster = ServerInfoContextKey.IsBigDataCluster.bindTo(contextKeyService);
this._engineEdition = ServerInfoContextKey.EngineEdition.bindTo(contextKeyService);
}
@@ -38,7 +35,6 @@ export class ServerInfoContextKey implements IContextKey<ServerInfo> {
this._serverMajorVersion.set(`${majorVersion}`);
}
this._isCloud.set(value && value.isCloud);
this._isBigDataCluster.set(value && value.options && value.options['isBigDataCluster']);
let engineEditionId = value && value.engineEditionId;
engineEditionId ? this._engineEdition.set(engineEditionId) : this._engineEdition.set(DatabaseEngineEdition.Unknown);
}
@@ -46,7 +42,6 @@ export class ServerInfoContextKey implements IContextKey<ServerInfo> {
reset(): void {
this._serverMajorVersion.reset();
this._isCloud.reset();
this._isBigDataCluster.reset();
this._engineEdition.reset();
}

View File

@@ -1338,7 +1338,7 @@ suite('SQL ConnectionManagementService tests', () => {
serverEdition: 'test_edition',
azureVersion: 0,
osVersion: 'test_version',
options: { isBigDataCluster: 'test' },
options: {},
isCloud: true,
cpuCount: 0,
physicalMemoryInMb: 0
@@ -1373,8 +1373,6 @@ suite('SQL ConnectionManagementService tests', () => {
if (connection.providerName === 'MSSQL') {
if (serverInfo.isCloud) {
iconName = 'mssql:cloud';
} else if (serverInfo.options['isBigDataCluster']) {
iconName = 'mssql:cluster';
}
}
called = true;

View File

@@ -3,7 +3,7 @@
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { nb, ServerInfo } from 'azdata';
import { nb } from 'azdata';
import { Event, Emitter } from 'vs/base/common/event';
import { URI } from 'vs/base/common/uri';
@@ -14,14 +14,13 @@ import { CellTypes, CellType, NotebookChangeType, TextCellEditModes } from 'sql/
import { NotebookModel } from 'sql/workbench/services/notebook/browser/models/notebookModel';
import { ICellModel, IOutputChangedEvent, CellExecutionState, ICellModelOptions, ITableUpdatedEvent, CellEditModes, ICaretPosition, ICellEdit, CellEditType } from 'sql/workbench/services/notebook/browser/models/modelInterfaces';
import { IConnectionManagementService } from 'sql/platform/connection/common/connectionManagement';
import { IConnectionProfile } from 'sql/platform/connection/common/interfaces';
import { INotificationService, Severity } from 'vs/platform/notification/common/notification';
import { Schemas } from 'vs/base/common/network';
import { INotebookService } from 'sql/workbench/services/notebook/browser/notebookService';
import { getErrorMessage, onUnexpectedError } from 'vs/base/common/errors';
import { generateUuid } from 'vs/base/common/uuid';
import { HideInputTag, ParametersTag, InjectedParametersTag } from 'sql/platform/notebooks/common/outputRegistry';
import { FutureInternal, notebookConstants } from 'sql/workbench/services/notebook/browser/interfaces';
import { FutureInternal } from 'sql/workbench/services/notebook/browser/interfaces';
import { ICommandService } from 'vs/platform/commands/common/commands';
import { tryMatchCellMagic, extractCellMagicCommandPlusArgs } from 'sql/workbench/services/notebook/browser/utils';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
@@ -69,7 +68,6 @@ export class CellModel extends Disposable implements ICellModel {
private _hover: boolean;
private _executionCount: number | undefined;
private _cellUri: URI;
private _connectionManagementService: IConnectionManagementService;
private _stdInHandler: nb.MessageHandler<nb.IStdinMessage>;
private _onCellLoaded = new Emitter<string>();
private _loaded: boolean;
@@ -615,9 +613,6 @@ export class CellModel extends Disposable implements ICellModel {
this.active = true;
}
if (connectionManagementService) {
this._connectionManagementService = connectionManagementService;
}
if (this.cellType !== CellTypes.Code) {
// TODO should change hidden state to false if we add support
// for this property
@@ -890,7 +885,7 @@ export class CellModel extends Disposable implements ICellModel {
// "Optional transient data introduced in 5.1. Information not to be persisted to a notebook or other documents."
// (https://jupyter-client.readthedocs.io/en/stable/messaging.html)
delete output['transient'];
this._outputs.splice(i, 0, this.rewriteOutputUrls(output));
this._outputs.splice(i, 0, output);
this.fireOutputsChanged();
added = true;
break;
@@ -931,45 +926,13 @@ export class CellModel extends Disposable implements ICellModel {
if (output && !added) {
// deletes transient node in the serialized JSON
delete output['transient'];
this._outputs.push(this.rewriteOutputUrls(output));
this._outputs.push(output);
// Only scroll on 1st output being added
let shouldScroll = this._outputs.length === 1;
this.fireOutputsChanged(shouldScroll);
}
}
private rewriteOutputUrls(output: nb.ICellOutput): nb.ICellOutput {
const driverLog = '/gateway/default/yarn/container';
const yarnUi = '/gateway/default/yarn/proxy';
const defaultPort = ':30433';
// Only rewrite if this is coming back during execution, not when loading from disk.
// A good approximation is that the model has a future (needed for execution)
if (this.future) {
try {
let result = output as nb.IDisplayResult;
if (result && result.data && result.data['text/html']) {
let model = this._options.notebook as NotebookModel;
if (model.context) {
let gatewayEndpointInfo = this.getGatewayEndpoint(model.context);
if (gatewayEndpointInfo) {
let hostAndIp = notebookUtils.getHostAndPortFromEndpoint(gatewayEndpointInfo.endpoint);
let host = hostAndIp.host ? hostAndIp.host : model.context.serverName;
let port = hostAndIp.port ? ':' + hostAndIp.port : defaultPort;
let html = result.data['text/html'];
// BDC Spark UI Link
html = notebookUtils.rewriteUrlUsingRegex(/(https?:\/\/sparkhead.*\/proxy)(.*)/g, html, host, port, yarnUi);
// Driver link
html = notebookUtils.rewriteUrlUsingRegex(/(https?:\/\/storage.*\/containerlogs)(.*)/g, html, host, port, driverLog);
(<nb.IDisplayResult>output).data['text/html'] = html;
}
}
}
}
catch (e) { }
}
return output;
}
public setStdInHandler(handler: nb.MessageHandler<nb.IStdinMessage>): void {
this._stdInHandler = handler;
}
@@ -1129,23 +1092,6 @@ export class CellModel extends Disposable implements ICellModel {
this.cellUri = uri;
}
// Get Knox endpoint from IConnectionProfile
// TODO: this will be refactored out into the notebooks extension as a contribution point
private getGatewayEndpoint(activeConnection: IConnectionProfile): notebookUtils.IEndpoint {
let endpoint;
if (this._connectionManagementService && activeConnection && activeConnection.providerName.toLowerCase() === notebookConstants.SQL_CONNECTION_PROVIDER.toLowerCase()) {
let serverInfo: ServerInfo = this._connectionManagementService.getServerInfo(activeConnection.id);
if (serverInfo) {
let endpoints: notebookUtils.IEndpoint[] = notebookUtils.getClusterEndpoints(serverInfo);
if (endpoints && endpoints.length > 0) {
endpoint = endpoints.find(ep => ep.serviceName.toLowerCase() === notebookUtils.hadoopEndpointNameGateway);
}
}
}
return endpoint;
}
private getMultilineSource(source: string | string[]): string | string[] {
if (source === undefined) {
return [];

View File

@@ -4,13 +4,10 @@
*--------------------------------------------------------------------------------------------*/
import * as path from 'vs/base/common/path';
import { nb, ServerInfo } from 'azdata';
import { nb } from 'azdata';
import { DEFAULT_NOTEBOOK_PROVIDER, INotebookService, SQL_NOTEBOOK_PROVIDER } from 'sql/workbench/services/notebook/browser/notebookService';
import { URI } from 'vs/base/common/uri';
import { DEFAULT_NOTEBOOK_FILETYPE, NotebookLanguage } from 'sql/workbench/common/constants';
export const clusterEndpointsProperty = 'clusterEndpoints';
export const hadoopEndpointNameGateway = 'gateway';
/**
* Test whether an output is from a stream.
*/
@@ -68,13 +65,6 @@ export interface IStandardKernelWithProvider {
readonly supportedFileExtensions?: string[];
}
export interface IEndpoint {
serviceName: string;
description: string;
endpoint: string;
protocol: string;
}
export async function asyncForEach(array: any[], callback: Function): Promise<any> {
if (array && callback) {
for (let index = 0; index < array.length; index++) {
@@ -82,67 +72,3 @@ export async function asyncForEach(array: any[], callback: Function): Promise<an
}
}
}
export function getClusterEndpoints(serverInfo: ServerInfo): IEndpoint[] | undefined {
let endpoints: RawEndpoint[] = serverInfo.options[clusterEndpointsProperty];
if (!endpoints || endpoints.length === 0) { return []; }
return endpoints.map(e => {
// If endpoint is missing, we're on CTP bits. All endpoints from the CTP serverInfo should be treated as HTTPS
let endpoint = e.endpoint ? e.endpoint : `https://${e.ipAddress}:${e.port}`;
let updatedEndpoint: IEndpoint = {
serviceName: e.serviceName,
description: e.description,
endpoint: endpoint,
protocol: e.protocol
};
return updatedEndpoint;
});
}
export type HostAndIp = { host: string, port: string };
export function getHostAndPortFromEndpoint(endpoint: string): HostAndIp {
let authority = URI.parse(endpoint).authority;
let hostAndPortRegex = /^(.*)([,:](\d+))/g;
let match = hostAndPortRegex.exec(authority);
if (match) {
return {
host: match[1],
port: match[3]
};
}
return {
host: authority,
port: undefined
};
}
export function rewriteUrlUsingRegex(regex: RegExp, html: string, host: string, port: string, target: string): string {
return html.replace(regex, function (a, b, c) {
let ret = '';
if (b !== '') {
ret = 'https://' + host + port + target;
}
if (c !== '') {
ret = ret + c;
}
return ret;
});
}
export interface RawEndpoint {
serviceName: string;
description?: string;
endpoint?: string;
protocol?: string;
ipAddress?: string;
port?: number;
}
export interface IEndpoint {
serviceName: string;
description: string;
endpoint: string;
protocol: string;
}

View File

@@ -18,9 +18,6 @@ export namespace nbversion {
export enum KernelsLanguage {
SQL = 'sql',
Python = 'python',
PySpark = 'python',
SparkScala = 'scala',
SparkR = 'sparkr',
PowerShell = 'powershell',
CSharp = 'csharp',
FSharp = 'fsharp'

View File

@@ -86,11 +86,11 @@ let notebookLanguageMagicType: IJSONSchema = {
type: 'string'
},
executionTarget: {
description: localize('carbon.extension.contributes.notebook.executionTarget', "Optional execution target this magic indicates, for example Spark vs SQL"),
description: localize('carbon.extension.contributes.notebook.executionTarget', "Optional execution target this magic indicates, for example Python vs SQL"),
type: 'string'
},
kernels: {
description: localize('carbon.extension.contributes.notebook.kernels', "Optional set of kernels this is valid for, e.g. python3, pyspark, sql"),
description: localize('carbon.extension.contributes.notebook.kernels', "Optional set of kernels this is valid for, e.g. python3, sql"),
type: 'array',
items: {
type: 'string'