ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 18:46:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove all Big Data Cluster features (#21369)

Browse Source
This commit is contained in:
Cory Rivera
2022-12-07 12:28:17 -08:00
committed by GitHub
parent bb1f5bfffe
commit e2327c393a
213 changed files with 346 additions and 46800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
extensions/notebook/src/test/common/notebookUtils.test.ts
View File

@@ -203,39 +203,4 @@ describe('notebookUtils Tests', function (): void {
should(notebookEditor.document.cells[0].contents.cell_type).equal(CellTypes.Markdown);
});
});
describe('analyzeNotebook', function () {
it('creates cell when oeContext exists', async function (): Promise<void> {
await azdata.nb.showNotebookDocument(vscode.Uri.from({ scheme: 'untitled' }));
const notebookEditor = azdata.nb.activeNotebookEditor;
sinon.replaceGetter(azdata.nb, 'activeNotebookEditor', () => notebookEditor);
sinon.stub(azdata.nb, 'showNotebookDocument').returns(Promise.resolve(notebookEditor));
const oeContext: azdata.ObjectExplorerContext = {
connectionProfile: undefined,
isConnectionNode: true,
nodeInfo: {
nodePath: 'path/HDFS/path2',
errorMessage: undefined,
isLeaf: false,
label: 'fakeLabel',
metadata: undefined,
nodeStatus: undefined,
nodeSubType: undefined,
nodeType: undefined
}
};
await notebookUtils.analyzeNotebook(oeContext);
should(notebookEditor.document.cells.length).equal(1, 'One cell should exist');
should(notebookEditor.document.cells[0].contents.cell_type).equal(CellTypes.Code, 'Cell was created with incorrect type');
});
it('does not create new cell when oeContext does not exist', async function (): Promise<void> {
await azdata.nb.showNotebookDocument(vscode.Uri.from({ scheme: 'untitled' }));
const notebookEditor = azdata.nb.activeNotebookEditor;
sinon.replaceGetter(azdata.nb, 'activeNotebookEditor', () => notebookEditor);
sinon.stub(azdata.nb, 'showNotebookDocument').returns(Promise.resolve(notebookEditor));
await notebookUtils.analyzeNotebook();
should(notebookEditor.document.cells.length).equal(0, 'No cells should exist');
});
});
});

77
extensions/notebook/src/test/common/utils.test.ts
View File

@@ -10,23 +10,9 @@ import * as os from 'os';
import * as path from 'path';
import * as utils from '../../common/utils';
import { MockOutputChannel } from './stubs';
import * as azdata from 'azdata';
import { sleep } from './testUtils';
describe('Utils Tests', function () {
it('getKnoxUrl', () => {
const host = '127.0.0.1';
const port = '8080';
should(utils.getKnoxUrl(host, port)).endWith('/gateway');
});
it('getLivyUrl', () => {
const host = '127.0.0.1';
const port = '8080';
should(utils.getLivyUrl(host, port)).endWith('/gateway/default/livy/v1/');
});
it('ensureDir', async () => {
const dirPath = path.join(os.tmpdir(), uuid.v4());
await should(fs.stat(dirPath)).be.rejected();
@@ -273,63 +259,6 @@ describe('Utils Tests', function () {
});
});
describe('getClusterEndpoints', () => {
const baseServerInfo: azdata.ServerInfo = {
serverMajorVersion: -1,
serverMinorVersion: -1,
serverReleaseVersion: -1,
engineEditionId: -1,
serverVersion: '',
serverLevel: '',
serverEdition: '',
isCloud: false,
azureVersion: -1,
osVersion: '',
options: {},
cpuCount: -1,
physicalMemoryInMb: -1
};
it('empty endpoints does not error', () => {
const serverInfo = Object.assign({}, baseServerInfo);
serverInfo.options['clusterEndpoints'] = [];
should(utils.getClusterEndpoints(serverInfo).length).equal(0);
});
it('endpoints without endpoint field are created successfully', () => {
const serverInfo = Object.assign({}, baseServerInfo);
const ipAddress = 'localhost';
const port = '123';
serverInfo.options['clusterEndpoints'] = [{ ipAddress: ipAddress, port: port }];
const endpoints = utils.getClusterEndpoints(serverInfo);
should(endpoints.length).equal(1);
should(endpoints[0].endpoint).equal('https://localhost:123');
});
it('endpoints with endpoint field are created successfully', () => {
const endpoint = 'https://myActualEndpoint:8080';
const serverInfo = Object.assign({}, baseServerInfo);
serverInfo.options['clusterEndpoints'] = [{ endpoint: endpoint, ipAddress: 'localhost', port: '123' }];
const endpoints = utils.getClusterEndpoints(serverInfo);
should(endpoints.length).equal(1);
should(endpoints[0].endpoint).equal(endpoint);
});
});
describe('getHostAndPortFromEndpoint', () => {
it('valid endpoint is parsed correctly', () => {
const host = 'localhost';
const port = '123';
const hostAndIp = utils.getHostAndPortFromEndpoint(`https://${host}:${port}`);
should(hostAndIp).deepEqual({ host: host, port: port });
});
it('invalid endpoint is returned as is', () => {
const host = 'localhost';
const hostAndIp = utils.getHostAndPortFromEndpoint(`https://${host}`);
should(hostAndIp).deepEqual({ host: host, port: undefined });
});
});
describe('exists', () => {
it('runs as expected', async () => {
const filename = path.join(os.tmpdir(), `NotebookUtilsTest_${uuid.v4()}`);
@@ -345,12 +274,6 @@ describe('Utils Tests', function () {
});
});
describe('getIgnoreSslVerificationConfigSetting', () => {
it('runs as expected', async () => {
should(utils.getIgnoreSslVerificationConfigSetting()).be.true();
});
});
describe('debounce', () => {
class DebounceTest {
public fnCalled = 0;

203
extensions/notebook/src/test/model/sessionManager.test.ts
View File

@@ -5,42 +5,18 @@
import * as should from 'should';
import * as TypeMoq from 'typemoq';
import * as utils from '../../common/utils';
import * as sinon from 'sinon';
import * as os from 'os';
import * as fs from 'fs';
import * as path from 'path';
import * as bdc from 'bdc';
import * as vscode from 'vscode';
import { nb, IConnectionProfile, connection, ConnectionOptionSpecialType, ServerInfo } from 'azdata';
import { nb } from 'azdata';
import { SessionManager, Session, Kernel } from '@jupyterlab/services';
import 'mocha';
import { JupyterSessionManager, JupyterSession } from '../../jupyter/jupyterSessionManager';
import { Deferred } from '../../common/promise';
import { SessionStub, KernelStub, FutureStub } from '../common';
import { noBDCConnectionError, providerNotValidError } from '../../common/localizedConstants';
import { ExtensionContextHelper } from '../../common/extensionContextHelper';
import { AppContext } from '../../common/appContext';
import uuid = require('uuid');
class TestClusterController implements bdc.IClusterController {
getClusterConfig(): Promise<any> {
return Promise.resolve({});
}
getKnoxUsername(clusterUsername: string): Promise<string> {
return Promise.resolve('knoxUsername');
}
getEndPoints(promptConnect?: boolean): Promise<bdc.IEndPointsResponse> {
return Promise.resolve( {
response: undefined,
endPoints: []
});
}
username: string;
password: string;
}
before(async function(): Promise<void> {
before(async function (): Promise<void> {
// We have to reset the extension context here since the test runner unloads the files before running the tests
// so the static state is lost
const api = await vscode.extensions.getExtension('Microsoft.notebook').activate();
@@ -207,181 +183,6 @@ describe('Jupyter Session', function (): void {
should(options.name).equal('python');
});
it('should write configuration to config.json file', async function (): Promise<void> {
let tempDir = os.tmpdir();
let configPath = path.join(tempDir, '.sparkmagic', 'config.json');
const expectedResult = {
'kernel_python_credentials': {
'url': 'http://localhost:8088'
},
'kernel_scala_credentials': {
'url': 'http://localhost:8088'
},
'kernel_r_credentials': {
'url': 'http://localhost:8088'
},
'livy_session_startup_timeout_seconds': 100,
'logging_config': {
'version': 1,
'formatters': {
'magicsFormatter': {
'format': '%(asctime)s\t%(levelname)s\t%(message)s',
'datefmt': ''
}
},
'handlers': {
'magicsHandler': {
'class': 'hdijupyterutils.filehandler.MagicsFileHandler',
'formatter': 'magicsFormatter',
'home_path': ''
}
},
'loggers': {
'magicsLogger': {
'handlers': ['magicsHandler'],
'level': 'DEBUG',
'propagate': 0
}
}
},
'ignore_ssl_errors': true,
};
expectedResult.logging_config.handlers.magicsHandler.home_path = path.join(tempDir, '.sparkmagic');
sinon.stub(utils, 'getUserHome').returns(tempDir);
await session.configureKernel();
let result = await fs.promises.readFile(configPath, 'utf-8');
should(JSON.parse(result) === expectedResult);
});
it('should configure connection correctly for MSSQL and SqlLogin auth type', async function (): Promise<void> {
const isLinux = os.platform() === 'linux';
if (!isLinux) {
let connectionProfile: IConnectionProfile = {
authenticationType: '',
connectionName: '',
databaseName: '',
id: 'id',
providerName: 'MSSQL',
options: {
authenticationType: connection.AuthenticationType.SqlLogin,
},
password: '',
savePassword: false,
saveProfile: false,
serverName: '',
userName: ''
};
let futureMock = TypeMoq.Mock.ofType(FutureStub);
let kernelMock = TypeMoq.Mock.ofType(KernelStub);
kernelMock.setup(k => k.name).returns(() => 'spark');
kernelMock.setup(m => m.requestExecute(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => futureMock.object);
mockJupyterSession.setup(s => s.kernel).returns(() => kernelMock.object);
let creds = { [ConnectionOptionSpecialType.password]: 'password' };
sinon.stub(connection, 'getCredentials').returns(Promise.resolve(creds));
// Set up connection info to big data cluster
const mockServerInfo: ServerInfo = {
serverMajorVersion: 0,
serverMinorVersion: 0,
serverReleaseVersion: 0,
engineEditionId: 0,
serverVersion: '',
serverLevel: '',
serverEdition: '',
isCloud: false,
azureVersion: 0,
osVersion: '',
cpuCount: 0,
physicalMemoryInMb: -1,
options: {
isBigDataCluster: true
}
};
const mockGatewayEndpoint: bdc.IEndpointModel = {
name: 'gateway',
description: '',
endpoint: '',
protocol: '',
};
const mockControllerEndpoint: bdc.IEndpointModel = {
name: 'controller',
description: '',
endpoint: '',
protocol: '',
};
const mockHostAndIp: utils.HostAndIp = {
host: '127.0.0.1',
port: '1337'
};
const mockClustercontroller = new TestClusterController();
mockClustercontroller.username = 'admin';
mockClustercontroller.password = uuid.v4();
let mockBdcExtension: TypeMoq.IMock<bdc.IExtension> = TypeMoq.Mock.ofType<bdc.IExtension>();
let mockExtension: TypeMoq.IMock<vscode.Extension<any>> = TypeMoq.Mock.ofType<vscode.Extension<any>>();
mockBdcExtension.setup(m => m.getClusterController(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => mockClustercontroller);
mockBdcExtension.setup((m: any) => m.then).returns(() => mockBdcExtension);
mockExtension.setup(m => m.activate()).returns(() => Promise.resolve(mockBdcExtension.object));
mockExtension.setup((m: any) => m.then).returns(() => mockExtension);
sinon.stub(vscode.extensions, 'getExtension').returns(mockExtension.object);
sinon.stub(connection, 'getServerInfo').returns(Promise.resolve(mockServerInfo));
sinon.stub(utils, 'getClusterEndpoints').returns([mockGatewayEndpoint, mockControllerEndpoint]);
sinon.stub(utils, 'getHostAndPortFromEndpoint').returns(mockHostAndIp);
await session.configureConnection(connectionProfile);
should(connectionProfile.options['host']).equal(mockHostAndIp.host);
should(connectionProfile.options['knoxport']).equal(mockHostAndIp.port);
}
});
it('configure connection should throw error if there is no connection to big data cluster', async function (): Promise<void> {
let connectionProfile: IConnectionProfile = {
authenticationType: '',
connectionName: '',
databaseName: '',
id: 'id',
providerName: 'MSSQL',
options: {
authenticationType: connection.AuthenticationType.SqlLogin,
},
password: '',
savePassword: false,
saveProfile: false,
serverName: '',
userName: ''
};
let futureMock = TypeMoq.Mock.ofType(FutureStub);
let kernelMock = TypeMoq.Mock.ofType(KernelStub);
kernelMock.setup(k => k.name).returns(() => 'spark');
kernelMock.setup(m => m.requestExecute(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => futureMock.object);
mockJupyterSession.setup(s => s.kernel).returns(() => kernelMock.object);
let credentials = { [ConnectionOptionSpecialType.password]: 'password' };
sinon.stub(connection, 'getCredentials').returns(Promise.resolve(credentials));
await should(session.configureConnection(connectionProfile)).be.rejectedWith(noBDCConnectionError);
});
it('configure connection should throw error if provider is not MSSQL for spark kernel', async function (): Promise<void> {
let connectionProfile: IConnectionProfile = {
authenticationType: '',
connectionName: '',
databaseName: '',
id: 'id',
providerName: 'provider',
options: {
authenticationType: connection.AuthenticationType.SqlLogin,
},
password: '',
savePassword: false,
saveProfile: false,
serverName: '',
userName: ''
};
let futureMock = TypeMoq.Mock.ofType(FutureStub);
let kernelMock = TypeMoq.Mock.ofType(KernelStub);
kernelMock.setup(k => k.name).returns(() => 'spark');
kernelMock.setup(m => m.requestExecute(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => futureMock.object);
mockJupyterSession.setup(s => s.kernel).returns(() => kernelMock.object);
await should(session.configureConnection(connectionProfile)).be.rejectedWith(providerNotValidError);
});
it('should set environment variables correctly', function (): void {
let futureMock = TypeMoq.Mock.ofType(FutureStub);
let kernelMock = TypeMoq.Mock.ofType(KernelStub);

15
extensions/notebook/src/test/python/jupyterInstallation.test.ts
View File

@@ -11,8 +11,8 @@ import * as uuid from 'uuid';
import * as fs from 'fs-extra';
import * as request from 'request';
import * as utils from '../../common/utils';
import { requiredJupyterPkg, JupyterServerInstallation, requiredPowershellPkg, PythonInstallSettings, PythonPkgDetails, requiredSparkPackages } from '../../jupyter/jupyterServerInstallation';
import { powershellDisplayName, pysparkDisplayName, python3DisplayName, sparkRDisplayName, sparkScalaDisplayName, winPlatform } from '../../common/constants';
import { requiredJupyterPkg, JupyterServerInstallation, requiredPowershellPkg, PythonInstallSettings, PythonPkgDetails } from '../../jupyter/jupyterServerInstallation';
import { powershellDisplayName, python3DisplayName, winPlatform } from '../../common/constants';
describe('Jupyter Server Installation', function () {
let outputChannelStub: TypeMoq.IMock<vscode.OutputChannel>;
@@ -234,17 +234,6 @@ describe('Jupyter Server Installation', function () {
should(packages).be.deepEqual([requiredJupyterPkg, requiredPowershellPkg]);
});
it('Get required packages test - Spark kernels', async function () {
let packages = installation.getRequiredPackagesForKernel(pysparkDisplayName);
should(packages).be.deepEqual(requiredSparkPackages, 'Unexpected packages for PySpark kernel.');
packages = installation.getRequiredPackagesForKernel(sparkScalaDisplayName);
should(packages).be.deepEqual(requiredSparkPackages, 'Unexpected packages for Spark Scala kernel.');
packages = installation.getRequiredPackagesForKernel(sparkRDisplayName);
should(packages).be.deepEqual(requiredSparkPackages, 'Unexpected packages for Spark R kernel.');
});
it('Install python test - Run install while Python is already running', async function () {
// Should reject overwriting an existing python install if running on Windows and python is currently running.
if (process.platform === winPlatform) {