Split up NotebookProvider into separate providers for handling file serialization and cell execution. (#17176)

This commit is contained in:
Cory Rivera
2021-09-29 16:15:28 -07:00
committed by GitHub
parent dfc2635aa7
commit 14904bb671
51 changed files with 1426 additions and 971 deletions

View File

@@ -7,14 +7,14 @@ import { nb } from 'azdata';
import * as vscode from 'vscode';
import { charCountToJsCountDiff, jsIndexToCharIndex } from './text';
import { JupyterNotebookProvider } from '../jupyter/jupyterNotebookProvider';
import { JupyterExecuteProvider } from '../jupyter/jupyterExecuteProvider';
import { JupyterSessionManager } from '../jupyter/jupyterSessionManager';
const timeoutMilliseconds = 3000;
export class NotebookCompletionItemProvider implements vscode.CompletionItemProvider {
constructor(private _notebookProvider: JupyterNotebookProvider) {
constructor(private _notebookProvider: JupyterExecuteProvider) {
}
public provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken, context: vscode.CompletionContext)
@@ -45,9 +45,9 @@ export class NotebookCompletionItemProvider implements vscode.CompletionItemProv
private async tryFindKernelForDocument(document: vscode.TextDocument, info: INewIntellisenseInfo): Promise<nb.IKernel> {
try {
let notebookManager = await this._notebookProvider.getNotebookManager(document.uri);
if (notebookManager) {
let sessionManager: JupyterSessionManager = <JupyterSessionManager>(notebookManager.sessionManager);
let executeManager = await this._notebookProvider.getExecuteManager(document.uri);
if (executeManager) {
let sessionManager: JupyterSessionManager = <JupyterSessionManager>(executeManager.sessionManager);
let sessions = sessionManager.listRunning();
if (sessions && sessions.length > 0) {
let session = sessions.find(session => session.path === info.notebook.uri.path);

View File

@@ -19,7 +19,6 @@ import { IPrompter, IQuestion, QuestionTypes } from '../prompts/question';
import { AppContext } from '../common/appContext';
import { LocalJupyterServerManager, ServerInstanceFactory } from './jupyterServerManager';
import { NotebookCompletionItemProvider } from '../intellisense/completionItemProvider';
import { JupyterNotebookProvider } from './jupyterNotebookProvider';
import { ConfigurePythonWizard } from '../dialog/configurePython/configurePythonWizard';
import CodeAdapter from '../prompts/adapter';
import { ManagePackagesDialog } from '../dialog/managePackages/managePackagesDialog';
@@ -28,6 +27,7 @@ import { LocalPipPackageManageProvider } from './localPipPackageManageProvider';
import { LocalCondaPackageManageProvider } from './localCondaPackageManageProvider';
import { ManagePackagesDialogModel, ManagePackageDialogOptions } from '../dialog/managePackages/managePackagesDialogModel';
import { PyPiClient } from './pypiClient';
import { JupyterExecuteProvider } from './jupyterExecuteProvider';
let untitledCounter = 0;
@@ -37,7 +37,7 @@ export class JupyterController {
private _packageManageProviders = new Map<string, IPackageManageProvider>();
private prompter: IPrompter;
private _notebookProvider: JupyterNotebookProvider;
private _executeProvider: JupyterExecuteProvider;
constructor(private appContext: AppContext) {
this.prompter = new CodeAdapter();
@@ -47,8 +47,8 @@ export class JupyterController {
return this.appContext && this.appContext.extensionContext;
}
public get notebookProvider(): JupyterNotebookProvider {
return this._notebookProvider;
public get executeProvider(): JupyterExecuteProvider {
return this._executeProvider;
}
// PUBLIC METHODS //////////////////////////////////////////////////////
@@ -79,21 +79,19 @@ export class JupyterController {
let supportedFileFilter: vscode.DocumentFilter[] = [
{ scheme: 'untitled', language: '*' }
];
this.registerNotebookProvider();
this.extensionContext.subscriptions.push(vscode.languages.registerCompletionItemProvider(supportedFileFilter, new NotebookCompletionItemProvider(this._notebookProvider), '.'));
this.registerDefaultPackageManageProviders();
return true;
}
private registerNotebookProvider(): void {
this._notebookProvider = new JupyterNotebookProvider((documentUri: vscode.Uri) => new LocalJupyterServerManager({
this._executeProvider = new JupyterExecuteProvider((documentUri: vscode.Uri) => new LocalJupyterServerManager({
documentPath: documentUri.fsPath,
jupyterInstallation: this._jupyterInstallation,
extensionContext: this.extensionContext,
factory: this._serverInstanceFactory
}));
azdata.nb.registerNotebookProvider(this._notebookProvider);
azdata.nb.registerExecuteProvider(this._executeProvider);
this.extensionContext.subscriptions.push(vscode.languages.registerCompletionItemProvider(supportedFileFilter, new NotebookCompletionItemProvider(this._executeProvider), '.'));
this.registerDefaultPackageManageProviders();
return true;
}
private saveProfileAndCreateNotebook(profile: azdata.IConnectionProfile): Promise<void> {

View File

@@ -10,7 +10,7 @@ import { ServerConnection, SessionManager } from '@jupyterlab/services';
import { JupyterSessionManager } from './jupyterSessionManager';
import { LocalJupyterServerManager } from './jupyterServerManager';
export class JupyterNotebookManager implements nb.NotebookManager, vscode.Disposable {
export class JupyterExecuteManager implements nb.ExecuteManager, vscode.Disposable {
protected _serverSettings: ServerConnection.ISettings;
private _sessionManager: JupyterSessionManager;
@@ -21,9 +21,6 @@ export class JupyterNotebookManager implements nb.NotebookManager, vscode.Dispos
this._sessionManager.installation = this._serverManager.instanceOptions.install;
});
}
public get contentManager(): nb.ContentManager {
return undefined;
}
public get sessionManager(): nb.SessionManager {
return this._sessionManager;

View File

@@ -11,41 +11,41 @@ const localize = nls.loadMessageBundle();
import * as constants from '../common/constants';
import * as utils from '../common/utils';
import { JupyterNotebookManager } from './jupyterNotebookManager';
import { JupyterExecuteManager } from './jupyterExecuteManager';
import { LocalJupyterServerManager } from './jupyterServerManager';
import { JupyterSessionManager } from './jupyterSessionManager';
export type ServerManagerFactory = (documentUri: vscode.Uri) => LocalJupyterServerManager;
export class JupyterNotebookProvider implements nb.NotebookProvider {
export class JupyterExecuteProvider implements nb.NotebookExecuteProvider {
readonly providerId: string = constants.jupyterNotebookProviderId;
private managerTracker = new Map<string, JupyterNotebookManager>();
private executeManagerTracker = new Map<string, JupyterExecuteManager>();
constructor(private createServerManager: ServerManagerFactory) {
}
public getNotebookManager(notebookUri: vscode.Uri): Thenable<nb.NotebookManager> {
public getExecuteManager(notebookUri: vscode.Uri): Thenable<nb.ExecuteManager> {
if (!notebookUri) {
return Promise.reject(localize('errNotebookUriMissing', "A notebook path is required"));
}
return Promise.resolve(this.doGetNotebookManager(notebookUri));
return Promise.resolve(this.doGetExecuteManager(notebookUri));
}
public get notebookManagerCount(): number {
return this.managerTracker.size;
public get executeManagerCount(): number {
return this.executeManagerTracker.size;
}
private doGetNotebookManager(notebookUri: vscode.Uri): nb.NotebookManager {
private doGetExecuteManager(notebookUri: vscode.Uri): nb.ExecuteManager {
let baseFolder = this.transformToBaseFolder(notebookUri?.fsPath?.toString());
let manager = this.managerTracker.get(baseFolder);
let manager = this.executeManagerTracker.get(baseFolder);
if (!manager) {
let baseFolderUri = vscode.Uri.file(baseFolder);
if (!baseFolderUri) {
baseFolderUri = notebookUri;
}
let serverManager = this.createServerManager(baseFolderUri);
manager = new JupyterNotebookManager(serverManager);
this.managerTracker.set(baseFolder, manager);
manager = new JupyterExecuteManager(serverManager);
this.executeManagerTracker.set(baseFolder, manager);
}
return manager;
}
@@ -56,7 +56,7 @@ export class JupyterNotebookProvider implements nb.NotebookProvider {
return;
}
let baseFolder = this.transformToBaseFolder(notebookUri.fsPath.toString());
let manager = this.managerTracker.get(baseFolder);
let manager = this.executeManagerTracker.get(baseFolder);
if (manager) {
let sessionManager = (manager.sessionManager as JupyterSessionManager);
let session = sessionManager.listRunning().find(e => e.path === notebookUri.fsPath);
@@ -70,7 +70,7 @@ export class JupyterNotebookProvider implements nb.NotebookProvider {
const timeoutInMs = timeoutInMinutes * 60 * 1000;
setTimeout(() => {
if (sessionManager.listRunning().length === 0) {
this.managerTracker.delete(baseFolder);
this.executeManagerTracker.delete(baseFolder);
manager.dispose();
}
}, timeoutInMs);
@@ -79,10 +79,6 @@ export class JupyterNotebookProvider implements nb.NotebookProvider {
}
}
public get standardKernels(): nb.IStandardKernel[] {
return [];
}
private transformToBaseFolder(notebookPath: string): string {
let parsedPath = path.parse(notebookPath);
let userHome = utils.getUserHome();

View File

@@ -58,7 +58,7 @@ const configBase = {
}
};
export class JupyterSessionManager implements nb.SessionManager {
export class JupyterSessionManager implements nb.SessionManager, vscode.Disposable {
private _ready: Deferred<void>;
private _isReady: boolean;
private _sessionManager: Session.IManager;

View File

@@ -1,41 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { nb } from 'azdata';
import * as vscode from 'vscode';
import { Contents } from '@jupyterlab/services';
export class RemoteContentManager implements nb.ContentManager {
constructor(private contents: Contents.IManager) {
}
public getNotebookContents(notebookUri: vscode.Uri): Thenable<nb.INotebookContents> {
return this.getNotebookContentsAsync(notebookUri.fsPath);
}
private async getNotebookContentsAsync(path: string): Promise<nb.INotebookContents> {
if (!path) {
return undefined;
}
// Note: intentionally letting caller handle exceptions
let contentsModel = await this.contents.get(path);
if (!contentsModel) {
return undefined;
}
return <nb.INotebookContents>contentsModel.content;
}
public async save(notebookUri: vscode.Uri, notebook: nb.INotebookContents): Promise<nb.INotebookContents> {
let path = notebookUri.fsPath;
await this.contents.save(path, {
path: path,
content: notebook,
type: 'notebook',
format: 'json'
});
return notebook;
}
}

View File

@@ -9,19 +9,19 @@ import * as vscode from 'vscode';
import * as TypeMoq from 'typemoq';
import { NotebookCompletionItemProvider } from '../../intellisense/completionItemProvider';
import { JupyterNotebookProvider } from '../../jupyter/jupyterNotebookProvider';
import { NotebookUtils } from '../../common/notebookUtils';
import { JupyterNotebookManager } from '../../jupyter/jupyterNotebookManager';
import { JupyterExecuteManager } from '../../jupyter/jupyterExecuteManager';
import { JupyterSessionManager, JupyterSession } from '../../jupyter/jupyterSessionManager';
import { LocalJupyterServerManager } from '../../jupyter/jupyterServerManager';
import { TestKernel } from '../common';
import { sleep } from '../common/testUtils';
import { JupyterExecuteProvider } from '../../jupyter/jupyterExecuteProvider';
describe('Completion Item Provider', function () {
let completionItemProvider: NotebookCompletionItemProvider;
let notebookProviderMock: TypeMoq.IMock<JupyterNotebookProvider>;
let executeProviderMock: TypeMoq.IMock<JupyterExecuteProvider>;
let notebookUtils: NotebookUtils;
let notebookManager: JupyterNotebookManager;
let notebookManager: JupyterExecuteManager;
let mockSessionManager: TypeMoq.IMock<JupyterSessionManager>;
let mockServerManager: TypeMoq.IMock<LocalJupyterServerManager>;
let mockJupyterSession: TypeMoq.IMock<JupyterSession>;
@@ -45,10 +45,10 @@ describe('Completion Item Provider', function () {
mockSessionManager = TypeMoq.Mock.ofType<JupyterSessionManager>();
mockJupyterSession = TypeMoq.Mock.ofType<JupyterSession>();
kernel = new TestKernel(true, true);
notebookManager = new JupyterNotebookManager(mockServerManager.object, mockSessionManager.object);
notebookProviderMock = TypeMoq.Mock.ofType<JupyterNotebookProvider>();
notebookProviderMock.setup(n => n.getNotebookManager(TypeMoq.It.isAny())).returns(() => Promise.resolve(notebookManager));
completionItemProvider = new NotebookCompletionItemProvider(notebookProviderMock.object);
notebookManager = new JupyterExecuteManager(mockServerManager.object, mockSessionManager.object);
executeProviderMock = TypeMoq.Mock.ofType<JupyterExecuteProvider>();
executeProviderMock.setup(n => n.getExecuteManager(TypeMoq.It.isAny())).returns(() => Promise.resolve(notebookManager));
completionItemProvider = new NotebookCompletionItemProvider(executeProviderMock.object);
});
it('should not return items when undefined passed in for every parameter', async () => {

View File

@@ -1,94 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as should from 'should';
import * as TypeMoq from 'typemoq';
import * as path from 'path';
import { ContentsManager, Contents } from '@jupyterlab/services';
import { nb } from 'azdata';
import 'mocha';
import { INotebook, CellTypes } from '../../contracts/content';
import { RemoteContentManager } from '../../jupyter/remoteContentManager';
import * as testUtils from '../common/testUtils';
let expectedNotebookContent: INotebook = {
cells: [{
cell_type: CellTypes.Code,
source: 'insert into t1 values (c1, c2)',
metadata: { language: 'python' },
execution_count: 1
}],
metadata: {
kernelspec: {
name: 'mssql',
language: 'sql'
}
},
nbformat: 5,
nbformat_minor: 0
};
function verifyMatchesExpectedNotebook(notebook: nb.INotebookContents): void {
should(notebook.cells).have.length(1, 'Expected 1 cell');
should(notebook.cells[0].cell_type).equal(CellTypes.Code);
should(notebook.cells[0].source).equal(expectedNotebookContent.cells[0].source);
should(notebook.metadata.kernelspec.name).equal(expectedNotebookContent.metadata.kernelspec.name);
should(notebook.nbformat).equal(expectedNotebookContent.nbformat);
should(notebook.nbformat_minor).equal(expectedNotebookContent.nbformat_minor);
}
describe('Remote Content Manager', function (): void {
let mockJupyterManager = TypeMoq.Mock.ofType(ContentsManager);
let contentManager = new RemoteContentManager(mockJupyterManager.object);
// TODO re-enable when we bring in usage of remote content managers / binders
// it('Should return undefined if path is undefined', async function(): Promise<void> {
// let content = await contentManager.getNotebookContents(undefined);
// should(content).be.undefined();
// // tslint:disable-next-line:no-null-keyword
// content = await contentManager.getNotebookContents(null);
// should(content).be.undefined();
// content = await contentManager.getNotebookContents(vscode.Uri.file(''));
// should(content).be.undefined();
// });
it('Should throw if API call throws', async function (): Promise<void> {
let exception = new Error('Path was wrong');
mockJupyterManager.setup(c => c.get(TypeMoq.It.isAny(), TypeMoq.It.isAny())).throws(exception);
await testUtils.assertThrowsAsync(async () => await contentManager.getNotebookContents(vscode.Uri.file('/path/doesnot/exist.ipynb')), undefined);
});
it('Should return notebook contents parsed as INotebook when valid notebook file parsed', async function (): Promise<void> {
// Given a valid request to the notebook server
let remotePath = '/remote/path/that/exists.ipynb';
let contentsModel: Contents.IModel = {
name: path.basename(remotePath),
content: expectedNotebookContent,
path: remotePath,
type: 'notebook',
writable: false,
created: undefined,
last_modified: undefined,
mimetype: 'json',
format: 'json'
};
mockJupyterManager.setup(c => c.get(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve(contentsModel));
// when I read the content
let notebook = await contentManager.getNotebookContents(vscode.Uri.file(remotePath));
// then I expect notebook format to match
verifyMatchesExpectedNotebook(notebook);
});
it('Should return undefined if service does not return anything', async function (): Promise<void> {
// Given a valid request to the notebook server
let remotePath = '/remote/path/that/does/not/exist.ipynb';
mockJupyterManager.setup(c => c.get(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve(undefined));
// when I read the content
let notebook = await contentManager.getNotebookContents(vscode.Uri.file(remotePath));
// then I expect notebook format to match
should(notebook).be.undefined();
});
});

View File

@@ -14,10 +14,10 @@ import { JupyterServerInstallation } from '../../jupyter/jupyterServerInstallati
import { Deferred } from '../../common/promise';
import { MockExtensionContext } from '../common/stubs';
import { JupyterSessionManager } from '../../jupyter/jupyterSessionManager';
import { JupyterNotebookManager } from '../../jupyter/jupyterNotebookManager';
import { JupyterExecuteManager } from '../../jupyter/jupyterExecuteManager';
import { initInstallAndInstance } from './serverManager.test';
describe('Jupyter Notebook Manager', function (): void {
describe('Jupyter Execute Manager', function (): void {
const pythonKernelSpec: azdata.nb.IKernelSpec = {
name: 'python3',
display_name: 'Python 3'
@@ -25,7 +25,7 @@ describe('Jupyter Notebook Manager', function (): void {
let expectedPath = 'my/notebook.ipynb';
let serverManager: LocalJupyterServerManager;
let sessionManager: JupyterSessionManager;
let notebookManager: JupyterNotebookManager;
let executeManager: JupyterExecuteManager;
let deferredInstall: Deferred<void>;
let mockExtensionContext: MockExtensionContext;
let mockFactory: TypeMoq.IMock<ServerInstanceFactory>;
@@ -48,48 +48,37 @@ describe('Jupyter Notebook Manager', function (): void {
serverManager = new LocalJupyterServerManager(serverManagerOptions);
sessionManager = new JupyterSessionManager();
notebookManager = new JupyterNotebookManager(serverManager, sessionManager);
executeManager = new JupyterExecuteManager(serverManager, sessionManager);
});
it('Server settings should be set', async function (): Promise<void> {
should(notebookManager.serverSettings).be.undefined();
should(executeManager.serverSettings).be.undefined();
let expectedUri = vscode.Uri.parse('http://localhost:1234?token=abcdefghijk');
initInstallAndInstance(expectedUri, mockFactory);
deferredInstall.resolve();
// When I start the server
await serverManager.startServer(pythonKernelSpec);
should(notebookManager.serverSettings.baseUrl).equal('http://localhost:1234', 'Server settings did not match expected value');
should(executeManager.serverSettings.baseUrl).equal('http://localhost:1234', 'Server settings did not match expected value');
});
it('Session Manager should exist', async function (): Promise<void> {
should(notebookManager.sessionManager).deepEqual(sessionManager);
should(executeManager.sessionManager).deepEqual(sessionManager);
});
it('Server Manager should exist', async function (): Promise<void> {
should(notebookManager.serverManager).deepEqual(serverManager);
});
it('Content manager should always be undefined', async function (): Promise<void> {
should(notebookManager.contentManager).be.undefined();
let expectedUri = vscode.Uri.parse('http://localhost:1234?token=abcdefghijk');
initInstallAndInstance(expectedUri, mockFactory);
deferredInstall.resolve();
// When I start the server
await serverManager.startServer(pythonKernelSpec);
should(notebookManager.contentManager).be.undefined();
should(executeManager.serverManager).deepEqual(serverManager);
});
it('Session and server managers should be shutdown/stopped on dispose', async function(): Promise<void> {
let sessionManager = TypeMoq.Mock.ofType<JupyterSessionManager>();
let serverManager = TypeMoq.Mock.ofType<LocalJupyterServerManager>();
notebookManager = new JupyterNotebookManager(serverManager.object, sessionManager.object);
executeManager = new JupyterExecuteManager(serverManager.object, sessionManager.object);
sessionManager.setup(s => s.shutdownAll()).returns(() => new Promise((resolve) => resolve()));
serverManager.setup(s => s.stopServer()).returns(() => new Promise((resolve) => resolve()));
// After I dispose the notebook manager
notebookManager.dispose();
// After I dispose the execute manager
executeManager.dispose();
// Session and server managers should be shutdown/stopped
sessionManager.verify((s) => s.shutdownAll(), TypeMoq.Times.once());

View File

@@ -76,24 +76,23 @@ describe('Jupyter Controller', function () {
it('Returns expected values from notebook provider', async () => {
await controller.activate();
should(controller.notebookProvider.standardKernels).deepEqual([], 'Notebook provider standard kernels should return empty array');
should(controller.notebookProvider.providerId).equal('jupyter', 'Notebook provider should be jupyter');
await should(controller.notebookProvider.getNotebookManager(undefined)).be.rejected();
should(controller.notebookProvider.notebookManagerCount).equal(0);
controller.notebookProvider.handleNotebookClosed(undefined);
should(controller.executeProvider.providerId).equal('jupyter', 'Notebook provider should be jupyter');
await should(controller.executeProvider.getExecuteManager(undefined)).be.rejected();
should(controller.executeProvider.executeManagerCount).equal(0);
controller.executeProvider.handleNotebookClosed(undefined);
});
it('Returns notebook manager for real notebook editor', async () => {
it('Returns execute manager for real notebook editor', async () => {
await controller.activate();
let notebookUtils = new NotebookUtils();
const notebookEditor = await notebookUtils.newNotebook(undefined);
let notebookManager = await controller.notebookProvider.getNotebookManager(notebookEditor.document.uri);
should(controller.notebookProvider.notebookManagerCount).equal(1);
let notebookManager = await controller.executeProvider.getExecuteManager(notebookEditor.document.uri);
should(controller.executeProvider.executeManagerCount).equal(1);
// Session manager should not be immediately ready
should(notebookManager.sessionManager.isReady).equal(false);
// Session manager should not immediately have specs
should(notebookManager.sessionManager.specs).equal(undefined);
controller.notebookProvider.handleNotebookClosed(notebookEditor.document.uri);
controller.executeProvider.handleNotebookClosed(notebookEditor.document.uri);
});
});