ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enable VS Code notebooks with a built-in SQL kernel. (#21995)

Browse Source
This commit is contained in:
Cory Rivera
2023-02-23 16:22:46 -08:00
committed by GitHub
parent 290687a207
commit f53119c2a6
66 changed files with 4962 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
extensions/ipynb/src/cellIdService.ts Normal file
View File

@@ -0,0 +1,77 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { ExtensionContext, NotebookDocument, NotebookDocumentChangeEvent, workspace, WorkspaceEdit } from 'vscode';
import { v4 as uuid } from 'uuid';
import { getCellMetadata } from './serializers';
import { CellMetadata } from './common';
import { getNotebookMetadata } from './notebookSerializer';
import * as nbformat from '@jupyterlab/nbformat';
/**
* Ensure all new cells in notebooks with nbformat >= 4.5 have an id.
* Details of the spec can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#
*/
export function ensureAllNewCellsHaveCellIds(context: ExtensionContext) {
workspace.onDidChangeNotebookDocument(onDidChangeNotebookCells, undefined, context.subscriptions);
}
function onDidChangeNotebookCells(e: NotebookDocumentChangeEvent) {
const nbMetadata = getNotebookMetadata(e.notebook);
if (!isCellIdRequired(nbMetadata)) {
return;
}
e.contentChanges.forEach(change => {
change.addedCells.forEach(cell => {
const cellMetadata = getCellMetadata(cell);
if (cellMetadata?.id) {
return;
}
const id = generateCellId(e.notebook);
const edit = new WorkspaceEdit();
// Don't edit the metadata directly, always get a clone (prevents accidental singletons and directly editing the objects).
const updatedMetadata: CellMetadata = { ...JSON.parse(JSON.stringify(cellMetadata || {})) };
updatedMetadata.id = id;
edit.replaceNotebookCellMetadata(cell.notebook.uri, cell.index, { ...(cell.metadata), custom: updatedMetadata });
workspace.applyEdit(edit);
});
});
}
/**
* Cell ids are required in notebooks only in notebooks with nbformat >= 4.5
*/
function isCellIdRequired(metadata: Pick<Partial<nbformat.INotebookContent>, 'nbformat' | 'nbformat_minor'>) {
if ((metadata.nbformat || 0) >= 5) {
return true;
}
if ((metadata.nbformat || 0) === 4 && (metadata.nbformat_minor || 0) >= 5) {
return true;
}
return false;
}
function generateCellId(notebook: NotebookDocument) {
while (true) {
// Details of the id can be found here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#adding-an-id-field,
// & here https://jupyter.org/enhancement-proposals/62-cell-id/cell-id.html#updating-older-formats
const id = uuid().replace(/-/g, '').substring(0, 8);
let duplicate = false;
for (let index = 0; index < notebook.cellCount; index++) {
const cell = notebook.cellAt(index);
const existingId = getCellMetadata(cell)?.id;
if (!existingId) {
continue;
}
if (existingId === id) {
duplicate = true;
break;
}
}
if (!duplicate) {
return id;
}
}
}

62
extensions/ipynb/src/common.ts Normal file
View File

@@ -0,0 +1,62 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as nbformat from '@jupyterlab/nbformat';
/**
* Metadata we store in VS Code cell output items.
* This contains the original metadata from the Jupyter outputs.
*/
export interface CellOutputMetadata {
/**
* Cell output metadata.
*/
metadata?: any;
/**
* Transient data from Jupyter.
*/
transient?: {
/**
* This is used for updating the output in other cells.
* We don't know of other properties, but this is definitely used.
*/
display_id?: string;
} & any;
/**
* Original cell output type
*/
outputType: nbformat.OutputType | string;
executionCount?: nbformat.IExecuteResult['ExecutionCount'];
/**
* Whether the original Mime data is JSON or not.
* This properly only exists in metadata for NotebookCellOutputItems
* (this is something we have added)
*/
__isJson?: boolean;
}
/**
* Metadata we store in VS Code cells.
* This contains the original metadata from the Jupyuter cells.
*/
export interface CellMetadata {
/**
* Cell id for notebooks created with the new 4.5 version of nbformat.
*/
id?: string;
/**
* Stores attachments for cells.
*/
attachments?: nbformat.IAttachments;
/**
* Stores cell metadata.
*/
metadata?: Partial<nbformat.ICellMetadata>;
}

6
extensions/ipynb/src/constants.ts Normal file
View File

@@ -0,0 +1,6 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
export const defaultNotebookFormat = { major: 4, minor: 2 };

359
extensions/ipynb/src/deserializers.ts Normal file
View File

@@ -0,0 +1,359 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as nbformat from '@jupyterlab/nbformat';
import { extensions, NotebookCellData, NotebookCellExecutionSummary, NotebookCellKind, NotebookCellOutput, NotebookCellOutputItem, NotebookData } from 'vscode';
import { CellMetadata, CellOutputMetadata } from './common';
const jupyterLanguageToMonacoLanguageMapping = new Map([
['c#', 'csharp'],
['f#', 'fsharp'],
['q#', 'qsharp'],
['c++11', 'c++'],
['c++12', 'c++'],
['c++14', 'c++']
]);
export function getPreferredLanguage(metadata?: nbformat.INotebookMetadata) {
const jupyterLanguage =
metadata?.language_info?.name ||
(metadata?.kernelspec as any)?.language;
// Default to python language only if the Python extension is installed.
const defaultLanguage =
extensions.getExtension('ms-python.python')
? 'python'
: (extensions.getExtension('ms-dotnettools.dotnet-interactive-vscode') ? 'csharp' : 'python');
// Note, whatever language is returned here, when the user selects a kernel, the cells (of blank documents) get updated based on that kernel selection.
return translateKernelLanguageToMonaco(jupyterLanguage || defaultLanguage);
}
function translateKernelLanguageToMonaco(language: string): string {
language = language.toLowerCase();
if (language.length === 2 && language.endsWith('#')) {
return `${language.substring(0, 1)}sharp`;
}
return jupyterLanguageToMonacoLanguageMapping.get(language) || language;
}
const orderOfMimeTypes = [
'application/vnd.*',
'application/vdom.*',
'application/geo+json',
'application/x-nteract-model-debug+json',
'text/html',
'application/javascript',
'image/gif',
'text/latex',
'text/markdown',
'image/png',
'image/svg+xml',
'image/jpeg',
'application/json',
'text/plain'
];
function isEmptyVendoredMimeType(outputItem: NotebookCellOutputItem) {
if (outputItem.mime.startsWith('application/vnd.')) {
try {
return outputItem.data.byteLength === 0 || Buffer.from(outputItem.data).toString().length === 0;
} catch { }
}
return false;
}
function isMimeTypeMatch(value: string, compareWith: string) {
if (value.endsWith('.*')) {
value = value.substr(0, value.indexOf('.*'));
}
return compareWith.startsWith(value);
}
function sortOutputItemsBasedOnDisplayOrder(outputItems: NotebookCellOutputItem[]): NotebookCellOutputItem[] {
return outputItems
.map(item => {
let index = orderOfMimeTypes.findIndex((mime) => isMimeTypeMatch(mime, item.mime));
// Sometimes we can have mime types with empty data, e.g. when using holoview we can have `application/vnd.holoviews_load.v0+json` with empty value.
// & in these cases we have HTML/JS and those take precedence.
// https://github.com/microsoft/vscode-jupyter/issues/6109
if (isEmptyVendoredMimeType(item)) {
index = -1;
}
index = index === -1 ? 100 : index;
return {
item, index
};
})
.sort((outputItemA, outputItemB) => outputItemA.index - outputItemB.index).map(item => item.item);
}
enum CellOutputMimeTypes {
error = 'application/vnd.code.notebook.error',
stderr = 'application/vnd.code.notebook.stderr',
stdout = 'application/vnd.code.notebook.stdout'
}
export const textMimeTypes = ['text/plain', 'text/markdown', 'text/latex', CellOutputMimeTypes.stderr, CellOutputMimeTypes.stdout];
function concatMultilineString(str: string | string[], trim?: boolean): string {
const nonLineFeedWhiteSpaceTrim = /(^[\t\f\v\r ]+|[\t\f\v\r ]+$)/g;
if (Array.isArray(str)) {
let result = '';
for (let i = 0; i < str.length; i += 1) {
const s = str[i];
if (i < str.length - 1 && !s.endsWith('\n')) {
result = result.concat(`${s}\n`);
} else {
result = result.concat(s);
}
}
// Just trim whitespace. Leave \n in place
return trim ? result.replace(nonLineFeedWhiteSpaceTrim, '') : result;
}
return trim ? str.toString().replace(nonLineFeedWhiteSpaceTrim, '') : str.toString();
}
function convertJupyterOutputToBuffer(mime: string, value: unknown): NotebookCellOutputItem {
if (!value) {
return NotebookCellOutputItem.text('', mime);
}
try {
if (
(mime.startsWith('text/') || textMimeTypes.includes(mime)) &&
(Array.isArray(value) || typeof value === 'string')
) {
const stringValue = Array.isArray(value) ? concatMultilineString(value) : value;
return NotebookCellOutputItem.text(stringValue, mime);
} else if (mime.startsWith('image/') && typeof value === 'string' && mime !== 'image/svg+xml') {
// Images in Jupyter are stored in base64 encoded format.
// VS Code expects bytes when rendering images.
if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') {
return new NotebookCellOutputItem(Buffer.from(value, 'base64'), mime);
} else {
const data = Uint8Array.from(atob(value), c => c.charCodeAt(0));
return new NotebookCellOutputItem(data, mime);
}
} else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
return NotebookCellOutputItem.text(JSON.stringify(value), mime);
} else {
// For everything else, treat the data as strings (or multi-line strings).
value = Array.isArray(value) ? concatMultilineString(value) : value;
return NotebookCellOutputItem.text(value as string, mime);
}
} catch (ex) {
return NotebookCellOutputItem.error(ex);
}
}
function getNotebookCellMetadata(cell: nbformat.IBaseCell): CellMetadata {
// We put this only for VSC to display in diff view.
// Else we don't use this.
const propertiesToClone: (keyof CellMetadata)[] = ['metadata', 'attachments'];
const custom: CellMetadata = {};
propertiesToClone.forEach((propertyToClone) => {
if (cell[propertyToClone]) {
custom[propertyToClone] = JSON.parse(JSON.stringify(cell[propertyToClone]));
}
});
if ('id' in cell && typeof cell.id === 'string') {
custom.id = cell.id;
}
return custom;
}
function getOutputMetadata(output: nbformat.IOutput): CellOutputMetadata {
// Add on transient data if we have any. This should be removed by our save functions elsewhere.
const metadata: CellOutputMetadata = {
outputType: output.output_type
};
if (output.transient) {
metadata.transient = output.transient;
}
switch (output.output_type as nbformat.OutputType) {
case 'display_data':
case 'execute_result':
case 'update_display_data': {
metadata.executionCount = output.execution_count;
metadata.metadata = output.metadata ? JSON.parse(JSON.stringify(output.metadata)) : {};
break;
}
default:
break;
}
return metadata;
}
function translateDisplayDataOutput(
output: nbformat.IDisplayData | nbformat.IDisplayUpdate | nbformat.IExecuteResult
): NotebookCellOutput {
// Metadata could be as follows:
// We'll have metadata specific to each mime type as well as generic metadata.
/*
IDisplayData = {
output_type: 'display_data',
data: {
'image/jpg': '/////'
'image/png': '/////'
'text/plain': '/////'
},
metadata: {
'image/png': '/////',
'background': true,
'xyz': '///
}
}
*/
const metadata = getOutputMetadata(output);
const items: NotebookCellOutputItem[] = [];
if (output.data) {
for (const key in output.data) {
items.push(convertJupyterOutputToBuffer(key, output.data[key]));
}
}
return new NotebookCellOutput(sortOutputItemsBasedOnDisplayOrder(items), metadata);
}
function translateErrorOutput(output?: nbformat.IError): NotebookCellOutput {
output = output || { output_type: 'error', ename: '', evalue: '', traceback: [] };
return new NotebookCellOutput(
[
NotebookCellOutputItem.error({
name: output?.ename || '',
message: output?.evalue || '',
stack: (output?.traceback || []).join('\n')
})
],
{ ...getOutputMetadata(output), originalError: output }
);
}
function translateStreamOutput(output: nbformat.IStream): NotebookCellOutput {
const value = concatMultilineString(output.text);
const item = output.name === 'stderr' ? NotebookCellOutputItem.stderr(value) : NotebookCellOutputItem.stdout(value);
return new NotebookCellOutput([item], getOutputMetadata(output));
}
const cellOutputMappers = new Map<nbformat.OutputType, (output: any) => NotebookCellOutput>();
cellOutputMappers.set('display_data', translateDisplayDataOutput);
cellOutputMappers.set('execute_result', translateDisplayDataOutput);
cellOutputMappers.set('update_display_data', translateDisplayDataOutput);
cellOutputMappers.set('error', translateErrorOutput);
cellOutputMappers.set('stream', translateStreamOutput);
export function jupyterCellOutputToCellOutput(output: nbformat.IOutput): NotebookCellOutput {
/**
* Stream, `application/x.notebook.stream`
* Error, `application/x.notebook.error-traceback`
* Rich, { mime: value }
*
* outputs: [
new vscode.NotebookCellOutput([
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 2),
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 3),
]),
new vscode.NotebookCellOutput([
new vscode.NotebookCellOutputItem('text/markdown', '## header 2'),
new vscode.NotebookCellOutputItem('image/svg+xml', [
"<svg baseProfile=\"full\" height=\"200\" version=\"1.1\" width=\"300\" xmlns=\"http://www.w3.org/2000/svg\">\n",
" <rect fill=\"blue\" height=\"100%\" width=\"100%\"/>\n",
" <circle cx=\"150\" cy=\"100\" fill=\"green\" r=\"80\"/>\n",
" <text fill=\"white\" font-size=\"60\" text-anchor=\"middle\" x=\"150\" y=\"125\">SVG</text>\n",
"</svg>"
]),
]),
]
*
*/
const fn = cellOutputMappers.get(output.output_type as nbformat.OutputType);
let result: NotebookCellOutput;
if (fn) {
result = fn(output);
} else {
result = translateDisplayDataOutput(output as any);
}
return result;
}
function createNotebookCellDataFromRawCell(cell: nbformat.IRawCell): NotebookCellData {
const cellData = new NotebookCellData(NotebookCellKind.Code, concatMultilineString(cell.source), 'raw');
cellData.outputs = [];
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
return cellData;
}
function createNotebookCellDataFromMarkdownCell(cell: nbformat.IMarkdownCell): NotebookCellData {
const cellData = new NotebookCellData(
NotebookCellKind.Markup,
concatMultilineString(cell.source),
'markdown'
);
cellData.outputs = [];
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
return cellData;
}
function createNotebookCellDataFromCodeCell(cell: nbformat.ICodeCell, cellLanguage: string): NotebookCellData {
const cellOutputs = Array.isArray(cell.outputs) ? cell.outputs : [];
const outputs = cellOutputs.map(jupyterCellOutputToCellOutput);
const hasExecutionCount = typeof cell.execution_count === 'number' && cell.execution_count > 0;
const source = concatMultilineString(cell.source);
const executionSummary: NotebookCellExecutionSummary = hasExecutionCount
? { executionOrder: cell.execution_count as number }
: {};
const vscodeCustomMetadata = cell.metadata['vscode'] as { [key: string]: any } | undefined;
const cellLanguageId = vscodeCustomMetadata && vscodeCustomMetadata.languageId && typeof vscodeCustomMetadata.languageId === 'string' ? vscodeCustomMetadata.languageId : cellLanguage;
const cellData = new NotebookCellData(NotebookCellKind.Code, source, cellLanguageId);
cellData.outputs = outputs;
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
cellData.executionSummary = executionSummary;
return cellData;
}
function createNotebookCellDataFromJupyterCell(
cellLanguage: string,
cell: nbformat.IBaseCell
): NotebookCellData | undefined {
switch (cell.cell_type) {
case 'raw': {
return createNotebookCellDataFromRawCell(cell as nbformat.IRawCell);
}
case 'markdown': {
return createNotebookCellDataFromMarkdownCell(cell as nbformat.IMarkdownCell);
}
case 'code': {
return createNotebookCellDataFromCodeCell(cell as nbformat.ICodeCell, cellLanguage);
}
}
return;
}
/**
* Converts a NotebookModel into VS Code format.
*/
export function jupyterNotebookModelToNotebookData(
notebookContent: Partial<nbformat.INotebookContent>,
preferredLanguage: string
): NotebookData {
const notebookContentWithoutCells = { ...notebookContent, cells: [] };
if (!notebookContent.cells || notebookContent.cells.length === 0) {
throw new Error('Notebook content is missing cells');
}
const cells = notebookContent.cells
.map(cell => createNotebookCellDataFromJupyterCell(preferredLanguage, cell))
.filter((item): item is NotebookCellData => !!item);
const notebookData = new NotebookData(cells);
notebookData.metadata = { custom: notebookContentWithoutCells };
return notebookData;
}

116
extensions/ipynb/src/ipynbMain.ts Normal file
View File

@@ -0,0 +1,116 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { ensureAllNewCellsHaveCellIds } from './cellIdService';
import { NotebookSerializer } from './notebookSerializer';
// From {nbformat.INotebookMetadata} in @jupyterlab/coreutils
type NotebookMetadata = {
kernelspec?: {
name: string;
display_name: string;
[propName: string]: unknown;
};
language_info?: {
name: string;
codemirror_mode?: string | {};
file_extension?: string;
mimetype?: string;
pygments_lexer?: string;
[propName: string]: unknown;
};
orig_nbformat: number;
[propName: string]: unknown;
};
export function activate(context: vscode.ExtensionContext) {
const serializer = new NotebookSerializer(context);
ensureAllNewCellsHaveCellIds(context);
context.subscriptions.push(vscode.workspace.registerNotebookSerializer('jupyter-notebook', serializer, {
transientOutputs: false,
transientCellMetadata: {
breakpointMargin: true,
custom: false
}
}));
vscode.languages.registerCodeLensProvider({ pattern: '**/*.ipynb' }, {
provideCodeLenses: (document) => {
if (
document.uri.scheme === 'vscode-notebook-cell' ||
document.uri.scheme === 'vscode-notebook-cell-metadata' ||
document.uri.scheme === 'vscode-notebook-cell-output'
) {
return [];
}
const codelens = new vscode.CodeLens(new vscode.Range(0, 0, 0, 0), { title: 'Open in Notebook Editor', command: 'ipynb.openIpynbInNotebookEditor', arguments: [document.uri] });
return [codelens];
}
});
context.subscriptions.push(vscode.commands.registerCommand('ipynb.newUntitledIpynb', async () => {
const language = 'python';
const cell = new vscode.NotebookCellData(vscode.NotebookCellKind.Code, '', language);
const data = new vscode.NotebookData([cell]);
data.metadata = {
custom: {
cells: [],
metadata: {
orig_nbformat: 4
},
nbformat: 4,
nbformat_minor: 2
}
};
const doc = await vscode.workspace.openNotebookDocument('jupyter-notebook', data);
await vscode.window.showNotebookDocument(doc);
}));
context.subscriptions.push(vscode.commands.registerCommand('ipynb.openIpynbInNotebookEditor', async (uri: vscode.Uri) => {
if (vscode.window.activeTextEditor?.document.uri.toString() === uri.toString()) {
await vscode.commands.executeCommand('workbench.action.closeActiveEditor');
}
const document = await vscode.workspace.openNotebookDocument(uri);
await vscode.window.showNotebookDocument(document);
}));
// Update new file contribution
vscode.extensions.onDidChange(() => {
vscode.commands.executeCommand('setContext', 'jupyterEnabled', vscode.extensions.getExtension('ms-toolsai.jupyter'));
});
vscode.commands.executeCommand('setContext', 'jupyterEnabled', vscode.extensions.getExtension('ms-toolsai.jupyter'));
return {
exportNotebook: (notebook: vscode.NotebookData): string => {
return exportNotebook(notebook, serializer);
},
setNotebookMetadata: async (resource: vscode.Uri, metadata: Partial<NotebookMetadata>): Promise<boolean> => {
const document = vscode.workspace.notebookDocuments.find(doc => doc.uri.toString() === resource.toString());
if (!document) {
return false;
}
const edit = new vscode.WorkspaceEdit();
edit.replaceNotebookMetadata(resource, {
...document.metadata,
custom: {
...(document.metadata.custom ?? {}),
metadata: <NotebookMetadata>{
...(document.metadata.custom?.metadata ?? {}),
...metadata
},
}
});
return vscode.workspace.applyEdit(edit);
},
};
}
function exportNotebook(notebook: vscode.NotebookData, serializer: NotebookSerializer): string {
return serializer.serializeNotebookToString(notebook);
}
export function deactivate() { }

108
extensions/ipynb/src/notebookSerializer.ts Normal file
View File

@@ -0,0 +1,108 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as nbformat from '@jupyterlab/nbformat';
import * as detectIndent from 'detect-indent';
import * as vscode from 'vscode';
import { defaultNotebookFormat } from './constants';
import { getPreferredLanguage, jupyterNotebookModelToNotebookData } from './deserializers';
import { createJupyterCellFromNotebookCell, pruneCell, sortObjectPropertiesRecursively } from './serializers';
import * as fnv from '@enonic/fnv-plus';
export class NotebookSerializer implements vscode.NotebookSerializer {
constructor(readonly context: vscode.ExtensionContext) {
}
public async deserializeNotebook(content: Uint8Array, _token: vscode.CancellationToken): Promise<vscode.NotebookData> {
let contents = '';
try {
contents = new TextDecoder().decode(content);
} catch {
}
let json = contents && /\S/.test(contents) ? (JSON.parse(contents) as Partial<nbformat.INotebookContent>) : {};
if (json.__webview_backup) {
const backupId = json.__webview_backup;
const uri = this.context.globalStorageUri;
const folder = uri.with({ path: this.context.globalStorageUri.path.replace('vscode.ipynb', 'ms-toolsai.jupyter') });
const fileHash = fnv.fast1a32hex(backupId) as string;
const fileName = `${fileHash}.ipynb`;
const file = vscode.Uri.joinPath(folder, fileName);
const data = await vscode.workspace.fs.readFile(file);
json = data ? JSON.parse(data.toString()) : {};
if (json.contents && typeof json.contents === 'string') {
contents = json.contents;
json = JSON.parse(contents) as Partial<nbformat.INotebookContent>;
}
}
if (json.nbformat && json.nbformat < 4) {
throw new Error('Only Jupyter notebooks version 4+ are supported');
}
// Then compute indent from the contents (only use first 1K characters as a perf optimization)
const indentAmount = contents ? detectIndent(contents.substring(0, 1_000)).indent : ' ';
const preferredCellLanguage = getPreferredLanguage(json.metadata);
// Ensure we always have a blank cell.
if ((json.cells || []).length === 0) {
json.cells = [
{
cell_type: 'code',
execution_count: null,
metadata: {},
outputs: [],
source: ''
}
];
}
// For notebooks without metadata default the language in metadata to the preferred language.
if (!json.metadata || (!json.metadata.kernelspec && !json.metadata.language_info)) {
json.metadata = json.metadata || { orig_nbformat: defaultNotebookFormat.major };
json.metadata.language_info = json.metadata.language_info || { name: preferredCellLanguage };
}
const data = jupyterNotebookModelToNotebookData(
json,
preferredCellLanguage
);
data.metadata = data.metadata || {};
data.metadata.indentAmount = indentAmount;
return data;
}
public serializeNotebook(data: vscode.NotebookData, _token: vscode.CancellationToken): Uint8Array {
return new TextEncoder().encode(this.serializeNotebookToString(data));
}
public serializeNotebookToString(data: vscode.NotebookData): string {
const notebookContent = getNotebookMetadata(data);
// use the preferred language from document metadata or the first cell language as the notebook preferred cell language
const preferredCellLanguage = notebookContent.metadata?.language_info?.name ?? data.cells[0].languageId;
notebookContent.cells = data.cells
.map(cell => createJupyterCellFromNotebookCell(cell, preferredCellLanguage))
.map(pruneCell);
const indentAmount = data.metadata && 'indentAmount' in data.metadata && typeof data.metadata.indentAmount === 'string' ?
data.metadata.indentAmount :
' ';
// ipynb always ends with a trailing new line (we add this so that SCMs do not show unnecesary changes, resulting from a missing trailing new line).
return JSON.stringify(sortObjectPropertiesRecursively(notebookContent), undefined, indentAmount) + '\n';
}
}
export function getNotebookMetadata(document: vscode.NotebookDocument | vscode.NotebookData) {
const notebookContent: Partial<nbformat.INotebookContent> = document.metadata?.custom || {};
notebookContent.cells = notebookContent.cells || [];
notebookContent.nbformat = notebookContent.nbformat || 4;
notebookContent.nbformat_minor = notebookContent.nbformat_minor ?? 2;
notebookContent.metadata = notebookContent.metadata || { orig_nbformat: 4 };
return notebookContent;
}

421
extensions/ipynb/src/serializers.ts Normal file
View File

@@ -0,0 +1,421 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as nbformat from '@jupyterlab/nbformat';
import { NotebookCell, NotebookCellData, NotebookCellKind, NotebookCellOutput } from 'vscode';
import { CellMetadata, CellOutputMetadata } from './common';
import { textMimeTypes } from './deserializers';
const textDecoder = new TextDecoder();
enum CellOutputMimeTypes {
error = 'application/vnd.code.notebook.error',
stderr = 'application/vnd.code.notebook.stderr',
stdout = 'application/vnd.code.notebook.stdout'
}
export function createJupyterCellFromNotebookCell(
vscCell: NotebookCellData,
preferredLanguage: string | undefined
): nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell {
let cell: nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell;
if (vscCell.kind === NotebookCellKind.Markup) {
cell = createMarkdownCellFromNotebookCell(vscCell);
} else if (vscCell.languageId === 'raw') {
cell = createRawCellFromNotebookCell(vscCell);
} else {
cell = createCodeCellFromNotebookCell(vscCell, preferredLanguage);
}
return cell;
}
/**
* Sort the JSON to minimize unnecessary SCM changes.
* Jupyter notbeooks/labs sorts the JSON keys in alphabetical order.
* https://github.com/microsoft/vscode-python/issues/13155
*/
export function sortObjectPropertiesRecursively(obj: any): any {
if (Array.isArray(obj)) {
return obj.map(sortObjectPropertiesRecursively);
}
if (obj !== undefined && obj !== null && typeof obj === 'object' && Object.keys(obj).length > 0) {
return (
Object.keys(obj)
.sort()
.reduce<Record<string, any>>((sortedObj, prop) => {
sortedObj[prop] = sortObjectPropertiesRecursively(obj[prop]);
return sortedObj;
}, {}) as any
);
}
return obj;
}
export function getCellMetadata(cell: NotebookCell | NotebookCellData) {
return cell.metadata?.custom as CellMetadata | undefined;
}
function createCodeCellFromNotebookCell(cell: NotebookCellData, preferredLanguage: string | undefined): nbformat.ICodeCell {
const cellMetadata = getCellMetadata(cell);
let metadata = cellMetadata?.metadata || {}; // This cannot be empty.
if (cell.languageId !== preferredLanguage) {
metadata = {
...metadata,
vscode: {
languageId: cell.languageId
}
};
} else {
// cell current language is the same as the preferred cell language in the document, flush the vscode custom language id metadata
metadata.vscode = undefined;
}
const codeCell: any = {
cell_type: 'code',
execution_count: cell.executionSummary?.executionOrder ?? null,
source: splitMultilineString(cell.value.replace(/\r\n/g, '\n')),
outputs: (cell.outputs || []).map(translateCellDisplayOutput),
metadata: metadata
};
if (cellMetadata?.id) {
codeCell.id = cellMetadata.id;
}
return codeCell;
}
function createRawCellFromNotebookCell(cell: NotebookCellData): nbformat.IRawCell {
const cellMetadata = getCellMetadata(cell);
const rawCell: any = {
cell_type: 'raw',
source: splitMultilineString(cell.value.replace(/\r\n/g, '\n')),
metadata: cellMetadata?.metadata || {} // This cannot be empty.
};
if (cellMetadata?.attachments) {
rawCell.attachments = cellMetadata.attachments;
}
if (cellMetadata?.id) {
rawCell.id = cellMetadata.id;
}
return rawCell;
}
function splitMultilineString(source: nbformat.MultilineString): string[] {
if (Array.isArray(source)) {
return source as string[];
}
const str = source.toString();
if (str.length > 0) {
// Each line should be a separate entry, but end with a \n if not last entry
const arr = str.split('\n');
return arr
.map((s, i) => {
if (i < arr.length - 1) {
return `${s}\n`;
}
return s;
})
.filter(s => s.length > 0); // Skip last one if empty (it's the only one that could be length 0)
}
return [];
}
function translateCellDisplayOutput(output: NotebookCellOutput): JupyterOutput {
const customMetadata = output.metadata as CellOutputMetadata | undefined;
let result: JupyterOutput;
// Possible some other extension added some output (do best effort to translate & save in ipynb).
// In which case metadata might not contain `outputType`.
const outputType = customMetadata?.outputType as nbformat.OutputType;
switch (outputType) {
case 'error': {
result = translateCellErrorOutput(output);
break;
}
case 'stream': {
result = convertStreamOutput(output);
break;
}
case 'display_data': {
result = {
output_type: 'display_data',
data: output.items.reduce((prev: any, curr) => {
prev[curr.mime] = convertOutputMimeToJupyterOutput(curr.mime, curr.data as Uint8Array);
return prev;
}, {}),
metadata: customMetadata?.metadata || {} // This can never be undefined.
};
break;
}
case 'execute_result': {
result = {
output_type: 'execute_result',
data: output.items.reduce((prev: any, curr) => {
prev[curr.mime] = convertOutputMimeToJupyterOutput(curr.mime, curr.data as Uint8Array);
return prev;
}, {}),
metadata: customMetadata?.metadata || {}, // This can never be undefined.
execution_count:
typeof customMetadata?.executionCount === 'number' ? customMetadata?.executionCount : null // This can never be undefined, only a number or `null`.
};
break;
}
case 'update_display_data': {
result = {
output_type: 'update_display_data',
data: output.items.reduce((prev: any, curr) => {
prev[curr.mime] = convertOutputMimeToJupyterOutput(curr.mime, curr.data as Uint8Array);
return prev;
}, {}),
metadata: customMetadata?.metadata || {} // This can never be undefined.
};
break;
}
default: {
const isError =
output.items.length === 1 && output.items.every((item) => item.mime === CellOutputMimeTypes.error);
const isStream = output.items.every(
(item) => item.mime === CellOutputMimeTypes.stderr || item.mime === CellOutputMimeTypes.stdout
);
if (isError) {
return translateCellErrorOutput(output);
}
// In the case of .NET & other kernels, we need to ensure we save ipynb correctly.
// Hence if we have stream output, save the output as Jupyter `stream` else `display_data`
// Unless we already know its an unknown output type.
const outputType: nbformat.OutputType =
<nbformat.OutputType>customMetadata?.outputType || (isStream ? 'stream' : 'display_data');
let unknownOutput: nbformat.IUnrecognizedOutput | nbformat.IDisplayData | nbformat.IStream;
if (outputType === 'stream') {
// If saving as `stream` ensure the mandatory properties are set.
unknownOutput = convertStreamOutput(output);
} else if (outputType === 'display_data') {
// If saving as `display_data` ensure the mandatory properties are set.
const displayData: nbformat.IDisplayData = {
data: {},
metadata: {},
output_type: 'display_data'
};
unknownOutput = displayData;
} else {
unknownOutput = {
output_type: outputType
};
}
if (customMetadata?.metadata) {
unknownOutput.metadata = customMetadata.metadata;
}
if (output.items.length > 0) {
unknownOutput.data = output.items.reduce((prev: any, curr) => {
prev[curr.mime] = convertOutputMimeToJupyterOutput(curr.mime, curr.data as Uint8Array);
return prev;
}, {});
}
result = unknownOutput;
break;
}
}
// Account for transient data as well
// `transient.display_id` is used to update cell output in other cells, at least thats one use case we know of.
if (result && customMetadata && customMetadata.transient) {
result.transient = customMetadata.transient;
}
return result;
}
function translateCellErrorOutput(output: NotebookCellOutput): nbformat.IError {
// it should have at least one output item
const firstItem = output.items[0];
// Bug in VS Code.
if (!firstItem.data) {
return {
output_type: 'error',
ename: '',
evalue: '',
traceback: []
};
}
const originalError: undefined | nbformat.IError = output.metadata?.originalError;
const value: Error = JSON.parse(textDecoder.decode(firstItem.data));
return {
output_type: 'error',
ename: value.name,
evalue: value.message,
// VS Code needs an `Error` object which requires a `stack` property as a string.
// Its possible the format could change when converting from `traceback` to `string` and back again to `string`
// When .NET stores errors in output (with their .NET kernel),
// stack is empty, hence store the message instead of stack (so that somethign gets displayed in ipynb).
traceback: originalError?.traceback || splitMultilineString(value.stack || value.message || '')
};
}
function getOutputStreamType(output: NotebookCellOutput): string | undefined {
if (output.items.length > 0) {
return output.items[0].mime === CellOutputMimeTypes.stderr ? 'stderr' : 'stdout';
}
return;
}
type JupyterOutput =
| nbformat.IUnrecognizedOutput
| nbformat.IExecuteResult
| nbformat.IDisplayData
| nbformat.IStream
| nbformat.IError;
function convertStreamOutput(output: NotebookCellOutput): JupyterOutput {
const outputs: string[] = [];
output.items
.filter((opit) => opit.mime === CellOutputMimeTypes.stderr || opit.mime === CellOutputMimeTypes.stdout)
.map((opit) => textDecoder.decode(opit.data))
.forEach(value => {
// Ensure each line is a seprate entry in an array (ending with \n).
const lines = value.split('\n');
// If the last item in `outputs` is not empty and the first item in `lines` is not empty, then concate them.
// As they are part of the same line.
if (outputs.length && lines.length && lines[0].length > 0) {
outputs[outputs.length - 1] = `${outputs[outputs.length - 1]}${lines.shift()!}`;
}
for (const line of lines) {
outputs.push(line);
}
});
for (let index = 0; index < (outputs.length - 1); index++) {
outputs[index] = `${outputs[index]}\n`;
}
// Skip last one if empty (it's the only one that could be length 0)
if (outputs.length && outputs[outputs.length - 1].length === 0) {
outputs.pop();
}
const streamType = getOutputStreamType(output) || 'stdout';
return {
output_type: 'stream',
name: streamType,
text: outputs
};
}
function convertOutputMimeToJupyterOutput(mime: string, value: Uint8Array) {
if (!value) {
return '';
}
try {
if (mime === CellOutputMimeTypes.error) {
const stringValue = textDecoder.decode(value);
return JSON.parse(stringValue);
} else if (mime.startsWith('text/') || textMimeTypes.includes(mime)) {
const stringValue = textDecoder.decode(value);
return splitMultilineString(stringValue);
} else if (mime.startsWith('image/') && mime !== 'image/svg+xml') {
// Images in Jupyter are stored in base64 encoded format.
// VS Code expects bytes when rendering images.
if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') {
return Buffer.from(value).toString('base64');
} else {
return btoa(value.reduce((s: string, b: number) => s + String.fromCharCode(b), ''));
}
} else if (mime.toLowerCase().includes('json')) {
const stringValue = textDecoder.decode(value);
return stringValue.length > 0 ? JSON.parse(stringValue) : stringValue;
} else {
const stringValue = textDecoder.decode(value);
return stringValue;
}
} catch (ex) {
return '';
}
}
function createMarkdownCellFromNotebookCell(cell: NotebookCellData): nbformat.IMarkdownCell {
const cellMetadata = getCellMetadata(cell);
const markdownCell: any = {
cell_type: 'markdown',
source: splitMultilineString(cell.value.replace(/\r\n/g, '\n')),
metadata: cellMetadata?.metadata || {} // This cannot be empty.
};
if (cellMetadata?.attachments) {
markdownCell.attachments = cellMetadata.attachments;
}
if (cellMetadata?.id) {
markdownCell.id = cellMetadata.id;
}
return markdownCell;
}
export function pruneCell(cell: nbformat.ICell): nbformat.ICell {
// Source is usually a single string on input. Convert back to an array
const result = {
...cell,
source: splitMultilineString(cell.source)
} as nbformat.ICell;
// Remove outputs and execution_count from non code cells
if (result.cell_type !== 'code') {
delete (<any>result).outputs;
delete (<any>result).execution_count;
} else {
// Clean outputs from code cells
result.outputs = result.outputs ? (result.outputs as nbformat.IOutput[]).map(fixupOutput) : [];
}
return result;
}
const dummyStreamObj: nbformat.IStream = {
output_type: 'stream',
name: 'stdout',
text: ''
};
const dummyErrorObj: nbformat.IError = {
output_type: 'error',
ename: '',
evalue: '',
traceback: ['']
};
const dummyDisplayObj: nbformat.IDisplayData = {
output_type: 'display_data',
data: {},
metadata: {}
};
const dummyExecuteResultObj: nbformat.IExecuteResult = {
output_type: 'execute_result',
name: '',
execution_count: 0,
data: {},
metadata: {}
};
const AllowedCellOutputKeys = {
['stream']: new Set(Object.keys(dummyStreamObj)),
['error']: new Set(Object.keys(dummyErrorObj)),
['display_data']: new Set(Object.keys(dummyDisplayObj)),
['execute_result']: new Set(Object.keys(dummyExecuteResultObj))
};
function fixupOutput(output: nbformat.IOutput): nbformat.IOutput {
let allowedKeys: Set<string>;
switch (output.output_type) {
case 'stream':
case 'error':
case 'execute_result':
case 'display_data':
allowedKeys = AllowedCellOutputKeys[output.output_type];
break;
default:
return output;
}
const result = { ...output };
for (const k of Object.keys(output)) {
if (!allowedKeys.has(k)) {
delete result[k];
}
}
return result;
}

40
extensions/ipynb/src/test/index.ts Normal file
View File

@@ -0,0 +1,40 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
const path = require('path');
const testRunner = require('../../../../test/integration/electron/testrunner');
const options: any = {
ui: 'tdd',
color: true,
timeout: 60000
};
// These integration tests is being run in multiple environments (electron, web, remote)
// so we need to set the suite name based on the environment as the suite name is used
// for the test results file name
let suite = '';
if (process.env.VSCODE_BROWSER) {
suite = `${process.env.VSCODE_BROWSER} Browser Integration .ipynb Tests`;
} else if (process.env.REMOTE_VSCODE) {
suite = 'Remote Integration .ipynb Tests';
} else {
suite = 'Integration .ipynb Tests';
}
if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
options.reporter = 'mocha-multi-reporters';
options.reporterOptions = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
testsuitesTitle: `${suite} ${process.platform}`,
mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
}
};
}
testRunner.configure(options);
export = testRunner;

637
extensions/ipynb/src/test/serializers.test.ts Normal file
View File

@@ -0,0 +1,637 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as nbformat from '@jupyterlab/nbformat';
import * as assert from 'assert';
import * as vscode from 'vscode';
import { jupyterCellOutputToCellOutput, jupyterNotebookModelToNotebookData } from '../deserializers';
function deepStripProperties(obj: any, props: string[]) {
for (let prop in obj) {
if (obj[prop]) {
delete obj[prop];
} else if (typeof obj[prop] === 'object') {
deepStripProperties(obj[prop], props);
}
}
}
suite('ipynb serializer', () => {
const base64EncodedImage =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOUlZL6DwAB/wFSU1jVmgAAAABJRU5ErkJggg==';
test('Deserialize', async () => {
const cells: nbformat.ICell[] = [
{
cell_type: 'code',
execution_count: 10,
outputs: [],
source: 'print(1)',
metadata: {}
},
{
cell_type: 'markdown',
source: '# HEAD',
metadata: {}
}
];
const notebook = jupyterNotebookModelToNotebookData({ cells }, 'python');
assert.ok(notebook);
const expectedCodeCell = new vscode.NotebookCellData(vscode.NotebookCellKind.Code, 'print(1)', 'python');
expectedCodeCell.outputs = [];
expectedCodeCell.metadata = { custom: { metadata: {} } };
expectedCodeCell.executionSummary = { executionOrder: 10 };
const expectedMarkdownCell = new vscode.NotebookCellData(vscode.NotebookCellKind.Markup, '# HEAD', 'markdown');
expectedMarkdownCell.outputs = [];
expectedMarkdownCell.metadata = {
custom: { metadata: {} }
};
assert.deepStrictEqual(notebook.cells, [expectedCodeCell, expectedMarkdownCell]);
});
suite('Outputs', () => {
function validateCellOutputTranslation(
outputs: nbformat.IOutput[],
expectedOutputs: vscode.NotebookCellOutput[],
propertiesToExcludeFromComparison: string[] = []
) {
const cells: nbformat.ICell[] = [
{
cell_type: 'code',
execution_count: 10,
outputs,
source: 'print(1)',
metadata: {}
}
];
const notebook = jupyterNotebookModelToNotebookData({ cells }, 'python');
// OutputItems contain an `id` property generated by VSC.
// Exclude that property when comparing.
const propertiesToExclude = propertiesToExcludeFromComparison.concat(['id']);
const actualOuts = notebook.cells[0].outputs;
deepStripProperties(actualOuts, propertiesToExclude);
deepStripProperties(expectedOutputs, propertiesToExclude);
assert.deepStrictEqual(actualOuts, expectedOutputs);
}
test('Empty output', () => {
validateCellOutputTranslation([], []);
});
test('Stream output', () => {
validateCellOutputTranslation(
[
{
output_type: 'stream',
name: 'stderr',
text: 'Error'
},
{
output_type: 'stream',
name: 'stdout',
text: 'NoError'
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stderr('Error')], {
outputType: 'stream'
}),
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stdout('NoError')], {
outputType: 'stream'
})
]
);
});
test('Stream output and line endings', () => {
validateCellOutputTranslation(
[
{
output_type: 'stream',
name: 'stdout',
text: [
'Line1\n',
'\n',
'Line3\n',
'Line4'
]
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stdout('Line1\n\nLine3\nLine4')], {
outputType: 'stream'
})
]
);
validateCellOutputTranslation(
[
{
output_type: 'stream',
name: 'stdout',
text: [
'Hello\n',
'Hello\n',
'Hello\n',
'Hello\n',
'Hello\n',
'Hello\n'
]
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stdout('Hello\nHello\nHello\nHello\nHello\nHello\n')], {
outputType: 'stream'
})
]
);
});
test('Multi-line Stream output', () => {
validateCellOutputTranslation(
[
{
name: 'stdout',
output_type: 'stream',
text: [
'Epoch 1/5\n',
'...\n',
'Epoch 2/5\n',
'...\n',
'Epoch 3/5\n',
'...\n',
'Epoch 4/5\n',
'...\n',
'Epoch 5/5\n',
'...\n'
]
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stdout(['Epoch 1/5\n',
'...\n',
'Epoch 2/5\n',
'...\n',
'Epoch 3/5\n',
'...\n',
'Epoch 4/5\n',
'...\n',
'Epoch 5/5\n',
'...\n'].join(''))], {
outputType: 'stream'
})
]
);
});
test('Multi-line Stream output (last empty line should not be saved in ipynb)', () => {
validateCellOutputTranslation(
[
{
name: 'stderr',
output_type: 'stream',
text: [
'Epoch 1/5\n',
'...\n',
'Epoch 2/5\n',
'...\n',
'Epoch 3/5\n',
'...\n',
'Epoch 4/5\n',
'...\n',
'Epoch 5/5\n',
'...\n'
]
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stderr(['Epoch 1/5\n',
'...\n',
'Epoch 2/5\n',
'...\n',
'Epoch 3/5\n',
'...\n',
'Epoch 4/5\n',
'...\n',
'Epoch 5/5\n',
'...\n',
// This last empty line should not be saved in ipynb.
'\n'].join(''))], {
outputType: 'stream'
})
]
);
});
test('Streamed text with Ansi characters', async () => {
validateCellOutputTranslation(
[
{
name: 'stderr',
text: '\u001b[K\u001b[33m✅ \u001b[0m Loading\n',
output_type: 'stream'
}
],
[
new vscode.NotebookCellOutput(
[vscode.NotebookCellOutputItem.stderr('\u001b[K\u001b[33m✅ \u001b[0m Loading\n')],
{
outputType: 'stream'
}
)
]
);
});
test('Streamed text with angle bracket characters', async () => {
validateCellOutputTranslation(
[
{
name: 'stderr',
text: '1 is < 2',
output_type: 'stream'
}
],
[
new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.stderr('1 is < 2')], {
outputType: 'stream'
})
]
);
});
test('Streamed text with angle bracket characters and ansi chars', async () => {
validateCellOutputTranslation(
[
{
name: 'stderr',
text: '1 is < 2\u001b[K\u001b[33m✅ \u001b[0m Loading\n',
output_type: 'stream'
}
],
[
new vscode.NotebookCellOutput(
[vscode.NotebookCellOutputItem.stderr('1 is < 2\u001b[K\u001b[33m✅ \u001b[0m Loading\n')],
{
outputType: 'stream'
}
)
]
);
});
test('Error', async () => {
validateCellOutputTranslation(
[
{
ename: 'Error Name',
evalue: 'Error Value',
traceback: ['stack1', 'stack2', 'stack3'],
output_type: 'error'
}
],
[
new vscode.NotebookCellOutput(
[
vscode.NotebookCellOutputItem.error({
name: 'Error Name',
message: 'Error Value',
stack: ['stack1', 'stack2', 'stack3'].join('\n')
})
],
{
outputType: 'error',
originalError: {
ename: 'Error Name',
evalue: 'Error Value',
traceback: ['stack1', 'stack2', 'stack3'],
output_type: 'error'
}
}
)
]
);
});
['display_data', 'execute_result'].forEach(output_type => {
suite(`Rich output for output_type = ${output_type}`, () => {
// Properties to exclude when comparing.
let propertiesToExcludeFromComparison: string[] = [];
setup(() => {
if (output_type === 'display_data') {
// With display_data the execution_count property will never exist in the output.
// We can ignore that (as it will never exist).
// But we leave it in the case of `output_type === 'execute_result'`
propertiesToExcludeFromComparison = ['execution_count', 'executionCount'];
}
});
test('Text mimeType output', async () => {
validateCellOutputTranslation(
[
{
data: {
'text/plain': 'Hello World!'
},
output_type,
metadata: {},
execution_count: 1
}
],
[
new vscode.NotebookCellOutput(
[new vscode.NotebookCellOutputItem(Buffer.from('Hello World!', 'utf8'), 'text/plain')],
{
outputType: output_type,
metadata: {}, // display_data & execute_result always have metadata.
executionCount: 1
}
)
],
propertiesToExcludeFromComparison
);
});
test('png,jpeg images', async () => {
validateCellOutputTranslation(
[
{
execution_count: 1,
data: {
'image/png': base64EncodedImage,
'image/jpeg': base64EncodedImage
},
metadata: {},
output_type
}
],
[
new vscode.NotebookCellOutput(
[
new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/png'),
new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/jpeg')
],
{
executionCount: 1,
outputType: output_type,
metadata: {} // display_data & execute_result always have metadata.
}
)
],
propertiesToExcludeFromComparison
);
});
test('png image with a light background', async () => {
validateCellOutputTranslation(
[
{
execution_count: 1,
data: {
'image/png': base64EncodedImage
},
metadata: {
needs_background: 'light'
},
output_type
}
],
[
new vscode.NotebookCellOutput(
[new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/png')],
{
executionCount: 1,
metadata: {
needs_background: 'light'
},
outputType: output_type
}
)
],
propertiesToExcludeFromComparison
);
});
test('png image with a dark background', async () => {
validateCellOutputTranslation(
[
{
execution_count: 1,
data: {
'image/png': base64EncodedImage
},
metadata: {
needs_background: 'dark'
},
output_type
}
],
[
new vscode.NotebookCellOutput(
[new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/png')],
{
executionCount: 1,
metadata: {
needs_background: 'dark'
},
outputType: output_type
}
)
],
propertiesToExcludeFromComparison
);
});
test('png image with custom dimensions', async () => {
validateCellOutputTranslation(
[
{
execution_count: 1,
data: {
'image/png': base64EncodedImage
},
metadata: {
'image/png': { height: '111px', width: '999px' }
},
output_type
}
],
[
new vscode.NotebookCellOutput(
[new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/png')],
{
executionCount: 1,
metadata: {
'image/png': { height: '111px', width: '999px' }
},
outputType: output_type
}
)
],
propertiesToExcludeFromComparison
);
});
test('png allowed to scroll', async () => {
validateCellOutputTranslation(
[
{
execution_count: 1,
data: {
'image/png': base64EncodedImage
},
metadata: {
unconfined: true,
'image/png': { width: '999px' }
},
output_type
}
],
[
new vscode.NotebookCellOutput(
[new vscode.NotebookCellOutputItem(Buffer.from(base64EncodedImage, 'base64'), 'image/png')],
{
executionCount: 1,
metadata: {
unconfined: true,
'image/png': { width: '999px' }
},
outputType: output_type
}
)
],
propertiesToExcludeFromComparison
);
});
});
});
});
suite('Output Order', () => {
test('Verify order of outputs', async () => {
const dataAndExpectedOrder: { output: nbformat.IDisplayData; expectedMimeTypesOrder: string[] }[] = [
{
output: {
data: {
'application/vnd.vegalite.v4+json': 'some json',
'text/html': '<a>Hello</a>'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['application/vnd.vegalite.v4+json', 'text/html']
},
{
output: {
data: {
'application/vnd.vegalite.v4+json': 'some json',
'application/javascript': 'some js',
'text/plain': 'some text',
'text/html': '<a>Hello</a>'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: [
'application/vnd.vegalite.v4+json',
'text/html',
'application/javascript',
'text/plain'
]
},
{
output: {
data: {
'application/vnd.vegalite.v4+json': '', // Empty, should give preference to other mimetypes.
'application/javascript': 'some js',
'text/plain': 'some text',
'text/html': '<a>Hello</a>'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: [
'text/html',
'application/javascript',
'text/plain',
'application/vnd.vegalite.v4+json'
]
},
{
output: {
data: {
'text/plain': 'some text',
'text/html': '<a>Hello</a>'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['text/html', 'text/plain']
},
{
output: {
data: {
'application/javascript': 'some js',
'text/plain': 'some text'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['application/javascript', 'text/plain']
},
{
output: {
data: {
'image/svg+xml': 'some svg',
'text/plain': 'some text'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['image/svg+xml', 'text/plain']
},
{
output: {
data: {
'text/latex': 'some latex',
'text/plain': 'some text'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['text/latex', 'text/plain']
},
{
output: {
data: {
'application/vnd.jupyter.widget-view+json': 'some widget',
'text/plain': 'some text'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['application/vnd.jupyter.widget-view+json', 'text/plain']
},
{
output: {
data: {
'text/plain': 'some text',
'image/svg+xml': 'some svg',
'image/png': 'some png'
},
metadata: {},
output_type: 'display_data'
},
expectedMimeTypesOrder: ['image/png', 'image/svg+xml', 'text/plain']
}
];
dataAndExpectedOrder.forEach(({ output, expectedMimeTypesOrder }) => {
const sortedOutputs = jupyterCellOutputToCellOutput(output);
const mimeTypes = sortedOutputs.items.map((item) => item.mime).join(',');
assert.equal(mimeTypes, expectedMimeTypesOrder.join(','));
});
});
});
});

6
extensions/ipynb/src/types.d.ts vendored Normal file
View File

@@ -0,0 +1,6 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
declare module '@enonic/fnv-plus';