Notebooks: Add Support for Cell Attachment Images (#14449)

* Add to interfaces

* Works E2E

* Consolidate interface

* Add comments, cleanup

* Add some tests

* Cleanup

* interface cleanup

* Add more tests

* Add comments

* Add type for cell attachment

* wip

src/sql/azdata.proposed.d.ts

@@ -81,6 +81,12 @@ declare module 'azdata' {
 		export interface ICellMetadata {
 			connection_name?: string;
 		}
+
+		export interface ICellContents {
+			attachments?: ICellAttachment;
+		}
+
+		export type ICellAttachment = { [key: string]: { [key: string]: string } };
 	}
 
 	export type SqlDbType = 'BigInt' | 'Binary' | 'Bit' | 'Char' | 'DateTime' | 'Decimal'

src/sql/workbench/contrib/notebook/browser/cellViews/interfaces.ts

@@ -6,7 +6,10 @@
 import { OnDestroy } from '@angular/core';
 import { AngularDisposable } from 'sql/base/browser/lifecycle';
 import { ICellEditorProvider, NotebookRange } from 'sql/workbench/services/notebook/browser/notebookService';
+import { MarkdownRenderOptions } from 'vs/base/browser/markdownRenderer';
+import { IMarkdownString } from 'vs/base/common/htmlContent';
 import { BaseTextEditor } from 'vs/workbench/browser/parts/editor/textEditor';
+import { nb } from 'azdata';
 
 export abstract class CellView extends AngularDisposable implements OnDestroy, ICellEditorProvider {
 	constructor() {
@@ -29,3 +32,11 @@ export abstract class CellView extends AngularDisposable implements OnDestroy, I
 
 	}
 }
+
+export interface IMarkdownStringWithCellAttachments extends IMarkdownString {
+	readonly cellAttachments?: nb.ICellAttachment
+}
+
+export interface MarkdownRenderOptionsWithCellAttachments extends MarkdownRenderOptions {
+	readonly cellAttachments?: nb.ICellAttachment
+}

src/sql/workbench/contrib/notebook/browser/cellViews/textCell.component.ts

@@ -227,7 +227,8 @@ export class TextCellComponent extends CellView implements OnInit, OnChanges {
 			this.markdownRenderer.setNotebookURI(this.cellModel.notebookModel.notebookUri);
 			this.markdownResult = this.markdownRenderer.render({
 				isTrusted: true,
-				value: Array.isArray(this._content) ? this._content.join('') : this._content
+				value: Array.isArray(this._content) ? this._content.join('') : this._content,
+				cellAttachments: this.cellModel.attachments
 			});
 			this.markdownResult.element.innerHTML = this.sanitizeContent(this.markdownResult.element.innerHTML);
 			this.setLoading(false);

src/sql/workbench/contrib/notebook/browser/notebook.contribution.ts

@@ -50,6 +50,7 @@ import { NotebookExplorerViewletViewsContribution, OpenNotebookExplorerViewletAc
 import 'vs/css!./media/notebook.contribution';
 import { isMacintosh } from 'vs/base/common/platform';
 import { SearchSortOrder } from 'vs/workbench/services/search/common/search';
+import { ImageMimeTypes } from 'sql/workbench/services/notebook/common/contracts';
 
 Registry.as<IEditorInputFactoryRegistry>(EditorInputFactoryExtensions.EditorInputFactories)
 	.registerEditorInputFactory(FileNotebookInput.ID, FileNoteBookEditorInputFactory);
@@ -260,7 +261,7 @@ registerComponentType({
  * A mime renderer component for images.
  */
 registerComponentType({
-	mimeTypes: ['image/bmp', 'image/png', 'image/jpeg', 'image/gif'],
+	mimeTypes: ImageMimeTypes,
 	rank: 90,
 	safe: true,
 	ctor: MimeRendererComponent,

src/sql/workbench/contrib/notebook/browser/outputs/notebookMarkdown.ts

@@ -4,13 +4,15 @@
  *--------------------------------------------------------------------------------------------*/
 import * as path from 'vs/base/common/path';
 
+import { nb } from 'azdata';
 import { URI } from 'vs/base/common/uri';
 import { IMarkdownString, removeMarkdownEscapes } from 'vs/base/common/htmlContent';
 import { IMarkdownRenderResult } from 'vs/editor/browser/core/markdownRenderer';
 import * as marked from 'vs/base/common/marked/marked';
 import { defaultGenerator } from 'vs/base/common/idGenerator';
 import { revive } from 'vs/base/common/marshalling';
-import { MarkdownRenderOptions } from 'vs/base/browser/markdownRenderer';
+import { ImageMimeTypes } from 'sql/workbench/services/notebook/common/contracts';
+import { IMarkdownStringWithCellAttachments, MarkdownRenderOptionsWithCellAttachments } from 'sql/workbench/contrib/notebook/browser/cellViews/interfaces';
 
 // Based off of HtmlContentRenderer
 export class NotebookMarkdownRenderer {
@@ -21,15 +23,15 @@ export class NotebookMarkdownRenderer {
 
 	}
 
-	render(markdown: IMarkdownString): IMarkdownRenderResult {
+	render(markdown: IMarkdownStringWithCellAttachments): IMarkdownRenderResult {
-		const element: HTMLElement = markdown ? this.renderMarkdown(markdown, undefined) : document.createElement('span');
+		const element: HTMLElement = markdown ? this.renderMarkdown(markdown, { cellAttachments: markdown.cellAttachments }) : document.createElement('span');
 		return {
 			element,
 			dispose: () => { }
 		};
 	}
 
-	createElement(options: MarkdownRenderOptions): HTMLElement {
+	createElement(options: MarkdownRenderOptionsWithCellAttachments): HTMLElement {
 		const tagName = options.inline ? 'span' : 'div';
 		const element = document.createElement(tagName);
 		if (options.className) {
@@ -51,7 +53,7 @@ export class NotebookMarkdownRenderer {
 	 * respects the trusted state of a notebook, and allows command links to
 	 * be clickable.
 	 */
-	renderMarkdown(markdown: IMarkdownString, options: MarkdownRenderOptions = {}): HTMLElement {
+	renderMarkdown(markdown: IMarkdownString, options: MarkdownRenderOptionsWithCellAttachments = {}): HTMLElement {
 		const element = this.createElement(options);
 
 		// signal to code-block render that the element has been created
@@ -64,7 +66,11 @@ export class NotebookMarkdownRenderer {
 		}
 		const renderer = new marked.Renderer({ baseUrl: notebookFolder });
 		renderer.image = (href: string, title: string, text: string) => {
-			href = this.cleanUrl(!markdown.isTrusted, notebookFolder, href);
+			const attachment = findAttachmentIfExists(href, options.cellAttachments);
+			// Attachments are already properly formed, so do not need cleaning. Cleaning only takes into account relative/absolute
+			// paths issues, and encoding issues -- neither of which apply to cell attachments.
+			// Attachments are always shown, regardless of notebook trust
+			href = attachment ? attachment : this.cleanUrl(!markdown.isTrusted, notebookFolder, href);
 			let dimensions: string[] = [];
 			if (href) {
 				const splitted = href.split('|').map(s => s.trim());
@@ -246,3 +252,29 @@ export class NotebookMarkdownRenderer {
 		this._notebookURI = val;
 	}
 }
+
+/**
+* The following is a sample cell attachment from JSON:
+*  "attachments": {
+*     "test.png": {
+*        "image/png": "iVBORw0KGgoAAAANggg==="
+*     }
+*  }
+*
+* In a cell, the above attachment would be referenced in markdown like this:
+* ![altText](attachment:test.png)
+*/
+function findAttachmentIfExists(href: string, cellAttachments: nb.ICellAttachment): string {
+	if (href.startsWith('attachment:') && cellAttachments) {
+		const imageName = href.replace('attachment:', '');
+		const imageDefinition = cellAttachments[imageName];
+		if (imageDefinition) {
+			for (let i = 0; i < ImageMimeTypes.length; i++) {
+				if (imageDefinition[ImageMimeTypes[i]]) {
+					return `data:${ImageMimeTypes[i]};base64,${imageDefinition[ImageMimeTypes[i]]}`;
+				}
+			}
+		}
+	}
+	return '';
+}

src/sql/workbench/contrib/notebook/test/browser/notebookMarkdown.test.ts

@@ -61,4 +61,24 @@ suite('NotebookMarkdownRenderer', () => {
 			assert.strictEqual(result.innerHTML, `<p><a href="/maddy/test/.build/someimageurl" data-href="/maddy/test/.build/someimageurl" title="/maddy/test/.build/someimageurl">Link to relative path</a></p>`);
 		}
 	});
+
+	test('cell attachment image', () => {
+		let result: HTMLElement = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAggg==" alt="altText"></p>`, 'Cell attachment basic test failed when trusted');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: false }, { cellAttachments: JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAggg==" alt="altText"></p>`, 'Cell attachment basic test failed when not trusted');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment no attachment included failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment name not found failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachments:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachments:ads.png" alt="altText"></p>`, 'Cell attachment scheme mismatch failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":"image/png"}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment no image data failed');
+	});
 });

src/sql/workbench/contrib/notebook/test/electron-browser/cell.test.ts

@@ -1040,4 +1040,37 @@ suite('Cell Model', function (): void {
 		assert.equal(model.savedConnectionName, connectionName);
 	});
 
+	test('Should read attachments name from notebook attachments', async function () {
+		const cellAttachment = JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}');
+		let notebookModel = new NotebookModelStub({
+			name: '',
+			version: '',
+			mimetype: ''
+		});
+		let contents: nb.ICellContents = {
+			cell_type: CellTypes.Code,
+			source: '',
+			attachments: cellAttachment
+		};
+		let model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, contents.attachments);
+	});
+
+	test('Should read attachments name from notebook attachments', async function () {
+		let notebookModel = new NotebookModelStub({
+			name: '',
+			version: '',
+			mimetype: ''
+		});
+		let contents: nb.ICellContents = {
+			cell_type: CellTypes.Code,
+			source: ''
+		};
+		let model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, {});
+
+		contents.attachments = undefined;
+		model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, {});
+	});
 });

src/sql/workbench/services/notebook/browser/models/cell.ts

@@ -80,6 +80,7 @@ export class CellModel extends Disposable implements ICellModel {
 	private _isParameter: boolean;
 	private _onParameterStateChanged = new Emitter<boolean>();
 	private _isInjectedParameter: boolean;
+	private _attachments: nb.ICellAttachment;
 
 	constructor(cellData: nb.ICellContents,
 		private _options: ICellModelOptions,
@@ -140,6 +141,10 @@ export class CellModel extends Disposable implements ICellModel {
 		return this._metadata;
 	}
 
+	public get attachments() {
+		return this._attachments;
+	}
+
 	public get isEditMode(): boolean {
 		return this._isEditMode;
 	}
@@ -845,6 +850,8 @@ export class CellModel extends Disposable implements ICellModel {
 			if (this._configurationService?.getValue('notebook.saveConnectionName')) {
 				metadata.connection_name = this._savedConnectionName;
 			}
+		} else if (this._cellType === CellTypes.Markdown) {
+			cellJson.attachments = this._attachments;
 		}
 		return cellJson as nb.ICellContents;
 	}
@@ -866,7 +873,7 @@ export class CellModel extends Disposable implements ICellModel {
 			this._isParameter = false;
 			this._isInjectedParameter = false;
 		}
-
+		this._attachments = cell.attachments || {};
 		this._cellGuid = cell.metadata && cell.metadata.azdata_cell_guid ? cell.metadata.azdata_cell_guid : generateUuid();
 		this.setLanguageFromContents(cell);
 		this._savedConnectionName = this._metadata.connection_name;

src/sql/workbench/services/notebook/browser/models/modelInterfaces.ts

@@ -530,6 +530,7 @@ export interface ICellModel {
 	sendChangeToNotebook(change: NotebookChangeType): void;
 	cellSourceChanged: boolean;
 	readonly savedConnectionName: string | undefined;
+	readonly attachments: nb.ICellAttachment;
 	readonly currentMode: CellEditModes;
 }
 

src/sql/workbench/services/notebook/browser/outputs/factories.ts

@@ -4,6 +4,7 @@
 |----------------------------------------------------------------------------*/
 
 import * as widgets from 'sql/workbench/contrib/notebook/browser/outputs/widgets';
+import { ImageMimeTypes } from 'sql/workbench/services/notebook/common/contracts';
 import { IRenderMime } from './renderMimeInterfaces';
 
 /**
@@ -21,7 +22,7 @@ export const htmlRendererFactory: IRenderMime.IRendererFactory = {
  */
 export const imageRendererFactory: IRenderMime.IRendererFactory = {
 	safe: true,
-	mimeTypes: ['image/bmp', 'image/png', 'image/jpeg', 'image/gif'],
+	mimeTypes: ImageMimeTypes,
 	defaultRank: 90,
 	createRenderer: options => new widgets.RenderedImage(options)
 };

src/sql/workbench/services/notebook/common/contracts.ts

@@ -49,3 +49,5 @@ export enum NotebookChangeType {
 	CellOutputCleared,
 	CellMetadataUpdated
 }
+
+export const ImageMimeTypes = ['image/bmp', 'image/png', 'image/jpeg', 'image/gif'];

src/sql/workbench/services/notebook/common/localContentManager.ts

@@ -139,7 +139,8 @@ namespace v4 {
 		return {
 			cell_type: cell.cell_type,
 			source: cell.source,
-			metadata: cell.metadata
+			metadata: cell.metadata,
+			attachments: cell.attachments
 		};
 	}