Notebooks: Add Support for Cell Attachment Images (#14449)

* Add to interfaces

* Works E2E

* Consolidate interface

* Add comments, cleanup

* Add some tests

* Cleanup

* interface cleanup

* Add more tests

* Add comments

* Add type for cell attachment

* wip

src/sql/workbench/contrib/notebook/browser/cellViews/interfaces.ts

@@ -6,7 +6,10 @@
 import { OnDestroy } from '@angular/core';
 import { AngularDisposable } from 'sql/base/browser/lifecycle';
 import { ICellEditorProvider, NotebookRange } from 'sql/workbench/services/notebook/browser/notebookService';
+import { MarkdownRenderOptions } from 'vs/base/browser/markdownRenderer';
+import { IMarkdownString } from 'vs/base/common/htmlContent';
 import { BaseTextEditor } from 'vs/workbench/browser/parts/editor/textEditor';
+import { nb } from 'azdata';
 
 export abstract class CellView extends AngularDisposable implements OnDestroy, ICellEditorProvider {
 	constructor() {
@@ -29,3 +32,11 @@ export abstract class CellView extends AngularDisposable implements OnDestroy, I
 
 	}
 }
+
+export interface IMarkdownStringWithCellAttachments extends IMarkdownString {
+	readonly cellAttachments?: nb.ICellAttachment
+}
+
+export interface MarkdownRenderOptionsWithCellAttachments extends MarkdownRenderOptions {
+	readonly cellAttachments?: nb.ICellAttachment
+}

src/sql/workbench/contrib/notebook/browser/cellViews/textCell.component.ts

@@ -227,7 +227,8 @@ export class TextCellComponent extends CellView implements OnInit, OnChanges {
 			this.markdownRenderer.setNotebookURI(this.cellModel.notebookModel.notebookUri);
 			this.markdownResult = this.markdownRenderer.render({
 				isTrusted: true,
-				value: Array.isArray(this._content) ? this._content.join('') : this._content
+				value: Array.isArray(this._content) ? this._content.join('') : this._content,
+				cellAttachments: this.cellModel.attachments
 			});
 			this.markdownResult.element.innerHTML = this.sanitizeContent(this.markdownResult.element.innerHTML);
 			this.setLoading(false);

src/sql/workbench/contrib/notebook/browser/notebook.contribution.ts

@@ -50,6 +50,7 @@ import { NotebookExplorerViewletViewsContribution, OpenNotebookExplorerViewletAc
 import 'vs/css!./media/notebook.contribution';
 import { isMacintosh } from 'vs/base/common/platform';
 import { SearchSortOrder } from 'vs/workbench/services/search/common/search';
+import { ImageMimeTypes } from 'sql/workbench/services/notebook/common/contracts';
 
 Registry.as<IEditorInputFactoryRegistry>(EditorInputFactoryExtensions.EditorInputFactories)
 	.registerEditorInputFactory(FileNotebookInput.ID, FileNoteBookEditorInputFactory);
@@ -260,7 +261,7 @@ registerComponentType({
  * A mime renderer component for images.
  */
 registerComponentType({
-	mimeTypes: ['image/bmp', 'image/png', 'image/jpeg', 'image/gif'],
+	mimeTypes: ImageMimeTypes,
 	rank: 90,
 	safe: true,
 	ctor: MimeRendererComponent,

src/sql/workbench/contrib/notebook/browser/outputs/notebookMarkdown.ts

@@ -4,13 +4,15 @@
  *--------------------------------------------------------------------------------------------*/
 import * as path from 'vs/base/common/path';
 
+import { nb } from 'azdata';
 import { URI } from 'vs/base/common/uri';
 import { IMarkdownString, removeMarkdownEscapes } from 'vs/base/common/htmlContent';
 import { IMarkdownRenderResult } from 'vs/editor/browser/core/markdownRenderer';
 import * as marked from 'vs/base/common/marked/marked';
 import { defaultGenerator } from 'vs/base/common/idGenerator';
 import { revive } from 'vs/base/common/marshalling';
-import { MarkdownRenderOptions } from 'vs/base/browser/markdownRenderer';
+import { ImageMimeTypes } from 'sql/workbench/services/notebook/common/contracts';
+import { IMarkdownStringWithCellAttachments, MarkdownRenderOptionsWithCellAttachments } from 'sql/workbench/contrib/notebook/browser/cellViews/interfaces';
 
 // Based off of HtmlContentRenderer
 export class NotebookMarkdownRenderer {
@@ -21,15 +23,15 @@ export class NotebookMarkdownRenderer {
 
 	}
 
-	render(markdown: IMarkdownString): IMarkdownRenderResult {
-		const element: HTMLElement = markdown ? this.renderMarkdown(markdown, undefined) : document.createElement('span');
+	render(markdown: IMarkdownStringWithCellAttachments): IMarkdownRenderResult {
+		const element: HTMLElement = markdown ? this.renderMarkdown(markdown, { cellAttachments: markdown.cellAttachments }) : document.createElement('span');
 		return {
 			element,
 			dispose: () => { }
 		};
 	}
 
-	createElement(options: MarkdownRenderOptions): HTMLElement {
+	createElement(options: MarkdownRenderOptionsWithCellAttachments): HTMLElement {
 		const tagName = options.inline ? 'span' : 'div';
 		const element = document.createElement(tagName);
 		if (options.className) {
@@ -51,7 +53,7 @@ export class NotebookMarkdownRenderer {
 	 * respects the trusted state of a notebook, and allows command links to
 	 * be clickable.
 	 */
-	renderMarkdown(markdown: IMarkdownString, options: MarkdownRenderOptions = {}): HTMLElement {
+	renderMarkdown(markdown: IMarkdownString, options: MarkdownRenderOptionsWithCellAttachments = {}): HTMLElement {
 		const element = this.createElement(options);
 
 		// signal to code-block render that the element has been created
@@ -64,7 +66,11 @@ export class NotebookMarkdownRenderer {
 		}
 		const renderer = new marked.Renderer({ baseUrl: notebookFolder });
 		renderer.image = (href: string, title: string, text: string) => {
-			href = this.cleanUrl(!markdown.isTrusted, notebookFolder, href);
+			const attachment = findAttachmentIfExists(href, options.cellAttachments);
+			// Attachments are already properly formed, so do not need cleaning. Cleaning only takes into account relative/absolute
+			// paths issues, and encoding issues -- neither of which apply to cell attachments.
+			// Attachments are always shown, regardless of notebook trust
+			href = attachment ? attachment : this.cleanUrl(!markdown.isTrusted, notebookFolder, href);
 			let dimensions: string[] = [];
 			if (href) {
 				const splitted = href.split('|').map(s => s.trim());
@@ -246,3 +252,29 @@ export class NotebookMarkdownRenderer {
 		this._notebookURI = val;
 	}
 }
+
+/**
+* The following is a sample cell attachment from JSON:
+*  "attachments": {
+*     "test.png": {
+*        "image/png": "iVBORw0KGgoAAAANggg==="
+*     }
+*  }
+*
+* In a cell, the above attachment would be referenced in markdown like this:
+* ![altText](attachment:test.png)
+*/
+function findAttachmentIfExists(href: string, cellAttachments: nb.ICellAttachment): string {
+	if (href.startsWith('attachment:') && cellAttachments) {
+		const imageName = href.replace('attachment:', '');
+		const imageDefinition = cellAttachments[imageName];
+		if (imageDefinition) {
+			for (let i = 0; i < ImageMimeTypes.length; i++) {
+				if (imageDefinition[ImageMimeTypes[i]]) {
+					return `data:${ImageMimeTypes[i]};base64,${imageDefinition[ImageMimeTypes[i]]}`;
+				}
+			}
+		}
+	}
+	return '';
+}

src/sql/workbench/contrib/notebook/test/browser/notebookMarkdown.test.ts

@@ -61,4 +61,24 @@ suite('NotebookMarkdownRenderer', () => {
 			assert.strictEqual(result.innerHTML, `<p><a href="/maddy/test/.build/someimageurl" data-href="/maddy/test/.build/someimageurl" title="/maddy/test/.build/someimageurl">Link to relative path</a></p>`);
 		}
 	});
+
+	test('cell attachment image', () => {
+		let result: HTMLElement = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAggg==" alt="altText"></p>`, 'Cell attachment basic test failed when trusted');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: false }, { cellAttachments: JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAggg==" alt="altText"></p>`, 'Cell attachment basic test failed when not trusted');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment no attachment included failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment name not found failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachments:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachments:ads.png" alt="altText"></p>`, 'Cell attachment scheme mismatch failed');
+
+		result = notebookMarkdownRenderer.renderMarkdown({ value: `![altText](attachment:ads.png)`, isTrusted: true }, { cellAttachments: JSON.parse('{"ads2.png":"image/png"}') });
+		assert.strictEqual(result.innerHTML, `<p><img src="attachment:ads.png" alt="altText"></p>`, 'Cell attachment no image data failed');
+	});
 });

src/sql/workbench/contrib/notebook/test/electron-browser/cell.test.ts

@@ -1040,4 +1040,37 @@ suite('Cell Model', function (): void {
 		assert.equal(model.savedConnectionName, connectionName);
 	});
 
+	test('Should read attachments name from notebook attachments', async function () {
+		const cellAttachment = JSON.parse('{"ads.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAggg=="}}');
+		let notebookModel = new NotebookModelStub({
+			name: '',
+			version: '',
+			mimetype: ''
+		});
+		let contents: nb.ICellContents = {
+			cell_type: CellTypes.Code,
+			source: '',
+			attachments: cellAttachment
+		};
+		let model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, contents.attachments);
+	});
+
+	test('Should read attachments name from notebook attachments', async function () {
+		let notebookModel = new NotebookModelStub({
+			name: '',
+			version: '',
+			mimetype: ''
+		});
+		let contents: nb.ICellContents = {
+			cell_type: CellTypes.Code,
+			source: ''
+		};
+		let model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, {});
+
+		contents.attachments = undefined;
+		model = factory.createCell(contents, { notebook: notebookModel, isTrusted: false });
+		assert.deepEqual(model.attachments, {});
+	});
 });