Add support for new version of Jupyter Book (#12086)

* Add support for new jupyter book version

* Add changes to the jupyter notebook to create books

* Create config file

* Add support of new version of jupyter book on ADS

* Fix paths for opening folder with v1 and v2 books

* Add tests for jupyter book v2

* Update tests

* Fix tests

* Fix get parent issue

* Address PR comments

* Fix bookVersion condition in getSections and fix issue on create book notebook

* Fix search

* update python notebook

* Remove commented lines
This commit is contained in:
Barbara Valdez
2020-09-10 11:32:58 -07:00
committed by GitHub
parent ff61eae164
commit 3ff9df8e0b
6 changed files with 959 additions and 582 deletions

View File

@@ -5,22 +5,30 @@
import * as vscode from 'vscode';
import * as yaml from 'js-yaml';
import { BookTreeItem, BookTreeItemType } from './bookTreeItem';
import { BookTreeItem, BookTreeItemType, BookTreeItemFormat } from './bookTreeItem';
import * as constants from '../common/constants';
import * as path from 'path';
import * as fileServices from 'fs';
import * as fs from 'fs-extra';
import * as loc from '../common/localizedConstants';
import { IJupyterBookToc, IJupyterBookSection } from '../contracts/content';
import { IJupyterBookToc, JupyterBookSection, IJupyterBookSectionV2, IJupyterBookSectionV1 } from '../contracts/content';
const fsPromises = fileServices.promises;
const content = 'content';
export enum BookVersion {
v1 = 'v1',
v2 = 'v2'
}
export class BookModel {
private _bookItems: BookTreeItem[];
private _allNotebooks = new Map<string, BookTreeItem>();
private _tableOfContentsPath: string;
private _contentFolderPath: string;
private _configPath: string;
private _bookVersion: BookVersion;
private _rootPath: string;
private _errorMessage: string;
constructor(
@@ -37,11 +45,38 @@ export class BookModel {
if (this.isNotebook) {
this.readNotebook();
} else {
await this.loadTableOfContentFiles(this.bookPath);
await this.readBookStructure(this.bookPath);
await this.loadTableOfContentFiles();
await this.readBooks();
}
}
public async readBookStructure(folderPath: string): Promise<void> {
// check book structure to determine version
let isOlderVersion: boolean;
this._configPath = path.posix.join(folderPath, '_config.yml');
try {
isOlderVersion = (await fs.stat(path.posix.join(folderPath, '_data'))).isDirectory() && (await fs.stat(path.posix.join(folderPath, content))).isDirectory();
} catch {
isOlderVersion = false;
}
if (isOlderVersion) {
let isTocFile = (await fs.stat(path.posix.join(folderPath, '_data', 'toc.yml'))).isFile();
if (isTocFile) {
this._tableOfContentsPath = path.posix.join(folderPath, '_data', 'toc.yml');
}
this._bookVersion = BookVersion.v1;
this._contentFolderPath = path.posix.join(folderPath, content, '');
this._rootPath = path.dirname(path.dirname(this._tableOfContentsPath));
} else {
this._contentFolderPath = folderPath;
this._tableOfContentsPath = path.posix.join(folderPath, '_toc.yml');
this._rootPath = path.dirname(this._tableOfContentsPath);
this._bookVersion = BookVersion.v2;
}
}
public getAllNotebooks(): Map<string, BookTreeItem> {
return this._allNotebooks;
}
@@ -50,14 +85,12 @@ export class BookModel {
return this._allNotebooks.get(this.openAsUntitled ? path.basename(uri) : uri);
}
public async loadTableOfContentFiles(folderPath: string): Promise<void> {
public async loadTableOfContentFiles(): Promise<void> {
if (this.isNotebook) {
return;
}
let tableOfContentsPath: string = path.posix.join(folderPath, '_data', 'toc.yml');
if (await fs.pathExists(tableOfContentsPath)) {
this._tableOfContentsPath = tableOfContentsPath;
if (await fs.pathExists(this._tableOfContentsPath)) {
vscode.commands.executeCommand('setContext', 'bookOpened', true);
} else {
this._errorMessage = loc.missingTocError;
@@ -111,16 +144,16 @@ export class BookModel {
}
if (this._tableOfContentsPath) {
let root: string = path.dirname(path.dirname(this._tableOfContentsPath));
try {
let fileContents = await fsPromises.readFile(path.join(root, '_config.yml'), 'utf-8');
let fileContents = await fsPromises.readFile(this._configPath, 'utf-8');
const config = yaml.safeLoad(fileContents.toString());
fileContents = await fsPromises.readFile(this._tableOfContentsPath, 'utf-8');
const tableOfContents: any = yaml.safeLoad(fileContents.toString());
let tableOfContents: any = yaml.safeLoad(fileContents.toString());
let book: BookTreeItem = new BookTreeItem({
version: this._bookVersion,
title: config.title,
contentPath: this._tableOfContentsPath,
root: root,
root: this._rootPath,
tableOfContents: { sections: this.parseJupyterSections(tableOfContents) },
page: tableOfContents,
type: BookTreeItemType.Book,
@@ -145,11 +178,11 @@ export class BookModel {
return this._bookItems;
}
public async getSections(tableOfContents: IJupyterBookToc, sections: IJupyterBookSection[], root: string): Promise<BookTreeItem[]> {
public async getSections(tableOfContents: IJupyterBookToc, sections: JupyterBookSection[], root: string, book: BookTreeItemFormat): Promise<BookTreeItem[]> {
let notebooks: BookTreeItem[] = [];
for (let i = 0; i < sections.length; i++) {
if (sections[i].url) {
if (sections[i].external) {
if (sections[i].url || (sections[i] as IJupyterBookSectionV2).file) {
if (sections[i].url && ((sections[i] as IJupyterBookSectionV1).external || book.version === BookVersion.v2)) {
let externalLink: BookTreeItem = new BookTreeItem({
title: sections[i].title,
contentPath: undefined,
@@ -158,7 +191,8 @@ export class BookModel {
page: sections[i],
type: BookTreeItemType.ExternalLink,
treeItemCollapsibleState: vscode.TreeItemCollapsibleState.Collapsed,
isUntitled: this.openAsUntitled
isUntitled: this.openAsUntitled,
version: book.version
},
{
light: this._extensionContext.asAbsolutePath('resources/light/link.svg'),
@@ -168,20 +202,29 @@ export class BookModel {
notebooks.push(externalLink);
} else {
let pathToNotebook = path.join(root, 'content', sections[i].url.concat('.ipynb'));
let pathToMarkdown = path.join(root, 'content', sections[i].url.concat('.md'));
let pathToNotebook: string;
let pathToMarkdown: string;
if (book.version === BookVersion.v2) {
pathToNotebook = path.join(book.root, (sections[i] as IJupyterBookSectionV2).file.concat('.ipynb'));
pathToMarkdown = path.join(book.root, (sections[i] as IJupyterBookSectionV2).file.concat('.md'));
} else if (sections[i].url) {
pathToNotebook = path.join(book.root, content, (sections[i] as IJupyterBookSectionV1).url.concat('.ipynb'));
pathToMarkdown = path.join(book.root, content, (sections[i] as IJupyterBookSectionV1).url.concat('.md'));
}
// Note: Currently, if there is an ipynb and a md file with the same name, Jupyter Books only shows the notebook.
// Following Jupyter Books behavior for now
if (await fs.pathExists(pathToNotebook)) {
let notebook = new BookTreeItem({
title: sections[i].title,
title: sections[i].title ? sections[i].title : (sections[i] as IJupyterBookSectionV2).file,
contentPath: pathToNotebook,
root: root,
tableOfContents: tableOfContents,
page: sections[i],
type: BookTreeItemType.Notebook,
treeItemCollapsibleState: vscode.TreeItemCollapsibleState.Collapsed,
isUntitled: this.openAsUntitled
isUntitled: this.openAsUntitled,
version: book.version
},
{
light: this._extensionContext.asAbsolutePath('resources/light/notebook.svg'),
@@ -204,14 +247,15 @@ export class BookModel {
}
} else if (await fs.pathExists(pathToMarkdown)) {
let markdown: BookTreeItem = new BookTreeItem({
title: sections[i].title,
title: sections[i].title ? sections[i].title : (sections[i] as IJupyterBookSectionV2).file,
contentPath: pathToMarkdown,
root: root,
tableOfContents: tableOfContents,
page: sections[i],
type: BookTreeItemType.Markdown,
treeItemCollapsibleState: vscode.TreeItemCollapsibleState.Collapsed,
isUntitled: this.openAsUntitled
isUntitled: this.openAsUntitled,
version: book.version
},
{
light: this._extensionContext.asAbsolutePath('resources/light/markdown.svg'),
@@ -235,7 +279,7 @@ export class BookModel {
* Recursively parses out a section of a Jupyter Book.
* @param section The input data to parse
*/
private parseJupyterSections(section: any[]): IJupyterBookSection[] {
private parseJupyterSections(section: any[]): JupyterBookSection[] {
try {
return section.reduce((acc, val) => Array.isArray(val.sections) ?
acc.concat(val).concat(this.parseJupyterSections(val.sections)) : acc.concat(val), []);
@@ -246,15 +290,25 @@ export class BookModel {
}
throw this._errorMessage;
}
}
public get tableOfContentsPath(): string {
return this._tableOfContentsPath;
}
public get contentFolderPath(): string {
return this._contentFolderPath;
}
public get configPath(): string {
return this._configPath;
}
public get errorMessage(): string {
return this._errorMessage;
}
public get version(): string {
return this._bookVersion;
}
}

View File

@@ -6,9 +6,12 @@
import * as vscode from 'vscode';
import * as path from 'path';
import * as fs from 'fs';
import { IJupyterBookSection, IJupyterBookToc } from '../contracts/content';
import { JupyterBookSection, IJupyterBookToc, IJupyterBookSectionV2, IJupyterBookSectionV1 } from '../contracts/content';
import * as loc from '../common/localizedConstants';
import { isBookItemPinned } from '../common/utils';
import { BookVersion } from './bookModel';
const content = 'content';
export enum BookTreeItemType {
Book = 'Book',
@@ -26,13 +29,15 @@ export interface BookTreeItemFormat {
type: BookTreeItemType;
treeItemCollapsibleState: number;
isUntitled: boolean;
version?: string;
}
export class BookTreeItem extends vscode.TreeItem {
private _sections: IJupyterBookSection[];
private _sections: JupyterBookSection[];
private _uri: string | undefined;
private _previousUri: string;
private _nextUri: string;
public readonly version: string;
public command: vscode.Command;
constructor(public book: BookTreeItemFormat, icons: any) {
@@ -41,6 +46,7 @@ export class BookTreeItem extends vscode.TreeItem {
if (book.type === BookTreeItemType.Book) {
this.collapsibleState = book.treeItemCollapsibleState;
this._sections = book.page;
this.version = book.version;
if (book.isUntitled) {
this.contextValue = 'providedBook';
} else {
@@ -78,7 +84,7 @@ export class BookTreeItem extends vscode.TreeItem {
vscode.TreeItemCollapsibleState.Collapsed :
vscode.TreeItemCollapsibleState.None;
this._sections = this.book.page.sections || this.book.page.subsections;
this._uri = this.book.page.url;
this._uri = this.book.version === BookVersion.v1 ? this.book.page.url : this.book.page.file;
if (this.book.tableOfContents.sections) {
let index = (this.book.tableOfContents.sections.indexOf(this.book.page));
@@ -101,14 +107,18 @@ export class BookTreeItem extends vscode.TreeItem {
private setPreviousUri(index: number): void {
let i = --index;
while (i > -1) {
if (this.book.tableOfContents.sections[i].url) {
let pathToNotebook: string;
if (this.book.version === BookVersion.v2 && (this.book.tableOfContents.sections[i] as IJupyterBookSectionV2).file) {
// The Notebook editor expects a posix path for the resource (it will still resolve to the correct fsPath based on OS)
let pathToNotebook = path.posix.join(this.book.root, 'content', this.book.tableOfContents.sections[i].url.concat('.ipynb'));
// eslint-disable-next-line no-sync
if (fs.existsSync(pathToNotebook)) {
this._previousUri = pathToNotebook;
return;
}
pathToNotebook = path.posix.join(this.book.root, (this.book.tableOfContents.sections[i] as IJupyterBookSectionV2).file.concat('.ipynb'));
} else if ((this.book.tableOfContents.sections[i] as IJupyterBookSectionV1).url) {
pathToNotebook = path.posix.join(this.book.root, content, (this.book.tableOfContents.sections[i] as IJupyterBookSectionV1).url.concat('.ipynb'));
}
// eslint-disable-next-line no-sync
if (fs.existsSync(pathToNotebook)) {
this._previousUri = pathToNotebook;
return;
}
i--;
}
@@ -117,14 +127,18 @@ export class BookTreeItem extends vscode.TreeItem {
private setNextUri(index: number): void {
let i = ++index;
while (i < this.book.tableOfContents.sections.length) {
if (this.book.tableOfContents.sections[i].url) {
let pathToNotebook: string;
if (this.book.version === BookVersion.v2 && (this.book.tableOfContents.sections[i] as IJupyterBookSectionV2).file) {
// The Notebook editor expects a posix path for the resource (it will still resolve to the correct fsPath based on OS)
let pathToNotebook = path.posix.join(this.book.root, 'content', this.book.tableOfContents.sections[i].url.concat('.ipynb'));
// eslint-disable-next-line no-sync
if (fs.existsSync(pathToNotebook)) {
this._nextUri = pathToNotebook;
return;
}
pathToNotebook = path.posix.join(this.book.root, (this.book.tableOfContents.sections[i] as IJupyterBookSectionV2).file.concat('.ipynb'));
} else if ((this.book.tableOfContents.sections[i] as IJupyterBookSectionV1).url) {
pathToNotebook = path.posix.join(this.book.root, content, (this.book.tableOfContents.sections[i] as IJupyterBookSectionV1).url.concat('.ipynb'));
}
// eslint-disable-next-line no-sync
if (fs.existsSync(pathToNotebook)) {
this._nextUri = pathToNotebook;
return;
}
i++;
}
@@ -164,14 +178,14 @@ export class BookTreeItem extends vscode.TreeItem {
* Helper method to find a child section with a specified URL
* @param url The url of the section we're searching for
*/
public findChildSection(url?: string): IJupyterBookSection | undefined {
public findChildSection(url?: string): JupyterBookSection | undefined {
if (!url) {
return undefined;
}
return this.findChildSectionRecur(this, url);
return this.findChildSectionRecur(this as JupyterBookSection, url);
}
private findChildSectionRecur(section: IJupyterBookSection, url: string): IJupyterBookSection | undefined {
private findChildSectionRecur(section: JupyterBookSection, url: string): JupyterBookSection | undefined {
if (section.url && section.url === url) {
return section;
} else if (section.sections) {

View File

@@ -11,16 +11,17 @@ import * as constants from '../common/constants';
import { IPrompter, IQuestion, confirm } from '../prompts/question';
import CodeAdapter from '../prompts/adapter';
import { BookTreeItem, BookTreeItemType } from './bookTreeItem';
import { BookModel } from './bookModel';
import { BookModel, BookVersion } from './bookModel';
import { Deferred } from '../common/promise';
import { IBookTrustManager, BookTrustManager } from './bookTrustManager';
import * as loc from '../common/localizedConstants';
import * as glob from 'fast-glob';
import { isNullOrUndefined } from 'util';
import { IJupyterBookSectionV2, IJupyterBookSectionV1 } from '../contracts/content';
import { debounce, getPinnedNotebooks } from '../common/utils';
import { IBookPinManager, BookPinManager } from './bookPinManager';
const Content = 'content';
const content = 'content';
interface BookSearchResults {
notebookPaths: string[];
@@ -152,7 +153,7 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
// add file watcher on toc file.
if (!isNotebook) {
fs.watchFile(path.join(bookPath, '_data', 'toc.yml'), async (curr, prev) => {
fs.watchFile(this.currentBook.tableOfContentsPath, async (curr, prev) => {
if (curr.mtime > prev.mtime) {
let book = this.books.find(book => book.bookPath === bookPath);
if (book) {
@@ -206,7 +207,7 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
} finally {
// remove watch on toc file.
if (deletedBook && !deletedBook.isNotebook) {
fs.unwatchFile(path.join(deletedBook.bookPath, '_data', 'toc.yml'));
fs.unwatchFile(deletedBook.tableOfContentsPath);
}
}
}
@@ -247,7 +248,7 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
const sectionToOpen = bookRoot.findChildSection(urlToOpen);
urlPath = sectionToOpen?.url;
} else {
urlPath = this.currentBook.bookItems[0].tableOfContents.sections[0].url;
urlPath = this.currentBook.version === BookVersion.v1 ? (this.currentBook.bookItems[0].tableOfContents.sections[0] as IJupyterBookSectionV1).url : (this.currentBook.bookItems[0].tableOfContents.sections[0] as IJupyterBookSectionV2).file;
}
}
if (urlPath) {
@@ -260,8 +261,8 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
}
} else {
// The Notebook editor expects a posix path for the resource (it will still resolve to the correct fsPath based on OS)
const sectionToOpenMarkdown: string = path.posix.join(this.currentBook.bookPath, Content, urlPath.concat('.md'));
const sectionToOpenNotebook: string = path.posix.join(this.currentBook.bookPath, Content, urlPath.concat('.ipynb'));
const sectionToOpenMarkdown: string = path.posix.join(this.currentBook.contentFolderPath, urlPath.concat('.md'));
const sectionToOpenNotebook: string = path.posix.join(this.currentBook.contentFolderPath, urlPath.concat('.ipynb'));
if (await fs.pathExists(sectionToOpenMarkdown)) {
this.openMarkdown(sectionToOpenMarkdown);
}
@@ -411,13 +412,22 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
public async searchJupyterBooks(treeItem?: BookTreeItem): Promise<void> {
let folderToSearch: string;
if (treeItem && treeItem.sections !== undefined) {
if (treeItem.uri) {
folderToSearch = path.join(treeItem.root, Content, path.dirname(treeItem.uri));
} else {
folderToSearch = path.join(treeItem.root, Content);
if (treeItem.book.version === BookVersion.v1) {
if (treeItem.uri) {
folderToSearch = path.join(treeItem.book.root, content, path.dirname(treeItem.uri));
} else {
folderToSearch = path.join(treeItem.root, content);
}
} else if (treeItem.book.version === BookVersion.v2) {
if (treeItem.uri) {
folderToSearch = path.join(treeItem.book.root, path.dirname(treeItem.uri));
} else {
folderToSearch = path.join(treeItem.root);
}
}
} else if (this.currentBook && !this.currentBook.isNotebook) {
folderToSearch = path.join(this.currentBook.bookPath, Content);
folderToSearch = path.join(this.currentBook.contentFolderPath);
} else {
vscode.window.showErrorMessage(loc.noBooksSelectedError);
}
@@ -476,6 +486,9 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
}
private async getNotebooksInTree(folderPath: string): Promise<BookSearchResults> {
let tocTrimLength: number;
let ignorePaths: string[] = [];
let notebookConfig = vscode.workspace.getConfiguration(constants.notebookConfigKey);
let maxDepth = notebookConfig[constants.maxBookSearchDepth];
// Use default value if user enters an invalid value
@@ -486,13 +499,26 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
}
let escapedPath = glob.escapePath(folderPath.replace(/\\/g, '/'));
let bookFilter = path.posix.join(escapedPath, '**', '_data', 'toc.yml');
let bookPaths = await glob(bookFilter, { deep: maxDepth });
let tocTrimLength = '/_data/toc.yml'.length * -1;
bookPaths = bookPaths.map(path => path.slice(0, tocTrimLength));
let bookV1Filter = path.posix.join(escapedPath, '**', '_data', 'toc.yml');
let bookV2Filter = path.posix.join(escapedPath, '**', '_toc.yml');
let bookPaths = await glob([bookV1Filter, bookV2Filter], { deep: maxDepth });
let ignoreNotebook: string[];
bookPaths = bookPaths.map(function (path) {
if (path.includes('/_data/toc.yml')) {
tocTrimLength = '/_data/toc.yml'.length * -1;
ignoreNotebook = ['/**/*.ipynb'];
} else {
tocTrimLength = '/_toc.yml'.length * -1;
ignoreNotebook = ['/**/*.ipynb', '/*.ipynb'];
}
path = path.slice(0, tocTrimLength);
ignoreNotebook.map(notebook => ignorePaths.push(glob.escapePath(path) + notebook));
return path;
});
let notebookFilter = path.posix.join(escapedPath, '**', '*.ipynb');
let notebookPaths = await glob(notebookFilter, { ignore: bookPaths.map(path => glob.escapePath(path) + '/**/*.ipynb'), deep: maxDepth });
let notebookPaths = await glob(notebookFilter, { ignore: ignorePaths, deep: maxDepth });
return { notebookPaths: notebookPaths, bookPaths: bookPaths };
}
@@ -512,7 +538,7 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
getChildren(element?: BookTreeItem): Thenable<BookTreeItem[]> {
if (element) {
if (element.sections) {
return Promise.resolve(this.currentBook.getSections(element.tableOfContents, element.sections, element.root).then(sections => { return sections; }));
return Promise.resolve(this.currentBook.getSections(element.tableOfContents, element.sections, element.root, element.book).then(sections => { return sections; }));
} else {
return Promise.resolve([]);
}
@@ -528,7 +554,8 @@ export class BookTreeViewProvider implements vscode.TreeDataProvider<BookTreeIte
getParent(element?: BookTreeItem): vscode.ProviderResult<BookTreeItem> {
if (element?.uri) {
let parentPath: string;
parentPath = path.join(element.root, Content, element.uri.substring(0, element.uri.lastIndexOf(path.posix.sep)));
let contentFolder = element.book.version === BookVersion.v1 ? path.join(element.book.root, content) : element.book.root;
parentPath = path.join(contentFolder, element.uri.substring(0, element.uri.lastIndexOf(path.posix.sep)));
if (parentPath === element.root) {
return undefined;
}