Editing Books (#13535)

* start work on ui

* Move notebooks complete

* Simplify version handling

* fix issues with add section method

* fix issues with the edit experience and add the quick pick for editing

* add version handler for re-writing tocs

* fix book toc manager tests

* add notebook test

* fix localize constant

* normalize paths on windows

* check that a section has sections before attempting to mve files

* Implement method for renaming duplicated files

* moving last notebook from section converts the section into notebook

* Add recovery method restore original state of file system

* Add clean up, for files that are copied instead of renamed

* remove dir complexity

* divide edit book in methods for easier testing and remove promise chain

* Keep structure of toc

* normalize paths on windows

* fix external link

* Add other fields for versions

* fix paths in uri of findSection

* add section to section test

* Add error messages

* address pr comments and add tests

* check that the new path of a notebook is different from the original path before deleting
This commit is contained in:
Barbara Valdez
2021-02-02 20:39:11 -08:00
committed by GitHub
parent 41915bda8d
commit 9ac180d772
13 changed files with 1296 additions and 595 deletions

View File

@@ -6,14 +6,23 @@ import * as path from 'path';
import { BookTreeItem } from './bookTreeItem';
import * as yaml from 'js-yaml';
import * as fs from 'fs-extra';
import { IJupyterBookSectionV1, IJupyterBookSectionV2, JupyterBookSection } from '../contracts/content';
import { BookVersion } from './bookModel';
import { JupyterBookSection } from '../contracts/content';
import { BookVersion, convertTo } from './bookVersionHandler';
import * as vscode from 'vscode';
import * as loc from '../common/localizedConstants';
import { BookModel } from './bookModel';
export interface IBookTocManager {
updateBook(element: BookTreeItem, book: BookTreeItem): Promise<void>;
createBook(bookContentPath: string, contentFolder: string): Promise<void>
updateBook(element: BookTreeItem, book: BookTreeItem, targetSection?: JupyterBookSection): Promise<void>;
createBook(bookContentPath: string, contentFolder: string): Promise<void>;
recovery(): Promise<void>
}
export interface quickPickResults {
quickPickSection?: vscode.QuickPickItem,
book?: BookTreeItem
}
const allowedFileExtensions: string[] = ['.md', '.ipynb'];
const initMarkdown: string[] = ['index.md', 'introduction.md', 'intro.md', 'readme.md'];
@@ -22,13 +31,24 @@ export function hasSections(node: JupyterBookSection): boolean {
}
export class BookTocManager implements IBookTocManager {
public tableofContents: IJupyterBookSectionV2[];
public newSection: JupyterBookSection = {};
public tableofContents: JupyterBookSection[];
public newSection: JupyterBookSection;
private _movedFiles = new Map<string, string>();
private _modifiedDirectory = new Set<string>();
private _tocFiles = new Map<string, JupyterBookSection[]>();
private sourceBookContentPath: string;
private targetBookContentPath: string;
private _sourceBook: BookModel;
constructor() {
constructor(targetBook?: BookModel, sourceBook?: BookModel) {
this._sourceBook = sourceBook;
this.newSection = {};
this.tableofContents = [];
this.sourceBookContentPath = sourceBook?.bookItems[0].rootContentPath;
this.targetBookContentPath = targetBook?.bookItems[0].rootContentPath;
}
async getAllFiles(toc: IJupyterBookSectionV2[], directory: string, filesInDir: string[], rootDirectory: string): Promise<IJupyterBookSectionV2[]> {
async getAllFiles(toc: JupyterBookSection[], directory: string, filesInDir: string[], rootDirectory: string): Promise<JupyterBookSection[]> {
await Promise.all(filesInDir.map(async file => {
let isDirectory = (await fs.promises.stat(path.join(directory, file))).isDirectory();
if (isDirectory) {
@@ -41,7 +61,7 @@ export class BookTocManager implements IBookTocManager {
files.splice(index, 1);
}
});
let jupyterSection: IJupyterBookSectionV2 = {
let jupyterSection: JupyterBookSection = {
title: file,
file: path.join(file, initFile),
expand_sections: true,
@@ -53,7 +73,7 @@ export class BookTocManager implements IBookTocManager {
} else if (allowedFileExtensions.includes(path.extname(file))) {
// if the file is in the book root we don't include the directory.
const filePath = directory === rootDirectory ? path.parse(file).name : path.join(path.basename(directory), path.parse(file).name);
const addFile: IJupyterBookSectionV2 = {
const addFile: JupyterBookSection = {
title: path.parse(file).name,
file: filePath
};
@@ -74,21 +94,119 @@ export class BookTocManager implements IBookTocManager {
return toc;
}
updateToc(tableOfContents: JupyterBookSection[], findSection: BookTreeItem, addSection: JupyterBookSection): JupyterBookSection[] {
for (const section of tableOfContents) {
if ((section as IJupyterBookSectionV1).url && path.dirname(section.url) === path.join(path.sep, path.dirname(findSection.uri)) || (section as IJupyterBookSectionV2).file && path.dirname((section as IJupyterBookSectionV2).file) === path.join(path.sep, path.dirname(findSection.uri))) {
if (tableOfContents[tableOfContents.length - 1].sections) {
tableOfContents[tableOfContents.length - 1].sections.push(addSection);
} else {
tableOfContents[tableOfContents.length - 1].sections = [addSection];
/**
* Renames files if it already exists in the target book
* @param src The source file that will be moved.
* @param dest The destination path of the file, that it's been moved.
* Returns a new file name that does not exist in destination folder.
*/
async renameFile(src: string, dest: string): Promise<string> {
let newFileName = path.join(path.parse(dest).dir, path.parse(dest).name);
let counter = 2;
while (await fs.pathExists(path.join(newFileName.concat(' - ', counter.toString())).concat(path.parse(dest).ext))) {
counter++;
}
await fs.move(src, path.join(newFileName.concat(' - ', counter.toString())).concat(path.parse(dest).ext), { overwrite: true });
this._movedFiles.set(src, path.join(newFileName.concat(' - ', counter.toString())).concat(path.parse(dest).ext));
vscode.window.showInformationMessage(loc.duplicateFileError(path.parse(dest).base, src, newFileName.concat(' - ', counter.toString())));
return newFileName.concat(' - ', counter.toString());
}
/**
* Restore user's original state in case of error, when trying to move files.
* We keep track of all the moved files in the _movedFiles. The value of the map contains the current path of the file,
* while the key contains the original path.
*
* Rewrite the original table of contents of the book, in case of error as well.
*/
async recovery(): Promise<void> {
this._movedFiles.forEach(async (value, key) => {
await fs.move(value, key);
});
this._tocFiles.forEach(async (value, key) => {
await fs.writeFile(key, yaml.safeDump(value, { lineWidth: Infinity, noRefs: true, skipInvalid: true }));
});
}
async cleanUp(directory: string): Promise<void> {
let contents = await fs.readdir(directory);
if (contents.length === 0 && this._modifiedDirectory.has(directory)) {
// remove empty folders
await fs.rmdir(directory);
} else {
contents.forEach(async (content) => {
if ((await fs.stat(path.join(directory, content))).isFile) {
//check if the file is in the moved files
let isCopy = this._movedFiles.get(path.join(directory, content));
if (isCopy && this._movedFiles.get(path.join(directory, content)) !== path.join(directory, content)) {
// the file could not be renamed, so a copy was created.
// remove file only if the new path and old path are different
await fs.unlink(path.join(directory, content));
}
} else if ((await fs.stat(path.join(directory, content))).isDirectory) {
await this.cleanUp(path.join(directory, content));
}
break;
}
else if (hasSections(section)) {
return this.updateToc(section.sections, findSection, addSection);
});
}
}
/**
* Reads and modifies the table of contents file of the target book.
* @param version the version of the target book
* @param tocPath Path to the table of contents
* @param findSection The section that will be modified.
* @param addSection The section that'll be added to the target section. If it's undefined then the target section (findSection) is removed from the table of contents.
*/
async updateTOC(version: BookVersion, tocPath: string, findSection: JupyterBookSection, addSection?: JupyterBookSection): Promise<void> {
const toc = yaml.safeLoad((await fs.readFile(tocPath, 'utf8')));
this._tocFiles.set(tocPath, toc);
let newToc = new Array<JupyterBookSection>(toc.length);
for (const [index, section] of toc.entries()) {
let newSection = this.buildTOC(version, section, findSection, addSection);
if (newSection) {
newToc[index] = newSection;
}
}
return tableOfContents;
await fs.writeFile(tocPath, yaml.safeDump(newToc, { lineWidth: Infinity, noRefs: true, skipInvalid: true }));
this.tableofContents = newToc;
}
/**
* Creates a new table of contents structure containing the added section. This method is only called when we move a section to another section.
* Since the sections can be arranged in a tree structure we need to look for the section that will be modified in a recursively.
* @param version Version of the book
* @param section The current section that we are iterating
* @param findSection The section that will be modified.
* @param addSection The section that'll be added to the target section. If it's undefined then the target section (findSection) is removed from the table of contents.
*/
private buildTOC(version: BookVersion, section: JupyterBookSection, findSection: JupyterBookSection, addSection: JupyterBookSection): JupyterBookSection {
// condition to find the section to be modified
if (section.title === findSection.title && (section.file && section.file === findSection.file || section.url && section.url === findSection.file)) {
if (addSection) {
//if addSection is not undefined, then we added to the table of contents.
section.sections !== undefined && section.sections.length > 0 ? section.sections.push(addSection) : section.sections = [addSection];
return section;
}
// if addSection is undefined then we remove the whole section from the table of contents.
return addSection;
} else {
let newSection = convertTo(version, section);
if (section.sections && section.sections.length > 0) {
newSection.sections = [] as JupyterBookSection[];
for (let s of section.sections) {
const child = this.buildTOC(version, s, findSection, addSection);
if (child) {
newSection.sections.push(convertTo(version, child));
}
}
}
if (newSection.sections?.length === 0) {
// if sections is an empty array then assign it to undefined, so it's converted into a markdown file.
newSection.sections = undefined;
}
return newSection;
}
}
/**
@@ -107,75 +225,198 @@ export class BookTocManager implements IBookTocManager {
await vscode.commands.executeCommand('notebook.command.openNotebookFolder', bookContentPath, undefined, true);
}
async addSection(section: BookTreeItem, book: BookTreeItem, isSection: boolean): Promise<void> {
this.newSection.title = section.title;
//the book contentPath contains the first file of the section, we get the dirname to identify the section's root path
const rootPath = isSection ? path.dirname(book.book.contentPath) : book.rootContentPath;
// TODO: the uri contains the first notebook or markdown file in the TOC format. If we are in a section,
// we want to include the intermediary directories between the book's root and the section
const uri = isSection ? path.join(path.basename(rootPath), section.uri) : section.uri;
if (section.book.version === BookVersion.v1) {
this.newSection.url = uri;
let movedSections: IJupyterBookSectionV1[] = [];
const files = section.sections as IJupyterBookSectionV1[];
for (const elem of files) {
await fs.promises.mkdir(path.join(rootPath, path.dirname(elem.url)), { recursive: true });
await fs.move(path.join(path.dirname(section.book.contentPath), path.basename(elem.url)), path.join(rootPath, elem.url));
movedSections.push({ url: isSection ? path.join(path.basename(rootPath), elem.url) : elem.url, title: elem.title });
/**
* When moving a section, we need to modify every file path that it's within the section. Since sections is a tree like structure, we need to modify each of the file paths
* and move the files individually. The overwrite option is set to false to prevent any issues with duplicated file names.
* @param files Files in the section.
*/
async traverseSections(files: JupyterBookSection[]): Promise<JupyterBookSection[]> {
let movedSections: JupyterBookSection[] = [];
for (const elem of files) {
if (elem.file) {
let fileName = undefined;
try {
await fs.move(path.join(this.sourceBookContentPath, elem.file).concat('.ipynb'), path.join(this.targetBookContentPath, elem.file).concat('.ipynb'), { overwrite: false });
this._movedFiles.set(path.join(this.sourceBookContentPath, elem.file).concat('.ipynb'), path.join(this.targetBookContentPath, elem.file).concat('.ipynb'));
} catch (error) {
if (error.code === 'EEXIST') {
fileName = await this.renameFile(path.join(this.sourceBookContentPath, elem.file).concat('.ipynb'), path.join(this.targetBookContentPath, elem.file).concat('.ipynb'));
}
else if (error.code !== 'ENOENT') {
throw (error);
}
}
try {
await fs.move(path.join(this.sourceBookContentPath, elem.file).concat('.md'), path.join(this.targetBookContentPath, elem.file).concat('.md'), { overwrite: false });
this._movedFiles.set(path.join(this.sourceBookContentPath, elem.file).concat('.md'), path.join(this.targetBookContentPath, elem.file).concat('.md'));
} catch (error) {
if (error.code === 'EEXIST') {
fileName = await this.renameFile(path.join(this.sourceBookContentPath, elem.file).concat('.md'), path.join(this.targetBookContentPath, elem.file).concat('.md'));
}
else if (error.code !== 'ENOENT') {
throw (error);
}
}
elem.file = fileName === undefined ? elem.file : path.join(path.dirname(elem.file), path.parse(fileName).name);
elem.sections = elem.sections ? await this.traverseSections(elem.sections) : undefined;
}
this.newSection.sections = movedSections;
} else if (section.book.version === BookVersion.v2) {
(this.newSection as IJupyterBookSectionV2).file = uri;
let movedSections: IJupyterBookSectionV2[] = [];
const files = section.sections as IJupyterBookSectionV2[];
for (const elem of files) {
await fs.promises.mkdir(path.join(rootPath, path.dirname(elem.file)), { recursive: true });
await fs.move(path.join(path.dirname(section.book.contentPath), path.basename(elem.file)), path.join(rootPath, elem.file));
movedSections.push({ file: isSection ? path.join(path.basename(rootPath), elem.file) : elem.file, title: elem.title });
}
this.newSection.sections = movedSections;
movedSections.push(elem);
}
return movedSections;
}
async addNotebook(notebook: BookTreeItem, book: BookTreeItem, isSection: boolean): Promise<void> {
//the book's contentPath contains the first file of the section, we get the dirname to identify the section's root path
const rootPath = isSection ? path.dirname(book.book.contentPath) : book.rootContentPath;
let notebookName = path.basename(notebook.book.contentPath);
await fs.move(notebook.book.contentPath, path.join(rootPath, notebookName));
if (book.book.version === BookVersion.v1) {
this.newSection = { url: notebookName, title: notebookName };
} else if (book.book.version === BookVersion.v2) {
this.newSection = { file: notebookName, title: notebookName };
/**
* Moves a section to a book top level or another book's section. If there's a target section we add the the targetSection directory if it has one and append it to the
* notebook's path. The overwrite option is set to false to prevent any issues with duplicated file names.
* @param section The section that's been moved.
* @param book The target book.
*/
async addSection(section: BookTreeItem, book: BookTreeItem): Promise<void> {
const uri = path.sep.concat(path.relative(section.rootContentPath, section.book.contentPath));
let moveFile = path.join(path.parse(uri).dir, path.parse(uri).name);
let fileName = undefined;
try {
await fs.move(section.book.contentPath, path.join(this.targetBookContentPath, moveFile).concat(path.parse(uri).ext), { overwrite: false });
this._movedFiles.set(section.book.contentPath, path.join(this.targetBookContentPath, moveFile).concat(path.parse(uri).ext));
} catch (error) {
if (error.code === 'EEXIST') {
fileName = await this.renameFile(section.book.contentPath, path.join(this.targetBookContentPath, moveFile).concat(path.parse(uri).ext));
}
else if (error.code !== 'ENOENT') {
throw (error);
}
}
fileName = fileName === undefined ? path.parse(uri).name : path.parse(fileName).name;
if (this._sourceBook) {
const sectionTOC = this._sourceBook.bookItems[0].findChildSection(section.uri);
if (sectionTOC) {
this.newSection = sectionTOC;
}
}
this.newSection.title = section.title;
this.newSection.file = path.join(path.parse(uri).dir, fileName)?.replace(/\\/g, '/');
if (section.sections) {
const files = section.sections as JupyterBookSection[];
const movedSections = await this.traverseSections(files);
this.newSection.sections = movedSections;
this._modifiedDirectory.add(path.dirname(section.book.contentPath));
this.cleanUp(path.dirname(section.book.contentPath));
}
if (book.version === BookVersion.v1) {
// here we only convert if is v1 because we are already using the v2 notation for every book that we read.
this.newSection = convertTo(book.version, this.newSection);
}
}
/**
* Moves the element to the book's folder and adds it to the table of contents.
* Moves a notebook to a book top level or a book's section. If there's a target section we add the the targetSection directory if it has one and append it to the
* notebook's path. The overwrite option is set to false to prevent any issues with duplicated file names.
* @param element Notebook, Markdown File, or section that will be added to the book.
* @param book Book or a BookSection that will be modified.
* @param targetBook Book that will be modified.
*/
public async updateBook(element: BookTreeItem, book: BookTreeItem): Promise<void> {
if (element.contextValue === 'section' && book.book.version === element.book.version) {
if (book.contextValue === 'section') {
await this.addSection(element, book, true);
this.tableofContents = this.updateToc(book.tableOfContents.sections, book, this.newSection);
await fs.writeFile(book.tableOfContentsPath, yaml.safeDump(this.tableofContents, { lineWidth: Infinity }));
} else if (book.contextValue === 'savedBook') {
await this.addSection(element, book, false);
book.tableOfContents.sections.push(this.newSection);
await fs.writeFile(book.tableOfContentsPath, yaml.safeDump(book.tableOfContents, { lineWidth: Infinity }));
async addNotebook(notebook: BookTreeItem, book: BookTreeItem): Promise<void> {
const rootPath = book.rootContentPath;
const notebookPath = path.parse(notebook.book.contentPath);
let fileName = undefined;
try {
await fs.move(notebook.book.contentPath, path.join(rootPath, notebookPath.base), { overwrite: false });
} catch (error) {
if (error.code === 'EEXIST') {
fileName = await this.renameFile(notebook.book.contentPath, path.join(rootPath, notebookPath.base));
}
else {
throw (error);
}
}
if (this._sourceBook) {
const sectionTOC = this._sourceBook.bookItems[0].findChildSection(notebook.uri);
this.newSection = sectionTOC;
}
fileName = fileName === undefined ? notebookPath.name : path.parse(fileName).name;
this.newSection.file = path.sep.concat(fileName);
this.newSection.title = notebook.book.title;
if (book.version === BookVersion.v1) {
// here we only convert if is v1 because we are already using the v2 notation for every book that we read.
this.newSection = convertTo(book.version, this.newSection);
}
}
/**
* Moves the element to the target book's folder and adds it to the table of contents.
* @param element Notebook, Markdown File, or section that will be added to the book.
* @param targetBook Book that will be modified.
* @param targetSection Book section that'll be modified.
*/
public async updateBook(element: BookTreeItem, targetBook: BookTreeItem, targetSection?: JupyterBookSection): Promise<void> {
const targetBookVersion = targetBook.book.version === BookVersion.v1 ? BookVersion.v1 : BookVersion.v2;
if (element.contextValue === 'section') {
await this.addSection(element, targetBook);
const elementVersion = element.book.version === BookVersion.v1 ? BookVersion.v1 : BookVersion.v2;
// modify the sourceBook toc and remove the section
const findSection: JupyterBookSection = { file: element.book.page.file?.replace(/\\/g, '/'), title: element.book.page.title };
await this.updateTOC(elementVersion, element.tableOfContentsPath, findSection, undefined);
if (targetSection) {
// adding new section to the target book toc file
await this.updateTOC(targetBookVersion, targetBook.tableOfContentsPath, targetSection, this.newSection);
}
else {
//since there's not a target section, we just append the section at the end of the file
if (this.targetBookContentPath !== this.sourceBookContentPath) {
this.tableofContents = targetBook.sections.map(section => convertTo(targetBook.version, section));
}
this.tableofContents.push(this.newSection);
await fs.writeFile(targetBook.tableOfContentsPath, yaml.safeDump(this.tableofContents, { lineWidth: Infinity, noRefs: true, skipInvalid: true }));
}
}
else if (element.contextValue === 'savedNotebook') {
if (book.contextValue === 'savedBook') {
await this.addNotebook(element, book, false);
book.tableOfContents.sections.push(this.newSection);
await fs.writeFile(book.tableOfContentsPath, yaml.safeDump(book.tableOfContents, { lineWidth: Infinity }));
} else if (book.contextValue === 'section') {
await this.addNotebook(element, book, true);
this.tableofContents = this.updateToc(book.tableOfContents.sections, book, this.newSection);
await fs.writeFile(book.tableOfContentsPath, yaml.safeDump(this.tableofContents, { lineWidth: Infinity }));
await this.addNotebook(element, targetBook);
if (element.tableOfContentsPath) {
const elementVersion = element.book.version === BookVersion.v1 ? BookVersion.v1 : BookVersion.v2;
// the notebook is part of a book so we need to modify its toc as well
const findSection = { file: element.book.page.file?.replace(/\\/g, '/'), title: element.book.page.title };
await this.updateTOC(elementVersion, element.tableOfContentsPath, findSection, undefined);
} else {
// close the standalone notebook, so it doesn't throw an error when we move the notebook to new location.
await vscode.commands.executeCommand('notebook.command.closeNotebook', element);
}
if (!targetSection) {
if (this.targetBookContentPath !== this.sourceBookContentPath) {
this.tableofContents = targetBook.sections.map(section => convertTo(targetBook.version, section));
}
this.tableofContents.push(this.newSection);
await fs.writeFile(targetBook.tableOfContentsPath, yaml.safeDump(this.tableofContents, { lineWidth: Infinity, noRefs: true, skipInvalid: true }));
} else {
await this.updateTOC(targetBookVersion, targetBook.tableOfContentsPath, targetSection, this.newSection);
}
}
}
public get movedFiles(): Map<string, string> {
return this._movedFiles;
}
public get originalToc(): Map<string, JupyterBookSection[]> {
return this._tocFiles;
}
public get modifiedDir(): Set<string> {
return this._modifiedDirectory;
}
public set movedFiles(files: Map<string, string>) {
this._movedFiles = files;
}
public set originalToc(files: Map<string, JupyterBookSection[]>) {
this._tocFiles = files;
}
public set modifiedDir(files: Set<string>) {
this._modifiedDirectory = files;
}
}