Support HDFS Tiering (#7484)

This is the 1st step to supporting HDFS Tiering
Changes:

Add new mounted folder icon. Will have separate commit for file icon
Disable delete/mkdir/upload for mounted files and folders
Disable delete for root HDFS folder (this was added in error)

extensions/mssql/src/constants.ts

@@ -56,7 +56,9 @@ export enum MssqlClusterItems {
 }
 
 export enum MssqlClusterItemsSubType {
-	Spark = 'mssqlCluster:spark'
+	Mount = ':mount:',
+	MountChild = ':mountChild:',
+	Spark = ':spark:'
 }
 
 // SPARK JOB SUBMISSION //////////////////////////////////////////////////////////

extensions/mssql/src/hdfs/mount.ts

@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * Information about a HDFS mount to a remote directory
+ */
+export interface Mount {
+	mountPath: string;
+	mountStatus: string;
+	remotePath: string;
+}
+
+export enum MountStatus {
+	None = 0,
+	Mount = 1,
+	Mount_Child = 2
+}

extensions/mssql/src/hdfs/webhdfs.ts

@@ -12,6 +12,7 @@ import * as nls from 'vscode-nls';
 import * as auth from '../util/auth';
 import { IHdfsOptions, IRequestParams } from '../objectExplorerNodeProvider/fileSources';
 import { IAclStatus, AclEntry, parseAcl, AclPermissionType, parseAclPermissionFromOctal, AclEntryScope } from './aclEntry';
+import { Mount } from './mount';
 import { everyoneName } from '../localizedConstants';
 
 const localize = nls.loadMessageBundle();
@@ -472,6 +473,26 @@ export class WebHDFS {
 		});
 	}
 
+	/**
+	 * Get all mounts for a HDFS connection
+	 * @param callback Callback to handle the response
+	 * @returns void
+	 */
+	public getMounts(callback: (error: HdfsError, mounts: Mount[]) => void): void {
+		let endpoint = this.getOperationEndpoint('listmounts', '');
+		this.sendRequest('GET', endpoint, undefined, (error, response) => {
+			if (!callback) { return; }
+			if (error) {
+				callback(error, undefined);
+			} else if (response.body.hasOwnProperty('Mounts')) {
+				const mounts = response.body.Mounts;
+				callback(undefined, mounts);
+			} else {
+				callback(new HdfsError(ErrorMessageInvalidDataStructure), undefined);
+			}
+		});
+	}
+
 	/**
 	 * Check file existence
 	 * Wraps stat method

extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts

@@ -16,6 +16,7 @@ import * as nls from 'vscode-nls';
 import * as constants from '../constants';
 import { WebHDFS, HdfsError } from '../hdfs/webhdfs';
 import { AclEntry, IAclStatus } from '../hdfs/aclEntry';
+import { Mount, MountStatus } from '../hdfs/mount';
 
 const localize = nls.loadMessageBundle();
 
@@ -29,9 +30,11 @@ export function joinHdfsPath(parent: string, child: string): string {
 export interface IFile {
 	path: string;
 	isDirectory: boolean;
+	mountStatus?: MountStatus;
 }
 
 export class File implements IFile {
+	public mountStatus?: MountStatus;
 	constructor(public path: string, public isDirectory: boolean) {
 
 	}
@@ -58,7 +61,7 @@ export class File implements IFile {
 }
 
 export interface IFileSource {
-	enumerateFiles(path: string): Promise<IFile[]>;
+	enumerateFiles(path: string, refresh?: boolean): Promise<IFile[]>;
 	mkdir(dirName: string, remoteBasePath: string): Promise<void>;
 	createReadStream(path: string): fs.ReadStream;
 	readFile(path: string, maxBytes?: number): Promise<Buffer>;
@@ -159,18 +162,43 @@ export class FileSourceFactory {
 }
 
 export class HdfsFileSource implements IFileSource {
+	private mounts: Map<string, Mount>;
 	constructor(private client: WebHDFS) {
 	}
 
-	public enumerateFiles(path: string): Promise<IFile[]> {
+	public async enumerateFiles(path: string, refresh?: boolean): Promise<IFile[]> {
+		if (!this.mounts || refresh) {
+			await this.loadMounts();
+		}
+		return this.readdir(path);
+	}
+
+	private loadMounts(): Promise<void> {
+		return new Promise((resolve, reject) => {
+			this.client.getMounts((error, mounts) => {
+				this.mounts = new Map();
+				if (!error && mounts) {
+					mounts.forEach(m => this.mounts.set(m.mountPath, m));
+				}
+				resolve();
+			});
+		});
+	}
+
+	private readdir(path: string): Promise<IFile[]> {
 		return new Promise((resolve, reject) => {
 			this.client.readdir(path, (error, files) => {
 				if (error) {
 					reject(error);
-				} else {
-					let hdfsFiles: IFile[] = files.map(file => {
-						let hdfsFile = <IHdfsFileStatus>file;
-						return new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY');
+				}
+				else {
+					let hdfsFiles: IFile[] = files.map(fileStat => {
+						let hdfsFile = <IHdfsFileStatus>fileStat;
+						let file = new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY');
+						if (this.mounts && this.mounts.has(file.path)) {
+							file.mountStatus = MountStatus.Mount;
+						}
+						return file;
 					});
 					resolve(hdfsFiles);
 				}

extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts

@@ -16,6 +16,7 @@ import { CancelableStream } from './cancelableStream';
 import { TreeNode } from './treeNodes';
 import * as utils from '../utils';
 import { IFileNode } from './types';
+import { MountStatus } from '../hdfs/mount';
 
 export interface ITreeChangeHandler {
 	notifyNodeChanged(node: TreeNode): void;
@@ -103,7 +104,7 @@ export abstract class HdfsFileSourceNode extends TreeNode {
 export class FolderNode extends HdfsFileSourceNode {
 	private children: TreeNode[];
 	protected _nodeType: string;
-	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string) {
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string, private mountStatus?: MountStatus) {
 		super(context, path, fileSource);
 		this._nodeType = nodeType ? nodeType : Constants.MssqlClusterItems.Folder;
 	}
@@ -126,8 +127,8 @@ export class FolderNode extends HdfsFileSourceNode {
 				if (files) {
 					// Note: for now, assuming HDFS-provided sorting is sufficient
 					this.children = files.map((file) => {
-						let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource)
-							: new FileNode(this.context, file.path, this.fileSource);
+						let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource, Constants.MssqlClusterItems.Folder, this.getChildMountStatus(file))
+							: new FileNode(this.context, file.path, this.fileSource, this.getChildMountStatus(file));
 						node.parent = this;
 						return node;
 					});
@@ -139,6 +140,17 @@ export class FolderNode extends HdfsFileSourceNode {
 		return this.children;
 	}
 
+	private getChildMountStatus(file: IFile): MountStatus {
+		if (file.mountStatus !== undefined && file.mountStatus !== MountStatus.None) {
+			return file.mountStatus;
+		}
+		else if (this.mountStatus !== undefined && this.mountStatus !== MountStatus.None) {
+			// Any child node of a mount (or subtree) must be a mount child
+			return MountStatus.Mount_Child;
+		}
+		return MountStatus.None;
+	}
+
 	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
 		let item = new vscode.TreeItem(this.getDisplayName(), vscode.TreeItemCollapsibleState.Collapsed);
 		// For now, folder always looks the same. We're using SQL icons to differentiate remote vs local files
@@ -161,12 +173,26 @@ export class FolderNode extends HdfsFileSourceNode {
 			nodePath: this.generateNodePath(),
 			nodeStatus: undefined,
 			nodeType: this._nodeType,
-			nodeSubType: undefined,
-			iconType: 'Folder'
+			nodeSubType: this.getSubType(),
+			iconType: this.isMounted() ? 'Folder_mounted' : 'Folder'
 		};
 		return nodeInfo;
 	}
 
+	private isMounted(): boolean {
+		return this.mountStatus === MountStatus.Mount || this.mountStatus === MountStatus.Mount_Child;
+	}
+
+	private getSubType(): string | undefined {
+		if (this.mountStatus === MountStatus.Mount) {
+			return Constants.MssqlClusterItemsSubType.Mount;
+		} else if (this.mountStatus === MountStatus.Mount_Child) {
+			return Constants.MssqlClusterItemsSubType.MountChild;
+		}
+
+		return undefined;
+	}
+
 	public async writeFile(localFile: IFile): Promise<FileNode> {
 		return this.runChildAddAction<FileNode>(() => this.writeFileAsync(localFile));
 	}
@@ -243,7 +269,7 @@ export class ConnectionNode extends FolderNode {
 
 export class FileNode extends HdfsFileSourceNode implements IFileNode {
 
-	constructor(context: TreeDataContext, path: string, fileSource: IFileSource) {
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, private mountStatus?: MountStatus) {
 		super(context, path, fileSource);
 	}
 
@@ -322,12 +348,15 @@ export class FileNode extends HdfsFileSourceNode implements IFileNode {
 		});
 	}
 
-	private getSubType(): string {
+	private getSubType(): string | undefined {
+		let subType = '';
 		if (this.getDisplayName().toLowerCase().endsWith('.jar') || this.getDisplayName().toLowerCase().endsWith('.py')) {
-			return Constants.MssqlClusterItemsSubType.Spark;
+			subType += Constants.MssqlClusterItemsSubType.Spark;
+		} else if (this.mountStatus === MountStatus.Mount_Child) {
+			subType += Constants.MssqlClusterItemsSubType.MountChild;
 		}
 
-		return undefined;
+		return subType.length > 0 ? subType : undefined;
 	}
 }