Support HDFS Tiering (#7484)

This is the 1st step to supporting HDFS Tiering
Changes:

Add new mounted folder icon. Will have separate commit for file icon
Disable delete/mkdir/upload for mounted files and folders
Disable delete for root HDFS folder (this was added in error)

extensions/mssql/package.json

@@ -253,12 +253,12 @@
       "objectExplorer/item/context": [
         {
           "command": "mssqlCluster.uploadFiles",
-          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file && nodeSubType=~/^(?!:mount).*$/",
           "group": "1mssqlCluster@1"
         },
         {
           "command": "mssqlCluster.mkdir",
-          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file && nodeSubType=~/^(?!:mount).*$/",
           "group": "1mssqlCluster@1"
         },
         {
@@ -278,7 +278,7 @@
         },
         {
           "command": "mssqlCluster.deleteFiles",
-          "when": "nodeType=~/^mssqlCluster/ && viewItem != mssqlCluster:connection && nodeType != mssqlCluster:message",
+          "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:hdfs && nodeType != mssqlCluster:connection && viewItem != mssqlCluster:connection && nodeType != mssqlCluster:message && nodeSubType=~/^(?!:mount).*$/",
           "group": "1mssqlCluster@4"
         },
         {
@@ -288,7 +288,7 @@
         },
         {
           "command": "mssqlCluster.livy.cmd.submitFileToSparkJob",
-          "when": "nodeType == mssqlCluster:file && nodeSubType == mssqlCluster:spark",
+          "when": "nodeType == mssqlCluster:file && nodeSubType =~/:spark:/",
           "group": "1mssqlCluster@6"
         }
       ]

extensions/mssql/src/constants.ts

@@ -56,7 +56,9 @@ export enum MssqlClusterItems {
 }
 
 export enum MssqlClusterItemsSubType {
-	Spark = 'mssqlCluster:spark'
+	Mount = ':mount:',
+	MountChild = ':mountChild:',
+	Spark = ':spark:'
 }
 
 // SPARK JOB SUBMISSION //////////////////////////////////////////////////////////

extensions/mssql/src/hdfs/mount.ts

@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * Information about a HDFS mount to a remote directory
+ */
+export interface Mount {
+	mountPath: string;
+	mountStatus: string;
+	remotePath: string;
+}
+
+export enum MountStatus {
+	None = 0,
+	Mount = 1,
+	Mount_Child = 2
+}

extensions/mssql/src/hdfs/webhdfs.ts

@@ -12,6 +12,7 @@ import * as nls from 'vscode-nls';
 import * as auth from '../util/auth';
 import { IHdfsOptions, IRequestParams } from '../objectExplorerNodeProvider/fileSources';
 import { IAclStatus, AclEntry, parseAcl, AclPermissionType, parseAclPermissionFromOctal, AclEntryScope } from './aclEntry';
+import { Mount } from './mount';
 import { everyoneName } from '../localizedConstants';
 
 const localize = nls.loadMessageBundle();
@@ -472,6 +473,26 @@ export class WebHDFS {
 		});
 	}
 
+	/**
+	 * Get all mounts for a HDFS connection
+	 * @param callback Callback to handle the response
+	 * @returns void
+	 */
+	public getMounts(callback: (error: HdfsError, mounts: Mount[]) => void): void {
+		let endpoint = this.getOperationEndpoint('listmounts', '');
+		this.sendRequest('GET', endpoint, undefined, (error, response) => {
+			if (!callback) { return; }
+			if (error) {
+				callback(error, undefined);
+			} else if (response.body.hasOwnProperty('Mounts')) {
+				const mounts = response.body.Mounts;
+				callback(undefined, mounts);
+			} else {
+				callback(new HdfsError(ErrorMessageInvalidDataStructure), undefined);
+			}
+		});
+	}
+
 	/**
 	 * Check file existence
 	 * Wraps stat method

extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts

@@ -16,6 +16,7 @@ import * as nls from 'vscode-nls';
 import * as constants from '../constants';
 import { WebHDFS, HdfsError } from '../hdfs/webhdfs';
 import { AclEntry, IAclStatus } from '../hdfs/aclEntry';
+import { Mount, MountStatus } from '../hdfs/mount';
 
 const localize = nls.loadMessageBundle();
 
@@ -29,9 +30,11 @@ export function joinHdfsPath(parent: string, child: string): string {
 export interface IFile {
 	path: string;
 	isDirectory: boolean;
+	mountStatus?: MountStatus;
 }
 
 export class File implements IFile {
+	public mountStatus?: MountStatus;
 	constructor(public path: string, public isDirectory: boolean) {
 
 	}
@@ -58,7 +61,7 @@ export class File implements IFile {
 }
 
 export interface IFileSource {
-	enumerateFiles(path: string): Promise<IFile[]>;
+	enumerateFiles(path: string, refresh?: boolean): Promise<IFile[]>;
 	mkdir(dirName: string, remoteBasePath: string): Promise<void>;
 	createReadStream(path: string): fs.ReadStream;
 	readFile(path: string, maxBytes?: number): Promise<Buffer>;
@@ -159,18 +162,43 @@ export class FileSourceFactory {
 }
 
 export class HdfsFileSource implements IFileSource {
+	private mounts: Map<string, Mount>;
 	constructor(private client: WebHDFS) {
 	}
 
-	public enumerateFiles(path: string): Promise<IFile[]> {
+	public async enumerateFiles(path: string, refresh?: boolean): Promise<IFile[]> {
+		if (!this.mounts || refresh) {
+			await this.loadMounts();
+		}
+		return this.readdir(path);
+	}
+
+	private loadMounts(): Promise<void> {
+		return new Promise((resolve, reject) => {
+			this.client.getMounts((error, mounts) => {
+				this.mounts = new Map();
+				if (!error && mounts) {
+					mounts.forEach(m => this.mounts.set(m.mountPath, m));
+				}
+				resolve();
+			});
+		});
+	}
+
+	private readdir(path: string): Promise<IFile[]> {
 		return new Promise((resolve, reject) => {
 			this.client.readdir(path, (error, files) => {
 				if (error) {
 					reject(error);
-				} else {
+				}
-					let hdfsFiles: IFile[] = files.map(file => {
+				else {
-						let hdfsFile = <IHdfsFileStatus>file;
+					let hdfsFiles: IFile[] = files.map(fileStat => {
-						return new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY');
+						let hdfsFile = <IHdfsFileStatus>fileStat;
+						let file = new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY');
+						if (this.mounts && this.mounts.has(file.path)) {
+							file.mountStatus = MountStatus.Mount;
+						}
+						return file;
 					});
 					resolve(hdfsFiles);
 				}

extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts

@@ -16,6 +16,7 @@ import { CancelableStream } from './cancelableStream';
 import { TreeNode } from './treeNodes';
 import * as utils from '../utils';
 import { IFileNode } from './types';
+import { MountStatus } from '../hdfs/mount';
 
 export interface ITreeChangeHandler {
 	notifyNodeChanged(node: TreeNode): void;
@@ -103,7 +104,7 @@ export abstract class HdfsFileSourceNode extends TreeNode {
 export class FolderNode extends HdfsFileSourceNode {
 	private children: TreeNode[];
 	protected _nodeType: string;
-	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string) {
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string, private mountStatus?: MountStatus) {
 		super(context, path, fileSource);
 		this._nodeType = nodeType ? nodeType : Constants.MssqlClusterItems.Folder;
 	}
@@ -126,8 +127,8 @@ export class FolderNode extends HdfsFileSourceNode {
 				if (files) {
 					// Note: for now, assuming HDFS-provided sorting is sufficient
 					this.children = files.map((file) => {
-						let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource)
+						let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource, Constants.MssqlClusterItems.Folder, this.getChildMountStatus(file))
-							: new FileNode(this.context, file.path, this.fileSource);
+							: new FileNode(this.context, file.path, this.fileSource, this.getChildMountStatus(file));
 						node.parent = this;
 						return node;
 					});
@@ -139,6 +140,17 @@ export class FolderNode extends HdfsFileSourceNode {
 		return this.children;
 	}
 
+	private getChildMountStatus(file: IFile): MountStatus {
+		if (file.mountStatus !== undefined && file.mountStatus !== MountStatus.None) {
+			return file.mountStatus;
+		}
+		else if (this.mountStatus !== undefined && this.mountStatus !== MountStatus.None) {
+			// Any child node of a mount (or subtree) must be a mount child
+			return MountStatus.Mount_Child;
+		}
+		return MountStatus.None;
+	}
+
 	getTreeItem(): vscode.TreeItem | Promise<vscode.TreeItem> {
 		let item = new vscode.TreeItem(this.getDisplayName(), vscode.TreeItemCollapsibleState.Collapsed);
 		// For now, folder always looks the same. We're using SQL icons to differentiate remote vs local files
@@ -161,12 +173,26 @@ export class FolderNode extends HdfsFileSourceNode {
 			nodePath: this.generateNodePath(),
 			nodeStatus: undefined,
 			nodeType: this._nodeType,
-			nodeSubType: undefined,
+			nodeSubType: this.getSubType(),
-			iconType: 'Folder'
+			iconType: this.isMounted() ? 'Folder_mounted' : 'Folder'
 		};
 		return nodeInfo;
 	}
 
+	private isMounted(): boolean {
+		return this.mountStatus === MountStatus.Mount || this.mountStatus === MountStatus.Mount_Child;
+	}
+
+	private getSubType(): string | undefined {
+		if (this.mountStatus === MountStatus.Mount) {
+			return Constants.MssqlClusterItemsSubType.Mount;
+		} else if (this.mountStatus === MountStatus.Mount_Child) {
+			return Constants.MssqlClusterItemsSubType.MountChild;
+		}
+
+		return undefined;
+	}
+
 	public async writeFile(localFile: IFile): Promise<FileNode> {
 		return this.runChildAddAction<FileNode>(() => this.writeFileAsync(localFile));
 	}
@@ -243,7 +269,7 @@ export class ConnectionNode extends FolderNode {
 
 export class FileNode extends HdfsFileSourceNode implements IFileNode {
 
-	constructor(context: TreeDataContext, path: string, fileSource: IFileSource) {
+	constructor(context: TreeDataContext, path: string, fileSource: IFileSource, private mountStatus?: MountStatus) {
 		super(context, path, fileSource);
 	}
 
@@ -322,12 +348,15 @@ export class FileNode extends HdfsFileSourceNode implements IFileNode {
 		});
 	}
 
-	private getSubType(): string {
+	private getSubType(): string | undefined {
+		let subType = '';
 		if (this.getDisplayName().toLowerCase().endsWith('.jar') || this.getDisplayName().toLowerCase().endsWith('.py')) {
-			return Constants.MssqlClusterItemsSubType.Spark;
+			subType += Constants.MssqlClusterItemsSubType.Spark;
+		} else if (this.mountStatus === MountStatus.Mount_Child) {
+			subType += Constants.MssqlClusterItemsSubType.MountChild;
 		}
 
-		return undefined;
+		return subType.length > 0 ? subType : undefined;
 	}
 }
 

src/sql/media/objectTypes/Folder_mounted.svg

@@ -0,0 +1,21 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="15.984" height="15.984" viewBox="0 0 15.984 15.984">
+  <defs>
+    <clipPath id="a">
+      <rect width="15.984" height="15.984" fill="none"/>
+    </clipPath>
+  </defs>
+  <title>Folder_mounted_updatedSize</title>
+  <g clip-path="url(#a)">
+    <g>
+      <path d="M15.984,2.5v9.99a1.5,1.5,0,0,1-1.5,1.5H2.5a1.5,1.5,0,0,1-1.5-1.5V4.5A1.5,1.5,0,0,1,2.5,3H5.385l1-2h8.1A1.5,1.5,0,0,1,15.984,2.5Z" fill="#f6f6f6"/>
+      <path d="M14.486,2H7L6,4H2.5a.5.5,0,0,0-.5.5v7.992a.5.5,0,0,0,.5.5H14.486a.5.5,0,0,0,.5-.5V2.5A.5.5,0,0,0,14.486,2Zm-.5,2H7.493l.5-1h5.994Z" fill="#dcb67a"/>
+      <path d="M13.986,3V4H7.493l.5-1Z" fill="#f0eff1"/>
+    </g>
+  </g>
+  <path d="M7.143,10.989H6.474l.519,1c.38,0,1,0,1,1,0,.38,0,1-.779,1H5.574a3.735,3.735,0,0,0-.829-.939c-.289-.19-.749.29-.459.929a5.716,5.716,0,0,1,.529,1H6.993a1.8,1.8,0,0,0,2-1.573,1.777,1.777,0,0,0,0-.425C8.991,11.489,8.262,10.989,7.143,10.989Z" fill="#424242"/>
+  <path d="M2.9,14.985h.639l-.539-1c-.37,0-1,0-1-1,0-.37,0-1,.779-1H4.456a3.735,3.735,0,0,0,.829.939c.289.19.749-.29.459-.929a5.716,5.716,0,0,1-.529-1H3a1.8,1.8,0,0,0-2,1.573A1.777,1.777,0,0,0,1,13C1,14.486,1.768,14.985,2.9,14.985Z" fill="#424242"/>
+  <path d="M7.2,15.984a2.592,2.592,0,0,1-.264-.013h-2.8L3.9,15.354a4.752,4.752,0,0,0-.437-.825l-.044-.066-.039-.077a1.8,1.8,0,0,1,.081-1.727A1.38,1.38,0,0,1,4.606,12a1.248,1.248,0,0,1,.686.205l.081.058a4.675,4.675,0,0,1,.721.72h.3l-1.559-3H7.143A2.647,2.647,0,0,1,10,12.415a2.6,2.6,0,0,1-.007.507,2.8,2.8,0,0,1-2.52,3.049A2.617,2.617,0,0,1,7.2,15.984Z" fill="#f6f6f6"/>
+  <path d="M7.143,10.989H6.474l.519,1c.38,0,1,0,1,1,0,.38,0,1-.779,1H5.574a3.735,3.735,0,0,0-.829-.939c-.289-.19-.749.29-.459.929a5.716,5.716,0,0,1,.529,1H6.993a1.8,1.8,0,0,0,2-1.573,1.777,1.777,0,0,0,0-.425C8.991,11.489,8.262,10.989,7.143,10.989Z" fill="#424242"/>
+  <path d="M2.9,15.984A2.67,2.67,0,0,1,0,13.565a2.587,2.587,0,0,1,0-.513A2.8,2.8,0,0,1,2.52,10a2.707,2.707,0,0,1,.53,0H5.886l.254.621a4.738,4.738,0,0,0,.436.824l.044.066.034.075a1.8,1.8,0,0,1-.081,1.727,1.38,1.38,0,0,1-1.149.655,1.248,1.248,0,0,1-.686-.205l-.081-.058a4.65,4.65,0,0,1-.716-.721H3.6l1.619,3Z" fill="#f6f6f6"/>
+  <path d="M2.9,14.985h.639l-.539-1c-.37,0-1,0-1-1,0-.37,0-1,.779-1H4.456a3.735,3.735,0,0,0,.829.939c.289.19.749-.29.459-.929a5.716,5.716,0,0,1-.529-1H3a1.8,1.8,0,0,0-2,1.573A1.777,1.777,0,0,0,1,13C1,14.486,1.768,14.985,2.9,14.985Z" fill="#424242"/>
+</svg>

src/sql/media/objectTypes/objecttypes.css

@@ -177,6 +177,12 @@
 	background: url("Folder.svg") center center no-repeat;
 }
 
+.vs .icon.folder_mounted,
+.vs-dark .icon.folder_mounted,
+.hc-black .icon.folder_mounted {
+	background: url("Folder_mounted.svg") center center no-repeat;
+}
+
 .vs .icon.fulltextcatalog,
 .vs-dark .icon.fulltextcatalog,
 .hc-black .icon.fulltextcatalog {