From e85f93abeca603e4cb3d3a80e8db5ac50f578e6a Mon Sep 17 00:00:00 2001 From: Kevin Cunnane Date: Thu, 3 Oct 2019 14:48:19 -0700 Subject: [PATCH] Support HDFS Tiering (#7484) This is the 1st step to supporting HDFS Tiering Changes: Add new mounted folder icon. Will have separate commit for file icon Disable delete/mkdir/upload for mounted files and folders Disable delete for root HDFS folder (this was added in error) --- extensions/mssql/package.json | 8 ++-- extensions/mssql/src/constants.ts | 4 +- extensions/mssql/src/hdfs/mount.ts | 19 ++++++++ extensions/mssql/src/hdfs/webhdfs.ts | 21 +++++++++ .../objectExplorerNodeProvider/fileSources.ts | 40 +++++++++++++--- .../hdfsProvider.ts | 47 +++++++++++++++---- src/sql/media/objectTypes/Folder_mounted.svg | 21 +++++++++ src/sql/media/objectTypes/objecttypes.css | 6 +++ 8 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 extensions/mssql/src/hdfs/mount.ts create mode 100644 src/sql/media/objectTypes/Folder_mounted.svg diff --git a/extensions/mssql/package.json b/extensions/mssql/package.json index cbd6dfea0f..55f1a481c3 100644 --- a/extensions/mssql/package.json +++ b/extensions/mssql/package.json @@ -253,12 +253,12 @@ "objectExplorer/item/context": [ { "command": "mssqlCluster.uploadFiles", - "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file", + "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file && nodeSubType=~/^(?!:mount).*$/", "group": "1mssqlCluster@1" }, { "command": "mssqlCluster.mkdir", - "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file", + "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:message && nodeType != mssqlCluster:file && nodeSubType=~/^(?!:mount).*$/", "group": "1mssqlCluster@1" }, { @@ -278,7 +278,7 @@ }, { "command": "mssqlCluster.deleteFiles", - "when": "nodeType=~/^mssqlCluster/ && viewItem != mssqlCluster:connection && nodeType != mssqlCluster:message", + "when": "nodeType=~/^mssqlCluster/ && nodeType != mssqlCluster:hdfs && nodeType != mssqlCluster:connection && viewItem != mssqlCluster:connection && nodeType != mssqlCluster:message && nodeSubType=~/^(?!:mount).*$/", "group": "1mssqlCluster@4" }, { @@ -288,7 +288,7 @@ }, { "command": "mssqlCluster.livy.cmd.submitFileToSparkJob", - "when": "nodeType == mssqlCluster:file && nodeSubType == mssqlCluster:spark", + "when": "nodeType == mssqlCluster:file && nodeSubType =~/:spark:/", "group": "1mssqlCluster@6" } ] diff --git a/extensions/mssql/src/constants.ts b/extensions/mssql/src/constants.ts index 717e14ade0..e071911efc 100644 --- a/extensions/mssql/src/constants.ts +++ b/extensions/mssql/src/constants.ts @@ -56,7 +56,9 @@ export enum MssqlClusterItems { } export enum MssqlClusterItemsSubType { - Spark = 'mssqlCluster:spark' + Mount = ':mount:', + MountChild = ':mountChild:', + Spark = ':spark:' } // SPARK JOB SUBMISSION ////////////////////////////////////////////////////////// diff --git a/extensions/mssql/src/hdfs/mount.ts b/extensions/mssql/src/hdfs/mount.ts new file mode 100644 index 0000000000..a4c424ea69 --- /dev/null +++ b/extensions/mssql/src/hdfs/mount.ts @@ -0,0 +1,19 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Information about a HDFS mount to a remote directory + */ +export interface Mount { + mountPath: string; + mountStatus: string; + remotePath: string; +} + +export enum MountStatus { + None = 0, + Mount = 1, + Mount_Child = 2 +} diff --git a/extensions/mssql/src/hdfs/webhdfs.ts b/extensions/mssql/src/hdfs/webhdfs.ts index cd7b7fd708..baf4a15514 100644 --- a/extensions/mssql/src/hdfs/webhdfs.ts +++ b/extensions/mssql/src/hdfs/webhdfs.ts @@ -12,6 +12,7 @@ import * as nls from 'vscode-nls'; import * as auth from '../util/auth'; import { IHdfsOptions, IRequestParams } from '../objectExplorerNodeProvider/fileSources'; import { IAclStatus, AclEntry, parseAcl, AclPermissionType, parseAclPermissionFromOctal, AclEntryScope } from './aclEntry'; +import { Mount } from './mount'; import { everyoneName } from '../localizedConstants'; const localize = nls.loadMessageBundle(); @@ -472,6 +473,26 @@ export class WebHDFS { }); } + /** + * Get all mounts for a HDFS connection + * @param callback Callback to handle the response + * @returns void + */ + public getMounts(callback: (error: HdfsError, mounts: Mount[]) => void): void { + let endpoint = this.getOperationEndpoint('listmounts', ''); + this.sendRequest('GET', endpoint, undefined, (error, response) => { + if (!callback) { return; } + if (error) { + callback(error, undefined); + } else if (response.body.hasOwnProperty('Mounts')) { + const mounts = response.body.Mounts; + callback(undefined, mounts); + } else { + callback(new HdfsError(ErrorMessageInvalidDataStructure), undefined); + } + }); + } + /** * Check file existence * Wraps stat method diff --git a/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts b/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts index 4072373da1..d5d0db447e 100644 --- a/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts +++ b/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts @@ -16,6 +16,7 @@ import * as nls from 'vscode-nls'; import * as constants from '../constants'; import { WebHDFS, HdfsError } from '../hdfs/webhdfs'; import { AclEntry, IAclStatus } from '../hdfs/aclEntry'; +import { Mount, MountStatus } from '../hdfs/mount'; const localize = nls.loadMessageBundle(); @@ -29,9 +30,11 @@ export function joinHdfsPath(parent: string, child: string): string { export interface IFile { path: string; isDirectory: boolean; + mountStatus?: MountStatus; } export class File implements IFile { + public mountStatus?: MountStatus; constructor(public path: string, public isDirectory: boolean) { } @@ -58,7 +61,7 @@ export class File implements IFile { } export interface IFileSource { - enumerateFiles(path: string): Promise; + enumerateFiles(path: string, refresh?: boolean): Promise; mkdir(dirName: string, remoteBasePath: string): Promise; createReadStream(path: string): fs.ReadStream; readFile(path: string, maxBytes?: number): Promise; @@ -159,18 +162,43 @@ export class FileSourceFactory { } export class HdfsFileSource implements IFileSource { + private mounts: Map; constructor(private client: WebHDFS) { } - public enumerateFiles(path: string): Promise { + public async enumerateFiles(path: string, refresh?: boolean): Promise { + if (!this.mounts || refresh) { + await this.loadMounts(); + } + return this.readdir(path); + } + + private loadMounts(): Promise { + return new Promise((resolve, reject) => { + this.client.getMounts((error, mounts) => { + this.mounts = new Map(); + if (!error && mounts) { + mounts.forEach(m => this.mounts.set(m.mountPath, m)); + } + resolve(); + }); + }); + } + + private readdir(path: string): Promise { return new Promise((resolve, reject) => { this.client.readdir(path, (error, files) => { if (error) { reject(error); - } else { - let hdfsFiles: IFile[] = files.map(file => { - let hdfsFile = file; - return new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY'); + } + else { + let hdfsFiles: IFile[] = files.map(fileStat => { + let hdfsFile = fileStat; + let file = new File(File.createPath(path, hdfsFile.pathSuffix), hdfsFile.type === 'DIRECTORY'); + if (this.mounts && this.mounts.has(file.path)) { + file.mountStatus = MountStatus.Mount; + } + return file; }); resolve(hdfsFiles); } diff --git a/extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts b/extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts index b7c260ba5d..ed19cdda24 100644 --- a/extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts +++ b/extensions/mssql/src/objectExplorerNodeProvider/hdfsProvider.ts @@ -16,6 +16,7 @@ import { CancelableStream } from './cancelableStream'; import { TreeNode } from './treeNodes'; import * as utils from '../utils'; import { IFileNode } from './types'; +import { MountStatus } from '../hdfs/mount'; export interface ITreeChangeHandler { notifyNodeChanged(node: TreeNode): void; @@ -103,7 +104,7 @@ export abstract class HdfsFileSourceNode extends TreeNode { export class FolderNode extends HdfsFileSourceNode { private children: TreeNode[]; protected _nodeType: string; - constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string) { + constructor(context: TreeDataContext, path: string, fileSource: IFileSource, nodeType?: string, private mountStatus?: MountStatus) { super(context, path, fileSource); this._nodeType = nodeType ? nodeType : Constants.MssqlClusterItems.Folder; } @@ -126,8 +127,8 @@ export class FolderNode extends HdfsFileSourceNode { if (files) { // Note: for now, assuming HDFS-provided sorting is sufficient this.children = files.map((file) => { - let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource) - : new FileNode(this.context, file.path, this.fileSource); + let node: TreeNode = file.isDirectory ? new FolderNode(this.context, file.path, this.fileSource, Constants.MssqlClusterItems.Folder, this.getChildMountStatus(file)) + : new FileNode(this.context, file.path, this.fileSource, this.getChildMountStatus(file)); node.parent = this; return node; }); @@ -139,6 +140,17 @@ export class FolderNode extends HdfsFileSourceNode { return this.children; } + private getChildMountStatus(file: IFile): MountStatus { + if (file.mountStatus !== undefined && file.mountStatus !== MountStatus.None) { + return file.mountStatus; + } + else if (this.mountStatus !== undefined && this.mountStatus !== MountStatus.None) { + // Any child node of a mount (or subtree) must be a mount child + return MountStatus.Mount_Child; + } + return MountStatus.None; + } + getTreeItem(): vscode.TreeItem | Promise { let item = new vscode.TreeItem(this.getDisplayName(), vscode.TreeItemCollapsibleState.Collapsed); // For now, folder always looks the same. We're using SQL icons to differentiate remote vs local files @@ -161,12 +173,26 @@ export class FolderNode extends HdfsFileSourceNode { nodePath: this.generateNodePath(), nodeStatus: undefined, nodeType: this._nodeType, - nodeSubType: undefined, - iconType: 'Folder' + nodeSubType: this.getSubType(), + iconType: this.isMounted() ? 'Folder_mounted' : 'Folder' }; return nodeInfo; } + private isMounted(): boolean { + return this.mountStatus === MountStatus.Mount || this.mountStatus === MountStatus.Mount_Child; + } + + private getSubType(): string | undefined { + if (this.mountStatus === MountStatus.Mount) { + return Constants.MssqlClusterItemsSubType.Mount; + } else if (this.mountStatus === MountStatus.Mount_Child) { + return Constants.MssqlClusterItemsSubType.MountChild; + } + + return undefined; + } + public async writeFile(localFile: IFile): Promise { return this.runChildAddAction(() => this.writeFileAsync(localFile)); } @@ -243,7 +269,7 @@ export class ConnectionNode extends FolderNode { export class FileNode extends HdfsFileSourceNode implements IFileNode { - constructor(context: TreeDataContext, path: string, fileSource: IFileSource) { + constructor(context: TreeDataContext, path: string, fileSource: IFileSource, private mountStatus?: MountStatus) { super(context, path, fileSource); } @@ -322,12 +348,15 @@ export class FileNode extends HdfsFileSourceNode implements IFileNode { }); } - private getSubType(): string { + private getSubType(): string | undefined { + let subType = ''; if (this.getDisplayName().toLowerCase().endsWith('.jar') || this.getDisplayName().toLowerCase().endsWith('.py')) { - return Constants.MssqlClusterItemsSubType.Spark; + subType += Constants.MssqlClusterItemsSubType.Spark; + } else if (this.mountStatus === MountStatus.Mount_Child) { + subType += Constants.MssqlClusterItemsSubType.MountChild; } - return undefined; + return subType.length > 0 ? subType : undefined; } } diff --git a/src/sql/media/objectTypes/Folder_mounted.svg b/src/sql/media/objectTypes/Folder_mounted.svg new file mode 100644 index 0000000000..a402f0c63b --- /dev/null +++ b/src/sql/media/objectTypes/Folder_mounted.svg @@ -0,0 +1,21 @@ + + + + + + + Folder_mounted_updatedSize + + + + + + + + + + + + + + diff --git a/src/sql/media/objectTypes/objecttypes.css b/src/sql/media/objectTypes/objecttypes.css index 5a9bbe6f89..52558b1f0e 100644 --- a/src/sql/media/objectTypes/objecttypes.css +++ b/src/sql/media/objectTypes/objecttypes.css @@ -177,6 +177,12 @@ background: url("Folder.svg") center center no-repeat; } +.vs .icon.folder_mounted, +.vs-dark .icon.folder_mounted, +.hc-black .icon.folder_mounted { + background: url("Folder_mounted.svg") center center no-repeat; +} + .vs .icon.fulltextcatalog, .vs-dark .icon.fulltextcatalog, .hc-black .icon.fulltextcatalog {