From 63f3d9862fa20170655b05d1cda9bab4a00f89a3 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 27 Sep 2019 13:45:45 -0700 Subject: [PATCH] Add getaclstatus/setacl calls to WebHDFS API (#7378) * Add getaclstatus/setacl calls to WebHDFS API * Fix hygiene check --- build/gulpfile.hygiene.js | 2 +- extensions/mssql/src/hdfs/aclEntry.ts | 228 ++++++++++++++++++ .../webhdfs.ts | 57 ++++- .../objectExplorerNodeProvider/fileSources.ts | 40 ++- 4 files changed, 319 insertions(+), 8 deletions(-) create mode 100644 extensions/mssql/src/hdfs/aclEntry.ts rename extensions/mssql/src/{objectExplorerNodeProvider => hdfs}/webhdfs.ts (90%) diff --git a/build/gulpfile.hygiene.js b/build/gulpfile.hygiene.js index 4d329b6169..01e05c0025 100644 --- a/build/gulpfile.hygiene.js +++ b/build/gulpfile.hygiene.js @@ -136,7 +136,7 @@ const copyrightFilter = [ '!src/vs/editor/test/node/classification/typescript-test.ts', // {{SQL CARBON EDIT}} '!extensions/notebook/src/intellisense/text.ts', - '!extensions/mssql/src/objectExplorerNodeProvider/webhdfs.ts', + '!extensions/mssql/src/hdfs/webhdfs.ts', '!src/sql/workbench/parts/notebook/browser/outputs/tableRenderers.ts', '!src/sql/workbench/parts/notebook/common/models/url.ts', '!src/sql/workbench/parts/notebook/browser/models/renderMimeInterfaces.ts', diff --git a/extensions/mssql/src/hdfs/aclEntry.ts b/extensions/mssql/src/hdfs/aclEntry.ts new file mode 100644 index 0000000000..a8d0b43a7d --- /dev/null +++ b/extensions/mssql/src/hdfs/aclEntry.ts @@ -0,0 +1,228 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * The parsed result from calling getAclStatus on the controller + */ +export interface IAclStatus { + /** + * The ACL entries defined for the object + */ + entries: AclEntry[]; + /** + * The ACL entry object for the owner permissions + */ + owner: AclEntry; + /** + * The ACL entry object for the group permissions + */ + group: AclEntry; + /** + * The ACL entry object for the other permissions + */ + other: AclEntry; + /** + * The sticky bit status for the object. If true the owner/root are + * the only ones who can delete the resource or its contents (if a folder) + */ + stickyBit: boolean; +} + +/** + * The type of an ACL entry. Corresponds to the first (or second if a scope is present) field of + * an ACL entry - e.g. user:bob:rwx (user) or default:group::r-- (group) + */ +export enum AclEntryType { + /** + * An ACL entry applied to a specific user. + */ + user = 'user', + /** + * An ACL entry applied to a specific group. + */ + group = 'group', + /** + * An ACL mask entry. + */ + mask = 'mask', + /** + * An ACL entry that applies to all other users that were not covered by one of the more specific ACL entry types. + */ + other = 'other' +} + +/** + * The type of permission on a file - this corresponds to the field in the file status used in commands such as chmod. + * Typically this value is represented as a 3 digit octal - e.g. 740 - where the first digit is the owner, the second + * the group and the third other. @see parseAclPermissionFromOctal + */ +export enum AclPermissionType { + owner = 'owner', + group = 'group', + other = 'other' +} + +export enum AclEntryScope { + /** + * An ACL entry that is inspected during permission checks to enforce permissions. + */ + access = 'access', + /** + * An ACL entry to be applied to a directory's children that do not otherwise have their own ACL defined. + */ + default = 'default' +} + +/** + * The read, write and execute permissions for an ACL + */ +export class AclEntryPermission { + + constructor(public read: boolean, public write: boolean, public execute: boolean) { } + + /** + * Returns the string representation of the permissions in the form [r-][w-][x-]. + * e.g. + * rwx + * r-- + * --- + */ + public toString() { + return `${this.read ? 'r' : '-'}${this.write ? 'w' : '-'}${this.execute ? 'x' : '-'}`; + } +} + +/** + * Parses a string representation of a permission into an AclPermission object. The string must consist + * of 3 characters for the read, write and execute permissions where each character is either a r/w/x or + * a -. + * e.g. The following are all valid strings + * rwx + * --- + * -w- + * @param permissionString The string representation of the permission + */ +function parseAclPermission(permissionString: string): AclEntryPermission { + permissionString = permissionString.toLowerCase(); + if (!/^[r\-][w\-][x\-]$/i.test(permissionString)) { + throw new Error(`Invalid permission string ${permissionString}- must match /^[r\-][w\-][x\-]$/i`); + } + return new AclEntryPermission(permissionString[0] === 'r', permissionString[1] === 'w', permissionString[2] === 'x'); +} + +/** + * A single ACL Entry. This consists of up to 4 values + * scope - The scope of the entry @see AclEntryScope + * type - The type of the entry @see AclEntryType + * name - The name of the user/group. Optional. + * permission - The permission set for this ACL. @see AclPermission + */ +export class AclEntry { + constructor( + public readonly scope: AclEntryScope, + public readonly type: AclEntryType | AclPermissionType, + public readonly name: string, + public readonly permission: AclEntryPermission, + ) { } + + /** + * Returns the string representation of the ACL Entry in the form [SCOPE:]TYPE:NAME:PERMISSION. + * Note that SCOPE is only displayed if it's default - access is implied if there is no scope + * specified. + * The name is optional and so may be empty. + * Example strings : + * user:bob:rwx + * default:user:bob:rwx + * user::r-x + * default:group::r-- + */ + toString(): string { + return `${this.scope === AclEntryScope.default ? 'default:' : ''}${this.type}:${this.name}:${this.permission.toString()}`; + } +} + +/** + * Parses a complete ACL string into separate AclEntry objects for each entry. A valid string consists of multiple entries + * separated by a comma. + * + * A valid entry must match (default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3}) + * e.g. the following are all valid entries + * user:bob:rwx + * user::rwx + * default::bob:rwx + * group::r-x + * default:other:r-- + * + * So a valid ACL string might look like this + * user:bob:rwx,user::rwx,default::bob:rwx,group::r-x,default:other:r-- + * @param aclString The string representation of the ACL + */ +export function parseAcl(aclString: string): AclEntry[] { + if (!/^(default:)?(user|group|mask|other):([A-Za-z_][A-Za-z0-9._-]*)?:([rwx-]{3})?(,(default:)?(user|group|mask|other):([A-Za-z_][A-Za-z0-9._-]*)?:([rwx-]{3})?)*$/.test(aclString)) { + throw new Error(`Invalid ACL string ${aclString}. Expected to match ^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$`); + } + return aclString.split(',').map(aclEntryString => parseAclEntry(aclEntryString)); +} + +/** + * Parses a given string representation of an ACL Entry into an AclEntry object. This method + * assumes the string has already been checked for validity. + * @param aclString The string representation of the ACL entry + */ +export function parseAclEntry(aclString: string): AclEntry { + const parts: string[] = aclString.split(':'); + let i = 0; + const scope: AclEntryScope = parts.length === 4 && parts[i++] === 'default' ? AclEntryScope.default : AclEntryScope.access; + let type: AclEntryType; + switch (parts[i++]) { + case 'user': + type = AclEntryType.user; + break; + case 'group': + type = AclEntryType.group; + break; + case 'mask': + type = AclEntryType.mask; + break; + case 'other': + type = AclEntryType.other; + break; + default: + throw new Error(`Unknown ACL Entry type ${parts[i - 1]}`); + } + const name = parts[i++]; + const permission = parseAclPermission(parts[i++]); + return new AclEntry(scope, type, name, permission); +} + +/** + * Parses an octal in the form ### into a set of @see AclEntryPermission. Each digit in the octal corresponds + * to a particular user type - owner, group and other respectively. + * Each digit is then expected to be a value between 0 and 7 inclusive, which is a bitwise OR the permission flags + * for the file. + * 4 - Read + * 2 - Write + * 1 - Execute + * So an octal of 730 would map to : + * - The owner with rwx permissions + * - The group with -wx permissions + * - All others with --- permissions + * @param octal The octal string to parse + */ +export function parseAclPermissionFromOctal(octal: string): { owner: AclEntryPermission, group: AclEntryPermission, other: AclEntryPermission } { + if (!octal || octal.length !== 3) { + throw new Error(`Invalid octal ${octal} - it must be a 3 digit string`); + } + + const ownerPermissionDigit = parseInt(octal[0]); + const groupPermissionDigit = parseInt(octal[1]); + const otherPermissionDigit = parseInt(octal[2]); + + return { + owner: new AclEntryPermission((ownerPermissionDigit & 4) === 4, (ownerPermissionDigit & 2) === 2, (ownerPermissionDigit & 1) === 1), + group: new AclEntryPermission((groupPermissionDigit & 4) === 4, (groupPermissionDigit & 2) === 2, (groupPermissionDigit & 1) === 1), + other: new AclEntryPermission((otherPermissionDigit & 4) === 4, (otherPermissionDigit & 2) === 2, (otherPermissionDigit & 1) === 1) + }; +} diff --git a/extensions/mssql/src/objectExplorerNodeProvider/webhdfs.ts b/extensions/mssql/src/hdfs/webhdfs.ts similarity index 90% rename from extensions/mssql/src/objectExplorerNodeProvider/webhdfs.ts rename to extensions/mssql/src/hdfs/webhdfs.ts index b355b7e112..0ccef66019 100644 --- a/extensions/mssql/src/objectExplorerNodeProvider/webhdfs.ts +++ b/extensions/mssql/src/hdfs/webhdfs.ts @@ -10,7 +10,8 @@ import { Cookie } from 'tough-cookie'; import * as through from 'through2'; import * as nls from 'vscode-nls'; import * as auth from '../util/auth'; -import { IHdfsOptions, IRequestParams } from './fileSources'; +import { IHdfsOptions, IRequestParams } from '../objectExplorerNodeProvider/fileSources'; +import { IAclStatus, AclEntry, parseAcl, AclPermissionType, parseAclPermissionFromOctal, AclEntryScope } from './aclEntry'; const localize = nls.loadMessageBundle(); const ErrorMessageInvalidDataStructure = localize('webhdfs.invalidDataStructure', "Invalid Data Structure"); @@ -25,6 +26,7 @@ const emitError = (instance, err) => { instance.errorEmitted = true; }; + export class WebHDFS { private _requestParams: IRequestParams; private _opts: IHdfsOptions; @@ -75,7 +77,7 @@ export class WebHDFS { params || {} ); endpoint.search = querystring.stringify(searchOpts); - return encodeURI(url.format(endpoint)); + return url.format(endpoint); } /** @@ -87,7 +89,7 @@ export class WebHDFS { private toStatusMessage(statusCode: number): string { let statusMessage: string = undefined; switch (statusCode) { - case 400: statusMessage = localize('webhdfs.httpError400', "Bad Request");break; + case 400: statusMessage = localize('webhdfs.httpError400', "Bad Request"); break; case 401: statusMessage = localize('webhdfs.httpError401', "Unauthorized"); break; case 403: statusMessage = localize('webhdfs.httpError403', "Forbidden"); break; case 404: statusMessage = localize('webhdfs.httpError404', "Not Found"); break; @@ -416,6 +418,53 @@ export class WebHDFS { }); } + /** + * Get ACL status for given path + * @param path The path to the file/folder to get the status of + * @param callback Callback to handle the response + * @returns void + */ + public getAclStatus(path: string, callback: (error: HdfsError, aclStatus: IAclStatus) => void): void { + this.checkArgDefined('path', path); + + let endpoint = this.getOperationEndpoint('getaclstatus', path); + this.sendRequest('GET', endpoint, undefined, (error, response) => { + if (!callback) { return; } + if (error) { + callback(error, undefined); + } else if (response.body.hasOwnProperty('AclStatus')) { + const permissions = parseAclPermissionFromOctal(response.body.AclStatus.permission); + const aclStatus: IAclStatus = { + owner: new AclEntry(AclEntryScope.access, AclPermissionType.owner, response.body.AclStatus.owner || '', permissions.owner), + group: new AclEntry(AclEntryScope.access, AclPermissionType.group, response.body.AclStatus.group || '', permissions.group), + other: new AclEntry(AclEntryScope.access, AclPermissionType.other, response.body.AclStatus.other || '', permissions.other), + stickyBit: !!response.body.AclStatus.stickyBit, + entries: (response.body.AclStatus.entries).map(entry => parseAcl(entry)).reduce((acc, parsedEntries) => acc.concat(parsedEntries, [])) + }; + callback(undefined, aclStatus); + } else { + callback(new HdfsError(ErrorMessageInvalidDataStructure), undefined); + } + }); + } + + /** + * Set ACL for the given path + * @param path The path to the file/folder to set the ACL on + * @param aclEntries The ACL entries to set + * @param callback Callback to handle the response + * @returns void + */ + public setAcl(path: string, aclEntries: AclEntry[], callback: (error: HdfsError) => void): void { + this.checkArgDefined('path', path); + this.checkArgDefined('aclEntries', aclEntries); + const aclSpec = aclEntries.join(','); + let endpoint = this.getOperationEndpoint('setacl', path, { aclspec: aclSpec }); + this.sendRequest('PUT', endpoint, undefined, (error) => { + return callback && callback(error); + }); + } + /** * Check file existence * Wraps stat method @@ -649,7 +698,7 @@ export class WebHDFS { src.unpipe(req); req.end(); }); - return req; + return req; } /** diff --git a/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts b/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts index b2b2efe780..79014c8879 100644 --- a/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts +++ b/extensions/mssql/src/objectExplorerNodeProvider/fileSources.ts @@ -3,7 +3,6 @@ * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -'use strict'; import * as vscode from 'vscode'; import * as fspath from 'path'; import * as fs from 'fs'; @@ -15,7 +14,8 @@ import * as os from 'os'; import * as nls from 'vscode-nls'; import * as constants from '../constants'; -import { WebHDFS, HdfsError } from './webhdfs'; +import { WebHDFS, HdfsError } from '../hdfs/webhdfs'; +import { AclEntry, IAclStatus } from '../hdfs/aclEntry'; const localize = nls.loadMessageBundle(); @@ -58,7 +58,6 @@ export class File implements IFile { } export interface IFileSource { - enumerateFiles(path: string): Promise; mkdir(dirName: string, remoteBasePath: string): Promise; createReadStream(path: string): fs.ReadStream; @@ -66,6 +65,8 @@ export interface IFileSource { readFileLines(path: string, maxLines: number): Promise; writeFile(localFile: IFile, remoteDir: string): Promise; delete(path: string, recursive?: boolean): Promise; + getAclStatus(path: string): Promise; + setAcl(path: string, aclEntries: AclEntry[]): Promise; exists(path: string): Promise; } @@ -302,4 +303,37 @@ export class HdfsFileSource implements IFileSource { }); }); } + + /** + * Get ACL status for given path + * @param path The path to the file/folder to get the status of + */ + public getAclStatus(path: string): Promise { + return new Promise((resolve, reject) => { + this.client.getAclStatus(path, (error: HdfsError, aclStatus: IAclStatus) => { + if (error) { + reject(error); + } else { + resolve(aclStatus); + } + }); + }); + } + + /** + * Sets the ACL status for given path + * @param path The path to the file/folder to set the ACL on + * @param aclEntries The ACL entries to set + */ + public setAcl(path: string, aclEntries: AclEntry[]): Promise { + return new Promise((resolve, reject) => { + this.client.setAcl(path, aclEntries, (error: HdfsError) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + } }