Merge from vscode 2cd495805cf99b31b6926f08ff4348124b2cf73d

extensions/github-browser/src/github/api.ts

@@ -0,0 +1,491 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { authentication, AuthenticationSession, Disposable, Event, EventEmitter, Range, Uri } from 'vscode';
+import { graphql } from '@octokit/graphql';
+import { Octokit } from '@octokit/rest';
+import { fromGitHubUri } from './fs';
+import { ContextStore } from '../contextStore';
+import { Iterables } from '../iterables';
+
+export const shaRegex = /^[0-9a-f]{40}$/;
+
+export interface GitHubApiContext {
+	sha: string;
+	timestamp: number;
+}
+
+interface CreateCommitOperation {
+	type: 'created';
+	path: string;
+	content: string
+}
+
+interface ChangeCommitOperation {
+	type: 'changed';
+	path: string;
+	content: string
+}
+
+interface DeleteCommitOperation {
+	type: 'deleted';
+	path: string;
+	content: undefined
+}
+
+export type CommitOperation = CreateCommitOperation | ChangeCommitOperation | DeleteCommitOperation;
+
+type ArrayElement<T extends Array<unknown>> = T extends (infer U)[] ? U : never;
+type GitCreateTreeParamsTree = ArrayElement<NonNullable<Parameters<Octokit['git']['createTree']>[0]>['tree']>;
+
+function getGitHubRootUri(uri: Uri) {
+	const rootIndex = uri.path.indexOf('/', uri.path.indexOf('/', 1) + 1);
+	return uri.with({
+		path: uri.path.substring(0, rootIndex === -1 ? undefined : rootIndex),
+		query: ''
+	});
+}
+
+export class GitHubApi implements Disposable {
+	private _onDidChangeContext = new EventEmitter<Uri>();
+	get onDidChangeContext(): Event<Uri> {
+		return this._onDidChangeContext.event;
+	}
+
+	private readonly disposable: Disposable;
+
+	constructor(private readonly context: ContextStore<GitHubApiContext>) {
+		this.disposable = Disposable.from(
+			context.onDidChange(e => this._onDidChangeContext.fire(e))
+		);
+	}
+
+	dispose() {
+		this.disposable.dispose();
+	}
+
+	private _session: AuthenticationSession | undefined;
+	async ensureAuthenticated() {
+		if (this._session === undefined) {
+			const providers = await authentication.getProviderIds();
+			if (!providers.includes('github')) {
+				await new Promise(resolve => {
+					authentication.onDidChangeAuthenticationProviders(e => {
+						if (e.added.includes('github')) {
+							resolve();
+						}
+					});
+				});
+			}
+
+			this._session = await authentication.getSession('github', ['repo'], { createIfNone: true });
+		}
+
+		return this._session;
+	}
+
+	private _graphql: typeof graphql | undefined;
+	private async graphql() {
+		if (this._graphql === undefined) {
+			const session = await this.ensureAuthenticated();
+			this._graphql = graphql.defaults({
+				headers: {
+					Authorization: `Bearer ${session.accessToken}`,
+				}
+			});
+		}
+
+		return this._graphql;
+	}
+
+	private _octokit: typeof Octokit | undefined;
+	private async octokit(options?: ConstructorParameters<typeof Octokit>[0]) {
+		if (this._octokit === undefined) {
+			const session = await this.ensureAuthenticated();
+			this._octokit = Octokit.defaults({ auth: `token ${session.accessToken}` });
+		}
+		return new this._octokit(options);
+	}
+
+	async commit(rootUri: Uri, message: string, operations: CommitOperation[]): Promise<string | undefined> {
+		let { owner, repo, ref } = fromGitHubUri(rootUri);
+
+		try {
+			if (ref === undefined || ref === 'HEAD') {
+				ref = await this.defaultBranchQuery(rootUri);
+				if (ref === undefined) {
+					throw new Error('Cannot commit — invalid ref');
+				}
+			}
+
+			const context = await this.getContext(rootUri);
+			if (context.sha === undefined) {
+				throw new Error('Cannot commit — invalid context');
+			}
+
+			const hasDeletes = operations.some(op => op.type === 'deleted');
+
+			const github = await this.octokit();
+			const treeResp = await github.git.getTree({
+				owner: owner,
+				repo: repo,
+				tree_sha: context.sha,
+				recursive: hasDeletes ? 'true' : undefined,
+			});
+
+			// 0100000000000000 (040000): Directory
+			// 1000000110100100 (100644): Regular non-executable file
+			// 1000000110110100 (100664): Regular non-executable group-writeable file
+			// 1000000111101101 (100755): Regular executable file
+			// 1010000000000000 (120000): Symbolic link
+			// 1110000000000000 (160000): Gitlink
+			let updatedTree: GitCreateTreeParamsTree[];
+
+			if (hasDeletes) {
+				updatedTree = treeResp.data.tree as GitCreateTreeParamsTree[];
+
+				for (const operation of operations) {
+					switch (operation.type) {
+						case 'created':
+							updatedTree.push({ path: operation.path, mode: '100644', type: 'blob', content: operation.content });
+							break;
+
+						case 'changed': {
+							const index = updatedTree.findIndex(item => item.path === operation.path);
+							if (index !== -1) {
+								const { path, mode, type } = updatedTree[index];
+								updatedTree.splice(index, 1, { path: path, mode: mode, type: type, content: operation.content });
+							}
+							break;
+						}
+						case 'deleted': {
+							const index = updatedTree.findIndex(item => item.path === operation.path);
+							if (index !== -1) {
+								updatedTree.splice(index, 1);
+							}
+							break;
+						}
+					}
+				}
+			} else {
+				updatedTree = [];
+
+				for (const operation of operations) {
+					switch (operation.type) {
+						case 'created':
+							updatedTree.push({ path: operation.path, mode: '100644', type: 'blob', content: operation.content });
+							break;
+
+						case 'changed':
+							const item = treeResp.data.tree.find(item => item.path === operation.path) as GitCreateTreeParamsTree;
+							if (item !== undefined) {
+								const { path, mode, type } = item;
+								updatedTree.push({ path: path, mode: mode, type: type, content: operation.content });
+							}
+							break;
+					}
+				}
+			}
+
+			const updatedTreeResp = await github.git.createTree({
+				owner: owner,
+				repo: repo,
+				base_tree: hasDeletes ? undefined : treeResp.data.sha,
+				tree: updatedTree
+			});
+
+			const resp = await github.git.createCommit({
+				owner: owner,
+				repo: repo,
+				message: message,
+				tree: updatedTreeResp.data.sha,
+				parents: [context.sha]
+			});
+
+			this.updateContext(rootUri, { sha: resp.data.sha, timestamp: Date.now() });
+
+			// TODO@eamodio need to send a file change for any open files
+
+			await github.git.updateRef({
+				owner: owner,
+				repo: repo,
+				ref: `heads/${ref}`,
+				sha: resp.data.sha
+			});
+
+			return resp.data.sha;
+		} catch (ex) {
+			console.log(ex);
+			throw ex;
+		}
+	}
+
+	async defaultBranchQuery(uri: Uri) {
+		const { owner, repo } = fromGitHubUri(uri);
+
+		try {
+			const query = `query defaultBranch($owner: String!, $repo: String!) {
+	repository(owner: $owner, name: $repo) {
+		defaultBranchRef {
+			name
+		}
+	}
+}`;
+
+			const rsp = await this.gqlQuery<{
+				repository: { defaultBranchRef: { name: string; target: { oid: string } } | null | undefined };
+			}>(query, {
+				owner: owner,
+				repo: repo,
+			});
+			return rsp?.repository?.defaultBranchRef?.name ?? undefined;
+		} catch (ex) {
+			return undefined;
+		}
+	}
+
+	async filesQuery(uri: Uri) {
+		const { owner, repo, ref } = fromGitHubUri(uri);
+
+		try {
+			const context = await this.getContext(uri);
+
+			const resp = await (await this.octokit()).git.getTree({
+				owner: owner,
+				repo: repo,
+				recursive: '1',
+				tree_sha: context?.sha ?? ref ?? 'HEAD',
+			});
+			return Iterables.filterMap(resp.data.tree, p => p.type === 'blob' ? p.path : undefined);
+		} catch (ex) {
+			return [];
+		}
+	}
+
+	async fsQuery<T>(uri: Uri, innerQuery: string): Promise<T | undefined> {
+		const { owner, repo, path, ref } = fromGitHubUri(uri);
+
+		try {
+			const context = await this.getContext(uri);
+
+			const query = `query fs($owner: String!, $repo: String!, $path: String) {
+	repository(owner: $owner, name: $repo) {
+		object(expression: $path) {
+			${innerQuery}
+		}
+	}
+}`;
+
+			const rsp = await this.gqlQuery<{
+				repository: { object: T | null | undefined };
+			}>(query, {
+				owner: owner,
+				repo: repo,
+				path: `${context.sha ?? ref ?? 'HEAD'}:${path}`,
+			});
+			return rsp?.repository?.object ?? undefined;
+		} catch (ex) {
+			return undefined;
+		}
+	}
+
+	async latestCommitQuery(uri: Uri) {
+		const { owner, repo, ref } = fromGitHubUri(uri);
+
+		try {
+			if (ref === undefined || ref === 'HEAD') {
+				const query = `query latest($owner: String!, $repo: String!) {
+	repository(owner: $owner, name: $repo) {
+		defaultBranchRef {
+			target {
+				oid
+			}
+		}
+	}
+}`;
+
+				const rsp = await this.gqlQuery<{
+					repository: { defaultBranchRef: { name: string; target: { oid: string } } | null | undefined };
+				}>(query, {
+					owner: owner,
+					repo: repo,
+				});
+				return rsp?.repository?.defaultBranchRef?.target.oid ?? undefined;
+			}
+
+			const query = `query latest($owner: String!, $repo: String!, $ref: String!) {
+	repository(owner: $owner, name: $repo) {
+		ref(qualifiedName: $ref) {
+			target {
+				oid
+			}
+		}
+}`;
+
+			const rsp = await this.gqlQuery<{
+				repository: { ref: { target: { oid: string } } | null | undefined };
+			}>(query, {
+				owner: owner,
+				repo: repo,
+				ref: ref ?? 'HEAD',
+			});
+			return rsp?.repository?.ref?.target.oid ?? undefined;
+		} catch (ex) {
+			return undefined;
+		}
+	}
+
+	async searchQuery(
+		query: string,
+		uri: Uri,
+		options: { maxResults?: number; context?: { before?: number; after?: number } },
+	): Promise<SearchQueryResults> {
+		const { owner, repo, ref } = fromGitHubUri(uri);
+
+		// If we have a specific ref, don't try to search, because GitHub search only works against the default branch
+		if (ref === undefined) {
+			return { matches: [], limitHit: true };
+		}
+
+		try {
+			const resp = await (await this.octokit({
+				request: {
+					headers: {
+						accept: 'application/vnd.github.v3.text-match+json',
+					},
+				}
+			})).search.code({
+				q: `${query} repo:${owner}/${repo}`,
+			});
+
+			// Since GitHub doesn't return ANY line numbers just fake it at the top of the file 😢
+			const range = new Range(0, 0, 0, 0);
+
+			const matches: SearchQueryMatch[] = [];
+
+			let counter = 0;
+			let match: SearchQueryMatch;
+			for (const item of resp.data.items) {
+				for (const m of (item as typeof item & { text_matches: GitHubSearchTextMatch[] }).text_matches) {
+					counter++;
+					if (options.maxResults !== undefined && counter > options.maxResults) {
+						return { matches: matches, limitHit: true };
+					}
+
+					match = {
+						path: item.path,
+						ranges: [],
+						preview: m.fragment,
+						matches: [],
+					};
+
+					for (const lm of m.matches) {
+						let line = 0;
+						let shartChar = 0;
+						let endChar = 0;
+						for (let i = 0; i < lm.indices[1]; i++) {
+							if (i === lm.indices[0]) {
+								shartChar = endChar;
+							}
+
+							if (m.fragment[i] === '\n') {
+								line++;
+								endChar = 0;
+							} else {
+								endChar++;
+							}
+						}
+
+						match.ranges.push(range);
+						match.matches.push(new Range(line, shartChar, line, endChar));
+					}
+
+					matches.push(match);
+				}
+			}
+
+			return { matches: matches, limitHit: false };
+		} catch (ex) {
+			return { matches: [], limitHit: true };
+		}
+	}
+
+	private async gqlQuery<T>(query: string, variables: { [key: string]: string | number }): Promise<T | undefined> {
+		return (await this.graphql())<T>(query, variables);
+	}
+
+	private readonly pendingContextRequests = new Map<string, Promise<GitHubApiContext>>();
+	async getContext(uri: Uri): Promise<GitHubApiContext> {
+		const rootUri = getGitHubRootUri(uri);
+
+		let pending = this.pendingContextRequests.get(rootUri.toString());
+		if (pending === undefined) {
+			pending = this.getContextCore(rootUri);
+			this.pendingContextRequests.set(rootUri.toString(), pending);
+		}
+
+		try {
+			return await pending;
+		} finally {
+			this.pendingContextRequests.delete(rootUri.toString());
+		}
+	}
+
+	private readonly rootUriToContextMap = new Map<string, GitHubApiContext>();
+
+	private async getContextCore(rootUri: Uri): Promise<GitHubApiContext> {
+		let context = this.rootUriToContextMap.get(rootUri.toString());
+		if (context === undefined) {
+			const { ref } = fromGitHubUri(rootUri);
+			if (ref !== undefined && shaRegex.test(ref)) {
+				context = { sha: ref, timestamp: Date.now() };
+			} else {
+				context = this.context.get(rootUri);
+				if (context?.sha === undefined) {
+					const sha = await this.latestCommitQuery(rootUri);
+					if (sha !== undefined) {
+						context = { sha: sha, timestamp: Date.now() };
+					} else {
+						context = undefined;
+					}
+				}
+			}
+
+			if (context !== undefined) {
+				this.updateContext(rootUri, context);
+			}
+		}
+
+		return context ?? { sha: rootUri.authority, timestamp: Date.now() };
+	}
+
+	private updateContext(rootUri: Uri, context: GitHubApiContext) {
+		this.rootUriToContextMap.set(rootUri.toString(), context);
+		this.context.set(rootUri, context);
+	}
+}
+
+interface GitHubSearchTextMatch {
+	object_url: string;
+	object_type: string;
+	property: string;
+	fragment: string;
+	matches: {
+		text: string;
+		indices: number[];
+	}[];
+}
+
+interface SearchQueryMatch {
+	path: string;
+	ranges: Range[];
+	preview: string;
+	matches: Range[];
+}
+
+interface SearchQueryResults {
+	matches: SearchQueryMatch[];
+	limitHit: boolean;
+}

extensions/github-browser/src/github/fs.ts

@@ -0,0 +1,332 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+import {
+	CancellationToken,
+	Disposable,
+	Event,
+	EventEmitter,
+	FileChangeEvent,
+	FileSearchOptions,
+	FileSearchProvider,
+	FileSearchQuery,
+	FileStat,
+	FileSystemError,
+	FileSystemProvider,
+	FileType,
+	Progress,
+	TextSearchComplete,
+	TextSearchOptions,
+	TextSearchProvider,
+	TextSearchQuery,
+	TextSearchResult,
+	Uri,
+	workspace,
+} from 'vscode';
+import * as fuzzySort from 'fuzzysort';
+import fetch from 'node-fetch';
+import { GitHubApi } from './api';
+import { Iterables } from '../iterables';
+import { getRootUri } from '../extension';
+
+const emptyDisposable = { dispose: () => { /* noop */ } };
+const replaceBackslashRegex = /(\/|\\)/g;
+const textEncoder = new TextEncoder();
+
+interface Fuzzysort extends Fuzzysort.Fuzzysort {
+	prepareSlow(target: string): Fuzzysort.Prepared;
+	cleanup(): void;
+}
+
+export class GitHubFS implements FileSystemProvider, FileSearchProvider, TextSearchProvider, Disposable {
+	static scheme = 'github';
+
+	private _onDidChangeFile = new EventEmitter<FileChangeEvent[]>();
+	get onDidChangeFile(): Event<FileChangeEvent[]> {
+		return this._onDidChangeFile.event;
+	}
+
+	private readonly disposable: Disposable;
+	private fsCache = new Map<string, Map<string, any>>();
+
+	constructor(private readonly github: GitHubApi) {
+		this.disposable = Disposable.from(
+			workspace.registerFileSystemProvider(GitHubFS.scheme, this, {
+				isCaseSensitive: true,
+				isReadonly: true
+			}),
+			workspace.registerFileSearchProvider(GitHubFS.scheme, this),
+			workspace.registerTextSearchProvider(GitHubFS.scheme, this),
+			github.onDidChangeContext(e => this.fsCache.delete(e.toString()))
+		);
+	}
+
+	dispose() {
+		this.disposable?.dispose();
+	}
+
+	private getCache(uri: Uri) {
+		const rootUri = getRootUri(uri);
+		if (rootUri === undefined) {
+			return undefined;
+		}
+
+		let cache = this.fsCache.get(rootUri.toString());
+		if (cache === undefined) {
+			cache = new Map<string, any>();
+			this.fsCache.set(rootUri.toString(), cache);
+		}
+		return cache;
+	}
+
+	//#region FileSystemProvider
+
+	watch(): Disposable {
+		return emptyDisposable;
+	}
+
+	async stat(uri: Uri): Promise<FileStat> {
+		if (uri.path === '' || uri.path.lastIndexOf('/') === 0) {
+			const context = await this.github.getContext(uri);
+			return { type: FileType.Directory, size: 0, ctime: 0, mtime: context?.timestamp };
+		}
+
+		const data = await this.fsQuery<{
+			__typename: string;
+			byteSize: number | undefined;
+		}>(
+			uri,
+			`__typename
+			...on Blob {
+				byteSize
+			}`,
+			this.getCache(uri),
+		);
+
+		if (data === undefined) {
+			throw FileSystemError.FileNotFound();
+		}
+
+		const context = await this.github.getContext(uri);
+
+		return {
+			type: typenameToFileType(data.__typename),
+			size: data.byteSize ?? 0,
+			ctime: 0,
+			mtime: context?.timestamp,
+		};
+	}
+
+	async readDirectory(uri: Uri): Promise<[string, FileType][]> {
+		const data = await this.fsQuery<{
+			entries: { name: string; type: string }[];
+		}>(
+			uri,
+			`... on Tree {
+				entries {
+					name
+					type
+				}
+			}`,
+			this.getCache(uri),
+		);
+
+		return (data?.entries ?? []).map<[string, FileType]>(e => [
+			e.name,
+			typenameToFileType(e.type),
+		]);
+	}
+
+	createDirectory(_uri: Uri): void | Thenable<void> {
+		throw FileSystemError.NoPermissions();
+	}
+
+	async readFile(uri: Uri): Promise<Uint8Array> {
+		const data = await this.fsQuery<{
+			oid: string;
+			isBinary: boolean;
+			text: string;
+		}>(
+			uri,
+			`... on Blob {
+				oid,
+				isBinary,
+				text
+			}`,
+		);
+
+		if (data?.isBinary) {
+			const { owner, repo, path } = fromGitHubUri(uri);
+			// e.g. https://raw.githubusercontent.com/eamodio/vscode-gitlens/HEAD/images/gitlens-icon.png
+			const downloadUri = uri.with({
+				scheme: 'https',
+				authority: 'raw.githubusercontent.com',
+				path: `/${owner}/${repo}/HEAD/${path}`,
+			});
+
+			return downloadBinary(downloadUri);
+		}
+
+		return textEncoder.encode(data?.text ?? '');
+	}
+
+	async writeFile(_uri: Uri, _content: Uint8Array, _options: { create: boolean, overwrite: boolean }): Promise<void> {
+		throw FileSystemError.NoPermissions();
+	}
+
+	delete(_uri: Uri, _options: { recursive: boolean }): void | Thenable<void> {
+		throw FileSystemError.NoPermissions();
+	}
+
+	rename(_oldUri: Uri, _newUri: Uri, _options: { overwrite: boolean }): void | Thenable<void> {
+		throw FileSystemError.NoPermissions();
+	}
+
+	copy(_source: Uri, _destination: Uri, _options: { overwrite: boolean }): void | Thenable<void> {
+		throw FileSystemError.NoPermissions();
+	}
+
+	//#endregion
+
+	//#region FileSearchProvider
+
+	private fileSearchCache = new Map<string, Fuzzysort.Prepared[]>();
+
+	async provideFileSearchResults(
+		query: FileSearchQuery,
+		options: FileSearchOptions,
+		token: CancellationToken,
+	): Promise<Uri[]> {
+		let searchable = this.fileSearchCache.get(options.folder.toString(true));
+		if (searchable === undefined) {
+			const matches = await this.github.filesQuery(options.folder);
+			if (matches === undefined || token.isCancellationRequested) {
+				return [];
+			}
+
+			searchable = [...Iterables.map(matches, m => (fuzzySort as Fuzzysort).prepareSlow(m))];
+			this.fileSearchCache.set(options.folder.toString(true), searchable);
+		}
+
+		if (options.maxResults === undefined || options.maxResults === 0 || options.maxResults >= searchable.length) {
+			const results = searchable.map(m => Uri.joinPath(options.folder, m.target));
+			return results;
+		}
+
+		const results = fuzzySort
+			.go(query.pattern.replace(replaceBackslashRegex, '/'), searchable, {
+				allowTypo: true,
+				limit: options.maxResults,
+			})
+			.map(m => Uri.joinPath(options.folder, m.target));
+
+		(fuzzySort as Fuzzysort).cleanup();
+
+		return results;
+	}
+
+	//#endregion
+
+	//#region TextSearchProvider
+
+	async provideTextSearchResults(
+		query: TextSearchQuery,
+		options: TextSearchOptions,
+		progress: Progress<TextSearchResult>,
+		_token: CancellationToken,
+	): Promise<TextSearchComplete> {
+		const results = await this.github.searchQuery(
+			query.pattern,
+			options.folder,
+			{ maxResults: options.maxResults, context: { before: options.beforeContext, after: options.afterContext } },
+		);
+		if (results === undefined) { return { limitHit: true }; }
+
+		let uri;
+		for (const m of results.matches) {
+			uri = Uri.joinPath(options.folder, m.path);
+
+			progress.report({
+				uri: uri,
+				ranges: m.ranges,
+				preview: {
+					text: m.preview,
+					matches: m.matches,
+				},
+			});
+		}
+
+		return { limitHit: false };
+	}
+
+	//#endregion
+
+	private async fsQuery<T>(uri: Uri, query: string, cache?: Map<string, any>): Promise<T | undefined> {
+		const key = `${uri.toString()}:${getHashCode(query)}`;
+
+		let data = cache?.get(key);
+		if (data !== undefined) {
+			return data as T;
+		}
+
+		data = await this.github.fsQuery<T>(uri, query);
+		cache?.set(key, data);
+		return data;
+	}
+}
+
+async function downloadBinary(uri: Uri) {
+	const resp = await fetch(uri.toString());
+	const array = new Uint8Array(await resp.arrayBuffer());
+	return array;
+}
+
+function typenameToFileType(typename: string | undefined | null) {
+	if (typename) {
+		typename = typename.toLocaleLowerCase();
+	}
+
+	switch (typename) {
+		case 'blob':
+			return FileType.File;
+		case 'tree':
+			return FileType.Directory;
+		default:
+			return FileType.Unknown;
+	}
+}
+
+type RepoInfo = { owner: string; repo: string; path: string | undefined; ref?: string };
+export function fromGitHubUri(uri: Uri): RepoInfo {
+	const [, owner, repo, ...rest] = uri.path.split('/');
+
+	let ref;
+	if (uri.authority) {
+		ref = uri.authority;
+		// The casing of HEAD is important for the GitHub api to work
+		if (/HEAD/i.test(ref)) {
+			ref = 'HEAD';
+		}
+	}
+	return { owner: owner, repo: repo, path: rest.join('/'), ref: ref };
+}
+
+function getHashCode(s: string): number {
+	let hash = 0;
+
+	if (s.length === 0) {
+		return hash;
+	}
+
+	let char;
+	const len = s.length;
+	for (let i = 0; i < len; i++) {
+		char = s.charCodeAt(i);
+		hash = ((hash << 5) - hash) + char;
+		hash |= 0; // Convert to 32bit integer
+	}
+	return hash;
+}