Switch to 1DS endpoint (#20769)

* Update to 1DS client

* remove product config

* Update ai keys

* use our own event prefix

* re-enable telemetry

* Update distro and remove default enableTelemetry

* distro

* Remove asimovKey references

* add comment

* distro

* distro

* distro

* add files that break precommit hook

* cleanup/fixes

* distro

* distro + event prefix update

* distro

* more

src/vs/platform/telemetry/browser/1dsAppender.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { AbstractOneDataSystemAppender, IAppInsightsCore } from 'vs/platform/telemetry/common/1dsAppender';
+
+
+export class OneDataSystemWebAppender extends AbstractOneDataSystemAppender {
+	constructor(
+		isInternalTelemetry: boolean,
+		eventPrefix: string,
+		defaultData: { [key: string]: any } | null,
+		iKeyOrClientFactory: string | (() => IAppInsightsCore), // allow factory function for testing
+	) {
+		super(isInternalTelemetry, eventPrefix, defaultData, iKeyOrClientFactory);
+
+		// If we cannot fetch the endpoint it means it is down and we should not send any telemetry.
+		// This is most likely due to ad blockers
+		fetch(this.endPointUrl, { method: 'POST' }).catch(err => {
+			this._aiCoreOrKey = undefined;
+		});
+	}
+}

src/vs/platform/telemetry/common/1dsAppender.ts

@@ -0,0 +1,148 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { IExtendedConfiguration, IExtendedTelemetryItem, ITelemetryItem, ITelemetryUnloadState } from '@microsoft/1ds-core-js';
+import type { IChannelConfiguration, IXHROverride, PostChannel } from '@microsoft/1ds-post-js';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { mixin } from 'vs/base/common/objects';
+import { ITelemetryAppender, validateTelemetryData } from 'vs/platform/telemetry/common/telemetryUtils';
+
+// Interface type which is a subset of @microsoft/1ds-core-js AppInsightsCore.
+// Allows us to more easily build mock objects for testing as the interface is quite large and we only need a few properties.
+export interface IAppInsightsCore {
+	pluginVersionString: string;
+	track(item: ITelemetryItem | IExtendedTelemetryItem): void;
+	unload(isAsync: boolean, unloadComplete: (unloadState: ITelemetryUnloadState) => void): void;
+}
+
+const endpointUrl = 'https://mobile.events.data.microsoft.com/OneCollector/1.0';
+
+async function getClient(instrumentationKey: string, addInternalFlag?: boolean, xhrOverride?: IXHROverride): Promise<IAppInsightsCore> {
+	const oneDs = await import('@microsoft/1ds-core-js');
+	const postPlugin = await import('@microsoft/1ds-post-js');
+	const appInsightsCore = new oneDs.AppInsightsCore();
+	const collectorChannelPlugin: PostChannel = new postPlugin.PostChannel();
+	// Configure the app insights core to send to collector++ and disable logging of debug info
+	const coreConfig: IExtendedConfiguration = {
+		instrumentationKey,
+		endpointUrl,
+		loggingLevelTelemetry: 0,
+		loggingLevelConsole: 0,
+		disableCookiesUsage: true,
+		disableDbgExt: true,
+		disableInstrumentationKeyValidation: true,
+		channels: [[
+			collectorChannelPlugin
+		]]
+	};
+
+	if (xhrOverride) {
+		coreConfig.extensionConfig = {};
+		// Configure the channel to use a XHR Request override since it's not available in node
+		const channelConfig: IChannelConfiguration = {
+			alwaysUseXhrOverride: true,
+			httpXHROverride: xhrOverride
+		};
+		coreConfig.extensionConfig[collectorChannelPlugin.identifier] = channelConfig;
+	}
+
+	appInsightsCore.initialize(coreConfig, []);
+
+	appInsightsCore.addTelemetryInitializer((envelope) => {
+		if (addInternalFlag) {
+			envelope['ext'] = envelope['ext'] ?? {};
+			envelope['ext']['utc'] = envelope['ext']['utc'] ?? {};
+			// Sets it to be internal only based on Windows UTC flagging
+			envelope['ext']['utc']['flags'] = 0x0000811ECD;
+		}
+	});
+
+	return appInsightsCore;
+}
+
+// TODO @lramos15 maybe make more in line with src/vs/platform/telemetry/browser/appInsightsAppender.ts with caching support
+export abstract class AbstractOneDataSystemAppender implements ITelemetryAppender {
+
+	protected _aiCoreOrKey: IAppInsightsCore | string | undefined;
+	private _asyncAiCore: Promise<IAppInsightsCore> | null;
+	protected readonly endPointUrl = endpointUrl;
+
+	constructor(
+		private readonly _isInternalTelemetry: boolean,
+		private _eventPrefix: string,
+		private _defaultData: { [key: string]: any } | null,
+		iKeyOrClientFactory: string | (() => IAppInsightsCore), // allow factory function for testing
+		private _xhrOverride?: IXHROverride
+	) {
+		if (!this._defaultData) {
+			this._defaultData = {};
+		}
+
+		if (typeof iKeyOrClientFactory === 'function') {
+			this._aiCoreOrKey = iKeyOrClientFactory();
+		} else {
+			this._aiCoreOrKey = iKeyOrClientFactory;
+		}
+		this._asyncAiCore = null;
+	}
+
+	private _withAIClient(callback: (aiCore: IAppInsightsCore) => void): void {
+		if (!this._aiCoreOrKey) {
+			return;
+		}
+
+		if (typeof this._aiCoreOrKey !== 'string') {
+			callback(this._aiCoreOrKey);
+			return;
+		}
+
+		if (!this._asyncAiCore) {
+			this._asyncAiCore = getClient(this._aiCoreOrKey, this._isInternalTelemetry, this._xhrOverride);
+		}
+
+		this._asyncAiCore.then(
+			(aiClient) => {
+				callback(aiClient);
+			},
+			(err) => {
+				onUnexpectedError(err);
+				console.error(err);
+			}
+		);
+	}
+
+	log(eventName: string, data?: any): void {
+		if (!this._aiCoreOrKey) {
+			return;
+		}
+		data = mixin(data, this._defaultData);
+		data = validateTelemetryData(data);
+		const name = this._eventPrefix + '/' + eventName;
+
+		try {
+			this._withAIClient((aiClient) => {
+				aiClient.pluginVersionString = data?.properties.version ?? 'Unknown';
+				aiClient.track({
+					name,
+					baseData: { name, properties: data?.properties, measurements: data?.measurements }
+				});
+			});
+		} catch { }
+	}
+
+	flush(): Promise<any> {
+		if (this._aiCoreOrKey) {
+			return new Promise(resolve => {
+				this._withAIClient((aiClient) => {
+					aiClient.unload(true, () => {
+						this._aiCoreOrKey = undefined;
+						resolve(undefined);
+					});
+				});
+			});
+		}
+		return Promise.resolve(undefined);
+	}
+}

src/vs/platform/telemetry/common/remoteTelemetryChannel.ts

@@ -45,6 +45,10 @@ export class ServerTelemetryChannel extends Disposable implements IServerChannel
 
 				return Promise.resolve();
 			}
+
+			case 'ping': {
+				return;
+			}
 		}
 		// Command we cannot handle so we throw an error
 		throw new Error(`IPC Command ${command} not found`);

src/vs/platform/telemetry/common/serverTelemetryService.ts

@@ -6,7 +6,7 @@
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { refineServiceDecorator } from 'vs/platform/instantiation/common/instantiation';
 import { IProductService } from 'vs/platform/product/common/productService';
-import { ClassifiedEvent, GDPRClassification, StrictPropertyCheck } from 'vs/platform/telemetry/common/gdprTypings';
+import { ClassifiedEvent, IGDPRProperty, StrictPropertyCheck } from 'vs/platform/telemetry/common/gdprTypings';
 import { ITelemetryData, ITelemetryService, TelemetryLevel } from 'vs/platform/telemetry/common/telemetry';
 import { ITelemetryServiceConfig, TelemetryService } from 'vs/platform/telemetry/common/telemetryService';
 import { NullTelemetryServiceShape } from 'vs/platform/telemetry/common/telemetryUtils';
@@ -37,7 +37,7 @@ export class ServerTelemetryService extends TelemetryService implements IServerT
 		return super.publicLog(eventName, data, anonymizeFilePaths);
 	}
 
-	override publicLog2<E extends ClassifiedEvent<T> = never, T extends GDPRClassification<T> = never>(eventName: string, data?: StrictPropertyCheck<T, E>, anonymizeFilePaths?: boolean): Promise<void> {
+	override publicLog2<E extends ClassifiedEvent<T> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck<T, E>, anonymizeFilePaths?: boolean): Promise<void> {
 		return this.publicLog(eventName, data as ITelemetryData | undefined, anonymizeFilePaths);
 	}
 
@@ -48,7 +48,7 @@ export class ServerTelemetryService extends TelemetryService implements IServerT
 		return super.publicLogError(errorEventName, data);
 	}
 
-	override publicLogError2<E extends ClassifiedEvent<T> = never, T extends GDPRClassification<T> = never>(eventName: string, data?: StrictPropertyCheck<T, E>): Promise<void> {
+	override publicLogError2<E extends ClassifiedEvent<T> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck<T, E>): Promise<void> {
 		return this.publicLogError(eventName, data as ITelemetryData | undefined);
 	}
 

src/vs/platform/telemetry/common/telemetryUtils.ts

@@ -11,6 +11,7 @@ import { URI } from 'vs/base/common/uri';
 import { ConfigurationTarget, ConfigurationTargetToString, IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IEnvironmentService } from 'vs/platform/environment/common/environment';
 import { IProductService } from 'vs/platform/product/common/productService';
+import { verifyMicrosoftInternalDomain } from 'vs/platform/telemetry/common/commonProperties';
 import { ClassifiedEvent, GDPRClassification, StrictPropertyCheck } from 'vs/platform/telemetry/common/gdprTypings';
 import { ICustomEndpointTelemetryService, ITelemetryData, ITelemetryEndpoint, ITelemetryInfo, ITelemetryService, TelemetryConfiguration, TelemetryLevel, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from 'vs/platform/telemetry/common/telemetry';
 
@@ -250,6 +251,18 @@ function flatKeys(result: string[], prefix: string, value: { [key: string]: any
 	}
 }
 
+/**
+ * Whether or not this is an internal user
+ * @param productService The product service
+ * @param configService The config servivce
+ * @returns true if internal, false otherwise
+ */
+export function isInternalTelemetry(productService: IProductService, configService: IConfigurationService) {
+	const msftInternalDomains = productService.msftInternalDomains || [];
+	const internalTesting = configService.getValue<boolean>('telemetry.internalTesting');
+	return verifyMicrosoftInternalDomain(msftInternalDomains) || internalTesting;
+}
+
 interface IPathEnvironment {
 	appRoot: string;
 	extensionsPath: string;

src/vs/platform/telemetry/node/1dsAppender.ts

@@ -0,0 +1,51 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { IPayloadData, IXHROverride } from '@microsoft/1ds-post-js';
+import * as https from 'https';
+import { AbstractOneDataSystemAppender, IAppInsightsCore } from 'vs/platform/telemetry/common/1dsAppender';
+
+
+export class OneDataSystemAppender extends AbstractOneDataSystemAppender {
+
+	constructor(
+		isInternalTelemetry: boolean,
+		eventPrefix: string,
+		defaultData: { [key: string]: any } | null,
+		iKeyOrClientFactory: string | (() => IAppInsightsCore), // allow factory function for testing
+	) {
+		// Override the way events get sent since node doesn't have XHTMLRequest
+		const customHttpXHROverride: IXHROverride = {
+			sendPOST: (payload: IPayloadData, oncomplete) => {
+				const options = {
+					method: 'POST',
+					headers: {
+						...payload.headers,
+						'Content-Type': 'application/json',
+						'Content-Length': Buffer.byteLength(payload.data)
+					}
+				};
+				try {
+					const req = https.request(payload.urlString, options, res => {
+						res.on('data', function (responseData) {
+							oncomplete(res.statusCode ?? 200, res.headers as Record<string, any>, responseData.toString());
+						});
+						// On response with error send status of 0 and a blank response to oncomplete so we can retry events
+						res.on('error', function (err) {
+							oncomplete(0, {});
+						});
+					});
+					req.write(payload.data);
+					req.end();
+				} catch {
+					// If it errors out, send status of 0 and a blank response to oncomplete so we can retry events
+					oncomplete(0, {});
+				}
+			}
+		};
+
+		super(isInternalTelemetry, eventPrefix, defaultData, iKeyOrClientFactory, customHttpXHROverride);
+	}
+}

src/vs/platform/telemetry/node/appInsightsAppender.ts

@@ -1,121 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import type { TelemetryClient } from 'applicationinsights';
-import { onUnexpectedError } from 'vs/base/common/errors';
-import { mixin } from 'vs/base/common/objects';
-import { ITelemetryAppender, validateTelemetryData } from 'vs/platform/telemetry/common/telemetryUtils';
-
-async function getClient(aiKey: string): Promise<TelemetryClient> {
-	const appInsights = await import('applicationinsights');
-	let client: TelemetryClient;
-	if (appInsights.defaultClient) {
-		client = new appInsights.TelemetryClient(aiKey);
-		client.channel.setUseDiskRetryCaching(true);
-	} else {
-		appInsights.setup(aiKey)
-			.setAutoCollectRequests(false)
-			.setAutoCollectPerformance(false)
-			.setAutoCollectExceptions(false)
-			.setAutoCollectDependencies(false)
-			.setAutoDependencyCorrelation(false)
-			.setAutoCollectConsole(false)
-			.setInternalLogging(false, false)
-			.setUseDiskRetryCaching(true)
-			.start();
-		client = appInsights.defaultClient;
-	}
-
-	if (aiKey.indexOf('AIF-') === 0) {
-		client.config.endpointUrl = 'https://mobile.events.data.microsoft.com/collect/v1';
-	}
-	return client;
-}
-
-
-export class AppInsightsAppender implements ITelemetryAppender {
-
-	private _aiClient: string | TelemetryClient | undefined;
-	private _asyncAIClient: Promise<TelemetryClient> | null;
-
-	constructor(
-		private _eventPrefix: string,
-		private _defaultData: { [key: string]: any } | null,
-		aiKeyOrClientFactory: string | (() => TelemetryClient), // allow factory function for testing
-	) {
-		if (!this._defaultData) {
-			this._defaultData = Object.create(null);
-		}
-
-		if (typeof aiKeyOrClientFactory === 'function') {
-			this._aiClient = aiKeyOrClientFactory();
-		} else {
-			this._aiClient = aiKeyOrClientFactory;
-		}
-		this._asyncAIClient = null;
-	}
-
-	private _withAIClient(callback: (aiClient: TelemetryClient) => void): void {
-		if (!this._aiClient) {
-			return;
-		}
-
-		if (typeof this._aiClient !== 'string') {
-			callback(this._aiClient);
-			return;
-		}
-
-		if (!this._asyncAIClient) {
-			this._asyncAIClient = getClient(this._aiClient);
-		}
-
-		this._asyncAIClient.then(
-			(aiClient) => {
-				callback(aiClient);
-			},
-			(err) => {
-				onUnexpectedError(err);
-				console.error(err);
-			}
-		);
-	}
-
-	log(eventName: string, data?: any): void {
-		if (!this._aiClient) {
-			return;
-		}
-		data = mixin(data, this._defaultData);
-		data = validateTelemetryData(data);
-
-		// Attemps to suppress https://github.com/microsoft/vscode/issues/140624
-		try {
-			this._withAIClient((aiClient) => aiClient.trackEvent({
-				name: this._eventPrefix + '/' + eventName,
-				properties: data.properties,
-				measurements: data.measurements
-			}));
-		} catch { }
-	}
-
-	flush(): Promise<any> {
-		if (this._aiClient) {
-			return new Promise(resolve => {
-				this._withAIClient((aiClient) => {
-					// Attempts to suppress https://github.com/microsoft/vscode/issues/140624
-					try {
-						aiClient.flush({
-							callback: () => {
-								// all data flushed
-								this._aiClient = undefined;
-								resolve(undefined);
-							}
-						});
-					} catch { }
-				});
-			});
-		}
-		return Promise.resolve(undefined);
-	}
-}

src/vs/platform/telemetry/test/browser/1dsAppender.test.ts

@@ -2,39 +2,35 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { Contracts, TelemetryClient } from 'applicationinsights';
+import { ITelemetryItem, ITelemetryUnloadState } from '@microsoft/1ds-core-js';
 import * as assert from 'assert';
-import { AppInsightsAppender } from 'vs/platform/telemetry/node/appInsightsAppender';
+import { OneDataSystemWebAppender } from 'vs/platform/telemetry/browser/1dsAppender';
+import { IAppInsightsCore } from 'vs/platform/telemetry/common/1dsAppender';
 
-class AppInsightsMock extends TelemetryClient {
-	public override config: any;
-	public override channel: any;
-	public events: Contracts.EventTelemetry[] = [];
+class AppInsightsCoreMock implements IAppInsightsCore {
+	pluginVersionString: string = 'Test Runner';
+	public events: any[] = [];
 	public IsTrackingPageView: boolean = false;
 	public exceptions: any[] = [];
 
-	constructor() {
-		super('testKey');
+	public track(event: ITelemetryItem) {
+		this.events.push(event.baseData);
 	}
 
-	public override trackEvent(event: any) {
-		this.events.push(event);
-	}
-
-	public override flush(options: any): void {
-		// called on dispose
+	public unload(isAsync: boolean, unloadComplete: (unloadState: ITelemetryUnloadState) => void): void {
+		// No-op
 	}
 }
 
 suite('AIAdapter', () => {
-	let appInsightsMock: AppInsightsMock;
-	let adapter: AppInsightsAppender;
-	let prefix = 'prefix';
+	let appInsightsMock: AppInsightsCoreMock;
+	let adapter: OneDataSystemWebAppender;
+	const prefix = 'prefix';
 
 
 	setup(() => {
-		appInsightsMock = new AppInsightsMock();
-		adapter = new AppInsightsAppender(prefix, undefined!, () => appInsightsMock);
+		appInsightsMock = new AppInsightsCoreMock();
+		adapter = new OneDataSystemWebAppender(false, prefix, undefined!, () => appInsightsMock);
 	});
 
 	teardown(() => {
@@ -49,11 +45,11 @@ suite('AIAdapter', () => {
 	});
 
 	test('addional data', () => {
-		adapter = new AppInsightsAppender(prefix, { first: '1st', second: 2, third: true }, () => appInsightsMock);
+		adapter = new OneDataSystemWebAppender(false, prefix, { first: '1st', second: 2, third: true }, () => appInsightsMock);
 		adapter.log('testEvent');
 
 		assert.strictEqual(appInsightsMock.events.length, 1);
-		let [first] = appInsightsMock.events;
+		const [first] = appInsightsMock.events;
 		assert.strictEqual(first.name, `${prefix}/testEvent`);
 		assert.strictEqual(first.properties!['first'], '1st');
 		assert.strictEqual(first.measurements!['second'], 2);
@@ -73,21 +69,21 @@ suite('AIAdapter', () => {
 		}
 		assert(reallyLongPropertyValue.length > 8192);
 
-		let data = Object.create(null);
+		const data = Object.create(null);
 		data[reallyLongPropertyName] = '1234';
 		data['reallyLongPropertyValue'] = reallyLongPropertyValue;
 		adapter.log('testEvent', data);
 
 		assert.strictEqual(appInsightsMock.events.length, 1);
 
-		for (let prop in appInsightsMock.events[0].properties!) {
+		for (const prop in appInsightsMock.events[0].properties!) {
 			assert(prop.length < 150);
 			assert(appInsightsMock.events[0].properties![prop].length < 8192);
 		}
 	});
 
 	test('Different data types', () => {
-		let date = new Date();
+		const date = new Date();
 		adapter.log('testEvent', { favoriteDate: date, likeRed: false, likeBlue: true, favoriteNumber: 1, favoriteColor: 'blue', favoriteCars: ['bmw', 'audi', 'ford'] });
 
 		assert.strictEqual(appInsightsMock.events.length, 1);