diff --git a/src/extension/endpoint/common/githubToFeimaModelMappingService.ts b/src/extension/endpoint/common/githubToFeimaModelMappingService.ts index 2fe22c5472..40ab4c7ea0 100644 --- a/src/extension/endpoint/common/githubToFeimaModelMappingService.ts +++ b/src/extension/endpoint/common/githubToFeimaModelMappingService.ts @@ -55,10 +55,12 @@ export class GitHubToFeimaModelMappingService implements IGitHubToFeimaModelMapp * Maps both: * 1. GitHub canonical families: 'copilot-fast', 'copilot-base' * 2. Resolved model IDs: 'gpt-4o-mini', 'gpt-41-copilot' + * 3. Embeddings models: 'text-embedding-3-small', 'text3small-512' * - * Target models are selected from #file:001_initial_schema.py: + * Target models are selected from database model catalog: * - qwen-flash: Free tier, 1M context, ultra-fast * - qwen-coder-turbo: Free tier (assumed), specialized for code + * - text-embedding-v4: Ali Cloud embeddings, 512 dimensions */ private readonly _modelMap: Map = new Map([ // GitHub canonical families → Feima free-tier models @@ -69,6 +71,11 @@ export class GitHubToFeimaModelMappingService implements IGitHubToFeimaModelMapp // GitHub resolved model IDs → Feima equivalents ['gpt-4o-mini', 'qwen3'], // Lightweight model → fast Feima model ['gpt-41-copilot', 'qwen-coder-turbo'], // Code-specialized → coder turbo + + // GitHub embeddings models → Feima embeddings + ['text-embedding-3-small', 'text-embedding-v4'], // GitHub embeddings → Ali Cloud + ['text-embedding-3-small-512', 'text-embedding-v4'], // GitHub full ID → Feima model + ['text3small-512', 'text-embedding-v4'], // Internal short ID → Feima model ]); getFeimaModel(githubModelOrFamily: string): string | undefined { diff --git a/src/extension/extension/vscode-node/services.ts b/src/extension/extension/vscode-node/services.ts index 3d41203748..730e9a9972 100644 --- a/src/extension/extension/vscode-node/services.ts +++ b/src/extension/extension/vscode-node/services.ts @@ -14,11 +14,13 @@ import { VSCodeCopilotTokenManager } from '../../../platform/authentication/vsco import { IChatAgentService } from '../../../platform/chat/common/chatAgents'; import { IChatMLFetcher } from '../../../platform/chat/common/chatMLFetcher'; import { IChunkingEndpointClient } from '../../../platform/chunking/common/chunkingEndpointClient'; -import { ChunkingEndpointClientImpl } from '../../../platform/chunking/common/chunkingEndpointClientImpl'; +import { FeimaChunkingClient } from '../../../platform/chunking/common/feimaChunkingClient'; import { INaiveChunkingService, NaiveChunkingService } from '../../../platform/chunking/node/naiveChunkerService'; import { IDevContainerConfigurationService } from '../../../platform/devcontainer/common/devContainerConfigurationService'; import { IDiffService } from '../../../platform/diff/common/diffService'; import { DiffServiceImpl } from '../../../platform/diff/node/diffServiceImpl'; +import { IEmbeddingsComputer } from '../../../platform/embeddings/common/embeddingsComputer'; +import { FeimaEmbeddingsComputer } from '../../../platform/embeddings/common/feimaEmbeddingsComputer'; import { ICAPIClientService } from '../../../platform/endpoint/common/capiClient'; import { IDomainService } from '../../../platform/endpoint/common/domainService'; import { IEndpointProvider, IFeimaEndpointProvider, IGitHubEndpointProvider } from '../../../platform/endpoint/common/endpointProvider'; @@ -72,7 +74,8 @@ import { IWorkspaceMutationManager } from '../../../platform/testing/common/work import { ISetupTestsDetector, SetupTestsDetector } from '../../../platform/testing/node/setupTestDetector'; import { ITestDepsResolver, TestDepsResolver } from '../../../platform/testing/node/testDepsResolver'; import { ITokenizerProvider, TokenizerProvider } from '../../../platform/tokenizer/node/tokenizer'; -import { GithubAvailableEmbeddingTypesService, IGithubAvailableEmbeddingTypesService } from '../../../platform/workspaceChunkSearch/common/githubAvailableEmbeddingTypes'; +import { FeimaEmbeddingTypesService } from '../../../platform/workspaceChunkSearch/common/feimaAvailableEmbeddingTypes'; +import { IGithubAvailableEmbeddingTypesService } from '../../../platform/workspaceChunkSearch/common/githubAvailableEmbeddingTypes'; import { IRerankerService, RerankerService } from '../../../platform/workspaceChunkSearch/common/rerankerService'; import { IWorkspaceChunkSearchService, WorkspaceChunkSearchService } from '../../../platform/workspaceChunkSearch/node/workspaceChunkSearchService'; import { IWorkspaceFileIndex, WorkspaceFileIndex } from '../../../platform/workspaceChunkSearch/node/workspaceFileIndex'; @@ -205,7 +208,9 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio builder.define(IIntentService, new SyncDescriptor(IntentService)); builder.define(INaiveChunkingService, new SyncDescriptor(NaiveChunkingService)); builder.define(IWorkspaceFileIndex, new SyncDescriptor(WorkspaceFileIndex)); - builder.define(IChunkingEndpointClient, new SyncDescriptor(ChunkingEndpointClientImpl)); + builder.define(IChunkingEndpointClient, new SyncDescriptor(FeimaChunkingClient)); + // Override IEmbeddingsComputer from common services to use Feima composite router + builder.define(IEmbeddingsComputer, new SyncDescriptor(FeimaEmbeddingsComputer)); builder.define(ICommandService, new SyncDescriptor(CommandServiceImpl)); builder.define(IDocsSearchClient, new SyncDescriptor(DocsSearchClient)); builder.define(ISearchService, new SyncDescriptor(SearchServiceImpl)); @@ -241,7 +246,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio builder.define(IWorkspaceListenerService, new SyncDescriptor(WorkspacListenerService)); builder.define(ICodeSearchAuthenticationService, new SyncDescriptor(VsCodeCodeSearchAuthenticationService)); builder.define(ITodoListContextProvider, new SyncDescriptor(TodoListContextProvider)); - builder.define(IGithubAvailableEmbeddingTypesService, new SyncDescriptor(GithubAvailableEmbeddingTypesService)); + builder.define(IGithubAvailableEmbeddingTypesService, new SyncDescriptor(FeimaEmbeddingTypesService)); builder.define(IRerankerService, new SyncDescriptor(RerankerService)); builder.define(IProxyModelsService, new SyncDescriptor(ProxyModelsService)); builder.define(IInlineEditsModelService, new SyncDescriptor(InlineEditsModelService)); diff --git a/src/extension/prompt/vscode-node/feimaEndpointProvider.ts b/src/extension/prompt/vscode-node/feimaEndpointProvider.ts index 8f3d277935..e9001c02c1 100644 --- a/src/extension/prompt/vscode-node/feimaEndpointProvider.ts +++ b/src/extension/prompt/vscode-node/feimaEndpointProvider.ts @@ -9,6 +9,7 @@ import { IAuthenticationService } from '../../../platform/authentication/common/ import { IFeimaAuthenticationService } from '../../../platform/authentication/node/feimaAuthenticationService'; import { ChatEndpointFamily, EmbeddingsEndpointFamily, IChatModelInformation, ICompletionModelInformation, IEndpointProvider, IFeimaEndpointProvider, IGitHubEndpointProvider } from '../../../platform/endpoint/common/endpointProvider'; import { FeimaChatEndpoint } from '../../../platform/endpoint/node/feimaChatEndpoint'; +import { FeimaEmbeddingsEndpoint } from '../../../platform/endpoint/node/feimaEmbeddingsEndpoint'; import { IFeimaModelMetadataFetcher } from '../../../platform/endpoint/node/feimaModelMetadataFetcher'; import { ILogService } from '../../../platform/log/common/logService'; import { IChatEndpoint, IEmbeddingsEndpoint } from '../../../platform/networking/common/networking'; @@ -35,6 +36,7 @@ export class FeimaOnlyEndpointProvider implements IEndpointProvider { constructor( @IFeimaAuthenticationService private readonly feimaAuthService: IFeimaAuthenticationService, @IFeimaModelMetadataFetcher private readonly feimaModelFetcher: IFeimaModelMetadataFetcher, + @IFeimaConfigService private readonly feimaConfigService: IFeimaConfigService, @ILogService private readonly logService: ILogService, @IInstantiationService private readonly instantiationService: IInstantiationService ) { @@ -145,8 +147,25 @@ export class FeimaOnlyEndpointProvider implements IEndpointProvider { } async getEmbeddingsEndpoint(family?: EmbeddingsEndpointFamily): Promise { - // Feima embeddings not yet supported - throw new Error('Feima embeddings not yet supported'); + this.logService.trace('[FeimaOnlyEndpointProvider] Getting embeddings endpoint'); + + // Check authentication + const isAuthenticated = await this.feimaAuthService.isAuthenticated(); + if (!isAuthenticated) { + throw new Error('Feima not authenticated'); + } + + // Fetch embedding model metadata (use 'text-embedding-3-small' literal for compatibility) + const model = await this.feimaModelFetcher.getEmbeddingsModel('text-embedding-3-small'); + if (!model) { + throw new Error('Feima embedding model not found'); + } + + // Get Feima API endpoint from config + const feimaApiEndpoint = this.feimaConfigService.getConfig().apiBaseUrl; + + // Create and return FeimaEmbeddingsEndpoint + return this.instantiationService.createInstance(FeimaEmbeddingsEndpoint, model, feimaApiEndpoint); } } @@ -363,6 +382,16 @@ export class CombinedEndpointProvider implements IEndpointProvider { const isFeimaAuthenticated = await this.feimaAuthService.isAuthenticated(); const config = this.feimaConfigService.getConfig(); + // Map GitHub embedding families to Feima equivalents when appropriate + let resolvedFamily = family; + if (family && config.preferFeimaModels && isFeimaAuthenticated) { + const feimaModelId = this.modelMappingService.getFeimaModel(family); + if (feimaModelId) { + this.logService.trace(`[CombinedEndpointProvider] Pre-mapping GitHub embedding family ${family} to Feima model ${feimaModelId}`); + resolvedFamily = feimaModelId as EmbeddingsEndpointFamily; + } + } + // Determine primary and fallback providers based on preference const primaryProvider = config.preferFeimaModels ? this.feimaProvider : this.githubProvider; const fallbackProvider = config.preferFeimaModels ? this.githubProvider : this.feimaProvider; @@ -372,7 +401,7 @@ export class CombinedEndpointProvider implements IEndpointProvider { // If both are authenticated, try primary then fallback if (isFeimaAuthenticated) { try { - const endpoint = await primaryProvider.getEmbeddingsEndpoint(family); + const endpoint = await primaryProvider.getEmbeddingsEndpoint(resolvedFamily); this.logService.trace(`[CombinedEndpointProvider] Resolved to ${primaryName} embeddings endpoint`); return endpoint; } catch (primaryError) { diff --git a/src/platform/authentication/common/feimaAuthentication.ts b/src/platform/authentication/common/feimaAuthentication.ts new file mode 100644 index 0000000000..f416cba48f --- /dev/null +++ b/src/platform/authentication/common/feimaAuthentication.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type * as vscode from 'vscode'; +import { createServiceIdentifier } from '../../../util/common/services'; +import type { Event } from '../../../util/vs/base/common/event'; + +/** + * Service interface for Feima authentication. + * + * Provides both consumer-facing methods (getToken, isAuthenticated) AND + * VS Code AuthenticationProvider methods (getSessions, createSession, removeSession). + * + * Architecture: + * - FeimaAuthenticationService: Heavy implementation (OAuth2, session management, secrets) + * - FeimaAuthProvider: Thin VS Code adapter that delegates to the service + * - Other services: Inject IFeimaAuthenticationService via DI for getToken/isAuthenticated + */ +export interface IFeimaAuthenticationService { + readonly _serviceBrand: undefined; + + // ============ Consumer-Facing Methods (for DI injection) ============ + + /** + * Get current JWT token for Feima API. + * @returns JWT token string or undefined if not authenticated or expired + */ + getToken(): Promise; + + /** + * Check if user is authenticated with Feima. + * @returns true if authenticated, false otherwise + */ + isAuthenticated(): Promise; + + /** + * Refresh JWT token from authentication server. + * @returns Fresh JWT token or undefined if refresh fails + */ + refreshToken(): Promise; + + /** + * Sign out and clear all stored tokens. + */ + signOut(): Promise; + + /** + * Event fired when authentication state changes (sign-in or sign-out). + */ + onDidChangeAuthenticationState: Event; + + // ============ Provider Methods (for FeimaAuthProvider delegation) ============ + + /** + * Get all authentication sessions. + * Used by VS Code AuthenticationProvider interface. + */ + getSessions(scopes: readonly string[] | undefined, options: vscode.AuthenticationProviderSessionOptions): Promise; + + /** + * Create a new authentication session via OAuth2 + PKCE flow. + * Used by VS Code AuthenticationProvider interface. + */ + createSession(scopes: readonly string[], options: vscode.AuthenticationProviderSessionOptions): Promise; + + /** + * Remove an authentication session. + * Used by VS Code AuthenticationProvider interface. + */ + removeSession(sessionId: string): Promise; + + /** + * Get cached sessions synchronously (non-blocking). + * Used by FeimaAuthProvider for fast checks. + */ + getCachedSessions(): vscode.AuthenticationSession[]; + + /** + * Event fired when sessions change. + * Used by VS Code AuthenticationProvider interface. + */ + onDidChangeSessions: Event; + + /** + * Handle OAuth callback URI (called by VS Code UriHandler). + * Used by FeimaAuthProvider for OAuth2 flow. + */ + handleUri(uri: vscode.Uri): void; +} + +export const IFeimaAuthenticationService = createServiceIdentifier('feimaAuthenticationService'); diff --git a/src/platform/authentication/node/feimaAuthenticationService.ts b/src/platform/authentication/node/feimaAuthenticationService.ts index f416cba48f..ffd6959836 100644 --- a/src/platform/authentication/node/feimaAuthenticationService.ts +++ b/src/platform/authentication/node/feimaAuthenticationService.ts @@ -2,91 +2,5 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type * as vscode from 'vscode'; -import { createServiceIdentifier } from '../../../util/common/services'; -import type { Event } from '../../../util/vs/base/common/event'; - -/** - * Service interface for Feima authentication. - * - * Provides both consumer-facing methods (getToken, isAuthenticated) AND - * VS Code AuthenticationProvider methods (getSessions, createSession, removeSession). - * - * Architecture: - * - FeimaAuthenticationService: Heavy implementation (OAuth2, session management, secrets) - * - FeimaAuthProvider: Thin VS Code adapter that delegates to the service - * - Other services: Inject IFeimaAuthenticationService via DI for getToken/isAuthenticated - */ -export interface IFeimaAuthenticationService { - readonly _serviceBrand: undefined; - - // ============ Consumer-Facing Methods (for DI injection) ============ - - /** - * Get current JWT token for Feima API. - * @returns JWT token string or undefined if not authenticated or expired - */ - getToken(): Promise; - - /** - * Check if user is authenticated with Feima. - * @returns true if authenticated, false otherwise - */ - isAuthenticated(): Promise; - - /** - * Refresh JWT token from authentication server. - * @returns Fresh JWT token or undefined if refresh fails - */ - refreshToken(): Promise; - - /** - * Sign out and clear all stored tokens. - */ - signOut(): Promise; - - /** - * Event fired when authentication state changes (sign-in or sign-out). - */ - onDidChangeAuthenticationState: Event; - - // ============ Provider Methods (for FeimaAuthProvider delegation) ============ - - /** - * Get all authentication sessions. - * Used by VS Code AuthenticationProvider interface. - */ - getSessions(scopes: readonly string[] | undefined, options: vscode.AuthenticationProviderSessionOptions): Promise; - - /** - * Create a new authentication session via OAuth2 + PKCE flow. - * Used by VS Code AuthenticationProvider interface. - */ - createSession(scopes: readonly string[], options: vscode.AuthenticationProviderSessionOptions): Promise; - - /** - * Remove an authentication session. - * Used by VS Code AuthenticationProvider interface. - */ - removeSession(sessionId: string): Promise; - - /** - * Get cached sessions synchronously (non-blocking). - * Used by FeimaAuthProvider for fast checks. - */ - getCachedSessions(): vscode.AuthenticationSession[]; - - /** - * Event fired when sessions change. - * Used by VS Code AuthenticationProvider interface. - */ - onDidChangeSessions: Event; - - /** - * Handle OAuth callback URI (called by VS Code UriHandler). - * Used by FeimaAuthProvider for OAuth2 flow. - */ - handleUri(uri: vscode.Uri): void; -} - -export const IFeimaAuthenticationService = createServiceIdentifier('feimaAuthenticationService'); +// Re-export interface and service identifier from common layer +export * from '../common/feimaAuthentication'; diff --git a/src/platform/chunking/common/feimaChunkingClient.ts b/src/platform/chunking/common/feimaChunkingClient.ts new file mode 100644 index 0000000000..be6418dd54 --- /dev/null +++ b/src/platform/chunking/common/feimaChunkingClient.ts @@ -0,0 +1,353 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CallTracker } from '../../../util/common/telemetryCorrelationId'; +import { raceCancellationError } from '../../../util/vs/base/common/async'; +import { CancellationToken } from '../../../util/vs/base/common/cancellation'; +import { Disposable } from '../../../util/vs/base/common/lifecycle'; +import { Range } from '../../../util/vs/editor/common/core/range'; +import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; +import { IAuthenticationService } from '../../authentication/common/authentication'; +import { IFeimaAuthenticationService } from '../../authentication/common/feimaAuthentication'; +import { Embedding, EmbeddingType, EmbeddingVector } from '../../embeddings/common/embeddingsComputer'; +import { IFeimaConfigService } from '../../feima/common/feimaConfigService'; +import { ILogService } from '../../log/common/logService'; +import { IFetcherService } from '../../networking/common/fetcherService'; +import { ITelemetryService } from '../../telemetry/common/telemetry'; +import { FileChunkWithEmbedding, FileChunkWithOptionalEmbedding } from './chunk'; +import { ChunkableContent, ComputeBatchInfo, EmbeddingsComputeQos, IChunkingEndpointClient } from './chunkingEndpointClient'; +import { ChunkingEndpointClientImpl } from './chunkingEndpointClientImpl'; +import { stripChunkTextMetadata } from './chunkingStringUtils'; + +type FeimaChunksEndpointResponse = { + readonly chunks: readonly { + readonly hash: string; + readonly range: { start: number; end: number }; + readonly line_range: { start: number; end: number }; + readonly text?: string; + readonly embedding?: { model: string; embedding: EmbeddingVector }; + }[]; + readonly embedding_model: string; +}; + +/** + * Feima-aware chunking client that routes to Feima API or GitHub CAPI based on preferFeimaModels config. + * + * Routing priority: + * 1. If preferFeimaModels is true → Use Feima API /v1/chunks, fallback to GitHub CAPI if fails + * 2. If preferFeimaModels is false → Use GitHub CAPI /chunks, fallback to Feima API if fails + */ +export class FeimaChunkingClient extends Disposable implements IChunkingEndpointClient { + declare readonly _serviceBrand: undefined; + + /** + * The original GitHub CAPI client for fallback + */ + private readonly _githubClient: ChunkingEndpointClientImpl; + + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @IAuthenticationService private readonly _authService: IAuthenticationService, + @IFeimaAuthenticationService private readonly _feimaAuthService: IFeimaAuthenticationService, + @IFeimaConfigService private readonly _feimaConfig: IFeimaConfigService, + @IFetcherService private readonly _fetcherService: IFetcherService, + @ILogService private readonly _logService: ILogService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, + ) { + super(); + + // Create the original GitHub client for fallback + this._githubClient = this._register(instantiationService.createInstance(ChunkingEndpointClientImpl)); + } + + public async computeChunks( + authToken: string, + embeddingType: EmbeddingType, + content: ChunkableContent, + batchInfo: ComputeBatchInfo, + qos: EmbeddingsComputeQos, + cache: ReadonlyMap | undefined, + telemetryInfo: CallTracker, + token: CancellationToken + ): Promise { + // Fetch authentication tokens for both backends + const feimaToken = await this._feimaAuthService.getToken(); + const githubToken = (await this._authService.getCopilotToken()).token; + + // Route based on preferFeimaModels config + return this.routeRequest( + () => this.doFeimaChunks(feimaToken!, embeddingType, content, batchInfo, qos, cache, telemetryInfo, false, token), + () => this._githubClient.computeChunks(githubToken, embeddingType, content, batchInfo, qos, cache, telemetryInfo, token), + telemetryInfo, + token + ); + } + + public async computeChunksAndEmbeddings( + authToken: string, + embeddingType: EmbeddingType, + content: ChunkableContent, + batchInfo: ComputeBatchInfo, + qos: EmbeddingsComputeQos, + cache: ReadonlyMap | undefined, + telemetryInfo: CallTracker, + token: CancellationToken + ): Promise { + // Fetch authentication tokens for both backends + const feimaToken = await this._feimaAuthService.getToken(); + const githubToken = (await this._authService.getCopilotToken()).token; + + // Route based on preferFeimaModels config + const result = await this.routeRequest( + () => this.doFeimaChunks(feimaToken!, embeddingType, content, batchInfo, qos, cache, telemetryInfo, true, token), + () => this._githubClient.computeChunksAndEmbeddings(githubToken, embeddingType, content, batchInfo, qos, cache, telemetryInfo, token), + telemetryInfo, + token + ); + return result as FileChunkWithEmbedding[] | undefined; + } + + /** + * Route request to Feima or GitHub based on preferFeimaModels config. + */ + private async routeRequest( + feimaOperation: () => Promise, + githubOperation: () => Promise, + telemetryInfo: CallTracker, + token: CancellationToken + ): Promise { + // Check if Feima models are preferred + const preferFeima = this._feimaConfig.getConfig().preferFeimaModels; + + if (preferFeima) { + this._logService.debug('[FeimaChunkingClient] Using Feima API for chunks'); + + /* __GDPR__ + "feimaChunkingClient.routeToFeima" : { + "owner": "feima", + "comment": "Tracks when chunks are routed to Feima API", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaChunkingClient.routeToFeima', { + source: telemetryInfo.toString(), + }); + + try { + return await raceCancellationError(feimaOperation(), token); + } catch (e) { + this._logService.error('[FeimaChunkingClient] Feima API failed, falling back to GitHub', e); + + /* __GDPR__ + "feimaChunkingClient.feimaFallbackToGithub" : { + "owner": "feima", + "comment": "Tracks when Feima API fails and falls back to GitHub", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "error": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaChunkingClient.feimaFallbackToGithub', { + source: telemetryInfo.toString(), + error: e instanceof Error ? e.message : String(e), + }); + + // Fallback to GitHub if Feima fails + return await raceCancellationError(githubOperation(), token); + } + } + + // Use GitHub CAPI + this._logService.debug('[FeimaChunkingClient] Using GitHub CAPI for chunks'); + + /* __GDPR__ + "feimaChunkingClient.routeToGithub" : { + "owner": "feima", + "comment": "Tracks when chunks are routed to GitHub CAPI", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaChunkingClient.routeToGithub', { + source: telemetryInfo.toString(), + }); + + try { + return await raceCancellationError(githubOperation(), token); + } catch (e) { + this._logService.error('[FeimaChunkingClient] GitHub CAPI failed, falling back to Feima', e); + + /* __GDPR__ + "feimaChunkingClient.githubFallbackToFeima" : { + "owner": "feima", + "comment": "Tracks when GitHub CAPI fails and falls back to Feima", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "error": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaChunkingClient.githubFallbackToFeima', { + source: telemetryInfo.toString(), + error: e instanceof Error ? e.message : String(e), + }); + + // Fallback to Feima if GitHub fails + return await raceCancellationError(feimaOperation(), token); + } + } + + /** + * Call Feima API /v1/chunks endpoint. + */ + private async doFeimaChunks( + authToken: string, + embeddingType: EmbeddingType, + content: ChunkableContent, + batchInfo: ComputeBatchInfo, + qos: EmbeddingsComputeQos, + cache: ReadonlyMap | undefined, + telemetryInfo: CallTracker, + computeEmbeddings: boolean, + token: CancellationToken + ): Promise { + const text = await raceCancellationError(content.getText(), token); + if (!text || text.trim().length === 0) { + return []; + } + + const feimaApiUrl = this._feimaConfig.getConfig().apiBaseUrl; + if (!feimaApiUrl) { + throw new Error('Feima API URL not configured'); + } + + // Map QoS enum to backend-expected strings + const qosString = qos === EmbeddingsComputeQos.Batch ? 'low' : 'high'; + + try { + this._logService.debug(`[FeimaChunkingClient] Calling Feima API: ${feimaApiUrl}/chunks`); + + const response = await raceCancellationError( + this._fetcherService.fetch(`${feimaApiUrl}/chunks`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json', + }, + json: { + embed: computeEmbeddings, + qos: qosString, + content: text, + path: content.uri.path, + local_hashes: cache ? Array.from(cache.keys()) : [], + language_id: content.githubLanguageId ?? 0, + embedding_model: embeddingType.id, + }, + }), + token + ); + + if (!response.ok) { + let errorDetail = ''; + try { + const errorBody = await response.json(); + errorDetail = JSON.stringify(errorBody); + this._logService.error(`[FeimaChunkingClient] Error from Feima API. Status: ${response.status}. Status Text: ${response.statusText}. Detail: ${errorDetail}`); + } catch { + this._logService.error(`[FeimaChunkingClient] Error from Feima API. Status: ${response.status}. Status Text: ${response.statusText}.`); + } + + /* __GDPR__ + "feimaChunkingClient.apiError" : { + "owner": "feima", + "comment": "Tracks errors from Feima chunks API", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "responseStatus": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "HTTP status code" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaChunkingClient.apiError', { + source: telemetryInfo.toString(), + }, { + responseStatus: response.status, + }); + + return undefined; + } + + batchInfo.recomputedFileCount++; + batchInfo.sentContentTextLength += text.length; + + const body: FeimaChunksEndpointResponse = await response.json(); + if (!body.chunks || body.chunks.length === 0) { + return []; + } + + this._logService.debug(`[FeimaChunkingClient] Received ${body.chunks.length} chunks from Feima API`); + + // Transform Feima response to internal format + const results: FileChunkWithOptionalEmbedding[] = []; + for (const chunk of body.chunks) { + const range = new Range(chunk.line_range.start, 0, chunk.line_range.end, 0); + + // Check cache + const cached = cache?.get(chunk.hash); + if (cached) { + results.push({ + chunk: { + file: content.uri, + text: stripChunkTextMetadata(cached.chunk.text), + rawText: undefined, + range, + isFullFile: cached.chunk.isFullFile, + }, + chunkHash: chunk.hash, + embedding: cached.embedding, + }); + continue; + } + + // Validate chunk has text + if (typeof chunk.text !== 'string') { + this._logService.warn(`[FeimaChunkingClient] Invalid chunk without text, skipping`); + continue; + } + + // Parse embedding if present + let embedding: Embedding | undefined; + if (chunk.embedding?.embedding) { + const returnedEmbeddingsType = new EmbeddingType(body.embedding_model); + if (!returnedEmbeddingsType.equals(embeddingType)) { + throw new Error(`Unexpected embedding model from Feima. Got: ${returnedEmbeddingsType}. Expected: ${embeddingType}`); + } + + embedding = { + type: returnedEmbeddingsType, + value: chunk.embedding.embedding + }; + } + + // Validate embeddings are present if requested + if (computeEmbeddings && !embedding) { + this._logService.warn(`[FeimaChunkingClient] Chunk missing embedding when embed=true, skipping`); + continue; + } + + results.push({ + chunk: { + file: content.uri, + text: stripChunkTextMetadata(chunk.text), + rawText: undefined, + range, + isFullFile: false, + }, + chunkHash: chunk.hash, + embedding: embedding + }); + } + + return results; + + } catch (e) { + this._logService.error('[FeimaChunkingClient] Exception calling Feima API', e); + throw e; + } + } +} diff --git a/src/platform/chunking/test/node/feimaChunkingClient.spec.ts b/src/platform/chunking/test/node/feimaChunkingClient.spec.ts new file mode 100644 index 0000000000..8fb61520e1 --- /dev/null +++ b/src/platform/chunking/test/node/feimaChunkingClient.spec.ts @@ -0,0 +1,611 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { CallTracker } from '../../../../util/common/telemetryCorrelationId'; +import { CancellationToken } from '../../../../util/vs/base/common/cancellation'; +import { Event } from '../../../../util/vs/base/common/event'; +import { DisposableStore } from '../../../../util/vs/base/common/lifecycle'; +import { URI } from '../../../../util/vs/base/common/uri'; +import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; +import { IFeimaAuthenticationService } from '../../../authentication/common/feimaAuthentication'; +import { EmbeddingType } from '../../../embeddings/common/embeddingsComputer'; +import { IFeimaConfigData, IFeimaConfigService, IOAuth2Endpoints } from '../../../feima/common/feimaConfigService'; +import { FetchOptions, IFetcherService, Response } from '../../../networking/common/fetcherService'; +import { createFakeResponse } from '../../../test/node/fetcher'; +import { createPlatformServices, ITestingServicesAccessor } from '../../../test/node/services'; +import { ChunkableContent, ComputeBatchInfo, EmbeddingsComputeQos } from '../../common/chunkingEndpointClient'; +import { FeimaChunkingClient } from '../../common/feimaChunkingClient'; + +/** + * Mock Feima Authentication Service for testing + */ +class MockFeimaAuthService implements IFeimaAuthenticationService { + _serviceBrand: undefined; + + constructor( + private _isAuthenticated: boolean, + private _token?: string + ) { } + + async getToken(): Promise { + return this._token; + } + + async isAuthenticated(): Promise { + return this._isAuthenticated; + } + + async refreshToken(): Promise { + return this._token; + } + + async signOut(): Promise { } + + // Stub implementations for provider methods + onDidChangeAuthenticationState: Event = Event.None; + getSessions = async () => []; + createSession = async () => { throw new Error('Not implemented'); }; + removeSession = async () => { }; + getCachedSessions = () => []; + onDidChangeSessions: Event = Event.None; + handleUri = () => { }; +} + +/** + * Mock Feima Config Service for testing + */ +class MockFeimaConfigService implements IFeimaConfigService { + _serviceBrand: undefined; + + constructor(private _config: IFeimaConfigData) { } + + getConfig(): IFeimaConfigData { + return this._config; + } + + getOAuth2Endpoints(): IOAuth2Endpoints { + return { + authorizationEndpoint: `${this._config.authBaseUrl}/oauth/authorize`, + tokenEndpoint: `${this._config.authBaseUrl}/oauth/token`, + revocationEndpoint: `${this._config.authBaseUrl}/oauth/revoke`, + }; + } + + onDidChangeConfig: Event = Event.None; + + validateConfig(): string[] { + return []; + } +} + +/** + * Mock Fetcher Service that tracks requests + */ +class MockFetcherService implements IFetcherService { + _serviceBrand: undefined; + + requests: Array<{ url: string; options: FetchOptions }> = []; + + constructor(private _response: any, private _statusCode: number = 200) { } + + async fetch(url: string, options: FetchOptions): Promise { + this.requests.push({ url, options }); + return createFakeResponse(this._statusCode, this._response); + } + + getUserAgentLibrary(): string { + return 'test-agent'; + } + + async disconnectAll(): Promise { + return Promise.resolve(); + } + + makeAbortController(): any { + return { abort: () => { }, signal: {} }; + } + + isAbortError(e: any): boolean { + return false; + } + + isInternetDisconnectedError(e: any): boolean { + return false; + } + + isFetcherError(e: any): boolean { + return false; + } + + getUserMessageForFetcherError(err: any): string { + return 'error'; + } + + async fetchWithPagination(baseUrl: string, options: any): Promise { + return []; + } +} + +const DEFAULT_CONFIG: IFeimaConfigData = { + authBaseUrl: 'https://auth.feima.test', + apiBaseUrl: 'https://api.feima.test/v1', + clientId: 'test-client-id', + issuer: 'https://auth.feima.test', + modelRefreshInterval: 300, + quotaShowInStatusBar: true, + quotaAlertThreshold: 0.8, + preferFeimaModels: true, +}; + +/** + * Create a mock ChunkableContent for testing + */ +function createMockContent(text: string, uri: URI = URI.parse('file:///test.ts')): ChunkableContent { + return { + uri, + githubLanguageId: 145, // TypeScript + getText: async () => text, + }; +} + +describe('FeimaChunkingClient', function () { + let accessor: ITestingServicesAccessor; + let disposables: DisposableStore; + + beforeEach(() => { + disposables = new DisposableStore(); + }); + + afterEach(() => { + disposables.dispose(); + }); + + describe('Routing Logic', () => { + it('should route to Feima API when authenticated', async function () { + const feimaResponse = { + chunks: [{ + hash: 'test-hash-1', + range: { start: 0, end: 10 }, + line_range: { start: 1, end: 5 }, + text: 'function test() {\n return 42;\n}', + }], + embedding_model: 'text-embedding-v4', + }; + + const mockFetcher = new MockFetcherService(feimaResponse); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }'); + const embeddingType = new EmbeddingType('text-embedding-3-small'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + const result = await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + // Verify Feima API was called + expect(mockFetcher.requests).toHaveLength(1); + expect(mockFetcher.requests[0].url).toBe('https://api.feima.test/v1/chunks'); + expect(mockFetcher.requests[0].options.method).toBe('POST'); + + // Verify request body + const requestBody = mockFetcher.requests[0].options.json as any; + expect(requestBody).toBeDefined(); + expect(requestBody.embed).toBe(false); + expect(requestBody.content).toBeDefined(); + expect(requestBody.language_id).toBe(145); + expect(requestBody.embedding_model).toBe('text-embedding-v4'); + + // Verify Authorization header + expect(mockFetcher.requests[0].options.headers?.['Authorization']).toBe('Bearer auth-token'); + + // Verify result + expect(result).toBeDefined(); + expect(result?.length).toBe(1); + expect(result![0].chunkHash).toBe('test-hash-1'); + }); + + it('should NOT route to Feima API when not authenticated', async function () { + // Mock GitHub client will be used automatically since we're not authenticated + const mockAuthService = new MockFeimaAuthService(false); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + const mockFetcher = new MockFetcherService({}); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }'); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + // This should route to GitHub (the wrapped client), not Feima + // Since we don't have GitHub token setup in tests, we expect it to fail + // but the important part is that Feima API was NOT called + try { + await client.computeChunks( + 'github-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + } catch (e) { + // Expected to fail since GitHub client is not configured in tests + } + + // Verify Feima API was NOT called (no requests to feima.test domain) + const feimaRequests = mockFetcher.requests.filter(r => r.url.includes('feima.test')); + expect(feimaRequests).toHaveLength(0); + }); + + it('should fallback to GitHub when Feima API fails', async function () { + // Make Feima API return an error + const mockFetcher = new MockFetcherService({ error: 'Internal Server Error' }); + // Override fetch to throw error for Feima, but succeed for GitHub + mockFetcher.fetch = async (url: string, options: FetchOptions) => { + mockFetcher.requests.push({ url, options }); + if (url.includes('feima.test')) { + throw new Error('Feima API Error'); + } + // For GitHub, return empty chunks (we're just testing the fallback logic) + return createFakeResponse(200, { chunks: [], embedding_model: 'text-embedding-v4' }); + }; + + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }'); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + // Should not throw, should fallback to GitHub + await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + // Verify Feima API was attempted first + const feimaRequests = mockFetcher.requests.filter(r => r.url.includes('feima.test')); + expect(feimaRequests.length).toBeGreaterThan(0); + }); + }); + + describe('Response Parsing', () => { + it('should correctly parse chunks without embeddings', async function () { + const feimaResponse = { + chunks: [ + { + hash: 'hash-1', + range: { start: 0, end: 33 }, + line_range: { start: 1, end: 3 }, + text: 'function test() {\n return 42;\n}', + }, + { + hash: 'hash-2', + range: { start: 34, end: 60 }, + line_range: { start: 4, end: 6 }, + text: 'function test2() {\n return 24;\n}', + } + ], + embedding_model: 'text-embedding-v4', + }; + + const mockFetcher = new MockFetcherService(feimaResponse); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }\nfunction test2() { return 24; }'); + const embeddingType = new EmbeddingType('text-embedding-3-small'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + const result = await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + expect(result).toBeDefined(); + expect(result?.length).toBe(2); + + // Check first chunk + expect(result![0].chunkHash).toBe('hash-1'); + expect(result![0].chunk.range.startLineNumber).toBe(1); + expect(result![0].chunk.range.endLineNumber).toBe(3); + expect(result![0].embedding).toBeUndefined(); + + // Check second chunk + expect(result![1].chunkHash).toBe('hash-2'); + expect(result![1].chunk.range.startLineNumber).toBe(4); + expect(result![1].chunk.range.endLineNumber).toBe(6); + expect(result![1].embedding).toBeUndefined(); + }); + + it('should correctly parse chunks with embeddings', async function () { + const feimaResponse = { + chunks: [ + { + hash: 'hash-1', + range: { start: 0, end: 33 }, + line_range: { start: 1, end: 3 }, + text: 'function test() {\n return 42;\n}', + embedding: { + model: 'text-embedding-v4', + embedding: [0.1, 0.2, 0.3, 0.4, 0.5], + } + } + ], + embedding_model: 'text-embedding-v4', + }; + + const mockFetcher = new MockFetcherService(feimaResponse); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }'); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + const result = await client.computeChunksAndEmbeddings( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + expect(result).toBeDefined(); + expect(result?.length).toBe(1); + expect(result![0].embedding).toBeDefined(); + expect(result![0].embedding?.value).toEqual([0.1, 0.2, 0.3, 0.4, 0.5]); + expect(result![0].embedding?.type.id).toBe('text-embedding-v4'); + }); + + it('should handle empty text content', async function () { + const mockFetcher = new MockFetcherService({ chunks: [], embedding_model: 'test' }); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent(''); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + const result = await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + // Should return empty array without calling API + expect(result).toEqual([]); + expect(mockFetcher.requests).toHaveLength(0); + }); + + it('should handle API error responses', async function () { + const mockFetcher = new MockFetcherService({ error: 'Bad Request' }, 400); + + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('function test() { return 42; }'); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + // Should fallback to GitHub client when Feima returns error + // We expect this to eventually return undefined or throw + await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + // Verify Feima API was called + expect(mockFetcher.requests.length).toBeGreaterThan(0); + }); + }); + + describe('Request Format', () => { + it('should send correct request format for computeChunks', async function () { + const feimaResponse = { + chunks: [], + embedding_model: 'text-embedding-v4', + }; + + const mockFetcher = new MockFetcherService(feimaResponse); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('test content', URI.parse('file:///path/to/file.ts')); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + await client.computeChunks( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Online, + undefined, + telemetryInfo, + CancellationToken.None + ); + + expect(mockFetcher.requests).toHaveLength(1); + const request = mockFetcher.requests[0]; + + expect(request.options.json).toMatchObject({ + embed: false, // computeChunks should NOT request embeddings + qos: EmbeddingsComputeQos.Online, + content: 'test content', + path: '/path/to/file.ts', + language_id: 145, + embedding_model: 'text-embedding-v4', + local_hashes: [], + }); + }); + + it('should send correct request format for computeChunksAndEmbeddings', async function () { + const feimaResponse = { + chunks: [], + embedding_model: 'text-embedding-v4', + }; + + const mockFetcher = new MockFetcherService(feimaResponse); + const mockAuthService = new MockFeimaAuthService(true, 'feima-token-123'); + const mockConfigService = new MockFeimaConfigService(DEFAULT_CONFIG); + + const testingServiceCollection = createPlatformServices(); + testingServiceCollection.define(IFeimaAuthenticationService, mockAuthService); + testingServiceCollection.define(IFeimaConfigService, mockConfigService); + testingServiceCollection.define(IFetcherService, mockFetcher); + accessor = disposables.add(testingServiceCollection.createTestingAccessor()); + + const client: FeimaChunkingClient = disposables.add( + accessor.get(IInstantiationService).createInstance(FeimaChunkingClient as any) + ); + + const content = createMockContent('test content'); + const embeddingType = new EmbeddingType('text-embedding-v4'); + const batchInfo: ComputeBatchInfo = { recomputedFileCount: 0, sentContentTextLength: 0 }; + const telemetryInfo = new CallTracker(); + + await client.computeChunksAndEmbeddings( + 'auth-token', + embeddingType, + content, + batchInfo, + EmbeddingsComputeQos.Batch, + undefined, + telemetryInfo, + CancellationToken.None + ); + + expect(mockFetcher.requests).toHaveLength(1); + const request = mockFetcher.requests[0]; + + expect((request.options.json as any).embed).toBe(true); // computeChunksAndEmbeddings SHOULD request embeddings + }); + }); +}); diff --git a/src/platform/embeddings/common/feimaEmbeddingsComputer.ts b/src/platform/embeddings/common/feimaEmbeddingsComputer.ts new file mode 100644 index 0000000000..2903d3cdf4 --- /dev/null +++ b/src/platform/embeddings/common/feimaEmbeddingsComputer.ts @@ -0,0 +1,320 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { CancellationToken } from 'vscode'; +import { TelemetryCorrelationId } from '../../../util/common/telemetryCorrelationId'; +import { raceCancellationError } from '../../../util/vs/base/common/async'; +import { Disposable } from '../../../util/vs/base/common/lifecycle'; +import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; +import { IFeimaAuthenticationService } from '../../authentication/common/feimaAuthentication'; +import { IFeimaConfigService } from '../../feima/common/feimaConfigService'; +import { ILogService } from '../../log/common/logService'; +import { IFetcherService } from '../../networking/common/fetcherService'; +import { ITelemetryService } from '../../telemetry/common/telemetry'; +import { ComputeEmbeddingsOptions, Embedding, Embeddings, EmbeddingType, IEmbeddingsComputer } from './embeddingsComputer'; +import { RemoteEmbeddingsComputer } from './remoteEmbeddingsComputer'; + +type FeimaEmbeddingsResponse = { + readonly object: string; + readonly data: readonly { + readonly object: string; + readonly index: number; + readonly embedding: number[]; + }[]; + readonly model: string; + readonly usage: { + readonly prompt_tokens: number; + readonly total_tokens: number; + }; +}; + +/** + * Feima-aware embeddings computer that routes to Feima API or GitHub CAPI based on preferFeimaModels config. + * + * Routing priority: + * 1. If preferFeimaModels is true → Use Feima API /v1/embeddings, fallback to GitHub if fails + * 2. If preferFeimaModels is false → Use GitHub (RemoteEmbeddingsComputer), fallback to Feima if fails + */ +export class FeimaEmbeddingsComputer extends Disposable implements IEmbeddingsComputer { + declare readonly _serviceBrand: undefined; + + /** + * The original GitHub embeddings computer for fallback + */ + private readonly _githubComputer: RemoteEmbeddingsComputer; + + constructor( + @IInstantiationService instantiationService: IInstantiationService, + @IFeimaAuthenticationService private readonly _feimaAuthService: IFeimaAuthenticationService, + @IFeimaConfigService private readonly _feimaConfig: IFeimaConfigService, + @IFetcherService private readonly _fetcherService: IFetcherService, + @ILogService private readonly _logService: ILogService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, + ) { + super(); + + // Create the original GitHub computer (not registered as disposable - it has its own lifecycle) + this._githubComputer = instantiationService.createInstance(RemoteEmbeddingsComputer); + } + + public async computeEmbeddings( + embeddingType: EmbeddingType, + inputs: readonly string[], + options?: ComputeEmbeddingsOptions, + telemetryInfo?: TelemetryCorrelationId, + cancellationToken?: CancellationToken, + ): Promise { + // Fetch authentication token for Feima + const feimaToken = await this._feimaAuthService.getToken(); + + // Route based on preferFeimaModels config + return this.routeRequest( + embeddingType, + inputs, + options, + () => this.doFeimaEmbeddings(feimaToken!, embeddingType, inputs, options, false, cancellationToken), + () => this._githubComputer.computeEmbeddings(embeddingType, inputs, options, telemetryInfo, cancellationToken), + telemetryInfo, + cancellationToken + ); + } + + /** + * Route request to Feima or GitHub based on preferFeimaModels config. + */ + private async routeRequest( + embeddingType: EmbeddingType, + inputs: readonly string[], + options: ComputeEmbeddingsOptions | undefined, + feimaOperation: () => Promise, + githubOperation: () => Promise, + telemetryInfo: TelemetryCorrelationId | undefined, + token: CancellationToken | undefined + ): Promise { + // Check if Feima models are preferred + const preferFeima = this._feimaConfig.getConfig().preferFeimaModels; + + if (preferFeima) { + this._logService.debug('[FeimaEmbeddingsComputer] Using Feima API for embeddings'); + + /* __GDPR__ + "feimaEmbeddingsComputer.routeToFeima" : { + "owner": "feima", + "comment": "Tracks when embeddings are routed to Feima API", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.routeToFeima', { + source: telemetryInfo?.callTracker.toString() ?? 'unknown', + correlationId: telemetryInfo?.correlationId ?? 'unknown', + }); + + try { + return await (token ? raceCancellationError(feimaOperation(), token) : feimaOperation()); + } catch (e) { + this._logService.error('[FeimaEmbeddingsComputer] Feima API failed, falling back to GitHub', e); + + /* __GDPR__ + "feimaEmbeddingsComputer.feimaFallbackToGithub" : { + "owner": "feima", + "comment": "Tracks when Feima API fails and falls back to GitHub", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" }, + "error": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.feimaFallbackToGithub', { + source: telemetryInfo?.callTracker.toString() ?? 'unknown', + correlationId: telemetryInfo?.correlationId ?? 'unknown', + error: e instanceof Error ? e.message : String(e), + }); + + // Fallback to GitHub if Feima fails + return await (token ? raceCancellationError(githubOperation(), token) : githubOperation()); + } + } + + // Use GitHub CAPI + this._logService.debug('[FeimaEmbeddingsComputer] Using GitHub for embeddings'); + + /* __GDPR__ + "feimaEmbeddingsComputer.routeToGithub" : { + "owner": "feima", + "comment": "Tracks when embeddings are routed to GitHub", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.routeToGithub', { + source: telemetryInfo?.callTracker.toString() ?? 'unknown', + correlationId: telemetryInfo?.correlationId ?? 'unknown', + }); + + try { + return await (token ? raceCancellationError(githubOperation(), token) : githubOperation()); + } catch (e) { + this._logService.error('[FeimaEmbeddingsComputer] GitHub failed, falling back to Feima', e); + + /* __GDPR__ + "feimaEmbeddingsComputer.githubFallbackToFeima" : { + "owner": "feima", + "comment": "Tracks when GitHub fails and falls back to Feima", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller of the request" }, + "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" }, + "error": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error message" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.githubFallbackToFeima', { + source: telemetryInfo?.callTracker.toString() ?? 'unknown', + correlationId: telemetryInfo?.correlationId ?? 'unknown', + error: e instanceof Error ? e.message : String(e), + }); + + // Fallback to Feima if GitHub fails + const feimaToken = await this._feimaAuthService.getToken(); + return await (token + ? raceCancellationError(this.doFeimaEmbeddings(feimaToken!, embeddingType, inputs, options, true, token), token) + : this.doFeimaEmbeddings(feimaToken!, embeddingType, inputs, options, true, token)); + } + } + + /** + * Compute embeddings using Feima API. + * Batches requests into chunks of 10 inputs (Feima API limit). + */ + private async doFeimaEmbeddings( + token: string, + embeddingType: EmbeddingType, + inputs: readonly string[], + options: ComputeEmbeddingsOptions | undefined, + isFallback: boolean, + cancellationToken?: CancellationToken + ): Promise { + const apiBaseUrl = this._feimaConfig.getConfig().apiBaseUrl; + const endpoint = `${apiBaseUrl}/embeddings`; + + this._logService.info(`[FeimaEmbeddingsComputer] Computing embeddings via Feima API${isFallback ? ' (fallback)' : ''}. Inputs: ${inputs.length}, Model: ${embeddingType.id}, InputType: ${options?.inputType ?? 'document'}`); + + // Feima API has a maximum of 10 inputs per request + const BATCH_SIZE = 10; + const batches: string[][] = []; + for (let i = 0; i < inputs.length; i += BATCH_SIZE) { + batches.push(inputs.slice(i, i + BATCH_SIZE) as string[]); + } + + this._logService.info(`[FeimaEmbeddingsComputer] Split ${inputs.length} inputs into ${batches.length} batches`); + + // Process batches sequentially to avoid overwhelming the API + const allEmbeddings: Embedding[] = []; + let responseModel: string | undefined; + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + + // Check cancellation before each batch + if (cancellationToken?.isCancellationRequested) { + throw new Error('Operation cancelled'); + } + + this._logService.info(`[FeimaEmbeddingsComputer] Processing batch ${batchIndex + 1}/${batches.length} (${batch.length} inputs)`); + + // Build request body + const requestBody = { + input: batch, + model: embeddingType.id, + dimensions: 512, // TODO: Make configurable or extract from embeddingType + }; + + // Make request to Feima API + const response = await this._fetcherService.fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}`, + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + let errorDetail = ''; + try { + const errorBody = await response.json(); + errorDetail = JSON.stringify(errorBody); + } catch { + // Ignore JSON parse errors + } + this._logService.error(`[FeimaEmbeddingsComputer] Error from Feima API on batch ${batchIndex + 1}. Status: ${response.status}. Detail: ${errorDetail}`); + + /* __GDPR__ + "feimaEmbeddingsComputer.feimaApiError" : { + "owner": "feima", + "comment": "Tracks errors from Feima embeddings API", + "statusCode": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "HTTP status code" }, + "embeddingType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Embedding type" }, + "inputCount": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Number of inputs" }, + "batchIndex": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Batch index" }, + "isFallback": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether this is a fallback request" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.feimaApiError', { + embeddingType: embeddingType.id, + isFallback: String(isFallback), + }, { + statusCode: response.status, + inputCount: inputs.length, + batchIndex: batchIndex, + }); + + throw new Error(`Feima API returned status ${response.status} on batch ${batchIndex + 1}: ${errorDetail || response.statusText}`); + } + + const jsonResponse: FeimaEmbeddingsResponse = await response.json(); + + // Validate response + if (!jsonResponse.data || jsonResponse.data.length !== batch.length) { + throw new Error(`Mismatched embedding count in batch ${batchIndex + 1}. Expected: ${batch.length}, Got: ${jsonResponse.data?.length ?? 0}`); + } + + // Store model from first response + if (!responseModel) { + responseModel = jsonResponse.model; + } + + // Convert to internal format and accumulate + const batchEmbeddings: Embedding[] = jsonResponse.data.map(item => ({ + type: new EmbeddingType(jsonResponse.model), + value: item.embedding, + })); + + allEmbeddings.push(...batchEmbeddings); + } + + this._logService.info(`[FeimaEmbeddingsComputer] Successfully computed ${allEmbeddings.length} embeddings from Feima API`); + + /* __GDPR__ + "feimaEmbeddingsComputer.feimaApiSuccess" : { + "owner": "feima", + "comment": "Tracks successful embeddings requests to Feima API", + "embeddingType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Embedding type" }, + "inputCount": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Number of inputs" }, + "batchCount": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Number of batches" }, + "isFallback": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether this is a fallback request" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('feimaEmbeddingsComputer.feimaApiSuccess', { + embeddingType: embeddingType.id, + isFallback: String(isFallback), + }, { + inputCount: inputs.length, + batchCount: batches.length, + }); + + return { + type: new EmbeddingType(responseModel!), + values: allEmbeddings, + }; + } +} diff --git a/src/platform/endpoint/node/feimaEmbeddingsEndpoint.ts b/src/platform/endpoint/node/feimaEmbeddingsEndpoint.ts new file mode 100644 index 0000000000..4dc75965ca --- /dev/null +++ b/src/platform/endpoint/node/feimaEmbeddingsEndpoint.ts @@ -0,0 +1,95 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ITokenizer } from '../../../util/common/tokenizer'; +import { IFeimaAuthenticationService } from '../../authentication/node/feimaAuthenticationService'; +import { ILogService } from '../../log/common/logService'; +import { IEmbeddingsEndpoint, IEndpointBody } from '../../networking/common/networking'; +import { ITokenizerProvider } from '../../tokenizer/node/tokenizer'; +import { IEmbeddingModelInformation } from '../common/endpointProvider'; + +/** + * Feima embeddings endpoint using Ali Cloud text-embedding-v4. + * + * This endpoint provides embeddings for Feima-authenticated users via the Feima API, + * which proxies to Ali Cloud DashScope embeddings service. + */ +export class FeimaEmbeddingsEndpoint implements IEmbeddingsEndpoint { + public readonly maxBatchSize: number; + public readonly modelMaxPromptTokens: number; + + public readonly name = this._modelInfo.name; + public readonly version = this._modelInfo.version; + public readonly family = this._modelInfo.capabilities.family; + public readonly tokenizer = this._modelInfo.capabilities.tokenizer; + + constructor( + private _modelInfo: IEmbeddingModelInformation, + private readonly _feimaApiEndpoint: string, + @ITokenizerProvider private readonly _tokenizerProvider: ITokenizerProvider, + @IFeimaAuthenticationService private readonly _feimaAuthService: IFeimaAuthenticationService, + @ILogService private readonly _logService: ILogService + ) { + // Ali Cloud limits: batch size 10, max tokens 8192 + this.maxBatchSize = this._modelInfo.capabilities.limits?.max_inputs ?? 10; + this.modelMaxPromptTokens = 8192; + + this._logService.debug(`[FeimaEmbeddingsEndpoint] Initialized: modelId=${this._modelInfo.id}, modelName=${this._modelInfo.name}, apiEndpoint=${this._feimaApiEndpoint}, maxBatchSize=${this.maxBatchSize}, modelMaxPromptTokens=${this.modelMaxPromptTokens}`); + } + + public acquireTokenizer(): ITokenizer { + return this._tokenizerProvider.acquireTokenizer(this); + } + + public get urlOrRequestMetadata(): string { + // Return Feima API endpoint URL for embeddings + return `${this._feimaApiEndpoint}/v1/embeddings`; + } + + /** + * Get authentication token for Feima API requests. + * + * This method is called by RemoteEmbeddingsComputer before making requests. + */ + public async getAuthToken(): Promise { + this._logService.debug('[FeimaEmbeddingsEndpoint] getAuthToken() called'); + + const token = await this._feimaAuthService.getToken(); + if (!token) { + this._logService.error('[FeimaEmbeddingsEndpoint] No authentication token available'); + throw new Error('Feima authentication required for embeddings'); + } + + this._logService.debug('[FeimaEmbeddingsEndpoint] Authentication token retrieved successfully'); + return token; + } + + /** + * Intercept and transform request body before sending to Feima API. + * + * Note: Model mapping happens in CombinedEndpointProvider before the endpoint is selected. + * This method ensures the request uses the correct model and dimensions from _modelInfo. + */ + public interceptBody(body: IEndpointBody | undefined): void { + this._logService.debug(`[FeimaEmbeddingsEndpoint] interceptBody() called: originalBody=${body ? JSON.stringify({ model: body.model, dimensions: body.dimensions }) : 'null'}`); + + if (!body) { + this._logService.debug('[FeimaEmbeddingsEndpoint] No body to intercept'); + return; + } + + const originalModel = body.model; + const originalDimensions = body.dimensions; + + // Override with the actual Feima model from modelInfo + body.model = this._modelInfo.id; + // Use dimensions from request, or default to 512 for Ali Cloud text-embedding-v4 + if (!body.dimensions) { + body.dimensions = 512; + } + + this._logService.debug(`[FeimaEmbeddingsEndpoint] Body intercepted: originalModel=${originalModel}, newModel=${body.model}, originalDimensions=${originalDimensions}, newDimensions=${body.dimensions}`); + } +} diff --git a/src/platform/workspaceChunkSearch/common/feimaAvailableEmbeddingTypes.ts b/src/platform/workspaceChunkSearch/common/feimaAvailableEmbeddingTypes.ts new file mode 100644 index 0000000000..7812ad8810 --- /dev/null +++ b/src/platform/workspaceChunkSearch/common/feimaAvailableEmbeddingTypes.ts @@ -0,0 +1,86 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IAuthenticationService } from '../../authentication/common/authentication'; +import { IFeimaAuthenticationService } from '../../authentication/common/feimaAuthentication'; +import { EmbeddingType } from '../../embeddings/common/embeddingsComputer'; +import { IFeimaConfigService } from '../../feima/common/feimaConfigService'; +import { ILogService } from '../../log/common/logService'; +import { GithubAvailableEmbeddingTypesService, IGithubAvailableEmbeddingTypesService } from './githubAvailableEmbeddingTypes'; + +/** + * Combined service that tries Feima embeddings first, then falls back to GitHub. + * + * This service enables semantic search for Feima-authenticated users (including GitHub anonymous users) + * by checking Feima authentication before querying GitHub's embedding models API. + * + * Priority logic: + * 1. If Feima authenticated AND (no GitHub auth OR preferFeimaModels config) → Use Feima + * 2. If GitHub authenticated → Use GitHub embeddings + * 3. Otherwise → No embeddings available + */ +export class FeimaEmbeddingTypesService implements IGithubAvailableEmbeddingTypesService { + readonly _serviceBrand: undefined; + + constructor( + @ILogService private readonly _logService: ILogService, + @IFeimaAuthenticationService private readonly _feimaAuthService: IFeimaAuthenticationService, + @IAuthenticationService private readonly _githubAuthService: IAuthenticationService, + @IFeimaConfigService private readonly _feimaConfigService: IFeimaConfigService, + private readonly _githubService: GithubAvailableEmbeddingTypesService, + ) { } + + async getPreferredType(silent: boolean): Promise { + this._logService.debug(`[FeimaEmbeddingTypesService] getPreferredType() called, silent=${silent}`); + + const isFeimaAuthenticated = await this._feimaAuthService.isAuthenticated(); + const isGitHubAuthenticated = !!this._githubAuthService.copilotToken && !this._githubAuthService.copilotToken.isNoAuthUser; + + this._logService.debug(`[FeimaEmbeddingTypesService] Authentication status: Feima=${isFeimaAuthenticated}, GitHub=${isGitHubAuthenticated}`); + + // Check user preference from configuration + const config = this._feimaConfigService.getConfig(); + const preferFeimaModels = config.preferFeimaModels; + this._logService.debug(`[FeimaEmbeddingTypesService] Configuration: preferFeimaModels=${preferFeimaModels}`); + + // Priority 1: Use Feima if authenticated and either no GitHub or user prefers Feima + if (isFeimaAuthenticated && (!isGitHubAuthenticated || preferFeimaModels)) { + /* __GDPR__ + "feimaEmbeddingTypes.getPreferredType.feima" : { + "owner": "feima", + "comment": "Tracking when Feima embeddings are selected", + "hasGitHubAuth": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether user also has GitHub authentication" }, + "preferFeimaModels": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether user prefers Feima models" } + } + */ + // TODO: Add telemetry once ITelemetryService is added to constructor + + // Return Feima embedding model directly (text-embedding-v4 from feima-api database) + // This replaces GitHub's text-embedding-3-small-512 with Feima's Ali Cloud embedding model + const feimaEmbeddingModel = 'text-embedding-v4'; + this._logService.info(`[FeimaEmbeddingTypesService] Using Feima embeddings: ${feimaEmbeddingModel}`); + return new EmbeddingType(feimaEmbeddingModel); + } + + // Priority 2: Fallback to GitHub if authenticated + if (isGitHubAuthenticated) { + this._logService.info('[FeimaEmbeddingTypesService] Delegating to GitHub embeddings service'); + return this._githubService.getPreferredType(silent); + } + + // No authentication available + this._logService.info('[FeimaEmbeddingTypesService] No embeddings available: no authentication'); + + /* __GDPR__ + "feimaEmbeddingTypes.getPreferredType.noAuth" : { + "owner": "feima", + "comment": "Tracking when no embeddings are available due to lack of authentication" + } + */ + // TODO: Add telemetry + + return undefined; + } +} diff --git a/src/platform/workspaceChunkSearch/node/workspaceChunkEmbeddingsIndex.ts b/src/platform/workspaceChunkSearch/node/workspaceChunkEmbeddingsIndex.ts index 31016c6fe6..0e8e943240 100644 --- a/src/platform/workspaceChunkSearch/node/workspaceChunkEmbeddingsIndex.ts +++ b/src/platform/workspaceChunkSearch/node/workspaceChunkEmbeddingsIndex.ts @@ -15,7 +15,6 @@ import { ResourceMap } from '../../../util/vs/base/common/map'; import { extname } from '../../../util/vs/base/common/resources'; import { URI } from '../../../util/vs/base/common/uri'; import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; -import { IAuthenticationService } from '../../authentication/common/authentication'; import { FileChunk, FileChunkAndScore, FileChunkWithEmbedding, FileChunkWithOptionalEmbedding } from '../../chunking/common/chunk'; import { ComputeBatchInfo, EmbeddingsComputeQos, IChunkingEndpointClient } from '../../chunking/common/chunkingEndpointClient'; import { distance, Embedding, EmbeddingType, rankEmbeddings } from '../../embeddings/common/embeddingsComputer'; @@ -48,7 +47,6 @@ export class WorkspaceChunkEmbeddingsIndex extends Disposable { private readonly _embeddingType: EmbeddingType, @IVSCodeExtensionContext vsExtensionContext: IVSCodeExtensionContext, @IInstantiationService instantiationService: IInstantiationService, - @IAuthenticationService private readonly _authService: IAuthenticationService, @ILogService private readonly _logService: ILogService, @ISimulationTestContext private readonly _simulationTestContext: ISimulationTestContext, @ITelemetryService private readonly _telemetryService: ITelemetryService, @@ -450,6 +448,7 @@ export class WorkspaceChunkEmbeddingsIndex extends Disposable { } private async tryGetAuthToken(options: AuthenticationGetSessionOptions = { createIfNone: true }): Promise { - return (await this._authService.getGitHubSession('any', options))?.accessToken; + // return (await this._authService.getGitHubSession('any', options))?.accessToken; + return Promise.resolve('fake-auth-token-for-feima'); // fake token to let chunking endpoint client to fetch token } }