feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)

close CLOUD-137

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination.
  - Added pluggable search providers: Elasticsearch and Manticoresearch.
  - New GraphQL queries, schema types, and resolver support for search and aggregation.
  - Enhanced configuration options for search providers in self-hosted and cloud deployments.
  - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch.
  - Integrated indexer service into deployment and CI workflows.

- **Bug Fixes**
  - Improved error handling with new user-friendly error messages for search provider and indexer issues.

- **Documentation**
  - Updated configuration examples and environment variable references for indexer and search providers.

- **Tests**
  - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases.
  - Included snapshot tests and test fixtures for search providers.

- **Chores**
  - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/base/error/def.ts

@@ -861,4 +861,21 @@ export const USER_FRIENDLY_ERRORS = {
     type: 'invalid_input',
     message: 'Invalid app config.',
   },
+
+  // indexer errors
+  search_provider_not_found: {
+    type: 'resource_not_found',
+    message: 'Search provider not found.',
+  },
+  invalid_search_provider_request: {
+    type: 'invalid_input',
+    args: { reason: 'string', type: 'string' },
+    message: ({ reason }) =>
+      `Invalid request argument to search provider: ${reason}`,
+  },
+  invalid_indexer_input: {
+    type: 'invalid_input',
+    args: { reason: 'string' },
+    message: ({ reason }) => `Invalid indexer input: ${reason}`,
+  },
 } satisfies Record<string, UserFriendlyErrorOptions>;

packages/backend/server/src/base/error/errors.gen.ts

@@ -991,6 +991,33 @@ export class InvalidAppConfig extends UserFriendlyError {
     super('invalid_input', 'invalid_app_config', message);
   }
 }
+
+export class SearchProviderNotFound extends UserFriendlyError {
+  constructor(message?: string) {
+    super('resource_not_found', 'search_provider_not_found', message);
+  }
+}
+@ObjectType()
+class InvalidSearchProviderRequestDataType {
+  @Field() reason!: string
+  @Field() type!: string
+}
+
+export class InvalidSearchProviderRequest extends UserFriendlyError {
+  constructor(args: InvalidSearchProviderRequestDataType, message?: string | ((args: InvalidSearchProviderRequestDataType) => string)) {
+    super('invalid_input', 'invalid_search_provider_request', message, args);
+  }
+}
+@ObjectType()
+class InvalidIndexerInputDataType {
+  @Field() reason!: string
+}
+
+export class InvalidIndexerInput extends UserFriendlyError {
+  constructor(args: InvalidIndexerInputDataType, message?: string | ((args: InvalidIndexerInputDataType) => string)) {
+    super('invalid_input', 'invalid_indexer_input', message, args);
+  }
+}
 export enum ErrorNames {
   INTERNAL_SERVER_ERROR,
   NETWORK_ERROR,
@@ -1118,7 +1145,10 @@ export enum ErrorNames {
   NOTIFICATION_NOT_FOUND,
   MENTION_USER_DOC_ACCESS_DENIED,
   MENTION_USER_ONESELF_DENIED,
-  INVALID_APP_CONFIG
+  INVALID_APP_CONFIG,
+  SEARCH_PROVIDER_NOT_FOUND,
+  INVALID_SEARCH_PROVIDER_REQUEST,
+  INVALID_INDEXER_INPUT
 }
 registerEnumType(ErrorNames, {
   name: 'ErrorNames'
@@ -1127,5 +1157,5 @@ registerEnumType(ErrorNames, {
 export const ErrorDataUnionType = createUnionType({
   name: 'ErrorDataUnion',
   types: () =>
-    [GraphqlBadRequestDataType, HttpRequestErrorDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType] as const,
+    [GraphqlBadRequestDataType, HttpRequestErrorDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
 });

packages/backend/server/src/base/graphql/pagination.ts

@@ -15,7 +15,7 @@ export class PaginationInput {
     transform: value => {
       return {
         ...value,
-        after: decode(value.after),
+        after: decode(value?.after),
         // before: decode(value.before),
       };
     },

packages/backend/server/src/base/metrics/opentelemetry.ts

@@ -105,6 +105,9 @@ export class OpentelemetryProvider {
 
   @OnEvent('config.init')
   async init(event: Events['config.init']) {
+    if (env.flavors.script) {
+      return;
+    }
     if (event.config.metrics.enabled) {
       await this.setup();
       registerCustomMetrics();