diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 07950fe411..a94ef36f70 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -9,106 +9,94 @@ import { groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' -import { getServiceChildLogger } from '@crowd/logging' +import { getServiceLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' -import { - ALL_PLATFORM_TYPES, - MemberAttributeType, - MemberIdentityType, - PageData, - SegmentType, -} from '@crowd/types' - -import { findManyLfxMemberships } from '../lfx_memberships' +import { ALL_PLATFORM_TYPES, MemberAttributeType, PageData, SegmentType } from '@crowd/types' + import { findMaintainerRoles } from '../maintainers' import { IDbMemberCreateData, IDbMemberUpdateData, } from '../old/apps/data_sink_worker/repo/member.data' -import { OrganizationField, queryOrgs } from '../organizations' import { QueryExecutor } from '../queryExecutor' -import { fetchManySegments, findSegmentById } from '../segments' +import { fetchManySegments } from '../segments' import { QueryOptions, QueryResult, queryTable, queryTableById } from '../utils' import { getMemberAttributeSettings } from './attributeSettings' +import { fetchOrganizationData, fetchSegmentData, sortActiveOrganizations } from './dataProcessor' +import { buildCountQuery, buildQuery, buildSearchCTE } from './queryBuilder' import { IDbMemberAttributeSetting, IDbMemberData } from './types' import { fetchManyMemberIdentities, fetchManyMemberOrgs, fetchManyMemberSegments } from '.' -/* eslint-disable @typescript-eslint/no-explicit-any */ - -const log = getServiceChildLogger('db/members') +const log = getServiceLogger() export enum MemberField { - // meta - ID = 'id', ATTRIBUTES = 'attributes', - DISPLAY_NAME = 'displayName', - SCORE = 'score', - JOINED_AT = 'joinedAt', - IMPORT_HASH = 'importHash', - REACH = 'reach', CONTRIBUTIONS = 'contributions', - CREATED_AT = 'createdAt', - UPDATED_AT = 'updatedAt', + CREATED_BY_ID = 'createdById', DELETED_AT = 'deletedAt', - + DISPLAY_NAME = 'displayName', + ID = 'id', + IMPORT_HASH = 'importHash', + JOINED_AT = 'joinedAt', + MANUALLY_CHANGED_FIELDS = 'manuallyChangedFields', + MANUALLY_CREATED = 'manuallyCreated', + REACH = 'reach', + SCORE = 'score', TENANT_ID = 'tenantId', - CREATED_BY_ID = 'createdById', + UPDATED_AT = 'updatedAt', UPDATED_BY_ID = 'updatedById', - - MANUALLY_CREATED = 'manuallyCreated', - MANUALLY_CHANGED_FIELDS = 'manuallyChangedFields', } export const MEMBER_MERGE_FIELDS = [ + 'affiliations', + 'attributes', + 'contributions', + 'displayName', 'id', - 'tags', + 'joinedAt', + 'manuallyChangedFields', + 'manuallyCreated', 'reach', + 'tags', 'tasks', - 'joinedAt', 'tenantId', - 'attributes', - 'displayName', - 'affiliations', - 'contributions', - 'manuallyCreated', - 'manuallyChangedFields', ] export const MEMBER_UPDATE_COLUMNS = [ - MemberField.DISPLAY_NAME, MemberField.ATTRIBUTES, MemberField.CONTRIBUTIONS, - MemberField.SCORE, - MemberField.REACH, + MemberField.DISPLAY_NAME, MemberField.IMPORT_HASH, + MemberField.REACH, + MemberField.SCORE, ] export const MEMBER_SELECT_COLUMNS = [ - 'id', - 'score', - 'joinedAt', - 'reach', 'attributes', 'displayName', + 'id', + 'joinedAt', 'manuallyChangedFields', + 'reach', + 'score', ] export const MEMBER_INSERT_COLUMNS = [ - 'id', 'attributes', + 'createdAt', 'displayName', + 'id', 'joinedAt', - 'tenantId', 'reach', - 'createdAt', + 'tenantId', 'updatedAt', ] const QUERY_FILTER_COLUMN_MAP: Map = new Map([ - ['activityCount', { name: 'coalesce(msa."activityCount", 0)::integer' }], + ['activityCount', { name: 'msa."activityCount"' }], ['attributes', { name: 'm.attributes' }], ['averageSentiment', { name: 'coalesce(msa."averageSentiment", 0)::decimal' }], ['displayName', { name: 'm."displayName"' }], @@ -126,17 +114,15 @@ const QUERY_FILTER_COLUMN_MAP: Map> { - if (!attributeSettings || attributeSettings.length === 0) { - attributeSettings = await getMemberAttributeSettings(qx, redis) - } - const withAggregates = !!segmentId - let segment - if (withAggregates) { - segment = (await findSegmentById(qx, segmentId)) as any - - if (segment === null) { - log.info('No segment found for member query. Returning empty result.') - return { - rows: [], - count: 0, - limit, - offset, - } - } - - segmentId = segment.id - } + const searchConfig = buildSearchCTE(search) const params = { limit, offset, segmentId, + ...searchConfig.params, } const filterString = RawQueryParser.parseFilters( @@ -194,7 +162,9 @@ export async function queryMembersAdvanced( property: 'attributes', column: 'm.attributes', attributeInfos: [ - ...attributeSettings, + ...(attributeSettings?.length > 0 + ? attributeSettings + : await getMemberAttributeSettings(qx, redis)), { name: 'jobTitle', type: MemberAttributeType.STRING, @@ -204,8 +174,8 @@ export async function queryMembersAdvanced( { property: 'username', column: 'aggs.username', - attributeInfos: ALL_PLATFORM_TYPES.map((p) => ({ - name: p, + attributeInfos: ALL_PLATFORM_TYPES.map((name) => ({ + name, type: MemberAttributeType.STRING, })), }, @@ -214,163 +184,88 @@ export async function queryMembersAdvanced( { pgPromiseFormat: true }, ) - const effectiveOrderBy = typeof orderBy === 'string' && orderBy.length ? orderBy : 'joinedAt_DESC' - - const order = (function prepareOrderBy( - orderBy = withAggregates ? 'activityCount_DESC' : 'id_DESC', - ) { - const orderSplit = orderBy.split('_') - - const orderField = QUERY_FILTER_COLUMN_MAP.get(orderSplit[0])?.name - if (!orderField) { - return withAggregates ? 'msa."activityCount" DESC' : 'm.id DESC' - } - const orderDirection = ['DESC', 'ASC'].includes(orderSplit[1]) ? orderSplit[1] : 'DESC' - - return `${orderField} ${orderDirection}` - })(effectiveOrderBy) - - const withSearch = !!search - let searchCTE = '' - let searchJoin = '' - - if (withSearch) { - search = search.toLowerCase() - searchCTE = ` - , - member_search AS ( - SELECT - "memberId" - FROM "memberIdentities" mi - join members m on m.id = mi."memberId" - where (verified and type = '${MemberIdentityType.EMAIL}' and lower("value") ilike '%${search}%') or m."displayName" ilike '%${search}%' - GROUP BY 1 - ) - ` - searchJoin = ` JOIN member_search ms ON ms."memberId" = m.id ` - } - - const createQuery = (fields) => ` - WITH member_orgs AS ( - SELECT - "memberId", - ARRAY_AGG("organizationId")::TEXT[] AS "organizationId" - FROM "memberOrganizations" - WHERE "deletedAt" IS NULL - GROUP BY 1 - ) - ${searchCTE} - SELECT - ${fields} - FROM members m - ${ - withAggregates - ? ` INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` - : '' - } - LEFT JOIN member_orgs mo ON mo."memberId" = m.id - LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id - ${searchJoin} - WHERE (${filterString}) - ` - - const countQuery = createQuery('COUNT(*)') + const countQuery = buildCountQuery({ + withAggregates, + searchConfig, + filterString, + includeMemberOrgs: include.memberOrganizations, + }) if (countOnly) { + const result = await qx.selectOne(countQuery, params) return { rows: [], - count: parseInt((await qx.selectOne(countQuery, params)).count, 10), + count: parseInt(result.count, 10), limit, offset, } } - const query = ` - ${createQuery( - (function prepareFields(fields) { - return `${fields - .map((f) => { - const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) - if (!mappedField) { - throw new Error400('en', `Invalid field: ${f}`) - } - - return { - alias: f, - ...mappedField, - } - }) - .filter((mappedField) => mappedField.queryable !== false) - .filter((mappedField) => { - if (!withAggregates && mappedField.name.includes('msa.')) { - return false - } - if (!include.memberOrganizations && mappedField.name.includes('mo.')) { - return false - } - return true - }) - .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) - .join(',\n')}` - })(fields), - )} - ORDER BY ${order} NULLS LAST - LIMIT $(limit) - OFFSET $(offset) - ` - - const results = await Promise.all([qx.select(query, params), qx.selectOne(countQuery, params)]) - - const rows = results[0] - - rows.forEach((row) => { - if (row.attributes && typeof row.attributes === 'object') { - const filteredAttributes = {} - QUERY_FILTER_ATTRIBUTE_MAP.forEach((attr) => { - if (row.attributes[attr] !== undefined) { - filteredAttributes[attr] = row.attributes[attr] - } - }) - row.attributes = filteredAttributes - } + // Prepare fields for main query + const preparedFields = fields + .map((f) => { + const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) + if (!mappedField) { + throw new Error400('en', `Invalid field: ${f}`) + } + return { alias: f, ...mappedField } + }) + .filter((mappedField) => mappedField.queryable !== false) + // Exclude fields from SELECT if their source table isn't joined: + // - skip msa.* when aggregates aren't included (no join with memberSegmentsAgg) + // - skip mo.* when member organizations aren't included (no join with member_orgs) + .filter((mappedField) => { + if (!withAggregates && mappedField.name.includes('msa.')) return false + if (!include.memberOrganizations && mappedField.name.includes('mo.')) return false + return true + }) + .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) + .join(',\n') + + const mainQuery = buildQuery({ + fields: preparedFields, + withAggregates, + includeMemberOrgs: include.memberOrganizations, + searchConfig, + filterString, + orderBy, + limit, + offset, }) - const count = parseInt(results[1].count, 10) + log.info(`main query: ${mainQuery} with params ${JSON.stringify(params)}`) + const [rows, countResult] = await Promise.all([ + qx.select(mainQuery, params), + qx.selectOne(countQuery, params), + ]) + + const count = parseInt(countResult.count, 10) const memberIds = rows.map((org) => org.id) + if (memberIds.length === 0) { return { rows: [], count, limit, offset } } - if (include.memberOrganizations) { - const memberOrganizations = await fetchManyMemberOrgs(qx, memberIds) - - const orgIds = uniq( - memberOrganizations.reduce((acc, mo) => { - acc.push(...mo.organizations.map((o) => o.organizationId)) - return acc - }, []), - ) - - const orgExtra = orgIds.length - ? await queryOrgs(qx, { - filter: { - [OrganizationField.ID]: { - in: orgIds, - }, - }, - fields: [ - OrganizationField.ID, - OrganizationField.DISPLAY_NAME, - OrganizationField.LOGO, - OrganizationField.CREATED_AT, - ], - }) - : [] + const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ + include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), + include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), + include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), + include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), + ]) + + const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ + include.memberOrganizations + ? fetchOrganizationData(qx, memberOrganizations) + : Promise.resolve({ orgs: [], lfx: [] }), + include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), + include.maintainers && maintainerRoles.length > 0 + ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) + : Promise.resolve([]), + ]) - const lfxMemberships = orgIds.length - ? await findManyLfxMemberships(qx, { organizationIds: orgIds }) - : [] + if (include.memberOrganizations) { + const { orgs = [], lfx = [] } = orgExtra for (const member of rows) { member.organizations = [] @@ -378,74 +273,22 @@ export async function queryMembersAdvanced( const memberOrgs = memberOrganizations.find((o) => o.memberId === member.id)?.organizations || [] - // Filter only organizations with null dateEnd (active organizations) const activeOrgs = memberOrgs.filter((org) => !org.dateEnd) - // Apply the same sorting logic as in the list function - const sortedActiveOrgs = activeOrgs.sort((a, b) => { - if (!a || !b) { - return 0 - } - - // First priority: isPrimaryWorkExperience - const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true - const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true + const sortedActiveOrgs = sortActiveOrganizations(activeOrgs, orgs) - if (aPrimary && !bPrimary) { - return -1 - } - if (!aPrimary && bPrimary) { - return 1 - } - - // Second priority: has dateStart (only for non-primary organizations) - const aHasDate = !!a.dateStart - const bHasDate = !!b.dateStart - - if (aHasDate && !bHasDate) { - return -1 - } - if (!aHasDate && bHasDate) { - return 1 - } - - // Third priority: if both have same dateStart status, sort by createdAt and alphabetically - if (!a.dateStart && !b.dateStart) { - // Get createdAt from organization data - const aOrgInfo = orgExtra.find((odn) => odn.id === a.organizationId) - const bOrgInfo = orgExtra.find((odn) => odn.id === b.organizationId) - - const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 - const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 - - if (aCreatedAt !== bCreatedAt) { - return bCreatedAt - aCreatedAt // Newest createdAt first - } - - // If createdAt is also the same, sort alphabetically by organization displayName - const aName = (aOrgInfo?.displayName || '').toLowerCase() - const bName = (bOrgInfo?.displayName || '').toLowerCase() - return aName.localeCompare(bName) - } - - return 0 - }) - - const activeOrg = sortedActiveOrgs[0] || null + const activeOrg = sortedActiveOrgs[0] if (activeOrg) { - const orgInfo = orgExtra.find((odn) => odn.id === activeOrg.organizationId) + const orgInfo = orgs.find((odn) => odn.id === activeOrg.organizationId) if (orgInfo) { - const lfxMembership = lfxMemberships.find( - (m) => m.organizationId === activeOrg.organizationId, - ) - + const lfxMembership = lfx.find((m) => m.organizationId === activeOrg.organizationId) member.organizations = [ { id: activeOrg.organizationId, - displayName: orgInfo?.displayName || '', - logo: orgInfo?.logo || '', + displayName: orgInfo.displayName || '', + logo: orgInfo.logo || '', lfxMembership: !!lfxMembership, }, ] @@ -454,38 +297,14 @@ export async function queryMembersAdvanced( } } - if (include.identities) { - const identities = await fetchManyMemberIdentities(qx, memberIds) - - rows.forEach((member) => { - const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] - - // Simplify the identities structure to include only necessary fields - member.identities = memberIdentities.map((identity) => ({ - type: identity.type, - value: identity.value, - platform: identity.platform, - verified: identity.verified, - })) - }) - } - if (include.segments) { - const memberSegments = await fetchManyMemberSegments(qx, memberIds) - const segmentIds = uniq( - memberSegments.reduce((acc, ms) => { - acc.push(...ms.segments.map((s) => s.segmentId)) - return acc - }, []), - ) - const segmentsInfo = await fetchManySegments(qx, segmentIds) + const segments = segmentsInfo || [] rows.forEach((member) => { member.segments = (memberSegments.find((i) => i.memberId === member.id)?.segments || []) .map((segment) => { - const segmentInfo = segmentsInfo.find((s) => s.id === segment.segmentId) + const segmentInfo = segments.find((s) => s.id === segment.segmentId) - // include only subprojects if flag is set if (include.onlySubProjects && segmentInfo?.type !== SegmentType.SUB_PROJECT) { return null } @@ -501,14 +320,10 @@ export async function queryMembersAdvanced( } if (include.maintainers) { - const maintainerRoles = await findMaintainerRoles(qx, memberIds) - const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) - const segmentsInfo = await fetchManySegments(qx, segmentIds) - const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) rows.forEach((member) => { member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { - const segmentInfo = segmentsInfo.find((s) => s.id === role.segmentId) + const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) return { ...role, segmentName: segmentInfo?.name, @@ -517,6 +332,19 @@ export async function queryMembersAdvanced( }) } + if (include.identities) { + rows.forEach((member) => { + const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] + + member.identities = memberIdentities.map((identity) => ({ + type: identity.type, + value: identity.value, + platform: identity.platform, + verified: identity.verified, + })) + }) + } + return { rows, count, limit, offset } } @@ -540,7 +368,7 @@ export async function moveAffiliationsBetweenMembers( fromMemberId: string, toMemberId: string, ): Promise { - const params: any = { + const params: Record = { fromMemberId, toMemberId, } diff --git a/services/libs/data-access-layer/src/members/dataProcessor.ts b/services/libs/data-access-layer/src/members/dataProcessor.ts new file mode 100644 index 0000000000..e48d871057 --- /dev/null +++ b/services/libs/data-access-layer/src/members/dataProcessor.ts @@ -0,0 +1,129 @@ +import { uniq } from 'lodash' + +import { SegmentData } from '@crowd/types' + +import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' +import { OrganizationField, queryOrgs } from '../organizations' +import { QueryExecutor } from '../queryExecutor' +import { fetchManySegments } from '../segments' + +interface MemberOrganization { + id: string + organizationId: string + dateStart?: string + dateEnd?: string + affiliationOverride?: { + isPrimaryWorkExperience?: boolean + } +} + +interface MemberOrganizationData { + memberId: string + organizations: MemberOrganization[] +} + +interface OrganizationInfo { + id: string + displayName: string + logo: string + createdAt: string +} + +interface MemberSegmentData { + memberId: string + segments: Array<{ + segmentId: string + activityCount: number + }> +} + +export const sortActiveOrganizations = ( + activeOrgs: MemberOrganization[], + organizationsInfo: OrganizationInfo[], +): MemberOrganization[] => { + return activeOrgs.sort((a, b) => { + if (!a || !b) return 0 + + // First priority: isPrimaryWorkExperience + const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true + const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true + + if (aPrimary !== bPrimary) return aPrimary ? -1 : 1 + + // Second priority: has dateStart + const aHasDate = !!a.dateStart + const bHasDate = !!b.dateStart + + if (aHasDate !== bHasDate) return aHasDate ? -1 : 1 + + // Third priority: createdAt and alphabetical + if (!a.dateStart && !b.dateStart) { + const aOrgInfo = organizationsInfo.find((odn) => odn.id === a.organizationId) + const bOrgInfo = organizationsInfo.find((odn) => odn.id === b.organizationId) + + const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 + const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 + + if (aCreatedAt !== bCreatedAt) return bCreatedAt - aCreatedAt + + const aName = (aOrgInfo?.displayName || '').toLowerCase() + const bName = (bOrgInfo?.displayName || '').toLowerCase() + return aName.localeCompare(bName) + } + + return 0 + }) +} + +export const fetchOrganizationData = async ( + qx: QueryExecutor, + memberOrganizations: MemberOrganizationData[], +): Promise<{ orgs: OrganizationInfo[]; lfx: LfxMembership[] }> => { + if (memberOrganizations.length === 0) { + return { orgs: [], lfx: [] } + } + + const orgIds = uniq( + memberOrganizations.reduce((acc, mo) => { + acc.push(...mo.organizations.map((o) => o.organizationId)) + return acc + }, []), + ) + + if (orgIds.length === 0) { + return { orgs: [], lfx: [] } + } + + const [orgs, lfx] = await Promise.all([ + queryOrgs(qx, { + filter: { [OrganizationField.ID]: { in: orgIds } }, + fields: [ + OrganizationField.ID, + OrganizationField.DISPLAY_NAME, + OrganizationField.LOGO, + OrganizationField.CREATED_AT, + ], + }), + findManyLfxMemberships(qx, { organizationIds: orgIds }), + ]) + + return { orgs, lfx } +} + +export const fetchSegmentData = async ( + qx: QueryExecutor, + memberSegments: MemberSegmentData[], +): Promise => { + if (memberSegments.length === 0) { + return [] + } + + const segmentIds = uniq( + memberSegments.reduce((acc, ms) => { + acc.push(...ms.segments.map((s) => s.segmentId)) + return acc + }, []), + ) + + return segmentIds.length > 0 ? fetchManySegments(qx, segmentIds) : [] +} diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts new file mode 100644 index 0000000000..eccc408cf7 --- /dev/null +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -0,0 +1,482 @@ +import { getServiceLogger } from '@crowd/logging' +import { MemberIdentityType } from '@crowd/types' + +const log = getServiceLogger() + +type OrderDirection = 'ASC' | 'DESC' + +interface SearchConfig { + cte: string + join: string +} + +interface BuildQueryArgs { + fields: string + withAggregates: boolean + includeMemberOrgs: boolean + searchConfig: SearchConfig + filterString: string + orderBy?: string + orderDirection?: OrderDirection + limit?: number + offset?: number +} + +interface BuildCountQueryArgs { + withAggregates: boolean + searchConfig: SearchConfig + filterString: string + includeMemberOrgs?: boolean +} + +const ORDER_FIELD_MAP: Record = { + activityCount: 'msa."activityCount"', + score: 'm."score"', + joinedAt: 'm."joinedAt"', + displayName: 'm."displayName"', +} + +export const buildSearchCTE = ( + search: string, +): { cte: string; join: string; params: Record } => { + if (!search?.trim()) { + return { cte: '', join: '', params: {} } + } + + const searchTerm = search.toLowerCase().trim() + + return { + cte: ` + member_search AS ( + SELECT DISTINCT mi."memberId" + FROM "memberIdentities" mi + INNER JOIN members m ON m.id = mi."memberId" + WHERE ( + (mi.verified = true AND mi.type = $(emailType) AND LOWER(mi."value") LIKE $(searchPattern)) + OR LOWER(m."displayName") LIKE $(searchPattern) + ) + ) + `, + join: `INNER JOIN member_search ms ON ms."memberId" = m.id`, + params: { + emailType: MemberIdentityType.EMAIL, + searchPattern: `%${searchTerm}%`, + }, + } +} + +export const buildMemberOrgsCTE = (includeMemberOrgs: boolean): string => { + if (!includeMemberOrgs) return '' + + return ` + member_orgs AS ( + SELECT + "memberId", + ARRAY_AGG("organizationId"::TEXT) AS "organizationId" + FROM "memberOrganizations" + WHERE "deletedAt" IS NULL + GROUP BY "memberId" + ) + ` +} + +const parseOrderBy = ( + orderBy: string | undefined, + fallbackDirection: OrderDirection, +): { field?: string; direction: OrderDirection } => { + const defaultDirection: OrderDirection = fallbackDirection || 'DESC' + + if (!orderBy || !orderBy.trim()) { + return { field: undefined, direction: defaultDirection } + } + + const [rawField, rawDir] = orderBy.trim().split('_') + const field = rawField?.trim() || undefined + + const dir = (rawDir || '').toUpperCase() + const direction: OrderDirection = + dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : defaultDirection + + return { field, direction } +} + +const getOrderClause = ( + parsedField: string | undefined, + direction: OrderDirection, + withAggregates: boolean, +): string => { + const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' + + if (!parsedField) return defaultOrder + + const fieldExpr = ORDER_FIELD_MAP[parsedField] + + if (!fieldExpr) return defaultOrder + + if (!withAggregates && fieldExpr.includes('msa.')) { + return defaultOrder + } + + return `${fieldExpr} ${direction}` +} + +/** + * Analyzes the filter string to determine if it targets specific member IDs, + * which allows for query optimization by skipping expensive sorting operations. + * + * @param filterString - The SQL filter condition string + * @returns Object indicating if members are pinned to specific IDs and if the list is small + */ +const analyzeMemberIdTargeting = ( + filterString: string, +): { + isTargetingSpecificMembers: boolean + hasSmallMemberSet: boolean +} => { + if (!filterString?.trim()) { + return { isTargetingSpecificMembers: false, hasSmallMemberSet: false } + } + + // Check for single member ID equality: m.id = '...' or m.id = $(param) or m.id = :param + const singleIdPattern = /\bm\.id\s*=\s*(?:'[^']+'|\$\([^)]+\)|:[a-zA-Z_]\w*|\?)/i + if (singleIdPattern.test(filterString)) { + return { isTargetingSpecificMembers: true, hasSmallMemberSet: true } + } + + // Check for member ID list: m.id IN (...) + const idListPattern = /\bm\.id\s+IN\s*\(([^)]+)\)/i + const listMatch = idListPattern.exec(filterString) + + if (listMatch?.length > 1) { + // Count items in the IN clause by splitting on commas + const itemsInList = listMatch[1] + .split(',') + .map((item) => item.trim()) + .filter((item) => item.length > 0) + + // Consider lists with <= 100 items as "small" for optimization purposes + const isSmallList = itemsInList.length <= 100 + + return { isTargetingSpecificMembers: true, hasSmallMemberSet: isSmallList } + } + + // No specific member targeting found + return { isTargetingSpecificMembers: false, hasSmallMemberSet: false } +} + +/** + * Checks if the filter string contains references to members table fields + * excluding m.id (which is handled separately for optimization purposes). + * + * @param filterString - The SQL filter condition string + * @returns true if filter uses members table fields other than m.id + */ +const hasNonIdMemberFieldReferences = (filterString: string): boolean => { + if (!filterString.includes('m.')) { + return false + } + + // Remove all m.id references from the filter string + const filterWithoutMemberIds = filterString.replace(/\bm\.id\b/g, '') + + // Check if there are still any m.* field references remaining + return /\bm\.\w+/.test(filterWithoutMemberIds) +} + +/** + * Determines if we can use the activityCount optimization strategy. + * This optimization creates a CTE with top members by activity count, + * which is much faster than sorting the entire dataset. + * + * @param withAggregates - Whether aggregates are available + * @param sortField - The field being sorted by (undefined means default activityCount) + * @param hasEnrichmentFilters - Whether filter references me.* fields + * @param hasOrganizationFilters - Whether filter references mo.* fields + * @returns true if activityCount optimization can be used + */ +const canUseActivityCountOptimization = ({ + filterHasMe, + filterHasMo, + sortField, + withAggregates, +}: { + filterHasMe: boolean + filterHasMo: boolean + sortField: string | undefined + withAggregates: boolean +}): boolean => { + // Need aggregates to access activityCount + if (!withAggregates) return false + + // Only works when sorting by activityCount (or using default sort) + if (sortField && sortField !== 'activityCount') return false + + // Cannot use if filter requires expensive joins (me.*, mo.*) + if (filterHasMe || filterHasMo) return false + + return true +} + +/** + * Builds optimized query for when we're targeting specific member IDs. + * This path starts from memberSegmentsAgg and avoids expensive sorting operations. + * + * @param fields - The SELECT fields to return + * @param filterString - The WHERE clause filter + * @param orderClause - The ORDER BY clause + * @param searchConfig - Search CTE configuration + * @param needsMemberOrgs - Whether to include member organizations + * @param limit - Query limit + * @param offset - Query offset + * @returns The optimized SQL query string + */ +const buildDirectIdPathQuery = ({ + fields, + filterString, + orderClause, + searchConfig, + needsMemberOrgs, + limit, + offset, +}: { + fields: string + filterString: string + orderClause: string + searchConfig: SearchConfig + needsMemberOrgs: boolean + limit: number + offset: number +}): string => { + // Build CTEs array + const ctes: string[] = [] + if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + + const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' + + // Build JOIN clauses + const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + const searchJoin = searchConfig.join || '' + + return ` + ${withClause} + SELECT ${fields} + FROM "memberSegmentsAgg" msa + JOIN members m + ON m.id = msa."memberId" + ${memberOrgsJoin} + ${searchJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +/** + * Builds optimized query using top-N activity count strategy. + * This creates a CTE with the most active members first, then joins back for full data. + * Much faster than sorting the entire member dataset. + * + * @param fields - The SELECT fields to return + * @param filterString - The WHERE clause filter + * @param searchConfig - Search CTE configuration + * @param direction - Sort direction for activityCount + * @param hasNonIdMemberFields - Whether filter uses m.* fields (requires oversampling) + * @param limit - Query limit + * @param offset - Query offset + * @returns The optimized SQL query string + */ +const buildActivityCountOptimizedQuery = ({ + fields, + filterString, + searchConfig, + direction, + hasNonIdMemberFields, + limit, + offset, +}: { + fields: string + filterString: string + searchConfig: SearchConfig + direction: OrderDirection + hasNonIdMemberFields: boolean + limit: number + offset: number +}): string => { + const ctes: string[] = [] + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + + const searchJoinForTopMembers = searchConfig.cte + ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` + : '' + + const baseNeeded = limit + offset + const oversampleMultiplier = hasNonIdMemberFields ? 10 : 1 // 10x oversampling for m.* filters + const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k + + ctes.push( + ` + top_members AS ( + SELECT + msa."memberId", + msa."activityCount" + FROM "memberSegmentsAgg" msa + INNER JOIN members m ON m.id = msa."memberId" + ${searchJoinForTopMembers} + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${totalNeeded} + ) + `.trim(), + ) + + const withClause = `WITH ${ctes.join(',\n')}` + + // Outer query is much simpler now - no more filtering needed + return ` + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m + ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +export const buildQuery = ({ + fields, + withAggregates, + includeMemberOrgs, + searchConfig, + filterString, + orderBy, + orderDirection, + limit = 20, + offset = 0, +}: BuildQueryArgs): string => { + const { field: sortField, direction } = parseOrderBy(orderBy, orderDirection) + + // Detect alias usage in filters (to decide joins/CTEs needed outside) + const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') + const filterHasNonIdMemberFields = hasNonIdMemberFieldReferences(filterString) + + log.info( + `filterHasMo=${filterHasMo}, filterHasMe=${filterHasMe}, filterHasNonIdMemberFields=${filterHasNonIdMemberFields}`, + ) + + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. + const { isTargetingSpecificMembers, hasSmallMemberSet } = analyzeMemberIdTargeting(filterString) + const useDirectIdPath = withAggregates && isTargetingSpecificMembers && hasSmallMemberSet + + // Default sort clause for fallback/outer queries + const orderClause = getOrderClause(sortField, direction, withAggregates) + + log.info(`useDirectIdPath=${useDirectIdPath}`) + if (useDirectIdPath) { + return buildDirectIdPathQuery({ + fields, + filterString, + orderClause, + searchConfig, + needsMemberOrgs, + limit, + offset, + }) + } + + const useActivityCountOptimized = canUseActivityCountOptimization({ + filterHasMe, + filterHasMo, + sortField, + withAggregates, + }) + + log.info(`useActivityCountOptimized=${useActivityCountOptimized}`) + + if (useActivityCountOptimized) { + return buildActivityCountOptimizedQuery({ + fields, + filterString, + searchConfig, + direction, + hasNonIdMemberFields: filterHasNonIdMemberFields, + limit, + offset, + }) + } + + log.info('Using fallback query path') + // Fallback path (other sorts / non-aggregate / filtered queries) + const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( + Boolean, + ) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, + searchConfig.join, + ].filter(Boolean) + + return ` + ${baseCtes.length ? `WITH ${baseCtes.join(',\n')}` : ''} + SELECT ${fields} + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +export const buildCountQuery = ({ + withAggregates, + searchConfig, + filterString, + includeMemberOrgs = false, +}: BuildCountQueryArgs): string => { + const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + const ctes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter(Boolean) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + filterHasMe ? `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id` : '', + searchConfig.join, + ].filter(Boolean) + + return ` + ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} + SELECT COUNT(DISTINCT m.id) AS count + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) + `.trim() +}