From 4e850c4ee761032d5266e1da0c34c5a0d6950d64 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 6 Nov 2025 15:42:42 +0100 Subject: [PATCH 01/28] feat: refactoring member query --- .../data-access-layer/src/members/base.ts | 599 +++++++++--------- 1 file changed, 313 insertions(+), 286 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 07950fe411..6f91d978b5 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -9,7 +9,7 @@ import { groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' -import { getServiceChildLogger } from '@crowd/logging' +import { getServiceChildLogger, getServiceLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' import { ALL_PLATFORM_TYPES, @@ -35,79 +35,74 @@ import { IDbMemberAttributeSetting, IDbMemberData } from './types' import { fetchManyMemberIdentities, fetchManyMemberOrgs, fetchManyMemberSegments } from '.' -/* eslint-disable @typescript-eslint/no-explicit-any */ - -const log = getServiceChildLogger('db/members') +const log = getServiceLogger() export enum MemberField { - // meta - ID = 'id', ATTRIBUTES = 'attributes', - DISPLAY_NAME = 'displayName', - SCORE = 'score', - JOINED_AT = 'joinedAt', - IMPORT_HASH = 'importHash', - REACH = 'reach', CONTRIBUTIONS = 'contributions', - CREATED_AT = 'createdAt', - UPDATED_AT = 'updatedAt', + CREATED_BY_ID = 'createdById', DELETED_AT = 'deletedAt', - + DISPLAY_NAME = 'displayName', + ID = 'id', + IMPORT_HASH = 'importHash', + JOINED_AT = 'joinedAt', + MANUALLY_CHANGED_FIELDS = 'manuallyChangedFields', + MANUALLY_CREATED = 'manuallyCreated', + REACH = 'reach', + SCORE = 'score', TENANT_ID = 'tenantId', - CREATED_BY_ID = 'createdById', + UPDATED_AT = 'updatedAt', UPDATED_BY_ID = 'updatedById', - - MANUALLY_CREATED = 'manuallyCreated', - MANUALLY_CHANGED_FIELDS = 'manuallyChangedFields', } export const MEMBER_MERGE_FIELDS = [ + 'affiliations', + 'attributes', + 'contributions', + 'displayName', 'id', - 'tags', + 'joinedAt', + 'manuallyChangedFields', + 'manuallyCreated', 'reach', + 'tags', 'tasks', - 'joinedAt', 'tenantId', - 'attributes', - 'displayName', - 'affiliations', - 'contributions', - 'manuallyCreated', - 'manuallyChangedFields', ] export const MEMBER_UPDATE_COLUMNS = [ - MemberField.DISPLAY_NAME, MemberField.ATTRIBUTES, MemberField.CONTRIBUTIONS, - MemberField.SCORE, - MemberField.REACH, + MemberField.DISPLAY_NAME, MemberField.IMPORT_HASH, + MemberField.REACH, + MemberField.SCORE, ] export const MEMBER_SELECT_COLUMNS = [ - 'id', - 'score', - 'joinedAt', - 'reach', 'attributes', 'displayName', + 'id', + 'joinedAt', 'manuallyChangedFields', + 'reach', + 'score', ] export const MEMBER_INSERT_COLUMNS = [ - 'id', 'attributes', + 'createdAt', 'displayName', + 'id', 'joinedAt', - 'tenantId', 'reach', - 'createdAt', + 'tenantId', 'updatedAt', ] const QUERY_FILTER_COLUMN_MAP: Map = new Map([ + ['activityCount', { name: 'coalesce(msa."activityCount", 0)::integer' }], ['activityCount', { name: 'coalesce(msa."activityCount", 0)::integer' }], ['attributes', { name: 'm.attributes' }], ['averageSentiment', { name: 'coalesce(msa."averageSentiment", 0)::decimal' }], @@ -126,7 +121,179 @@ const QUERY_FILTER_COLUMN_MAP: Map { + const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' + + if (!orderBy || typeof orderBy !== 'string' || !orderBy.length) { + return defaultOrder + } + + const [fieldName, direction = 'DESC'] = orderBy.split('_') + const orderField = QUERY_FILTER_COLUMN_MAP.get(fieldName)?.name + + if (!orderField) { + return defaultOrder + } + + const orderDirection = ['DESC', 'ASC'].includes(direction) ? direction : 'DESC' + return `${orderField} ${orderDirection}` +} + +const buildSearchCTE = ( + search: string, +): { cte: string; join: string; params: Record } => { + if (!search?.trim()) { + return { cte: '', join: '', params: {} } + } + + const searchTerm = search.toLowerCase().trim() + + return { + cte: ` + member_search AS ( + SELECT DISTINCT mi."memberId" + FROM "memberIdentities" mi + INNER JOIN members m ON m.id = mi."memberId" + WHERE ( + (mi.verified = true AND mi.type = $(emailType) AND LOWER(mi."value") LIKE $(searchPattern)) + OR LOWER(m."displayName") LIKE $(searchPattern) + ) + )`, + join: `INNER JOIN member_search ms ON ms."memberId" = m.id`, + params: { + emailType: MemberIdentityType.EMAIL, + searchPattern: `%${searchTerm}%`, + }, + } +} + +const buildMemberOrgsCTE = (includeMemberOrgs: boolean): string => { + if (!includeMemberOrgs) return '' + + return ` + member_orgs AS ( + SELECT + "memberId", + ARRAY_AGG("organizationId"::TEXT) AS "organizationId" + FROM "memberOrganizations" + WHERE "deletedAt" IS NULL + GROUP BY "memberId" + )` +} + +const buildQuery = ( + fields: string, + withAggregates: boolean, + includeMemberOrgs: boolean, + searchConfig: { cte: string; join: string }, + filterString: string, +): string => { + const ctes = [buildMemberOrgsCTE(includeMemberOrgs), searchConfig.cte].filter(Boolean) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + includeMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, + searchConfig.join, + ].filter(Boolean) + + return ` + ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} + SELECT ${fields} + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) + `.trim() +} + +const sortActiveOrganizations = (activeOrgs: any[], organizationsInfo: any[]): any[] => { + return activeOrgs.sort((a, b) => { + if (!a || !b) return 0 + + // First priority: isPrimaryWorkExperience + const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true + const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true + + if (aPrimary !== bPrimary) return aPrimary ? -1 : 1 + + // Second priority: has dateStart + const aHasDate = !!a.dateStart + const bHasDate = !!b.dateStart + + if (aHasDate !== bHasDate) return aHasDate ? -1 : 1 + + // Third priority: createdAt and alphabetical + if (!a.dateStart && !b.dateStart) { + const aOrgInfo = organizationsInfo.find((odn) => odn.id === a.organizationId) + const bOrgInfo = organizationsInfo.find((odn) => odn.id === b.organizationId) + + const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 + const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 + + if (aCreatedAt !== bCreatedAt) return bCreatedAt - aCreatedAt + + const aName = (aOrgInfo?.displayName || '').toLowerCase() + const bName = (bOrgInfo?.displayName || '').toLowerCase() + return aName.localeCompare(bName) + } + + return 0 + }) +} + +const fetchOrganizationData = async ( + qx: QueryExecutor, + memberOrganizations: any[], +): Promise<{ orgs: any[]; lfx: any[] }> => { + if (memberOrganizations.length === 0) { + return { orgs: [], lfx: [] } + } + + const orgIds = uniq( + memberOrganizations.reduce((acc, mo) => { + acc.push(...mo.organizations.map((o) => o.organizationId)) + return acc + }, []), + ) + + if (orgIds.length === 0) { + return { orgs: [], lfx: [] } + } + + const [orgs, lfx] = await Promise.all([ + queryOrgs(qx, { + filter: { [OrganizationField.ID]: { in: orgIds } }, + fields: [ + OrganizationField.ID, + OrganizationField.DISPLAY_NAME, + OrganizationField.LOGO, + OrganizationField.CREATED_AT, + ], + }), + findManyLfxMemberships(qx, { organizationIds: orgIds }), + ]) + + return { orgs, lfx } +} + +const fetchSegmentData = async (qx: QueryExecutor, memberSegments: any[]): Promise => { + if (memberSegments.length === 0) { + return [] + } + + const segmentIds = uniq( + memberSegments.reduce((acc, ms) => { + acc.push(...ms.segments.map((s) => s.segmentId)) + return acc + }, []), + ) + + return segmentIds.length > 0 ? fetchManySegments(qx, segmentIds) : [] +} export async function queryMembersAdvanced( qx: QueryExecutor, @@ -158,32 +325,14 @@ export async function queryMembersAdvanced( attributeSettings = [] as IDbMemberAttributeSetting[], }, ): Promise> { - if (!attributeSettings || attributeSettings.length === 0) { - attributeSettings = await getMemberAttributeSettings(qx, redis) - } - const withAggregates = !!segmentId - let segment - if (withAggregates) { - segment = (await findSegmentById(qx, segmentId)) as any - - if (segment === null) { - log.info('No segment found for member query. Returning empty result.') - return { - rows: [], - count: 0, - limit, - offset, - } - } - - segmentId = segment.id - } + const searchConfig = buildSearchCTE(search) const params = { limit, offset, segmentId, + ...searchConfig.params, } const filterString = RawQueryParser.parseFilters( @@ -194,7 +343,10 @@ export async function queryMembersAdvanced( property: 'attributes', column: 'm.attributes', attributeInfos: [ - ...attributeSettings, + ...(attributeSettings?.length > 0 + ? attributeSettings + : await getMemberAttributeSettings(qx, redis)), + // TODO: ci serve questo ? { name: 'jobTitle', type: MemberAttributeType.STRING, @@ -204,8 +356,8 @@ export async function queryMembersAdvanced( { property: 'username', column: 'aggs.username', - attributeInfos: ALL_PLATFORM_TYPES.map((p) => ({ - name: p, + attributeInfos: ALL_PLATFORM_TYPES.map((name) => ({ + name, type: MemberAttributeType.STRING, })), }, @@ -214,163 +366,101 @@ export async function queryMembersAdvanced( { pgPromiseFormat: true }, ) - const effectiveOrderBy = typeof orderBy === 'string' && orderBy.length ? orderBy : 'joinedAt_DESC' - - const order = (function prepareOrderBy( - orderBy = withAggregates ? 'activityCount_DESC' : 'id_DESC', - ) { - const orderSplit = orderBy.split('_') - - const orderField = QUERY_FILTER_COLUMN_MAP.get(orderSplit[0])?.name - if (!orderField) { - return withAggregates ? 'msa."activityCount" DESC' : 'm.id DESC' - } - const orderDirection = ['DESC', 'ASC'].includes(orderSplit[1]) ? orderSplit[1] : 'DESC' - - return `${orderField} ${orderDirection}` - })(effectiveOrderBy) - - const withSearch = !!search - let searchCTE = '' - let searchJoin = '' - - if (withSearch) { - search = search.toLowerCase() - searchCTE = ` - , - member_search AS ( - SELECT - "memberId" - FROM "memberIdentities" mi - join members m on m.id = mi."memberId" - where (verified and type = '${MemberIdentityType.EMAIL}' and lower("value") ilike '%${search}%') or m."displayName" ilike '%${search}%' - GROUP BY 1 - ) - ` - searchJoin = ` JOIN member_search ms ON ms."memberId" = m.id ` - } - - const createQuery = (fields) => ` - WITH member_orgs AS ( - SELECT - "memberId", - ARRAY_AGG("organizationId")::TEXT[] AS "organizationId" - FROM "memberOrganizations" - WHERE "deletedAt" IS NULL - GROUP BY 1 - ) - ${searchCTE} - SELECT - ${fields} - FROM members m - ${ - withAggregates - ? ` INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` - : '' - } - LEFT JOIN member_orgs mo ON mo."memberId" = m.id - LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id - ${searchJoin} - WHERE (${filterString}) - ` - - const countQuery = createQuery('COUNT(*)') + // Build queries + const countQuery = buildQuery( + 'COUNT(*) as count', + withAggregates, + include.memberOrganizations, + searchConfig, + filterString, + ) if (countOnly) { + const result = await qx.selectOne(countQuery, params) return { rows: [], - count: parseInt((await qx.selectOne(countQuery, params)).count, 10), + count: parseInt(result.count, 10), limit, offset, } } - const query = ` - ${createQuery( - (function prepareFields(fields) { - return `${fields - .map((f) => { - const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) - if (!mappedField) { - throw new Error400('en', `Invalid field: ${f}`) - } - - return { - alias: f, - ...mappedField, - } - }) - .filter((mappedField) => mappedField.queryable !== false) - .filter((mappedField) => { - if (!withAggregates && mappedField.name.includes('msa.')) { - return false - } - if (!include.memberOrganizations && mappedField.name.includes('mo.')) { - return false - } - return true - }) - .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) - .join(',\n')}` - })(fields), - )} - ORDER BY ${order} NULLS LAST - LIMIT $(limit) - OFFSET $(offset) - ` - - const results = await Promise.all([qx.select(query, params), qx.selectOne(countQuery, params)]) - - const rows = results[0] - - rows.forEach((row) => { - if (row.attributes && typeof row.attributes === 'object') { - const filteredAttributes = {} - QUERY_FILTER_ATTRIBUTE_MAP.forEach((attr) => { - if (row.attributes[attr] !== undefined) { - filteredAttributes[attr] = row.attributes[attr] - } - }) - row.attributes = filteredAttributes - } - }) - - const count = parseInt(results[1].count, 10) - + // Prepare fields for main query + const preparedFields = fields + .map((f) => { + const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) + if (!mappedField) { + throw new Error400('en', `Invalid field: ${f}`) + } + return { alias: f, ...mappedField } + }) + .filter((mappedField) => mappedField.queryable !== false) + .filter((mappedField) => { + if (!withAggregates && mappedField.name.includes('msa.')) return false + if (!include.memberOrganizations && mappedField.name.includes('mo.')) return false + return true + }) + .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) + .join(',\n') + + const mainQuery = ` + ${buildQuery( + preparedFields, + withAggregates, + include.memberOrganizations, + searchConfig, + filterString, + )} + ORDER BY ${getOrderClause(orderBy, withAggregates)} NULLS LAST + LIMIT $(limit) + OFFSET $(offset) + ` + + log.info(`main query: ${formatSql(mainQuery, params)}`) + log.info(`count query: ${formatSql(countQuery, params)}`) + + // Execute queries in parallel + const [rows, countResult] = await Promise.all([ + qx.select(mainQuery, params), + qx.selectOne(countQuery, params), + ]) + + // TODO: ci serve davvero questo filtro ? + // rows.forEach((row) => { + // if (row.attributes && typeof row.attributes === 'object') { + // const filteredAttributes = {} + // QUERY_FILTER_ATTRIBUTE_MAP.forEach((attr) => { + // if (row.attributes[attr] !== undefined) { + // filteredAttributes[attr] = row.attributes[attr] + // } + // }) + // row.attributes = filteredAttributes + // } + // }) + + const count = parseInt(countResult.count, 10) const memberIds = rows.map((org) => org.id) + if (memberIds.length === 0) { + // TODO: if memberIds is empty the count is 0 ?, if yes we can skip the count query return { rows: [], count, limit, offset } } - if (include.memberOrganizations) { - const memberOrganizations = await fetchManyMemberOrgs(qx, memberIds) - - const orgIds = uniq( - memberOrganizations.reduce((acc, mo) => { - acc.push(...mo.organizations.map((o) => o.organizationId)) - return acc - }, []), - ) - - const orgExtra = orgIds.length - ? await queryOrgs(qx, { - filter: { - [OrganizationField.ID]: { - in: orgIds, - }, - }, - fields: [ - OrganizationField.ID, - OrganizationField.DISPLAY_NAME, - OrganizationField.LOGO, - OrganizationField.CREATED_AT, - ], - }) - : [] + const [memberOrganizations, identities, memberSegments] = await Promise.all([ + include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), + include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), + include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), + ]) - const lfxMemberships = orgIds.length - ? await findManyLfxMemberships(qx, { organizationIds: orgIds }) - : [] + const [orgExtra, segmentsInfo] = await Promise.all([ + include.memberOrganizations + ? fetchOrganizationData(qx, memberOrganizations) + : Promise.resolve({ orgs: [], lfx: [] }), + include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), + ]) + + if (include.memberOrganizations) { + const { orgs = [], lfx = [] } = orgExtra as { orgs: any[]; lfx: any[] } for (const member of rows) { member.organizations = [] @@ -378,74 +468,22 @@ export async function queryMembersAdvanced( const memberOrgs = memberOrganizations.find((o) => o.memberId === member.id)?.organizations || [] - // Filter only organizations with null dateEnd (active organizations) const activeOrgs = memberOrgs.filter((org) => !org.dateEnd) - // Apply the same sorting logic as in the list function - const sortedActiveOrgs = activeOrgs.sort((a, b) => { - if (!a || !b) { - return 0 - } - - // First priority: isPrimaryWorkExperience - const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true - const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true - - if (aPrimary && !bPrimary) { - return -1 - } - if (!aPrimary && bPrimary) { - return 1 - } - - // Second priority: has dateStart (only for non-primary organizations) - const aHasDate = !!a.dateStart - const bHasDate = !!b.dateStart - - if (aHasDate && !bHasDate) { - return -1 - } - if (!aHasDate && bHasDate) { - return 1 - } - - // Third priority: if both have same dateStart status, sort by createdAt and alphabetically - if (!a.dateStart && !b.dateStart) { - // Get createdAt from organization data - const aOrgInfo = orgExtra.find((odn) => odn.id === a.organizationId) - const bOrgInfo = orgExtra.find((odn) => odn.id === b.organizationId) + const sortedActiveOrgs = sortActiveOrganizations(activeOrgs, orgs) - const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 - const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 - - if (aCreatedAt !== bCreatedAt) { - return bCreatedAt - aCreatedAt // Newest createdAt first - } - - // If createdAt is also the same, sort alphabetically by organization displayName - const aName = (aOrgInfo?.displayName || '').toLowerCase() - const bName = (bOrgInfo?.displayName || '').toLowerCase() - return aName.localeCompare(bName) - } - - return 0 - }) - - const activeOrg = sortedActiveOrgs[0] || null + const activeOrg = sortedActiveOrgs[0] if (activeOrg) { - const orgInfo = orgExtra.find((odn) => odn.id === activeOrg.organizationId) + const orgInfo = orgs.find((odn) => odn.id === activeOrg.organizationId) if (orgInfo) { - const lfxMembership = lfxMemberships.find( - (m) => m.organizationId === activeOrg.organizationId, - ) - + const lfxMembership = lfx.find((m) => m.organizationId === activeOrg.organizationId) member.organizations = [ { id: activeOrg.organizationId, - displayName: orgInfo?.displayName || '', - logo: orgInfo?.logo || '', + displayName: orgInfo.displayName || '', + logo: orgInfo.logo || '', lfxMembership: !!lfxMembership, }, ] @@ -454,38 +492,14 @@ export async function queryMembersAdvanced( } } - if (include.identities) { - const identities = await fetchManyMemberIdentities(qx, memberIds) - - rows.forEach((member) => { - const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] - - // Simplify the identities structure to include only necessary fields - member.identities = memberIdentities.map((identity) => ({ - type: identity.type, - value: identity.value, - platform: identity.platform, - verified: identity.verified, - })) - }) - } - if (include.segments) { - const memberSegments = await fetchManyMemberSegments(qx, memberIds) - const segmentIds = uniq( - memberSegments.reduce((acc, ms) => { - acc.push(...ms.segments.map((s) => s.segmentId)) - return acc - }, []), - ) - const segmentsInfo = await fetchManySegments(qx, segmentIds) + const segments = segmentsInfo || [] rows.forEach((member) => { member.segments = (memberSegments.find((i) => i.memberId === member.id)?.segments || []) .map((segment) => { - const segmentInfo = segmentsInfo.find((s) => s.id === segment.segmentId) + const segmentInfo = segments.find((s) => s.id === segment.segmentId) - // include only subprojects if flag is set if (include.onlySubProjects && segmentInfo?.type !== SegmentType.SUB_PROJECT) { return null } @@ -517,6 +531,19 @@ export async function queryMembersAdvanced( }) } + if (include.identities) { + rows.forEach((member) => { + const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] + + member.identities = memberIdentities.map((identity) => ({ + type: identity.type, + value: identity.value, + platform: identity.platform, + verified: identity.verified, + })) + }) + } + return { rows, count, limit, offset } } From 85bbd4f7a41b32d429298eb3d942a6568a59179b Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Nov 2025 14:48:53 +0100 Subject: [PATCH 02/28] feat: make request in parallel --- .../libs/data-access-layer/src/members/base.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 6f91d978b5..dad4173c5c 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -446,17 +446,21 @@ export async function queryMembersAdvanced( return { rows: [], count, limit, offset } } - const [memberOrganizations, identities, memberSegments] = await Promise.all([ + const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), + include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), ]) - const [orgExtra, segmentsInfo] = await Promise.all([ + const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ include.memberOrganizations ? fetchOrganizationData(qx, memberOrganizations) : Promise.resolve({ orgs: [], lfx: [] }), include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), + include.maintainers && maintainerRoles.length > 0 + ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) + : Promise.resolve([]), ]) if (include.memberOrganizations) { @@ -515,14 +519,10 @@ export async function queryMembersAdvanced( } if (include.maintainers) { - const maintainerRoles = await findMaintainerRoles(qx, memberIds) - const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) - const segmentsInfo = await fetchManySegments(qx, segmentIds) - const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) rows.forEach((member) => { member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { - const segmentInfo = segmentsInfo.find((s) => s.id === role.segmentId) + const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) return { ...role, segmentName: segmentInfo?.name, From 3c8e9ab8e25f335516063a67a49e05e0d8aa551c Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Nov 2025 15:08:35 +0100 Subject: [PATCH 03/28] fix: lint --- .../data-access-layer/src/members/base.ts | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index dad4173c5c..e62a3777fb 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -9,17 +9,18 @@ import { groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' -import { getServiceChildLogger, getServiceLogger } from '@crowd/logging' +import { getServiceLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' import { ALL_PLATFORM_TYPES, MemberAttributeType, MemberIdentityType, PageData, + SegmentData, SegmentType, } from '@crowd/types' -import { findManyLfxMemberships } from '../lfx_memberships' +import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' import { findMaintainerRoles } from '../maintainers' import { IDbMemberCreateData, @@ -27,7 +28,7 @@ import { } from '../old/apps/data_sink_worker/repo/member.data' import { OrganizationField, queryOrgs } from '../organizations' import { QueryExecutor } from '../queryExecutor' -import { fetchManySegments, findSegmentById } from '../segments' +import { fetchManySegments } from '../segments' import { QueryOptions, QueryResult, queryTable, queryTableById } from '../utils' import { getMemberAttributeSettings } from './attributeSettings' @@ -36,6 +37,35 @@ import { IDbMemberAttributeSetting, IDbMemberData } from './types' import { fetchManyMemberIdentities, fetchManyMemberOrgs, fetchManyMemberSegments } from '.' const log = getServiceLogger() +interface MemberOrganization { + id: string + organizationId: string + dateStart?: string + dateEnd?: string + affiliationOverride?: { + isPrimaryWorkExperience?: boolean + } +} + +interface MemberOrganizationData { + memberId: string + organizations: MemberOrganization[] +} + +interface OrganizationInfo { + id: string + displayName: string + logo: string + createdAt: string +} + +interface MemberSegmentData { + memberId: string + segments: Array<{ + segmentId: string + activityCount: number + }> +} export enum MemberField { ATTRIBUTES = 'attributes', @@ -210,7 +240,10 @@ const buildQuery = ( `.trim() } -const sortActiveOrganizations = (activeOrgs: any[], organizationsInfo: any[]): any[] => { +const sortActiveOrganizations = ( + activeOrgs: MemberOrganization[], + organizationsInfo: OrganizationInfo[], +): MemberOrganization[] => { return activeOrgs.sort((a, b) => { if (!a || !b) return 0 @@ -247,8 +280,8 @@ const sortActiveOrganizations = (activeOrgs: any[], organizationsInfo: any[]): a const fetchOrganizationData = async ( qx: QueryExecutor, - memberOrganizations: any[], -): Promise<{ orgs: any[]; lfx: any[] }> => { + memberOrganizations: MemberOrganizationData[], +): Promise<{ orgs: OrganizationInfo[]; lfx: LfxMembership[] }> => { if (memberOrganizations.length === 0) { return { orgs: [], lfx: [] } } @@ -280,7 +313,10 @@ const fetchOrganizationData = async ( return { orgs, lfx } } -const fetchSegmentData = async (qx: QueryExecutor, memberSegments: any[]): Promise => { +const fetchSegmentData = async ( + qx: QueryExecutor, + memberSegments: MemberSegmentData[], +): Promise => { if (memberSegments.length === 0) { return [] } @@ -464,7 +500,7 @@ export async function queryMembersAdvanced( ]) if (include.memberOrganizations) { - const { orgs = [], lfx = [] } = orgExtra as { orgs: any[]; lfx: any[] } + const { orgs = [], lfx = [] } = orgExtra for (const member of rows) { member.organizations = [] From b8a323094245edd45b824eec5318570e83ac4666 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Nov 2025 15:13:28 +0100 Subject: [PATCH 04/28] fix: lint --- services/libs/data-access-layer/src/members/base.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index e62a3777fb..f0e5acfc4a 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -173,7 +173,7 @@ const getOrderClause = (orderBy: string, withAggregates: boolean): string => { const buildSearchCTE = ( search: string, -): { cte: string; join: string; params: Record } => { +): { cte: string; join: string; params: Record } => { if (!search?.trim()) { return { cte: '', join: '', params: {} } } @@ -335,7 +335,7 @@ export async function queryMembersAdvanced( qx: QueryExecutor, redis: RedisClient, { - filter = {} as any, + filter = {}, search = null, limit = 20, offset = 0, @@ -603,7 +603,7 @@ export async function moveAffiliationsBetweenMembers( fromMemberId: string, toMemberId: string, ): Promise { - const params: any = { + const params: Record = { fromMemberId, toMemberId, } From 5e76a64f0a3974a5b9d6686a1df2568cc99d39d4 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Nov 2025 16:55:50 +0100 Subject: [PATCH 05/28] fix: adding logs --- .../data-access-layer/src/members/base.ts | 199 +++++++++++++++++- 1 file changed, 191 insertions(+), 8 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index f0e5acfc4a..05683d250d 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -361,6 +361,8 @@ export async function queryMembersAdvanced( attributeSettings = [] as IDbMemberAttributeSetting[], }, ): Promise> { + const startTime = Date.now() + const withAggregates = !!segmentId const searchConfig = buildSearchCTE(search) @@ -422,6 +424,7 @@ export async function queryMembersAdvanced( } // Prepare fields for main query + const fieldsStartTime = Date.now() const preparedFields = fields .map((f) => { const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) @@ -438,6 +441,7 @@ export async function queryMembersAdvanced( }) .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) .join(',\n') + log.info(`[PERF] Field preparation took: ${Date.now() - fieldsStartTime}ms`) const mainQuery = ` ${buildQuery( @@ -456,10 +460,16 @@ export async function queryMembersAdvanced( log.info(`count query: ${formatSql(countQuery, params)}`) // Execute queries in parallel + const mainQueryStartTime = Date.now() + const [rows, countResult] = await Promise.all([ qx.select(mainQuery, params), qx.selectOne(countQuery, params), ]) + const mainQueryDuration = Date.now() - mainQueryStartTime + log.info( + `[PERF] Main queries (parallel) took: ${mainQueryDuration}ms - returned ${rows.length} rows`, + ) // TODO: ci serve davvero questo filtro ? // rows.forEach((row) => { @@ -482,24 +492,181 @@ export async function queryMembersAdvanced( return { rows: [], count, limit, offset } } + // const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ + // include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), + // include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), + // include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), + // include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), + // ]) + const firstBatchStartTime = Date.now() + const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ - include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), - include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), - include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), - include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), + include.memberOrganizations + ? (async () => { + const start = Date.now() + const result = await fetchManyMemberOrgs(qx, memberIds) + log.info(`[PERF] fetchManyMemberOrgs took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), + include.identities + ? (async () => { + const start = Date.now() + const result = await fetchManyMemberIdentities(qx, memberIds) + log.info(`[PERF] fetchManyMemberIdentities took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), + include.segments + ? (async () => { + const start = Date.now() + const result = await fetchManyMemberSegments(qx, memberIds) + log.info(`[PERF] fetchManyMemberSegments took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), + include.maintainers + ? (async () => { + const start = Date.now() + const result = await findMaintainerRoles(qx, memberIds) + log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), ]) - + const firstBatchDuration = Date.now() - firstBatchStartTime + log.info(`[PERF] First parallel batch took: ${firstBatchDuration}ms`) + + // const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ + // include.memberOrganizations + // ? fetchOrganizationData(qx, memberOrganizations) + // : Promise.resolve({ orgs: [], lfx: [] }), + // include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), + // include.maintainers && maintainerRoles.length > 0 + // ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) + // : Promise.resolve([]), + // ]) + + // if (include.memberOrganizations) { + // const { orgs = [], lfx = [] } = orgExtra + + // for (const member of rows) { + // member.organizations = [] + + // const memberOrgs = + // memberOrganizations.find((o) => o.memberId === member.id)?.organizations || [] + + // const activeOrgs = memberOrgs.filter((org) => !org.dateEnd) + + // const sortedActiveOrgs = sortActiveOrganizations(activeOrgs, orgs) + + // const activeOrg = sortedActiveOrgs[0] + + // if (activeOrg) { + // const orgInfo = orgs.find((odn) => odn.id === activeOrg.organizationId) + + // if (orgInfo) { + // const lfxMembership = lfx.find((m) => m.organizationId === activeOrg.organizationId) + // member.organizations = [ + // { + // id: activeOrg.organizationId, + // displayName: orgInfo.displayName || '', + // logo: orgInfo.logo || '', + // lfxMembership: !!lfxMembership, + // }, + // ] + // } + // } + // } + // } + + // if (include.segments) { + // const segments = segmentsInfo || [] + + // rows.forEach((member) => { + // member.segments = (memberSegments.find((i) => i.memberId === member.id)?.segments || []) + // .map((segment) => { + // const segmentInfo = segments.find((s) => s.id === segment.segmentId) + + // if (include.onlySubProjects && segmentInfo?.type !== SegmentType.SUB_PROJECT) { + // return null + // } + + // return { + // id: segment.segmentId, + // name: segmentInfo?.name, + // activityCount: segment.activityCount, + // } + // }) + // .filter(Boolean) + // }) + // } + + // if (include.maintainers) { + // const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) + // rows.forEach((member) => { + // member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { + // const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) + // return { + // ...role, + // segmentName: segmentInfo?.name, + // } + // }) + // }) + // } + + // if (include.identities) { + // rows.forEach((member) => { + // const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] + + // member.identities = memberIdentities.map((identity) => ({ + // type: identity.type, + // value: identity.value, + // platform: identity.platform, + // verified: identity.verified, + // })) + // }) + // } + + // Second parallel batch - fetch related data + const secondBatchStartTime = Date.now() const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ include.memberOrganizations - ? fetchOrganizationData(qx, memberOrganizations) + ? (async () => { + const start = Date.now() + const result = await fetchOrganizationData(qx, memberOrganizations) + log.info(`[PERF] fetchOrganizationData took: ${Date.now() - start}ms`) + return result + })() : Promise.resolve({ orgs: [], lfx: [] }), - include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), + include.segments + ? (async () => { + const start = Date.now() + const result = await fetchSegmentData(qx, memberSegments) + log.info(`[PERF] fetchSegmentData took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), include.maintainers && maintainerRoles.length > 0 - ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) + ? (async () => { + const start = Date.now() + const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) + const result = await fetchManySegments(qx, segmentIds) + log.info( + `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, + ) + return result + })() : Promise.resolve([]), ]) + const secondBatchDuration = Date.now() - secondBatchStartTime + log.info(`[PERF] Second parallel batch took: ${secondBatchDuration}ms`) + + // Data processing section + const processingStartTime = Date.now() if (include.memberOrganizations) { + const orgProcessingStart = Date.now() const { orgs = [], lfx = [] } = orgExtra for (const member of rows) { @@ -530,9 +697,11 @@ export async function queryMembersAdvanced( } } } + log.info(`[PERF] Member organizations processing took: ${Date.now() - orgProcessingStart}ms`) } if (include.segments) { + const segmentProcessingStart = Date.now() const segments = segmentsInfo || [] rows.forEach((member) => { @@ -552,9 +721,11 @@ export async function queryMembersAdvanced( }) .filter(Boolean) }) + log.info(`[PERF] Segments processing took: ${Date.now() - segmentProcessingStart}ms`) } if (include.maintainers) { + const maintainerProcessingStart = Date.now() const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) rows.forEach((member) => { member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { @@ -565,9 +736,11 @@ export async function queryMembersAdvanced( } }) }) + log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) } if (include.identities) { + const identityProcessingStart = Date.now() rows.forEach((member) => { const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] @@ -578,8 +751,18 @@ export async function queryMembersAdvanced( verified: identity.verified, })) }) + log.info(`[PERF] Identities processing took: ${Date.now() - identityProcessingStart}ms`) } + const processingDuration = Date.now() - processingStartTime + log.info(`[PERF] Total data processing took: ${processingDuration}ms`) + + const totalDuration = Date.now() - startTime + log.info(`[PERF] Total queryMembersAdvanced took: ${totalDuration}ms`) + log.info( + `[PERF] Breakdown - Main queries: ${mainQueryDuration}ms (${((mainQueryDuration / totalDuration) * 100).toFixed(1)}%), First batch: ${firstBatchDuration}ms (${((firstBatchDuration / totalDuration) * 100).toFixed(1)}%), Second batch: ${secondBatchDuration}ms (${((secondBatchDuration / totalDuration) * 100).toFixed(1)}%), Processing: ${processingDuration}ms (${((processingDuration / totalDuration) * 100).toFixed(1)}%)`, + ) + return { rows, count, limit, offset } } From 918ca4ad2fb9585ace82df86604c1f2fd0079928 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 10 Nov 2025 10:28:04 +0100 Subject: [PATCH 06/28] fix: temporarly remove the maintainers role --- .../data-access-layer/src/members/base.ts | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 05683d250d..79ed7fdc4b 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -6,7 +6,6 @@ import { RawQueryParser, generateUUIDv1, getProperDisplayName, - groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' import { getServiceLogger } from '@crowd/logging' @@ -21,7 +20,6 @@ import { } from '@crowd/types' import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' -import { findMaintainerRoles } from '../maintainers' import { IDbMemberCreateData, IDbMemberUpdateData, @@ -500,7 +498,8 @@ export async function queryMembersAdvanced( // ]) const firstBatchStartTime = Date.now() - const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ + // const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ + const [memberOrganizations, identities, memberSegments] = await Promise.all([ include.memberOrganizations ? (async () => { const start = Date.now() @@ -525,14 +524,14 @@ export async function queryMembersAdvanced( return result })() : Promise.resolve([]), - include.maintainers - ? (async () => { - const start = Date.now() - const result = await findMaintainerRoles(qx, memberIds) - log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), + // include.maintainers + // ? (async () => { + // const start = Date.now() + // const result = await findMaintainerRoles(qx, memberIds) + // log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) + // return result + // })() + // : Promise.resolve([]), ]) const firstBatchDuration = Date.now() - firstBatchStartTime log.info(`[PERF] First parallel batch took: ${firstBatchDuration}ms`) @@ -630,7 +629,8 @@ export async function queryMembersAdvanced( // Second parallel batch - fetch related data const secondBatchStartTime = Date.now() - const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ + // const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ + const [orgExtra, segmentsInfo] = await Promise.all([ include.memberOrganizations ? (async () => { const start = Date.now() @@ -647,17 +647,17 @@ export async function queryMembersAdvanced( return result })() : Promise.resolve([]), - include.maintainers && maintainerRoles.length > 0 - ? (async () => { - const start = Date.now() - const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) - const result = await fetchManySegments(qx, segmentIds) - log.info( - `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, - ) - return result - })() - : Promise.resolve([]), + // include.maintainers && maintainerRoles.length > 0 + // ? (async () => { + // const start = Date.now() + // const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) + // const result = await fetchManySegments(qx, segmentIds) + // log.info( + // `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, + // ) + // return result + // })() + // : Promise.resolve([]), ]) const secondBatchDuration = Date.now() - secondBatchStartTime log.info(`[PERF] Second parallel batch took: ${secondBatchDuration}ms`) @@ -724,20 +724,20 @@ export async function queryMembersAdvanced( log.info(`[PERF] Segments processing took: ${Date.now() - segmentProcessingStart}ms`) } - if (include.maintainers) { - const maintainerProcessingStart = Date.now() - const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) - rows.forEach((member) => { - member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { - const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) - return { - ...role, - segmentName: segmentInfo?.name, - } - }) - }) - log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) - } + // if (include.maintainers) { + // const maintainerProcessingStart = Date.now() + // const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) + // rows.forEach((member) => { + // member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { + // const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) + // return { + // ...role, + // segmentName: segmentInfo?.name, + // } + // }) + // }) + // log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) + // } if (include.identities) { const identityProcessingStart = Date.now() From dcf8a344ea3583b9ba11037b8eff50f2106acafe Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 10 Nov 2025 10:42:26 +0100 Subject: [PATCH 07/28] fix: readd the maintainers role --- .../data-access-layer/src/members/base.ts | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 79ed7fdc4b..05683d250d 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -6,6 +6,7 @@ import { RawQueryParser, generateUUIDv1, getProperDisplayName, + groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' import { getServiceLogger } from '@crowd/logging' @@ -20,6 +21,7 @@ import { } from '@crowd/types' import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' +import { findMaintainerRoles } from '../maintainers' import { IDbMemberCreateData, IDbMemberUpdateData, @@ -498,8 +500,7 @@ export async function queryMembersAdvanced( // ]) const firstBatchStartTime = Date.now() - // const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ - const [memberOrganizations, identities, memberSegments] = await Promise.all([ + const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ include.memberOrganizations ? (async () => { const start = Date.now() @@ -524,14 +525,14 @@ export async function queryMembersAdvanced( return result })() : Promise.resolve([]), - // include.maintainers - // ? (async () => { - // const start = Date.now() - // const result = await findMaintainerRoles(qx, memberIds) - // log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) - // return result - // })() - // : Promise.resolve([]), + include.maintainers + ? (async () => { + const start = Date.now() + const result = await findMaintainerRoles(qx, memberIds) + log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) + return result + })() + : Promise.resolve([]), ]) const firstBatchDuration = Date.now() - firstBatchStartTime log.info(`[PERF] First parallel batch took: ${firstBatchDuration}ms`) @@ -629,8 +630,7 @@ export async function queryMembersAdvanced( // Second parallel batch - fetch related data const secondBatchStartTime = Date.now() - // const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ - const [orgExtra, segmentsInfo] = await Promise.all([ + const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ include.memberOrganizations ? (async () => { const start = Date.now() @@ -647,17 +647,17 @@ export async function queryMembersAdvanced( return result })() : Promise.resolve([]), - // include.maintainers && maintainerRoles.length > 0 - // ? (async () => { - // const start = Date.now() - // const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) - // const result = await fetchManySegments(qx, segmentIds) - // log.info( - // `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, - // ) - // return result - // })() - // : Promise.resolve([]), + include.maintainers && maintainerRoles.length > 0 + ? (async () => { + const start = Date.now() + const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) + const result = await fetchManySegments(qx, segmentIds) + log.info( + `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, + ) + return result + })() + : Promise.resolve([]), ]) const secondBatchDuration = Date.now() - secondBatchStartTime log.info(`[PERF] Second parallel batch took: ${secondBatchDuration}ms`) @@ -724,20 +724,20 @@ export async function queryMembersAdvanced( log.info(`[PERF] Segments processing took: ${Date.now() - segmentProcessingStart}ms`) } - // if (include.maintainers) { - // const maintainerProcessingStart = Date.now() - // const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) - // rows.forEach((member) => { - // member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { - // const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) - // return { - // ...role, - // segmentName: segmentInfo?.name, - // } - // }) - // }) - // log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) - // } + if (include.maintainers) { + const maintainerProcessingStart = Date.now() + const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) + rows.forEach((member) => { + member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { + const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) + return { + ...role, + segmentName: segmentInfo?.name, + } + }) + }) + log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) + } if (include.identities) { const identityProcessingStart = Date.now() From b907e2439a7a922acd460c373531bf7bd2ee8620 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 10 Nov 2025 16:40:48 +0100 Subject: [PATCH 08/28] refactor: create new build query --- .../data-access-layer/src/members/base.ts | 291 ++++++++++++++---- 1 file changed, 236 insertions(+), 55 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 05683d250d..f4fcc20501 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -132,8 +132,7 @@ export const MEMBER_INSERT_COLUMNS = [ ] const QUERY_FILTER_COLUMN_MAP: Map = new Map([ - ['activityCount', { name: 'coalesce(msa."activityCount", 0)::integer' }], - ['activityCount', { name: 'coalesce(msa."activityCount", 0)::integer' }], + ['activityCount', { name: 'msa."activityCount"' }], ['attributes', { name: 'm.attributes' }], ['averageSentiment', { name: 'coalesce(msa."averageSentiment", 0)::decimal' }], ['displayName', { name: 'm."displayName"' }], @@ -151,26 +150,6 @@ const QUERY_FILTER_COLUMN_MAP: Map { - const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' - - if (!orderBy || typeof orderBy !== 'string' || !orderBy.length) { - return defaultOrder - } - - const [fieldName, direction = 'DESC'] = orderBy.split('_') - const orderField = QUERY_FILTER_COLUMN_MAP.get(fieldName)?.name - - if (!orderField) { - return defaultOrder - } - - const orderDirection = ['DESC', 'ASC'].includes(direction) ? direction : 'DESC' - return `${orderField} ${orderDirection}` -} - const buildSearchCTE = ( search: string, ): { cte: string; join: string; params: Record } => { @@ -190,7 +169,8 @@ const buildSearchCTE = ( (mi.verified = true AND mi.type = $(emailType) AND LOWER(mi."value") LIKE $(searchPattern)) OR LOWER(m."displayName") LIKE $(searchPattern) ) - )`, + ) + `, join: `INNER JOIN member_search ms ON ms."memberId" = m.id`, params: { emailType: MemberIdentityType.EMAIL, @@ -210,33 +190,236 @@ const buildMemberOrgsCTE = (includeMemberOrgs: boolean): string => { FROM "memberOrganizations" WHERE "deletedAt" IS NULL GROUP BY "memberId" - )` + ) + ` +} + +type OrderDirection = 'ASC' | 'DESC' + +interface SearchConfig { + cte: string + join: string +} + +interface BuildQueryArgs { + fields: string + withAggregates: boolean + includeMemberOrgs: boolean + searchConfig: SearchConfig + filterString: string + orderBy?: string // e.g. "activityCount_DESC", "score_ASC", "joinedAt" + orderDirection?: OrderDirection + limit?: number + offset?: number +} + +const ORDER_FIELD_MAP: Record = { + activityCount: 'msa."activityCount"', + score: 'm."score"', + joinedAt: 'm."joinedAt"', + displayName: 'm."displayName"', +} + +const parseOrderBy = ( + orderBy: string | undefined, + fallbackDirection: OrderDirection, +): { field?: string; direction: OrderDirection } => { + if (!orderBy || !orderBy.trim()) { + return { field: undefined, direction: fallbackDirection } + } + + const [rawField, rawDir] = orderBy.trim().split('_') + const field = rawField?.trim() || undefined + + const dir = (rawDir || '').toUpperCase() + const direction: OrderDirection = + dir === 'ASC' || dir === 'DESC' + ? (dir as OrderDirection) + : fallbackDirection + + return { field, direction } } -const buildQuery = ( - fields: string, +const getOrderClause = ( + parsedField: string | undefined, + direction: OrderDirection, withAggregates: boolean, - includeMemberOrgs: boolean, - searchConfig: { cte: string; join: string }, - filterString: string, ): string => { - const ctes = [buildMemberOrgsCTE(includeMemberOrgs), searchConfig.cte].filter(Boolean) + const defaultOrder = withAggregates + ? 'msa."activityCount" DESC' + : 'm."joinedAt" DESC' + + if (!parsedField) return defaultOrder + + const fieldExpr = ORDER_FIELD_MAP[parsedField] + if (!fieldExpr) return defaultOrder + + return `${fieldExpr} ${direction}` +} + +const buildQuery = ({ + fields, + withAggregates, + includeMemberOrgs, + searchConfig, + filterString, + orderBy, + orderDirection, + limit = 20, + offset = 0, +}: BuildQueryArgs): string => { + const fallbackDir: OrderDirection = orderDirection || 'DESC' + const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) + + // Detect if filters reference extra aliases. + const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') + + // If filter references mo.*, we must ensure member_orgs is joined. + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + // Optimized path is only safe if: + // - withAggregates is true + // - sort is by activityCount (or default) + // - filter does NOT reference mo. or me. (those aliases do not exist in top_members) + const useActivityCountOptimized = + withAggregates && + !filterHasMo && + !filterHasMe && + (!sortField || sortField === 'activityCount') + + if (useActivityCountOptimized) { + log.info(`Using optimized activityCount path`) + const ctes: string[] = [] + + // For optimized path: + // - We MAY include member_orgs CTE only if includeMemberOrgs is true. + // - But filterString is guaranteed not to reference mo/me here. + if (includeMemberOrgs) { + const memberOrgsCTE = buildMemberOrgsCTE(true) + ctes.push(memberOrgsCTE.trim()) + } + + if (searchConfig.cte) { + ctes.push(searchConfig.cte.trim()) + } + + const searchJoinInTopMembers = searchConfig.join + ? `\n ${searchConfig.join}` // INNER JOIN member_search ms ON ms."memberId" = m.id + : '' + + ctes.push(` + top_members AS ( + SELECT + msa."memberId" + FROM "memberSegmentsAgg" msa + JOIN members m ON m.id = msa."memberId" + ${searchJoinInTopMembers} + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} OFFSET ${offset} + ) + `.trim()) + + const withClause = `WITH ${ctes.join(',\n')}` + + const memberOrgsJoin = includeMemberOrgs + ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` + : '' + + return ` + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m + ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + ${memberOrgsJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + `.trim() + } + else { + log.info(`Not using optimized activityCount path`) + } + + // Fallback path: any case that is not safe/eligible for optimization. + // Here we MUST align joins with what filterString references. + const baseCtes = [ + needsMemberOrgs ? buildMemberOrgsCTE(true) : '', + searchConfig.cte, + ].filter(Boolean) const joins = [ withAggregates ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` : '', - includeMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + needsMemberOrgs + ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` + : '', `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, searchConfig.join, ].filter(Boolean) + const orderClause = getOrderClause(sortField, direction, withAggregates) + return ` - ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} + ${baseCtes.length > 0 ? `WITH ${baseCtes.join(',\n')}` : ''} SELECT ${fields} FROM members m ${joins.join('\n')} WHERE (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +interface BuildCountQueryArgs { + withAggregates: boolean + searchConfig: SearchConfig + filterString: string + includeMemberOrgs?: boolean +} + +const buildCountQuery = ({ + withAggregates, + searchConfig, + filterString, + includeMemberOrgs = false, +}: BuildCountQueryArgs): string => { + const filterHasMo = filterString.includes('mo.') + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + const ctes = [ + needsMemberOrgs ? buildMemberOrgsCTE(true) : '', + searchConfig.cte, + ].filter(Boolean) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + needsMemberOrgs + ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` + : '', + searchConfig.join, + ].filter(Boolean) + + return ` + ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} + SELECT COUNT(DISTINCT m.id) AS count + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) `.trim() } @@ -339,7 +522,7 @@ export async function queryMembersAdvanced( search = null, limit = 20, offset = 0, - orderBy = 'joinedAt_DESC', + orderBy = 'activityCount_DESC', segmentId = undefined, countOnly = false, fields = [...QUERY_FILTER_COLUMN_MAP.keys()], @@ -361,6 +544,7 @@ export async function queryMembersAdvanced( attributeSettings = [] as IDbMemberAttributeSetting[], }, ): Promise> { + const startTime = Date.now() const withAggregates = !!segmentId @@ -405,13 +589,12 @@ export async function queryMembersAdvanced( ) // Build queries - const countQuery = buildQuery( - 'COUNT(*) as count', - withAggregates, - include.memberOrganizations, - searchConfig, - filterString, - ) + const countQuery = buildCountQuery({ + withAggregates, + searchConfig, + filterString, + includeMemberOrgs: include.memberOrganizations, +}) if (countOnly) { const result = await qx.selectOne(countQuery, params) @@ -443,21 +626,19 @@ export async function queryMembersAdvanced( .join(',\n') log.info(`[PERF] Field preparation took: ${Date.now() - fieldsStartTime}ms`) - const mainQuery = ` - ${buildQuery( - preparedFields, - withAggregates, - include.memberOrganizations, - searchConfig, - filterString, - )} - ORDER BY ${getOrderClause(orderBy, withAggregates)} NULLS LAST - LIMIT $(limit) - OFFSET $(offset) - ` - - log.info(`main query: ${formatSql(mainQuery, params)}`) - log.info(`count query: ${formatSql(countQuery, params)}`) + const mainQuery = buildQuery({ + fields: preparedFields, + withAggregates, // true when you need memberSegmentsAgg + includeMemberOrgs: include.memberOrganizations, + searchConfig, + filterString, + orderBy, // e.g. 'activityCount' | 'score' | 'joinedAt' + limit, + offset, +}) + + // log.info(`main query: ${formatSql(mainQuery, params)}`) + // log.info(`count query: ${formatSql(countQuery, params)}`) // Execute queries in parallel const mainQueryStartTime = Date.now() From 1f1a38dcbe7e3e3077a825f7db622ed385a72b9b Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 10 Nov 2025 17:08:58 +0100 Subject: [PATCH 09/28] fix: lint --- .../data-access-layer/src/members/base.ts | 76 +++++++------------ 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index f4fcc20501..2ff1abeab0 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -207,7 +207,7 @@ interface BuildQueryArgs { includeMemberOrgs: boolean searchConfig: SearchConfig filterString: string - orderBy?: string // e.g. "activityCount_DESC", "score_ASC", "joinedAt" + orderBy?: string orderDirection?: OrderDirection limit?: number offset?: number @@ -233,9 +233,7 @@ const parseOrderBy = ( const dir = (rawDir || '').toUpperCase() const direction: OrderDirection = - dir === 'ASC' || dir === 'DESC' - ? (dir as OrderDirection) - : fallbackDirection + dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : fallbackDirection return { field, direction } } @@ -245,9 +243,7 @@ const getOrderClause = ( direction: OrderDirection, withAggregates: boolean, ): string => { - const defaultOrder = withAggregates - ? 'msa."activityCount" DESC' - : 'm."joinedAt" DESC' + const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' if (!parsedField) return defaultOrder @@ -283,13 +279,10 @@ const buildQuery = ({ // - sort is by activityCount (or default) // - filter does NOT reference mo. or me. (those aliases do not exist in top_members) const useActivityCountOptimized = - withAggregates && - !filterHasMo && - !filterHasMe && - (!sortField || sortField === 'activityCount') + withAggregates && !filterHasMo && !filterHasMe && (!sortField || sortField === 'activityCount') + log.info(`buildQuery: useActivityCountOptimized=${useActivityCountOptimized}`) if (useActivityCountOptimized) { - log.info(`Using optimized activityCount path`) const ctes: string[] = [] // For optimized path: @@ -308,7 +301,8 @@ const buildQuery = ({ ? `\n ${searchConfig.join}` // INNER JOIN member_search ms ON ms."memberId" = m.id : '' - ctes.push(` + ctes.push( + ` top_members AS ( SELECT msa."memberId" @@ -322,7 +316,8 @@ const buildQuery = ({ msa."activityCount" ${direction} NULLS LAST LIMIT ${limit} OFFSET ${offset} ) - `.trim()) + `.trim(), + ) const withClause = `WITH ${ctes.join(',\n')}` @@ -347,24 +342,17 @@ const buildQuery = ({ msa."activityCount" ${direction} NULLS LAST `.trim() } - else { - log.info(`Not using optimized activityCount path`) - } // Fallback path: any case that is not safe/eligible for optimization. - // Here we MUST align joins with what filterString references. - const baseCtes = [ - needsMemberOrgs ? buildMemberOrgsCTE(true) : '', - searchConfig.cte, - ].filter(Boolean) + const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( + Boolean, + ) const joins = [ withAggregates ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` : '', - needsMemberOrgs - ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` - : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, searchConfig.join, ].filter(Boolean) @@ -399,18 +387,13 @@ const buildCountQuery = ({ const filterHasMo = filterString.includes('mo.') const needsMemberOrgs = includeMemberOrgs || filterHasMo - const ctes = [ - needsMemberOrgs ? buildMemberOrgsCTE(true) : '', - searchConfig.cte, - ].filter(Boolean) + const ctes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter(Boolean) const joins = [ withAggregates ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` : '', - needsMemberOrgs - ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` - : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', searchConfig.join, ].filter(Boolean) @@ -544,7 +527,6 @@ export async function queryMembersAdvanced( attributeSettings = [] as IDbMemberAttributeSetting[], }, ): Promise> { - const startTime = Date.now() const withAggregates = !!segmentId @@ -590,11 +572,11 @@ export async function queryMembersAdvanced( // Build queries const countQuery = buildCountQuery({ - withAggregates, - searchConfig, - filterString, - includeMemberOrgs: include.memberOrganizations, -}) + withAggregates, + searchConfig, + filterString, + includeMemberOrgs: include.memberOrganizations, + }) if (countOnly) { const result = await qx.selectOne(countQuery, params) @@ -627,15 +609,15 @@ export async function queryMembersAdvanced( log.info(`[PERF] Field preparation took: ${Date.now() - fieldsStartTime}ms`) const mainQuery = buildQuery({ - fields: preparedFields, - withAggregates, // true when you need memberSegmentsAgg - includeMemberOrgs: include.memberOrganizations, - searchConfig, - filterString, - orderBy, // e.g. 'activityCount' | 'score' | 'joinedAt' - limit, - offset, -}) + fields: preparedFields, + withAggregates, // true when you need memberSegmentsAgg + includeMemberOrgs: include.memberOrganizations, + searchConfig, + filterString, + orderBy, // e.g. 'activityCount' | 'score' | 'joinedAt' + limit, + offset, + }) // log.info(`main query: ${formatSql(mainQuery, params)}`) // log.info(`count query: ${formatSql(countQuery, params)}`) From 06b2030ef66b09ab8c9d6073c78dcb96afb5e62a Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Mon, 10 Nov 2025 17:37:48 +0100 Subject: [PATCH 10/28] fix: refactor in different files --- .../data-access-layer/src/members/base.ts | 605 +----------------- .../src/members/dataProcessor.ts | 129 ++++ .../src/members/queryBuilder.ts | 260 ++++++++ 3 files changed, 402 insertions(+), 592 deletions(-) create mode 100644 services/libs/data-access-layer/src/members/dataProcessor.ts create mode 100644 services/libs/data-access-layer/src/members/queryBuilder.ts diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 2ff1abeab0..4ce5090676 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -9,64 +9,25 @@ import { groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' -import { getServiceLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' -import { - ALL_PLATFORM_TYPES, - MemberAttributeType, - MemberIdentityType, - PageData, - SegmentData, - SegmentType, -} from '@crowd/types' +import { ALL_PLATFORM_TYPES, MemberAttributeType, PageData, SegmentType } from '@crowd/types' -import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' import { findMaintainerRoles } from '../maintainers' import { IDbMemberCreateData, IDbMemberUpdateData, } from '../old/apps/data_sink_worker/repo/member.data' -import { OrganizationField, queryOrgs } from '../organizations' import { QueryExecutor } from '../queryExecutor' import { fetchManySegments } from '../segments' import { QueryOptions, QueryResult, queryTable, queryTableById } from '../utils' import { getMemberAttributeSettings } from './attributeSettings' +import { fetchOrganizationData, fetchSegmentData, sortActiveOrganizations } from './dataProcessor' +import { buildCountQuery, buildQuery, buildSearchCTE } from './queryBuilder' import { IDbMemberAttributeSetting, IDbMemberData } from './types' import { fetchManyMemberIdentities, fetchManyMemberOrgs, fetchManyMemberSegments } from '.' -const log = getServiceLogger() -interface MemberOrganization { - id: string - organizationId: string - dateStart?: string - dateEnd?: string - affiliationOverride?: { - isPrimaryWorkExperience?: boolean - } -} - -interface MemberOrganizationData { - memberId: string - organizations: MemberOrganization[] -} - -interface OrganizationInfo { - id: string - displayName: string - logo: string - createdAt: string -} - -interface MemberSegmentData { - memberId: string - segments: Array<{ - segmentId: string - activityCount: number - }> -} - export enum MemberField { ATTRIBUTES = 'attributes', CONTRIBUTIONS = 'contributions', @@ -150,353 +111,6 @@ const QUERY_FILTER_COLUMN_MAP: Map } => { - if (!search?.trim()) { - return { cte: '', join: '', params: {} } - } - - const searchTerm = search.toLowerCase().trim() - - return { - cte: ` - member_search AS ( - SELECT DISTINCT mi."memberId" - FROM "memberIdentities" mi - INNER JOIN members m ON m.id = mi."memberId" - WHERE ( - (mi.verified = true AND mi.type = $(emailType) AND LOWER(mi."value") LIKE $(searchPattern)) - OR LOWER(m."displayName") LIKE $(searchPattern) - ) - ) - `, - join: `INNER JOIN member_search ms ON ms."memberId" = m.id`, - params: { - emailType: MemberIdentityType.EMAIL, - searchPattern: `%${searchTerm}%`, - }, - } -} - -const buildMemberOrgsCTE = (includeMemberOrgs: boolean): string => { - if (!includeMemberOrgs) return '' - - return ` - member_orgs AS ( - SELECT - "memberId", - ARRAY_AGG("organizationId"::TEXT) AS "organizationId" - FROM "memberOrganizations" - WHERE "deletedAt" IS NULL - GROUP BY "memberId" - ) - ` -} - -type OrderDirection = 'ASC' | 'DESC' - -interface SearchConfig { - cte: string - join: string -} - -interface BuildQueryArgs { - fields: string - withAggregates: boolean - includeMemberOrgs: boolean - searchConfig: SearchConfig - filterString: string - orderBy?: string - orderDirection?: OrderDirection - limit?: number - offset?: number -} - -const ORDER_FIELD_MAP: Record = { - activityCount: 'msa."activityCount"', - score: 'm."score"', - joinedAt: 'm."joinedAt"', - displayName: 'm."displayName"', -} - -const parseOrderBy = ( - orderBy: string | undefined, - fallbackDirection: OrderDirection, -): { field?: string; direction: OrderDirection } => { - if (!orderBy || !orderBy.trim()) { - return { field: undefined, direction: fallbackDirection } - } - - const [rawField, rawDir] = orderBy.trim().split('_') - const field = rawField?.trim() || undefined - - const dir = (rawDir || '').toUpperCase() - const direction: OrderDirection = - dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : fallbackDirection - - return { field, direction } -} - -const getOrderClause = ( - parsedField: string | undefined, - direction: OrderDirection, - withAggregates: boolean, -): string => { - const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' - - if (!parsedField) return defaultOrder - - const fieldExpr = ORDER_FIELD_MAP[parsedField] - if (!fieldExpr) return defaultOrder - - return `${fieldExpr} ${direction}` -} - -const buildQuery = ({ - fields, - withAggregates, - includeMemberOrgs, - searchConfig, - filterString, - orderBy, - orderDirection, - limit = 20, - offset = 0, -}: BuildQueryArgs): string => { - const fallbackDir: OrderDirection = orderDirection || 'DESC' - const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) - - // Detect if filters reference extra aliases. - const filterHasMo = filterString.includes('mo.') - const filterHasMe = filterString.includes('me.') - - // If filter references mo.*, we must ensure member_orgs is joined. - const needsMemberOrgs = includeMemberOrgs || filterHasMo - - // Optimized path is only safe if: - // - withAggregates is true - // - sort is by activityCount (or default) - // - filter does NOT reference mo. or me. (those aliases do not exist in top_members) - const useActivityCountOptimized = - withAggregates && !filterHasMo && !filterHasMe && (!sortField || sortField === 'activityCount') - - log.info(`buildQuery: useActivityCountOptimized=${useActivityCountOptimized}`) - if (useActivityCountOptimized) { - const ctes: string[] = [] - - // For optimized path: - // - We MAY include member_orgs CTE only if includeMemberOrgs is true. - // - But filterString is guaranteed not to reference mo/me here. - if (includeMemberOrgs) { - const memberOrgsCTE = buildMemberOrgsCTE(true) - ctes.push(memberOrgsCTE.trim()) - } - - if (searchConfig.cte) { - ctes.push(searchConfig.cte.trim()) - } - - const searchJoinInTopMembers = searchConfig.join - ? `\n ${searchConfig.join}` // INNER JOIN member_search ms ON ms."memberId" = m.id - : '' - - ctes.push( - ` - top_members AS ( - SELECT - msa."memberId" - FROM "memberSegmentsAgg" msa - JOIN members m ON m.id = msa."memberId" - ${searchJoinInTopMembers} - WHERE - msa."segmentId" = $(segmentId) - AND (${filterString}) - ORDER BY - msa."activityCount" ${direction} NULLS LAST - LIMIT ${limit} OFFSET ${offset} - ) - `.trim(), - ) - - const withClause = `WITH ${ctes.join(',\n')}` - - const memberOrgsJoin = includeMemberOrgs - ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` - : '' - - return ` - ${withClause} - SELECT ${fields} - FROM top_members tm - JOIN members m - ON m.id = tm."memberId" - INNER JOIN "memberSegmentsAgg" msa - ON msa."memberId" = m.id - AND msa."segmentId" = $(segmentId) - ${memberOrgsJoin} - LEFT JOIN "memberEnrichments" me - ON me."memberId" = m.id - WHERE (${filterString}) - ORDER BY - msa."activityCount" ${direction} NULLS LAST - `.trim() - } - - // Fallback path: any case that is not safe/eligible for optimization. - const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( - Boolean, - ) - - const joins = [ - withAggregates - ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` - : '', - needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', - `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, - searchConfig.join, - ].filter(Boolean) - - const orderClause = getOrderClause(sortField, direction, withAggregates) - - return ` - ${baseCtes.length > 0 ? `WITH ${baseCtes.join(',\n')}` : ''} - SELECT ${fields} - FROM members m - ${joins.join('\n')} - WHERE (${filterString}) - ORDER BY ${orderClause} NULLS LAST - LIMIT ${limit} - OFFSET ${offset} - `.trim() -} - -interface BuildCountQueryArgs { - withAggregates: boolean - searchConfig: SearchConfig - filterString: string - includeMemberOrgs?: boolean -} - -const buildCountQuery = ({ - withAggregates, - searchConfig, - filterString, - includeMemberOrgs = false, -}: BuildCountQueryArgs): string => { - const filterHasMo = filterString.includes('mo.') - const needsMemberOrgs = includeMemberOrgs || filterHasMo - - const ctes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter(Boolean) - - const joins = [ - withAggregates - ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` - : '', - needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', - searchConfig.join, - ].filter(Boolean) - - return ` - ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} - SELECT COUNT(DISTINCT m.id) AS count - FROM members m - ${joins.join('\n')} - WHERE (${filterString}) - `.trim() -} - -const sortActiveOrganizations = ( - activeOrgs: MemberOrganization[], - organizationsInfo: OrganizationInfo[], -): MemberOrganization[] => { - return activeOrgs.sort((a, b) => { - if (!a || !b) return 0 - - // First priority: isPrimaryWorkExperience - const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true - const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true - - if (aPrimary !== bPrimary) return aPrimary ? -1 : 1 - - // Second priority: has dateStart - const aHasDate = !!a.dateStart - const bHasDate = !!b.dateStart - - if (aHasDate !== bHasDate) return aHasDate ? -1 : 1 - - // Third priority: createdAt and alphabetical - if (!a.dateStart && !b.dateStart) { - const aOrgInfo = organizationsInfo.find((odn) => odn.id === a.organizationId) - const bOrgInfo = organizationsInfo.find((odn) => odn.id === b.organizationId) - - const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 - const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 - - if (aCreatedAt !== bCreatedAt) return bCreatedAt - aCreatedAt - - const aName = (aOrgInfo?.displayName || '').toLowerCase() - const bName = (bOrgInfo?.displayName || '').toLowerCase() - return aName.localeCompare(bName) - } - - return 0 - }) -} - -const fetchOrganizationData = async ( - qx: QueryExecutor, - memberOrganizations: MemberOrganizationData[], -): Promise<{ orgs: OrganizationInfo[]; lfx: LfxMembership[] }> => { - if (memberOrganizations.length === 0) { - return { orgs: [], lfx: [] } - } - - const orgIds = uniq( - memberOrganizations.reduce((acc, mo) => { - acc.push(...mo.organizations.map((o) => o.organizationId)) - return acc - }, []), - ) - - if (orgIds.length === 0) { - return { orgs: [], lfx: [] } - } - - const [orgs, lfx] = await Promise.all([ - queryOrgs(qx, { - filter: { [OrganizationField.ID]: { in: orgIds } }, - fields: [ - OrganizationField.ID, - OrganizationField.DISPLAY_NAME, - OrganizationField.LOGO, - OrganizationField.CREATED_AT, - ], - }), - findManyLfxMemberships(qx, { organizationIds: orgIds }), - ]) - - return { orgs, lfx } -} - -const fetchSegmentData = async ( - qx: QueryExecutor, - memberSegments: MemberSegmentData[], -): Promise => { - if (memberSegments.length === 0) { - return [] - } - - const segmentIds = uniq( - memberSegments.reduce((acc, ms) => { - acc.push(...ms.segments.map((s) => s.segmentId)) - return acc - }, []), - ) - - return segmentIds.length > 0 ? fetchManySegments(qx, segmentIds) : [] -} - export async function queryMembersAdvanced( qx: QueryExecutor, redis: RedisClient, @@ -527,8 +141,6 @@ export async function queryMembersAdvanced( attributeSettings = [] as IDbMemberAttributeSetting[], }, ): Promise> { - const startTime = Date.now() - const withAggregates = !!segmentId const searchConfig = buildSearchCTE(search) @@ -589,7 +201,6 @@ export async function queryMembersAdvanced( } // Prepare fields for main query - const fieldsStartTime = Date.now() const preparedFields = fields .map((f) => { const mappedField = QUERY_FILTER_COLUMN_MAP.get(f) @@ -606,230 +217,52 @@ export async function queryMembersAdvanced( }) .map((mappedField) => `${mappedField.name} AS "${mappedField.alias}"`) .join(',\n') - log.info(`[PERF] Field preparation took: ${Date.now() - fieldsStartTime}ms`) const mainQuery = buildQuery({ fields: preparedFields, - withAggregates, // true when you need memberSegmentsAgg + withAggregates, includeMemberOrgs: include.memberOrganizations, searchConfig, filterString, - orderBy, // e.g. 'activityCount' | 'score' | 'joinedAt' + orderBy, limit, offset, }) - // log.info(`main query: ${formatSql(mainQuery, params)}`) - // log.info(`count query: ${formatSql(countQuery, params)}`) - // Execute queries in parallel - const mainQueryStartTime = Date.now() - const [rows, countResult] = await Promise.all([ qx.select(mainQuery, params), qx.selectOne(countQuery, params), ]) - const mainQueryDuration = Date.now() - mainQueryStartTime - log.info( - `[PERF] Main queries (parallel) took: ${mainQueryDuration}ms - returned ${rows.length} rows`, - ) - - // TODO: ci serve davvero questo filtro ? - // rows.forEach((row) => { - // if (row.attributes && typeof row.attributes === 'object') { - // const filteredAttributes = {} - // QUERY_FILTER_ATTRIBUTE_MAP.forEach((attr) => { - // if (row.attributes[attr] !== undefined) { - // filteredAttributes[attr] = row.attributes[attr] - // } - // }) - // row.attributes = filteredAttributes - // } - // }) const count = parseInt(countResult.count, 10) const memberIds = rows.map((org) => org.id) if (memberIds.length === 0) { - // TODO: if memberIds is empty the count is 0 ?, if yes we can skip the count query return { rows: [], count, limit, offset } } - // const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ - // include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), - // include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), - // include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), - // include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), - // ]) - const firstBatchStartTime = Date.now() - const [memberOrganizations, identities, memberSegments, maintainerRoles] = await Promise.all([ - include.memberOrganizations - ? (async () => { - const start = Date.now() - const result = await fetchManyMemberOrgs(qx, memberIds) - log.info(`[PERF] fetchManyMemberOrgs took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), - include.identities - ? (async () => { - const start = Date.now() - const result = await fetchManyMemberIdentities(qx, memberIds) - log.info(`[PERF] fetchManyMemberIdentities took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), - include.segments - ? (async () => { - const start = Date.now() - const result = await fetchManyMemberSegments(qx, memberIds) - log.info(`[PERF] fetchManyMemberSegments took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), - include.maintainers - ? (async () => { - const start = Date.now() - const result = await findMaintainerRoles(qx, memberIds) - log.info(`[PERF] findMaintainerRoles took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), + include.memberOrganizations ? fetchManyMemberOrgs(qx, memberIds) : Promise.resolve([]), + include.identities ? fetchManyMemberIdentities(qx, memberIds) : Promise.resolve([]), + include.segments ? fetchManyMemberSegments(qx, memberIds) : Promise.resolve([]), + include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), ]) - const firstBatchDuration = Date.now() - firstBatchStartTime - log.info(`[PERF] First parallel batch took: ${firstBatchDuration}ms`) - - // const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ - // include.memberOrganizations - // ? fetchOrganizationData(qx, memberOrganizations) - // : Promise.resolve({ orgs: [], lfx: [] }), - // include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), - // include.maintainers && maintainerRoles.length > 0 - // ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) - // : Promise.resolve([]), - // ]) - - // if (include.memberOrganizations) { - // const { orgs = [], lfx = [] } = orgExtra - - // for (const member of rows) { - // member.organizations = [] - - // const memberOrgs = - // memberOrganizations.find((o) => o.memberId === member.id)?.organizations || [] - - // const activeOrgs = memberOrgs.filter((org) => !org.dateEnd) - - // const sortedActiveOrgs = sortActiveOrganizations(activeOrgs, orgs) - - // const activeOrg = sortedActiveOrgs[0] - - // if (activeOrg) { - // const orgInfo = orgs.find((odn) => odn.id === activeOrg.organizationId) - - // if (orgInfo) { - // const lfxMembership = lfx.find((m) => m.organizationId === activeOrg.organizationId) - // member.organizations = [ - // { - // id: activeOrg.organizationId, - // displayName: orgInfo.displayName || '', - // logo: orgInfo.logo || '', - // lfxMembership: !!lfxMembership, - // }, - // ] - // } - // } - // } - // } - - // if (include.segments) { - // const segments = segmentsInfo || [] - - // rows.forEach((member) => { - // member.segments = (memberSegments.find((i) => i.memberId === member.id)?.segments || []) - // .map((segment) => { - // const segmentInfo = segments.find((s) => s.id === segment.segmentId) - - // if (include.onlySubProjects && segmentInfo?.type !== SegmentType.SUB_PROJECT) { - // return null - // } - - // return { - // id: segment.segmentId, - // name: segmentInfo?.name, - // activityCount: segment.activityCount, - // } - // }) - // .filter(Boolean) - // }) - // } - - // if (include.maintainers) { - // const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) - // rows.forEach((member) => { - // member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { - // const segmentInfo = maintainerSegmentsInfo.find((s) => s.id === role.segmentId) - // return { - // ...role, - // segmentName: segmentInfo?.name, - // } - // }) - // }) - // } - - // if (include.identities) { - // rows.forEach((member) => { - // const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] - - // member.identities = memberIdentities.map((identity) => ({ - // type: identity.type, - // value: identity.value, - // platform: identity.platform, - // verified: identity.verified, - // })) - // }) - // } // Second parallel batch - fetch related data - const secondBatchStartTime = Date.now() const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ include.memberOrganizations - ? (async () => { - const start = Date.now() - const result = await fetchOrganizationData(qx, memberOrganizations) - log.info(`[PERF] fetchOrganizationData took: ${Date.now() - start}ms`) - return result - })() + ? fetchOrganizationData(qx, memberOrganizations) : Promise.resolve({ orgs: [], lfx: [] }), - include.segments - ? (async () => { - const start = Date.now() - const result = await fetchSegmentData(qx, memberSegments) - log.info(`[PERF] fetchSegmentData took: ${Date.now() - start}ms`) - return result - })() - : Promise.resolve([]), + include.segments ? fetchSegmentData(qx, memberSegments) : Promise.resolve([]), include.maintainers && maintainerRoles.length > 0 - ? (async () => { - const start = Date.now() - const segmentIds = uniq(maintainerRoles.map((m) => m.segmentId)) - const result = await fetchManySegments(qx, segmentIds) - log.info( - `[PERF] fetchManySegments for maintainers (${segmentIds.length} segments) took: ${Date.now() - start}ms`, - ) - return result - })() + ? fetchManySegments(qx, uniq(maintainerRoles.map((m) => m.segmentId))) : Promise.resolve([]), ]) - const secondBatchDuration = Date.now() - secondBatchStartTime - log.info(`[PERF] Second parallel batch took: ${secondBatchDuration}ms`) // Data processing section - const processingStartTime = Date.now() if (include.memberOrganizations) { - const orgProcessingStart = Date.now() const { orgs = [], lfx = [] } = orgExtra for (const member of rows) { @@ -860,7 +293,6 @@ export async function queryMembersAdvanced( } } } - log.info(`[PERF] Member organizations processing took: ${Date.now() - orgProcessingStart}ms`) } if (include.segments) { @@ -884,7 +316,6 @@ export async function queryMembersAdvanced( }) .filter(Boolean) }) - log.info(`[PERF] Segments processing took: ${Date.now() - segmentProcessingStart}ms`) } if (include.maintainers) { @@ -899,7 +330,6 @@ export async function queryMembersAdvanced( } }) }) - log.info(`[PERF] Maintainer roles processing took: ${Date.now() - maintainerProcessingStart}ms`) } if (include.identities) { @@ -914,21 +344,12 @@ export async function queryMembersAdvanced( verified: identity.verified, })) }) - log.info(`[PERF] Identities processing took: ${Date.now() - identityProcessingStart}ms`) } - const processingDuration = Date.now() - processingStartTime - log.info(`[PERF] Total data processing took: ${processingDuration}ms`) - - const totalDuration = Date.now() - startTime - log.info(`[PERF] Total queryMembersAdvanced took: ${totalDuration}ms`) - log.info( - `[PERF] Breakdown - Main queries: ${mainQueryDuration}ms (${((mainQueryDuration / totalDuration) * 100).toFixed(1)}%), First batch: ${firstBatchDuration}ms (${((firstBatchDuration / totalDuration) * 100).toFixed(1)}%), Second batch: ${secondBatchDuration}ms (${((secondBatchDuration / totalDuration) * 100).toFixed(1)}%), Processing: ${processingDuration}ms (${((processingDuration / totalDuration) * 100).toFixed(1)}%)`, - ) - return { rows, count, limit, offset } } +// ...existing code... (resto delle funzioni rimangono uguali) export async function queryMembers( qx: QueryExecutor, opts: QueryOptions, diff --git a/services/libs/data-access-layer/src/members/dataProcessor.ts b/services/libs/data-access-layer/src/members/dataProcessor.ts new file mode 100644 index 0000000000..e48d871057 --- /dev/null +++ b/services/libs/data-access-layer/src/members/dataProcessor.ts @@ -0,0 +1,129 @@ +import { uniq } from 'lodash' + +import { SegmentData } from '@crowd/types' + +import { LfxMembership, findManyLfxMemberships } from '../lfx_memberships' +import { OrganizationField, queryOrgs } from '../organizations' +import { QueryExecutor } from '../queryExecutor' +import { fetchManySegments } from '../segments' + +interface MemberOrganization { + id: string + organizationId: string + dateStart?: string + dateEnd?: string + affiliationOverride?: { + isPrimaryWorkExperience?: boolean + } +} + +interface MemberOrganizationData { + memberId: string + organizations: MemberOrganization[] +} + +interface OrganizationInfo { + id: string + displayName: string + logo: string + createdAt: string +} + +interface MemberSegmentData { + memberId: string + segments: Array<{ + segmentId: string + activityCount: number + }> +} + +export const sortActiveOrganizations = ( + activeOrgs: MemberOrganization[], + organizationsInfo: OrganizationInfo[], +): MemberOrganization[] => { + return activeOrgs.sort((a, b) => { + if (!a || !b) return 0 + + // First priority: isPrimaryWorkExperience + const aPrimary = a.affiliationOverride?.isPrimaryWorkExperience === true + const bPrimary = b.affiliationOverride?.isPrimaryWorkExperience === true + + if (aPrimary !== bPrimary) return aPrimary ? -1 : 1 + + // Second priority: has dateStart + const aHasDate = !!a.dateStart + const bHasDate = !!b.dateStart + + if (aHasDate !== bHasDate) return aHasDate ? -1 : 1 + + // Third priority: createdAt and alphabetical + if (!a.dateStart && !b.dateStart) { + const aOrgInfo = organizationsInfo.find((odn) => odn.id === a.organizationId) + const bOrgInfo = organizationsInfo.find((odn) => odn.id === b.organizationId) + + const aCreatedAt = aOrgInfo?.createdAt ? new Date(aOrgInfo.createdAt).getTime() : 0 + const bCreatedAt = bOrgInfo?.createdAt ? new Date(bOrgInfo.createdAt).getTime() : 0 + + if (aCreatedAt !== bCreatedAt) return bCreatedAt - aCreatedAt + + const aName = (aOrgInfo?.displayName || '').toLowerCase() + const bName = (bOrgInfo?.displayName || '').toLowerCase() + return aName.localeCompare(bName) + } + + return 0 + }) +} + +export const fetchOrganizationData = async ( + qx: QueryExecutor, + memberOrganizations: MemberOrganizationData[], +): Promise<{ orgs: OrganizationInfo[]; lfx: LfxMembership[] }> => { + if (memberOrganizations.length === 0) { + return { orgs: [], lfx: [] } + } + + const orgIds = uniq( + memberOrganizations.reduce((acc, mo) => { + acc.push(...mo.organizations.map((o) => o.organizationId)) + return acc + }, []), + ) + + if (orgIds.length === 0) { + return { orgs: [], lfx: [] } + } + + const [orgs, lfx] = await Promise.all([ + queryOrgs(qx, { + filter: { [OrganizationField.ID]: { in: orgIds } }, + fields: [ + OrganizationField.ID, + OrganizationField.DISPLAY_NAME, + OrganizationField.LOGO, + OrganizationField.CREATED_AT, + ], + }), + findManyLfxMemberships(qx, { organizationIds: orgIds }), + ]) + + return { orgs, lfx } +} + +export const fetchSegmentData = async ( + qx: QueryExecutor, + memberSegments: MemberSegmentData[], +): Promise => { + if (memberSegments.length === 0) { + return [] + } + + const segmentIds = uniq( + memberSegments.reduce((acc, ms) => { + acc.push(...ms.segments.map((s) => s.segmentId)) + return acc + }, []), + ) + + return segmentIds.length > 0 ? fetchManySegments(qx, segmentIds) : [] +} diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts new file mode 100644 index 0000000000..1ce5f85de4 --- /dev/null +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -0,0 +1,260 @@ +import { getServiceLogger } from '@crowd/logging' +import { MemberIdentityType } from '@crowd/types' + +const log = getServiceLogger() + +type OrderDirection = 'ASC' | 'DESC' + +interface SearchConfig { + cte: string + join: string +} + +interface BuildQueryArgs { + fields: string + withAggregates: boolean + includeMemberOrgs: boolean + searchConfig: SearchConfig + filterString: string + orderBy?: string + orderDirection?: OrderDirection + limit?: number + offset?: number +} + +interface BuildCountQueryArgs { + withAggregates: boolean + searchConfig: SearchConfig + filterString: string + includeMemberOrgs?: boolean +} + +const ORDER_FIELD_MAP: Record = { + activityCount: 'msa."activityCount"', + score: 'm."score"', + joinedAt: 'm."joinedAt"', + displayName: 'm."displayName"', +} + +export const buildSearchCTE = ( + search: string, +): { cte: string; join: string; params: Record } => { + if (!search?.trim()) { + return { cte: '', join: '', params: {} } + } + + const searchTerm = search.toLowerCase().trim() + + return { + cte: ` + member_search AS ( + SELECT DISTINCT mi."memberId" + FROM "memberIdentities" mi + INNER JOIN members m ON m.id = mi."memberId" + WHERE ( + (mi.verified = true AND mi.type = $(emailType) AND LOWER(mi."value") LIKE $(searchPattern)) + OR LOWER(m."displayName") LIKE $(searchPattern) + ) + ) + `, + join: `INNER JOIN member_search ms ON ms."memberId" = m.id`, + params: { + emailType: MemberIdentityType.EMAIL, + searchPattern: `%${searchTerm}%`, + }, + } +} + +export const buildMemberOrgsCTE = (includeMemberOrgs: boolean): string => { + if (!includeMemberOrgs) return '' + + return ` + member_orgs AS ( + SELECT + "memberId", + ARRAY_AGG("organizationId"::TEXT) AS "organizationId" + FROM "memberOrganizations" + WHERE "deletedAt" IS NULL + GROUP BY "memberId" + ) + ` +} + +const parseOrderBy = ( + orderBy: string | undefined, + fallbackDirection: OrderDirection, +): { field?: string; direction: OrderDirection } => { + if (!orderBy || !orderBy.trim()) { + return { field: undefined, direction: fallbackDirection } + } + + const [rawField, rawDir] = orderBy.trim().split('_') + const field = rawField?.trim() || undefined + + const dir = (rawDir || '').toUpperCase() + const direction: OrderDirection = + dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : fallbackDirection + + return { field, direction } +} + +const getOrderClause = ( + parsedField: string | undefined, + direction: OrderDirection, + withAggregates: boolean, +): string => { + const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' + + if (!parsedField) return defaultOrder + + const fieldExpr = ORDER_FIELD_MAP[parsedField] + if (!fieldExpr) return defaultOrder + + return `${fieldExpr} ${direction}` +} + +export const buildQuery = ({ + fields, + withAggregates, + includeMemberOrgs, + searchConfig, + filterString, + orderBy, + orderDirection, + limit = 20, + offset = 0, +}: BuildQueryArgs): string => { + const fallbackDir: OrderDirection = orderDirection || 'DESC' + const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) + + // Detect if filters reference extra aliases. + const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') + + // If filter references mo.*, we must ensure member_orgs is joined. + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + // Optimized path is only safe if: + // - withAggregates is true + // - sort is by activityCount (or default) + // - filter does NOT reference mo. or me. (those aliases do not exist in top_members) + const useActivityCountOptimized = + withAggregates && !filterHasMo && !filterHasMe && (!sortField || sortField === 'activityCount') + + log.info(`buildQuery: useActivityCountOptimized=${useActivityCountOptimized}`) + if (useActivityCountOptimized) { + const ctes: string[] = [] + + // For optimized path: + // - We MAY include member_orgs CTE only if includeMemberOrgs is true. + // - But filterString is guaranteed not to reference mo/me here. + if (includeMemberOrgs) { + const memberOrgsCTE = buildMemberOrgsCTE(true) + ctes.push(memberOrgsCTE.trim()) + } + + if (searchConfig.cte) { + ctes.push(searchConfig.cte.trim()) + } + + const searchJoinInTopMembers = searchConfig.join + ? `\n ${searchConfig.join}` // INNER JOIN member_search ms ON ms."memberId" = m.id + : '' + + ctes.push( + ` + top_members AS ( + SELECT + msa."memberId" + FROM "memberSegmentsAgg" msa + JOIN members m ON m.id = msa."memberId" + ${searchJoinInTopMembers} + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} OFFSET ${offset} + ) + `.trim(), + ) + + const withClause = `WITH ${ctes.join(',\n')}` + + const memberOrgsJoin = includeMemberOrgs + ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` + : '' + + return ` + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m + ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + ${memberOrgsJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + `.trim() + } + + // Fallback path: any case that is not safe/eligible for optimization. + const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( + Boolean, + ) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id`, + searchConfig.join, + ].filter(Boolean) + + const orderClause = getOrderClause(sortField, direction, withAggregates) + + return ` + ${baseCtes.length > 0 ? `WITH ${baseCtes.join(',\n')}` : ''} + SELECT ${fields} + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +export const buildCountQuery = ({ + withAggregates, + searchConfig, + filterString, + includeMemberOrgs = false, +}: BuildCountQueryArgs): string => { + const filterHasMo = filterString.includes('mo.') + const needsMemberOrgs = includeMemberOrgs || filterHasMo + + const ctes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter(Boolean) + + const joins = [ + withAggregates + ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '', + needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + searchConfig.join, + ].filter(Boolean) + + return ` + ${ctes.length > 0 ? `WITH ${ctes.join(',\n')}` : ''} + SELECT COUNT(DISTINCT m.id) AS count + FROM members m + ${joins.join('\n')} + WHERE (${filterString}) + `.trim() +} From 2e34107e812390c3ade26c6de0b747f89ca85601 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 11 Nov 2025 09:28:01 +0100 Subject: [PATCH 11/28] fix: remove useless comments --- services/libs/data-access-layer/src/members/base.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 4ce5090676..07a13e2b4a 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -162,7 +162,6 @@ export async function queryMembersAdvanced( ...(attributeSettings?.length > 0 ? attributeSettings : await getMemberAttributeSettings(qx, redis)), - // TODO: ci serve questo ? { name: 'jobTitle', type: MemberAttributeType.STRING, @@ -333,7 +332,6 @@ export async function queryMembersAdvanced( } if (include.identities) { - const identityProcessingStart = Date.now() rows.forEach((member) => { const memberIdentities = identities.find((i) => i.memberId === member.id)?.identities || [] From 212f3939682ce5863d26906a03d052e2aebf620b Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 11 Nov 2025 09:30:19 +0100 Subject: [PATCH 12/28] fix: remove useless comments --- services/libs/data-access-layer/src/members/base.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 07a13e2b4a..ae16bba259 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -295,7 +295,6 @@ export async function queryMembersAdvanced( } if (include.segments) { - const segmentProcessingStart = Date.now() const segments = segmentsInfo || [] rows.forEach((member) => { @@ -318,7 +317,6 @@ export async function queryMembersAdvanced( } if (include.maintainers) { - const maintainerProcessingStart = Date.now() const groupedMaintainers = groupBy(maintainerRoles, (m) => m.memberId) rows.forEach((member) => { member.maintainerRoles = (groupedMaintainers.get(member.id) || []).map((role) => { From 1ffb3869c405e36a15adae1863fa096dde09a3fc Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 11 Nov 2025 09:49:34 +0100 Subject: [PATCH 13/28] fix: add logs --- services/libs/data-access-layer/src/members/base.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index ae16bba259..4c44464314 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -9,6 +9,7 @@ import { groupBy, } from '@crowd/common' import { formatSql, getDbInstance, prepareForModification } from '@crowd/database' +import { getServiceLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' import { ALL_PLATFORM_TYPES, MemberAttributeType, PageData, SegmentType } from '@crowd/types' @@ -28,6 +29,8 @@ import { IDbMemberAttributeSetting, IDbMemberData } from './types' import { fetchManyMemberIdentities, fetchManyMemberOrgs, fetchManyMemberSegments } from '.' +const log = getServiceLogger() + export enum MemberField { ATTRIBUTES = 'attributes', CONTRIBUTIONS = 'contributions', @@ -228,6 +231,8 @@ export async function queryMembersAdvanced( offset, }) + log.info(`main query: ${mainQuery}`) + // Execute queries in parallel const [rows, countResult] = await Promise.all([ qx.select(mainQuery, params), From 1ee37f2f489612aeecaf82360fd2f8e8e595a348 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 11 Nov 2025 11:31:05 +0100 Subject: [PATCH 14/28] fix: add logs --- services/libs/data-access-layer/src/members/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 4c44464314..20102ac3ae 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -231,7 +231,7 @@ export async function queryMembersAdvanced( offset, }) - log.info(`main query: ${mainQuery}`) + log.info(`main query: ${mainQuery} with params ${JSON.stringify(params)}`) // Execute queries in parallel const [rows, countResult] = await Promise.all([ From 7b5787f1c885279b62bd67df227167f5e93b37da Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 11 Nov 2025 16:25:58 +0100 Subject: [PATCH 15/28] fix: add safer default order --- .../libs/data-access-layer/src/members/base.ts | 9 +++------ .../src/members/queryBuilder.ts | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 20102ac3ae..a94ef36f70 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -184,7 +184,6 @@ export async function queryMembersAdvanced( { pgPromiseFormat: true }, ) - // Build queries const countQuery = buildCountQuery({ withAggregates, searchConfig, @@ -212,6 +211,9 @@ export async function queryMembersAdvanced( return { alias: f, ...mappedField } }) .filter((mappedField) => mappedField.queryable !== false) + // Exclude fields from SELECT if their source table isn't joined: + // - skip msa.* when aggregates aren't included (no join with memberSegmentsAgg) + // - skip mo.* when member organizations aren't included (no join with member_orgs) .filter((mappedField) => { if (!withAggregates && mappedField.name.includes('msa.')) return false if (!include.memberOrganizations && mappedField.name.includes('mo.')) return false @@ -233,7 +235,6 @@ export async function queryMembersAdvanced( log.info(`main query: ${mainQuery} with params ${JSON.stringify(params)}`) - // Execute queries in parallel const [rows, countResult] = await Promise.all([ qx.select(mainQuery, params), qx.selectOne(countQuery, params), @@ -253,7 +254,6 @@ export async function queryMembersAdvanced( include.maintainers ? findMaintainerRoles(qx, memberIds) : Promise.resolve([]), ]) - // Second parallel batch - fetch related data const [orgExtra, segmentsInfo, maintainerSegmentsInfo] = await Promise.all([ include.memberOrganizations ? fetchOrganizationData(qx, memberOrganizations) @@ -264,8 +264,6 @@ export async function queryMembersAdvanced( : Promise.resolve([]), ]) - // Data processing section - if (include.memberOrganizations) { const { orgs = [], lfx = [] } = orgExtra @@ -350,7 +348,6 @@ export async function queryMembersAdvanced( return { rows, count, limit, offset } } -// ...existing code... (resto delle funzioni rimangono uguali) export async function queryMembers( qx: QueryExecutor, opts: QueryOptions, diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 1ce5f85de4..745d1b4aef 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -103,13 +103,27 @@ const getOrderClause = ( direction: OrderDirection, withAggregates: boolean, ): string => { + // Default sort: + // - when aggregates are included → sort by activityCount (from msa) + // - otherwise → sort by joinedAt (from members) const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' + // If no specific order field is provided, use the default one if (!parsedField) return defaultOrder const fieldExpr = ORDER_FIELD_MAP[parsedField] + + // If the requested field is not mapped, fall back to default order if (!fieldExpr) return defaultOrder + // Safety check: + // If the order field refers to msa.* but aggregates are not included, + // fallback to the default safe order instead of generating invalid SQL. + if (!withAggregates && fieldExpr.includes('msa.')) { + return defaultOrder + } + + // Return the valid ORDER BY clause return `${fieldExpr} ${direction}` } @@ -128,6 +142,7 @@ export const buildQuery = ({ const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) // Detect if filters reference extra aliases. + // TODO: capire che cosa sono questi mo. me. const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') @@ -145,9 +160,6 @@ export const buildQuery = ({ if (useActivityCountOptimized) { const ctes: string[] = [] - // For optimized path: - // - We MAY include member_orgs CTE only if includeMemberOrgs is true. - // - But filterString is guaranteed not to reference mo/me here. if (includeMemberOrgs) { const memberOrgsCTE = buildMemberOrgsCTE(true) ctes.push(memberOrgsCTE.trim()) From e11570d2e33a5e5d7499ca6dce8000128cdf0aaa Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 12 Nov 2025 10:10:34 +0100 Subject: [PATCH 16/28] fix: force using the index --- .../src/members/queryBuilder.ts | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 745d1b4aef..5e71c3eb62 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -141,62 +141,66 @@ export const buildQuery = ({ const fallbackDir: OrderDirection = orderDirection || 'DESC' const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) - // Detect if filters reference extra aliases. - // TODO: capire che cosa sono questi mo. me. + // If filter references mo.*, we must ensure member_orgs is joined in the outer query. const filterHasMo = filterString.includes('mo.') - const filterHasMe = filterString.includes('me.') - - // If filter references mo.*, we must ensure member_orgs is joined. const needsMemberOrgs = includeMemberOrgs || filterHasMo - // Optimized path is only safe if: - // - withAggregates is true - // - sort is by activityCount (or default) - // - filter does NOT reference mo. or me. (those aliases do not exist in top_members) - const useActivityCountOptimized = - withAggregates && !filterHasMo && !filterHasMe && (!sortField || sortField === 'activityCount') + // We use the optimized path when: + // - aggregates are requested (msa available) + // - sorting is by activityCount (or not specified → default) + // NOTE: we DO NOT check for mo./me. here because we keep all filters OUTSIDE the CTE. + const useActivityCountOptimized = withAggregates && (!sortField || sortField === 'activityCount') log.info(`buildQuery: useActivityCountOptimized=${useActivityCountOptimized}`) + if (useActivityCountOptimized) { const ctes: string[] = [] - if (includeMemberOrgs) { - const memberOrgsCTE = buildMemberOrgsCTE(true) - ctes.push(memberOrgsCTE.trim()) + // Include member_orgs CTE only if we need it in the OUTER query (never inside top_members) + if (needsMemberOrgs) { + ctes.push(buildMemberOrgsCTE(true).trim()) } + // Include search CTE if present if (searchConfig.cte) { ctes.push(searchConfig.cte.trim()) } - const searchJoinInTopMembers = searchConfig.join - ? `\n ${searchConfig.join}` // INNER JOIN member_search ms ON ms."memberId" = m.id + // Join search to msa WITHOUT touching members (so index on msa can be used) + const searchJoinForTopMembers = searchConfig.cte + ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' + // Oversample: fetch more rows than needed before applying outer filters, + // then apply LIMIT/OFFSET on the final ordered result. + const oversampleMultiplier = 5 + const prefetch = Math.max(limit * oversampleMultiplier + offset, limit + offset) + ctes.push( ` top_members AS ( SELECT - msa."memberId" + msa."memberId", + msa."activityCount" FROM "memberSegmentsAgg" msa - JOIN members m ON m.id = msa."memberId" - ${searchJoinInTopMembers} + ${searchJoinForTopMembers} WHERE msa."segmentId" = $(segmentId) - AND (${filterString}) ORDER BY msa."activityCount" ${direction} NULLS LAST - LIMIT ${limit} OFFSET ${offset} + LIMIT ${prefetch} ) `.trim(), ) const withClause = `WITH ${ctes.join(',\n')}` - const memberOrgsJoin = includeMemberOrgs - ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` - : '' + const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + // IMPORTANT: + // - All filters on members/orgs/enrichments are applied OUTSIDE the CTE. + // - Final ORDER BY keeps activityCount (already aligned with top_members). + // - Final LIMIT/OFFSET ensure correct pagination after applying filters. return ` ${withClause} SELECT ${fields} @@ -212,10 +216,12 @@ export const buildQuery = ({ WHERE (${filterString}) ORDER BY msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} `.trim() } - // Fallback path: any case that is not safe/eligible for optimization. + // Fallback path: any other sort mode. const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( Boolean, ) @@ -242,7 +248,6 @@ export const buildQuery = ({ OFFSET ${offset} `.trim() } - export const buildCountQuery = ({ withAggregates, searchConfig, From f60d283926a5af3320c4e36a655d88fafcd6a453 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 12 Nov 2025 17:34:35 +0100 Subject: [PATCH 17/28] fix: optimize for single choice --- .../src/members/queryBuilder.ts | 101 +++++++++++++----- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 5e71c3eb62..10b5f52e07 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -127,6 +127,28 @@ const getOrderClause = ( return `${fieldExpr} ${direction}` } +const detectPinnedMemberId = (filterString: string): { pinned: boolean; smallList: boolean } => { + if (!filterString) return { pinned: false, smallList: false } + + // m.id = '...' + const eqRe = /\bm\.id\s*=\s*(?:'[^']+'|\$\([^)]+\)|:[a-zA-Z_]\w*|\?)/i + if (eqRe.test(filterString)) return { pinned: true, smallList: true } + + // m.id IN ( ... ) → estimate list size by counting commas (rough but effective) + const inRe = /\bm\.id\s+IN\s*\(([^)]+)\)/i + const m = inRe.exec(filterString) + if (m && m[1]) { + const items = m[1] + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + // Consider "small" lists up to ~100 items; tune if needed. + return { pinned: true, smallList: items.length <= 100 } + } + + return { pinned: false, smallList: false } +} + export const buildQuery = ({ fields, withAggregates, @@ -141,38 +163,68 @@ export const buildQuery = ({ const fallbackDir: OrderDirection = orderDirection || 'DESC' const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) - // If filter references mo.*, we must ensure member_orgs is joined in the outer query. + // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') const needsMemberOrgs = includeMemberOrgs || filterHasMo - // We use the optimized path when: - // - aggregates are requested (msa available) - // - sorting is by activityCount (or not specified → default) - // NOTE: we DO NOT check for mo./me. here because we keep all filters OUTSIDE the CTE. - const useActivityCountOptimized = withAggregates && (!sortField || sortField === 'activityCount') + // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. + const { pinned, smallList } = detectPinnedMemberId(filterString) + const useDirectIdPath = withAggregates && pinned && smallList + + // Default sort clause for fallback/outer queries + const orderClause = getOrderClause(sortField, direction, withAggregates) + + if (useDirectIdPath) { + // Direct path: start from memberSegmentsAgg keyed by (memberId, segmentId) + const ctes: string[] = [] + if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) + + const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' - log.info(`buildQuery: useActivityCountOptimized=${useActivityCountOptimized}`) + const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + + // NOTE: + // - We do NOT include member_search here; an ID-pin makes it redundant. + // - We keep the full filterString (it already contains the m.id predicate). + // - This path leverages the UNIQUE (memberId, segmentId) index for O(1) lookups. + return ` + ${withClause} + SELECT ${fields} + FROM "memberSegmentsAgg" msa + JOIN members m + ON m.id = msa."memberId" + ${memberOrgsJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() + } + + // Decide if we can use the activityCount-optimized path + const useActivityCountOptimized = + withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe + // (we do allow mo.* now, but only outside the CTE; see below) if (useActivityCountOptimized) { const ctes: string[] = [] - // Include member_orgs CTE only if we need it in the OUTER query (never inside top_members) - if (needsMemberOrgs) { - ctes.push(buildMemberOrgsCTE(true).trim()) - } + // Include member_orgs CTE only for the OUTER query (never filter inside the CTE) + if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) - // Include search CTE if present - if (searchConfig.cte) { - ctes.push(searchConfig.cte.trim()) - } + // Include search CTE if present, but join it to msa inside top_members via memberId + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) - // Join search to msa WITHOUT touching members (so index on msa can be used) const searchJoinForTopMembers = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' - // Oversample: fetch more rows than needed before applying outer filters, - // then apply LIMIT/OFFSET on the final ordered result. + // Oversample to keep page filled after outer filters; tune multiplier if needed const oversampleMultiplier = 5 const prefetch = Math.max(limit * oversampleMultiplier + offset, limit + offset) @@ -194,13 +246,9 @@ export const buildQuery = ({ ) const withClause = `WITH ${ctes.join(',\n')}` - const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' - // IMPORTANT: - // - All filters on members/orgs/enrichments are applied OUTSIDE the CTE. - // - Final ORDER BY keeps activityCount (already aligned with top_members). - // - Final LIMIT/OFFSET ensure correct pagination after applying filters. + // Outer filters (including mo./me.) applied here; index handles the CTE ranking return ` ${withClause} SELECT ${fields} @@ -221,7 +269,7 @@ export const buildQuery = ({ `.trim() } - // Fallback path: any other sort mode. + // Fallback path (other sorts / non-aggregate) const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( Boolean, ) @@ -235,10 +283,8 @@ export const buildQuery = ({ searchConfig.join, ].filter(Boolean) - const orderClause = getOrderClause(sortField, direction, withAggregates) - return ` - ${baseCtes.length > 0 ? `WITH ${baseCtes.join(',\n')}` : ''} + ${baseCtes.length ? `WITH ${baseCtes.join(',\n')}` : ''} SELECT ${fields} FROM members m ${joins.join('\n')} @@ -248,6 +294,7 @@ export const buildQuery = ({ OFFSET ${offset} `.trim() } + export const buildCountQuery = ({ withAggregates, searchConfig, From 9c9297546d7e8675c293f8e1622fc94e26efa38d Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 12 Nov 2025 17:41:05 +0100 Subject: [PATCH 18/28] fix: revert logs --- services/libs/data-access-layer/src/members/queryBuilder.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 10b5f52e07..89bbf5d980 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -175,6 +175,7 @@ export const buildQuery = ({ // Default sort clause for fallback/outer queries const orderClause = getOrderClause(sortField, direction, withAggregates) + log.info(`useDirectIdPath=${useDirectIdPath}`) if (useDirectIdPath) { // Direct path: start from memberSegmentsAgg keyed by (memberId, segmentId) const ctes: string[] = [] From 2e0aee88cdff5d102df72e22d7d676f61e3fcf0d Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 13 Nov 2025 16:19:55 +0100 Subject: [PATCH 19/28] fix: cursor bugs --- .../src/members/queryBuilder.ts | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 89bbf5d980..9c231ca9db 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -103,30 +103,22 @@ const getOrderClause = ( direction: OrderDirection, withAggregates: boolean, ): string => { - // Default sort: - // - when aggregates are included → sort by activityCount (from msa) - // - otherwise → sort by joinedAt (from members) const defaultOrder = withAggregates ? 'msa."activityCount" DESC' : 'm."joinedAt" DESC' - // If no specific order field is provided, use the default one if (!parsedField) return defaultOrder const fieldExpr = ORDER_FIELD_MAP[parsedField] - // If the requested field is not mapped, fall back to default order if (!fieldExpr) return defaultOrder - // Safety check: - // If the order field refers to msa.* but aggregates are not included, - // fallback to the default safe order instead of generating invalid SQL. if (!withAggregates && fieldExpr.includes('msa.')) { return defaultOrder } - // Return the valid ORDER BY clause return `${fieldExpr} ${direction}` } +// TODO: rework const detectPinnedMemberId = (filterString: string): { pinned: boolean; smallList: boolean } => { if (!filterString) return { pinned: false, smallList: false } @@ -211,7 +203,7 @@ export const buildQuery = ({ const useActivityCountOptimized = withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe // (we do allow mo.* now, but only outside the CTE; see below) - + log.info(`useActivityCountOptimized=${useActivityCountOptimized}`) if (useActivityCountOptimized) { const ctes: string[] = [] @@ -225,9 +217,8 @@ export const buildQuery = ({ ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' - // Oversample to keep page filled after outer filters; tune multiplier if needed - const oversampleMultiplier = 5 - const prefetch = Math.max(limit * oversampleMultiplier + offset, limit + offset) + // Fix pagination: fetch enough rows to handle the requested page correctly + const totalNeeded = limit + offset ctes.push( ` @@ -241,7 +232,7 @@ export const buildQuery = ({ msa."segmentId" = $(segmentId) ORDER BY msa."activityCount" ${direction} NULLS LAST - LIMIT ${prefetch} + LIMIT ${totalNeeded} ) `.trim(), ) @@ -249,7 +240,7 @@ export const buildQuery = ({ const withClause = `WITH ${ctes.join(',\n')}` const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' - // Outer filters (including mo./me.) applied here; index handles the CTE ranking + // Outer filters (including mo./me.) applied here; remove OFFSET/LIMIT since top_members already handles it return ` ${withClause} SELECT ${fields} @@ -303,6 +294,7 @@ export const buildCountQuery = ({ includeMemberOrgs = false, }: BuildCountQueryArgs): string => { const filterHasMo = filterString.includes('mo.') + const filterHasMe = filterString.includes('me.') const needsMemberOrgs = includeMemberOrgs || filterHasMo const ctes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter(Boolean) @@ -312,6 +304,7 @@ export const buildCountQuery = ({ ? `INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` : '', needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '', + filterHasMe ? `LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id` : '', searchConfig.join, ].filter(Boolean) From 51bdaf67279eea9b20afaf936c0a36a92f39d1d3 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 13 Nov 2025 17:00:16 +0100 Subject: [PATCH 20/28] fix: pages bug --- .../src/members/queryBuilder.ts | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 9c231ca9db..bf58e41b84 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -206,59 +206,51 @@ export const buildQuery = ({ log.info(`useActivityCountOptimized=${useActivityCountOptimized}`) if (useActivityCountOptimized) { const ctes: string[] = [] - - // Include member_orgs CTE only for the OUTER query (never filter inside the CTE) if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) - - // Include search CTE if present, but join it to msa inside top_members via memberId if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) const searchJoinForTopMembers = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' - // Fix pagination: fetch enough rows to handle the requested page correctly - const totalNeeded = limit + offset + const oversampleMultiplier = 5 + const prefetch = Math.max(offset + limit * oversampleMultiplier, offset + limit) ctes.push( ` - top_members AS ( - SELECT - msa."memberId", - msa."activityCount" - FROM "memberSegmentsAgg" msa - ${searchJoinForTopMembers} - WHERE - msa."segmentId" = $(segmentId) - ORDER BY - msa."activityCount" ${direction} NULLS LAST - LIMIT ${totalNeeded} - ) - `.trim(), + top_members AS ( + SELECT + msa."memberId", + msa."activityCount", + ROW_NUMBER() OVER ( + ORDER BY msa."activityCount" ${direction} NULLS LAST, msa."memberId" ASC + ) AS rn + FROM "memberSegmentsAgg" msa + ${searchJoinForTopMembers} + WHERE msa."segmentId" = $(segmentId) + ORDER BY msa."activityCount" ${direction} NULLS LAST + LIMIT ${prefetch} + ) + `.trim(), ) const withClause = `WITH ${ctes.join(',\n')}` const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' - // Outer filters (including mo./me.) applied here; remove OFFSET/LIMIT since top_members already handles it return ` - ${withClause} - SELECT ${fields} - FROM top_members tm - JOIN members m - ON m.id = tm."memberId" - INNER JOIN "memberSegmentsAgg" msa - ON msa."memberId" = m.id - AND msa."segmentId" = $(segmentId) - ${memberOrgsJoin} - LEFT JOIN "memberEnrichments" me - ON me."memberId" = m.id - WHERE (${filterString}) - ORDER BY - msa."activityCount" ${direction} NULLS LAST - LIMIT ${limit} - OFFSET ${offset} - `.trim() + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId) + ${memberOrgsJoin} + LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id + WHERE (${filterString}) + AND tm.rn > ${offset} + ORDER BY tm.rn ASC + LIMIT ${limit} + `.trim() } // Fallback path (other sorts / non-aggregate) From 92e3cb04641a32d6e44d4786c9af2c6eb001b3d4 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 13 Nov 2025 17:11:51 +0100 Subject: [PATCH 21/28] fix: pages bug --- .../src/members/queryBuilder.ts | 84 +++++++++++-------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index bf58e41b84..5f9c1cf303 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -158,6 +158,7 @@ export const buildQuery = ({ // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') + const filterHasM = filterString.includes('m.') && !filterString.match(/\bm\.id\b/) const needsMemberOrgs = includeMemberOrgs || filterHasMo // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. @@ -199,61 +200,72 @@ export const buildQuery = ({ `.trim() } - // Decide if we can use the activityCount-optimized path + // Only use activityCount optimization if: + // 1. We have aggregates and are sorting by activityCount + // 2. No filters on member attributes, enrichments, or organizations (only segment/search filters are safe) + // 3. Only basic filters that don't reduce the result set significantly + const hasUnsafeFilters = filterHasMe || filterHasM || filterHasMo const useActivityCountOptimized = - withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe - // (we do allow mo.* now, but only outside the CTE; see below) - log.info(`useActivityCountOptimized=${useActivityCountOptimized}`) + withAggregates && + (!sortField || sortField === 'activityCount') && + !hasUnsafeFilters && + // Only allow if filterString is just basic segment/id filters or empty + (!filterString || filterString.trim() === '' || filterString.match(/^\s*1\s*=\s*1\s*$/)) + + log.info( + `useActivityCountOptimized=${useActivityCountOptimized}, hasUnsafeFilters=${hasUnsafeFilters}`, + ) + if (useActivityCountOptimized) { const ctes: string[] = [] - if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) const searchJoinForTopMembers = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' - const oversampleMultiplier = 5 - const prefetch = Math.max(offset + limit * oversampleMultiplier, offset + limit) + // Fix pagination: fetch enough rows to handle the requested page correctly + const totalNeeded = limit + offset ctes.push( ` - top_members AS ( - SELECT - msa."memberId", - msa."activityCount", - ROW_NUMBER() OVER ( - ORDER BY msa."activityCount" ${direction} NULLS LAST, msa."memberId" ASC - ) AS rn - FROM "memberSegmentsAgg" msa - ${searchJoinForTopMembers} - WHERE msa."segmentId" = $(segmentId) - ORDER BY msa."activityCount" ${direction} NULLS LAST - LIMIT ${prefetch} - ) - `.trim(), + top_members AS ( + SELECT + msa."memberId", + msa."activityCount" + FROM "memberSegmentsAgg" msa + ${searchJoinForTopMembers} + WHERE + msa."segmentId" = $(segmentId) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${totalNeeded} + ) + `.trim(), ) const withClause = `WITH ${ctes.join(',\n')}` - const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' return ` - ${withClause} - SELECT ${fields} - FROM top_members tm - JOIN members m ON m.id = tm."memberId" - INNER JOIN "memberSegmentsAgg" msa - ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId) - ${memberOrgsJoin} - LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id - WHERE (${filterString}) - AND tm.rn > ${offset} - ORDER BY tm.rn ASC - LIMIT ${limit} - `.trim() + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m + ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() } - // Fallback path (other sorts / non-aggregate) + // Fallback path (other sorts / non-aggregate / filtered queries) const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( Boolean, ) From 40a86fbf5ee4e7237a2ca835385c72162db69c71 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 11:38:30 +0100 Subject: [PATCH 22/28] fix: activity count optimized too restrictive --- .../src/members/queryBuilder.ts | 64 +++++++------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 5f9c1cf303..057b8daa4d 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -155,10 +155,18 @@ export const buildQuery = ({ const fallbackDir: OrderDirection = orderDirection || 'DESC' const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) + console.log(`filterString in buildQuery: ${filterString}`) + // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') - const filterHasM = filterString.includes('m.') && !filterString.match(/\bm\.id\b/) + const filterHasM = (() => { + if (!filterString.includes('m.')) return false + + // Remove m.id references and see if there are still m.* references + const withoutMId = filterString.replace(/\bm\.id\b/g, '') + return /\bm\.\w+/.test(withoutMId) + })() const needsMemberOrgs = includeMemberOrgs || filterHasMo // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. @@ -170,50 +178,18 @@ export const buildQuery = ({ log.info(`useDirectIdPath=${useDirectIdPath}`) if (useDirectIdPath) { - // Direct path: start from memberSegmentsAgg keyed by (memberId, segmentId) - const ctes: string[] = [] - if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) - - const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' - - const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' - - // NOTE: - // - We do NOT include member_search here; an ID-pin makes it redundant. - // - We keep the full filterString (it already contains the m.id predicate). - // - This path leverages the UNIQUE (memberId, segmentId) index for O(1) lookups. - return ` - ${withClause} - SELECT ${fields} - FROM "memberSegmentsAgg" msa - JOIN members m - ON m.id = msa."memberId" - ${memberOrgsJoin} - LEFT JOIN "memberEnrichments" me - ON me."memberId" = m.id - WHERE - msa."segmentId" = $(segmentId) - AND (${filterString}) - ORDER BY ${orderClause} NULLS LAST - LIMIT ${limit} - OFFSET ${offset} - `.trim() + // ...existing direct path code... } - // Only use activityCount optimization if: + // Use activityCount optimization if: // 1. We have aggregates and are sorting by activityCount - // 2. No filters on member attributes, enrichments, or organizations (only segment/search filters are safe) - // 3. Only basic filters that don't reduce the result set significantly - const hasUnsafeFilters = filterHasMe || filterHasM || filterHasMo + // 2. No expensive joins needed (me.*, mo.* filters) + // 3. Only m.* filters (we can handle these in the CTE) const useActivityCountOptimized = - withAggregates && - (!sortField || sortField === 'activityCount') && - !hasUnsafeFilters && - // Only allow if filterString is just basic segment/id filters or empty - (!filterString || filterString.trim() === '' || filterString.match(/^\s*1\s*=\s*1\s*$/)) + withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe && !filterHasMo log.info( - `useActivityCountOptimized=${useActivityCountOptimized}, hasUnsafeFilters=${hasUnsafeFilters}`, + `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}`, ) if (useActivityCountOptimized) { @@ -224,8 +200,10 @@ export const buildQuery = ({ ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' - // Fix pagination: fetch enough rows to handle the requested page correctly - const totalNeeded = limit + offset + // Calculate how many rows we need - be generous with filtering + const baseNeeded = limit + offset + const oversampleMultiplier = filterHasM ? 10 : 1 // 10x oversampling for m.* filters + const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k ctes.push( ` @@ -234,9 +212,11 @@ export const buildQuery = ({ msa."memberId", msa."activityCount" FROM "memberSegmentsAgg" msa + INNER JOIN members m ON m.id = msa."memberId" ${searchJoinForTopMembers} WHERE msa."segmentId" = $(segmentId) + AND (${filterString}) ORDER BY msa."activityCount" ${direction} NULLS LAST LIMIT ${totalNeeded} @@ -246,6 +226,7 @@ export const buildQuery = ({ const withClause = `WITH ${ctes.join(',\n')}` + // Outer query is much simpler now - no more filtering needed return ` ${withClause} SELECT ${fields} @@ -257,7 +238,6 @@ export const buildQuery = ({ AND msa."segmentId" = $(segmentId) LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id - WHERE (${filterString}) ORDER BY msa."activityCount" ${direction} NULLS LAST LIMIT ${limit} From 0cd20c064ea85ccffa6068dd612f96286ced7dfc Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 12:10:38 +0100 Subject: [PATCH 23/28] fix: revert direct path --- .../src/members/queryBuilder.ts | 66 +++++++++++++++++-- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 057b8daa4d..0e5730114a 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -155,8 +155,6 @@ export const buildQuery = ({ const fallbackDir: OrderDirection = orderDirection || 'DESC' const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) - console.log(`filterString in buildQuery: ${filterString}`) - // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') @@ -178,18 +176,72 @@ export const buildQuery = ({ log.info(`useDirectIdPath=${useDirectIdPath}`) if (useDirectIdPath) { - // ...existing direct path code... + // Direct path: start from memberSegmentsAgg keyed by (memberId, segmentId) + const ctes: string[] = [] + if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) + + const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' + + const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + + return ` + ${withClause} + SELECT ${fields} + FROM "memberSegmentsAgg" msa + JOIN members m + ON m.id = msa."memberId" + ${memberOrgsJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() } + // Check if filters are safe for activityCount optimization + const hasSafeFilters = (() => { + if (!filterString || filterString.trim() === '' || filterString.match(/^\s*1\s*=\s*1\s*$/)) { + return true // No filters or trivial filters + } + + // Allow filters that are likely to have good correlation with activity or are very selective + const safePatterns = [ + /\bm\.id\s*[=IN]/i, // ID filters are always safe + /\(1\s*=\s*1\)/, // Trivial conditions + /AND\s*\(1\s*=\s*1\)/, // Trivial AND conditions + /\(1\s*=\s*1\)\s*AND/, // Trivial conditions at start + ] + + // Check if filter contains only safe patterns and basic logical operators + let cleanFilter = filterString + safePatterns.forEach((pattern) => { + cleanFilter = cleanFilter.replace(pattern, '') + }) + + // Remove whitespace, parentheses, and basic logical operators + cleanFilter = cleanFilter.replace(/[\s\(\)]/g, '').replace(/\b(and|or)\b/gi, '') + + // If nothing significant remains, it's safe + return cleanFilter.length === 0 + })() + // Use activityCount optimization if: // 1. We have aggregates and are sorting by activityCount - // 2. No expensive joins needed (me.*, mo.* filters) - // 3. Only m.* filters (we can handle these in the CTE) + // 2. No unsafe joins needed (me.*, mo.* filters) + // 3. Filters are safe (mostly ID-based or trivial) const useActivityCountOptimized = - withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe && !filterHasMo + withAggregates && + (!sortField || sortField === 'activityCount') && + !filterHasMe && + !filterHasMo && + hasSafeFilters log.info( - `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}`, + `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}, hasSafeFilters=${hasSafeFilters}`, ) if (useActivityCountOptimized) { From 25489498b160374a47d48e6390b1d9d2f32fba60 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 12:16:25 +0100 Subject: [PATCH 24/28] fix: remove useless escape char --- services/libs/data-access-layer/src/members/queryBuilder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 0e5730114a..7cd0d40202 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -223,7 +223,7 @@ export const buildQuery = ({ }) // Remove whitespace, parentheses, and basic logical operators - cleanFilter = cleanFilter.replace(/[\s\(\)]/g, '').replace(/\b(and|or)\b/gi, '') + cleanFilter = cleanFilter.replace(/[\s()]/g, '').replace(/\b(and|or)\b/gi, '') // If nothing significant remains, it's safe return cleanFilter.length === 0 From 97d9a056d70e1053af2749894ea2f349fb0064bb Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 12:23:17 +0100 Subject: [PATCH 25/28] fix: remove useless escape char --- .../libs/data-access-layer/src/members/queryBuilder.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 7cd0d40202..42214c5f84 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -180,10 +180,16 @@ export const buildQuery = ({ const ctes: string[] = [] if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) + // Add search CTE if present + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + // Add search join if present + const searchJoin = searchConfig.join || '' + return ` ${withClause} SELECT ${fields} @@ -191,6 +197,7 @@ export const buildQuery = ({ JOIN members m ON m.id = msa."memberId" ${memberOrgsJoin} + ${searchJoin} LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id WHERE From 5d295efe7b5cd59675f0d380b28c1351a0d1d96a Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 12:42:26 +0100 Subject: [PATCH 26/28] fix: revert use optimized query --- .../src/members/queryBuilder.ts | 39 ++----------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 42214c5f84..8ae63b1287 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -209,46 +209,15 @@ export const buildQuery = ({ `.trim() } - // Check if filters are safe for activityCount optimization - const hasSafeFilters = (() => { - if (!filterString || filterString.trim() === '' || filterString.match(/^\s*1\s*=\s*1\s*$/)) { - return true // No filters or trivial filters - } - - // Allow filters that are likely to have good correlation with activity or are very selective - const safePatterns = [ - /\bm\.id\s*[=IN]/i, // ID filters are always safe - /\(1\s*=\s*1\)/, // Trivial conditions - /AND\s*\(1\s*=\s*1\)/, // Trivial AND conditions - /\(1\s*=\s*1\)\s*AND/, // Trivial conditions at start - ] - - // Check if filter contains only safe patterns and basic logical operators - let cleanFilter = filterString - safePatterns.forEach((pattern) => { - cleanFilter = cleanFilter.replace(pattern, '') - }) - - // Remove whitespace, parentheses, and basic logical operators - cleanFilter = cleanFilter.replace(/[\s()]/g, '').replace(/\b(and|or)\b/gi, '') - - // If nothing significant remains, it's safe - return cleanFilter.length === 0 - })() - // Use activityCount optimization if: // 1. We have aggregates and are sorting by activityCount - // 2. No unsafe joins needed (me.*, mo.* filters) - // 3. Filters are safe (mostly ID-based or trivial) + // 2. No expensive joins needed (me.*, mo.* filters) + // 3. Only m.* filters (we can handle these in the CTE) const useActivityCountOptimized = - withAggregates && - (!sortField || sortField === 'activityCount') && - !filterHasMe && - !filterHasMo && - hasSafeFilters + withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe && !filterHasMo log.info( - `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}, hasSafeFilters=${hasSafeFilters}`, + `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}`, ) if (useActivityCountOptimized) { From 153c3757b6d71f5d4bb2bbe54360fde78e14763d Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 15:03:10 +0100 Subject: [PATCH 27/28] refactor: extract functions from querybuilders --- .../src/members/queryBuilder.ts | 376 ++++++++++++------ 1 file changed, 262 insertions(+), 114 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 8ae63b1287..1098aa974f 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -118,27 +118,243 @@ const getOrderClause = ( return `${fieldExpr} ${direction}` } -// TODO: rework -const detectPinnedMemberId = (filterString: string): { pinned: boolean; smallList: boolean } => { - if (!filterString) return { pinned: false, smallList: false } - - // m.id = '...' - const eqRe = /\bm\.id\s*=\s*(?:'[^']+'|\$\([^)]+\)|:[a-zA-Z_]\w*|\?)/i - if (eqRe.test(filterString)) return { pinned: true, smallList: true } - - // m.id IN ( ... ) → estimate list size by counting commas (rough but effective) - const inRe = /\bm\.id\s+IN\s*\(([^)]+)\)/i - const m = inRe.exec(filterString) - if (m && m[1]) { - const items = m[1] +/** + * Analyzes the filter string to determine if it targets specific member IDs, + * which allows for query optimization by skipping expensive sorting operations. + * + * @param filterString - The SQL filter condition string + * @returns Object indicating if members are pinned to specific IDs and if the list is small + */ +const analyzeMemberIdTargeting = ( + filterString: string, +): { + isTargetingSpecificMembers: boolean + hasSmallMemberSet: boolean +} => { + if (!filterString?.trim()) { + return { isTargetingSpecificMembers: false, hasSmallMemberSet: false } + } + + // Check for single member ID equality: m.id = '...' or m.id = $(param) or m.id = :param + const singleIdPattern = /\bm\.id\s*=\s*(?:'[^']+'|\$\([^)]+\)|:[a-zA-Z_]\w*|\?)/i + if (singleIdPattern.test(filterString)) { + return { isTargetingSpecificMembers: true, hasSmallMemberSet: true } + } + + // Check for member ID list: m.id IN (...) + const idListPattern = /\bm\.id\s+IN\s*\(([^)]+)\)/i + const listMatch = idListPattern.exec(filterString) + + if (listMatch?.length > 1) { + // Count items in the IN clause by splitting on commas + const itemsInList = listMatch[1] .split(',') - .map((s) => s.trim()) - .filter(Boolean) - // Consider "small" lists up to ~100 items; tune if needed. - return { pinned: true, smallList: items.length <= 100 } + .map((item) => item.trim()) + .filter((item) => item.length > 0) + + // Consider lists with <= 100 items as "small" for optimization purposes + const isSmallList = itemsInList.length <= 100 + + return { isTargetingSpecificMembers: true, hasSmallMemberSet: isSmallList } } - return { pinned: false, smallList: false } + // No specific member targeting found + return { isTargetingSpecificMembers: false, hasSmallMemberSet: false } +} + +/** + * Checks if the filter string contains references to members table fields + * excluding m.id (which is handled separately for optimization purposes). + * + * @param filterString - The SQL filter condition string + * @returns true if filter uses members table fields other than m.id + */ +const hasNonIdMemberFieldReferences = (filterString: string): boolean => { + if (!filterString.includes('m.')) { + return false + } + + // Remove all m.id references from the filter string + const filterWithoutMemberIds = filterString.replace(/\bm\.id\b/g, '') + + // Check if there are still any m.* field references remaining + return /\bm\.\w+/.test(filterWithoutMemberIds) +} + +/** + * Determines if we can use the activityCount optimization strategy. + * This optimization creates a CTE with top members by activity count, + * which is much faster than sorting the entire dataset. + * + * @param withAggregates - Whether aggregates are available + * @param sortField - The field being sorted by (undefined means default activityCount) + * @param hasEnrichmentFilters - Whether filter references me.* fields + * @param hasOrganizationFilters - Whether filter references mo.* fields + * @returns true if activityCount optimization can be used + */ +const canUseActivityCountOptimization = ({ + filterHasMe, + filterHasMo, + sortField, + withAggregates, +}: { + filterHasMe: boolean + filterHasMo: boolean + sortField: string | undefined + withAggregates: boolean +}): boolean => { + // Need aggregates to access activityCount + if (!withAggregates) return false + + // Only works when sorting by activityCount (or using default sort) + if (sortField && sortField !== 'activityCount') return false + + // Cannot use if filter requires expensive joins (me.*, mo.*) + if (filterHasMe || filterHasMo) return false + + return true +} + +/** + * Builds optimized query for when we're targeting specific member IDs. + * This path starts from memberSegmentsAgg and avoids expensive sorting operations. + * + * @param fields - The SELECT fields to return + * @param filterString - The WHERE clause filter + * @param orderClause - The ORDER BY clause + * @param searchConfig - Search CTE configuration + * @param needsMemberOrgs - Whether to include member organizations + * @param limit - Query limit + * @param offset - Query offset + * @returns The optimized SQL query string + */ +const buildDirectIdPathQuery = ({ + fields, + filterString, + orderClause, + searchConfig, + needsMemberOrgs, + limit, + offset, +}: { + fields: string + filterString: string + orderClause: string + searchConfig: SearchConfig + needsMemberOrgs: boolean + limit: number + offset: number +}): string => { + // Build CTEs array + const ctes: string[] = [] + if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + + const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' + + // Build JOIN clauses + const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' + const searchJoin = searchConfig.join || '' + + return ` + ${withClause} + SELECT ${fields} + FROM "memberSegmentsAgg" msa + JOIN members m + ON m.id = msa."memberId" + ${memberOrgsJoin} + ${searchJoin} + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY ${orderClause} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() +} + +/** + * Builds optimized query using top-N activity count strategy. + * This creates a CTE with the most active members first, then joins back for full data. + * Much faster than sorting the entire member dataset. + * + * @param fields - The SELECT fields to return + * @param filterString - The WHERE clause filter + * @param searchConfig - Search CTE configuration + * @param direction - Sort direction for activityCount + * @param hasNonIdMemberFields - Whether filter uses m.* fields (requires oversampling) + * @param limit - Query limit + * @param offset - Query offset + * @returns The optimized SQL query string + */ +const buildActivityCountOptimizedQuery = ({ + fields, + filterString, + searchConfig, + direction, + hasNonIdMemberFields, + limit, + offset, +}: { + fields: string + filterString: string + searchConfig: SearchConfig + direction: OrderDirection + hasNonIdMemberFields: boolean + limit: number + offset: number +}): string => { + const ctes: string[] = [] + if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + + const searchJoinForTopMembers = searchConfig.cte + ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` + : '' + + const baseNeeded = limit + offset + const oversampleMultiplier = hasNonIdMemberFields ? 10 : 1 // 10x oversampling for m.* filters + const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k + + ctes.push( + ` + top_members AS ( + SELECT + msa."memberId", + msa."activityCount" + FROM "memberSegmentsAgg" msa + INNER JOIN members m ON m.id = msa."memberId" + ${searchJoinForTopMembers} + WHERE + msa."segmentId" = $(segmentId) + AND (${filterString}) + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${totalNeeded} + ) + `.trim(), + ) + + const withClause = `WITH ${ctes.join(',\n')}` + + // Outer query is much simpler now - no more filtering needed + return ` + ${withClause} + SELECT ${fields} + FROM top_members tm + JOIN members m + ON m.id = tm."memberId" + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + LEFT JOIN "memberEnrichments" me + ON me."memberId" = m.id + ORDER BY + msa."activityCount" ${direction} NULLS LAST + LIMIT ${limit} + OFFSET ${offset} + `.trim() } export const buildQuery = ({ @@ -158,119 +374,51 @@ export const buildQuery = ({ // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') - const filterHasM = (() => { - if (!filterString.includes('m.')) return false + const filterHasNonIdMemberFields = hasNonIdMemberFieldReferences(filterString) - // Remove m.id references and see if there are still m.* references - const withoutMId = filterString.replace(/\bm\.id\b/g, '') - return /\bm\.\w+/.test(withoutMId) - })() const needsMemberOrgs = includeMemberOrgs || filterHasMo // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. - const { pinned, smallList } = detectPinnedMemberId(filterString) - const useDirectIdPath = withAggregates && pinned && smallList + const { isTargetingSpecificMembers, hasSmallMemberSet } = analyzeMemberIdTargeting(filterString) + const useDirectIdPath = withAggregates && isTargetingSpecificMembers && hasSmallMemberSet // Default sort clause for fallback/outer queries const orderClause = getOrderClause(sortField, direction, withAggregates) log.info(`useDirectIdPath=${useDirectIdPath}`) if (useDirectIdPath) { - // Direct path: start from memberSegmentsAgg keyed by (memberId, segmentId) - const ctes: string[] = [] - if (needsMemberOrgs) ctes.push(buildMemberOrgsCTE(true).trim()) - - // Add search CTE if present - if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) - - const withClause = ctes.length ? `WITH ${ctes.join(',\n')}` : '' - - const memberOrgsJoin = needsMemberOrgs ? `LEFT JOIN member_orgs mo ON mo."memberId" = m.id` : '' - - // Add search join if present - const searchJoin = searchConfig.join || '' - - return ` - ${withClause} - SELECT ${fields} - FROM "memberSegmentsAgg" msa - JOIN members m - ON m.id = msa."memberId" - ${memberOrgsJoin} - ${searchJoin} - LEFT JOIN "memberEnrichments" me - ON me."memberId" = m.id - WHERE - msa."segmentId" = $(segmentId) - AND (${filterString}) - ORDER BY ${orderClause} NULLS LAST - LIMIT ${limit} - OFFSET ${offset} - `.trim() + return buildDirectIdPathQuery({ + fields, + filterString, + orderClause, + searchConfig, + needsMemberOrgs, + limit, + offset, + }) } - // Use activityCount optimization if: - // 1. We have aggregates and are sorting by activityCount - // 2. No expensive joins needed (me.*, mo.* filters) - // 3. Only m.* filters (we can handle these in the CTE) - const useActivityCountOptimized = - withAggregates && (!sortField || sortField === 'activityCount') && !filterHasMe && !filterHasMo + const useActivityCountOptimized = canUseActivityCountOptimization({ + filterHasMe, + filterHasMo, + sortField, + withAggregates, + }) log.info( - `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasM=${filterHasM}`, + `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasNonIdMemberFields=${filterHasNonIdMemberFields}`, ) if (useActivityCountOptimized) { - const ctes: string[] = [] - if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) - - const searchJoinForTopMembers = searchConfig.cte - ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` - : '' - - // Calculate how many rows we need - be generous with filtering - const baseNeeded = limit + offset - const oversampleMultiplier = filterHasM ? 10 : 1 // 10x oversampling for m.* filters - const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k - - ctes.push( - ` - top_members AS ( - SELECT - msa."memberId", - msa."activityCount" - FROM "memberSegmentsAgg" msa - INNER JOIN members m ON m.id = msa."memberId" - ${searchJoinForTopMembers} - WHERE - msa."segmentId" = $(segmentId) - AND (${filterString}) - ORDER BY - msa."activityCount" ${direction} NULLS LAST - LIMIT ${totalNeeded} - ) - `.trim(), - ) - - const withClause = `WITH ${ctes.join(',\n')}` - - // Outer query is much simpler now - no more filtering needed - return ` - ${withClause} - SELECT ${fields} - FROM top_members tm - JOIN members m - ON m.id = tm."memberId" - INNER JOIN "memberSegmentsAgg" msa - ON msa."memberId" = m.id - AND msa."segmentId" = $(segmentId) - LEFT JOIN "memberEnrichments" me - ON me."memberId" = m.id - ORDER BY - msa."activityCount" ${direction} NULLS LAST - LIMIT ${limit} - OFFSET ${offset} - `.trim() + return buildActivityCountOptimizedQuery({ + fields, + filterString, + searchConfig, + direction, + hasNonIdMemberFields: filterHasNonIdMemberFields, + limit, + offset, + }) } // Fallback path (other sorts / non-aggregate / filtered queries) From c65997dfa9cda4435fff0521da9e53ec468b22ed Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 14 Nov 2025 15:13:56 +0100 Subject: [PATCH 28/28] refactor: extract functions adding logs --- .../src/members/queryBuilder.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 1098aa974f..eccc408cf7 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -84,8 +84,10 @@ const parseOrderBy = ( orderBy: string | undefined, fallbackDirection: OrderDirection, ): { field?: string; direction: OrderDirection } => { + const defaultDirection: OrderDirection = fallbackDirection || 'DESC' + if (!orderBy || !orderBy.trim()) { - return { field: undefined, direction: fallbackDirection } + return { field: undefined, direction: defaultDirection } } const [rawField, rawDir] = orderBy.trim().split('_') @@ -93,7 +95,7 @@ const parseOrderBy = ( const dir = (rawDir || '').toUpperCase() const direction: OrderDirection = - dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : fallbackDirection + dir === 'ASC' || dir === 'DESC' ? (dir as OrderDirection) : defaultDirection return { field, direction } } @@ -368,14 +370,17 @@ export const buildQuery = ({ limit = 20, offset = 0, }: BuildQueryArgs): string => { - const fallbackDir: OrderDirection = orderDirection || 'DESC' - const { field: sortField, direction } = parseOrderBy(orderBy, fallbackDir) + const { field: sortField, direction } = parseOrderBy(orderBy, orderDirection) // Detect alias usage in filters (to decide joins/CTEs needed outside) const filterHasMo = filterString.includes('mo.') const filterHasMe = filterString.includes('me.') const filterHasNonIdMemberFields = hasNonIdMemberFieldReferences(filterString) + log.info( + `filterHasMo=${filterHasMo}, filterHasMe=${filterHasMe}, filterHasNonIdMemberFields=${filterHasNonIdMemberFields}`, + ) + const needsMemberOrgs = includeMemberOrgs || filterHasMo // If filters pin m.id to a single value or a small IN-list, skip top-N entirely. @@ -405,9 +410,7 @@ export const buildQuery = ({ withAggregates, }) - log.info( - `useActivityCountOptimized=${useActivityCountOptimized}, filterHasMe=${filterHasMe}, filterHasMo=${filterHasMo}, filterHasNonIdMemberFields=${filterHasNonIdMemberFields}`, - ) + log.info(`useActivityCountOptimized=${useActivityCountOptimized}`) if (useActivityCountOptimized) { return buildActivityCountOptimizedQuery({ @@ -421,6 +424,7 @@ export const buildQuery = ({ }) } + log.info('Using fallback query path') // Fallback path (other sorts / non-aggregate / filtered queries) const baseCtes = [needsMemberOrgs ? buildMemberOrgsCTE(true) : '', searchConfig.cte].filter( Boolean,