chore: github integration sync (CM-710)

backend/src/api/integration/helpers/githubOrgRepos.ts

@@ -1,5 +1,6 @@
+import GithubIntegrationService from '@crowd/common_services/src/services/github.integration.service'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {

backend/src/api/integration/helpers/githubSearchOrgs.ts

@@ -1,5 +1,6 @@
+import GithubIntegrationService from '@crowd/common_services/src/services/github.integration.service'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {

backend/src/api/integration/helpers/githubSearchRepos.ts

@@ -1,11 +1,12 @@
+import GithubIntegrationService from '@crowd/common_services/src/services/github.integration.service'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
 
-  const payload = await new GithubIntegrationService(req).findGithubRepos(
+  const payload = await new GithubIntegrationService(req.log).findGithubRepos(
     req.query.query,
     req.query.limit,
     req.query.offset,

backend/src/bin/jobs/index.ts

@@ -4,7 +4,7 @@ import autoImportGroups from './autoImportGroupsioGroups'
 import checkStuckIntegrationRuns from './checkStuckIntegrationRuns'
 import cleanUp from './cleanUp'
 import integrationTicks from './integrationTicks'
-// import refreshGithubRepoSettingsJob from './refreshGithubRepoSettings'
+import refreshGithubRepoSettingsJob from './refreshGithubRepoSettings'
 import refreshGitlabToken from './refreshGitlabToken'
 import refreshGroupsioToken from './refreshGroupsioToken'
 import refreshMaterializedViews from './refreshMaterializedViews'
@@ -16,7 +16,7 @@ const jobs: CrowdJob[] = [
   checkStuckIntegrationRuns,
   refreshGroupsioToken,
   refreshGitlabToken,
-  // refreshGithubRepoSettingsJob,
+  refreshGithubRepoSettingsJob,
   autoImportGroups,
 ]
 

backend/src/bin/jobs/refreshGithubRepoSettings.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-continue */
 import cronGenerator from 'cron-time-generator'
 
-import { timeout } from '@crowd/common'
+import { IS_DEV_ENV, timeout } from '@crowd/common'
 import { getServiceChildLogger } from '@crowd/logging'
 
 import SequelizeRepository from '../../database/repositories/sequelizeRepository'
@@ -59,7 +59,7 @@ export const refreshGithubRepoSettings = async () => {
 const job: CrowdJob = {
   name: 'Refresh Github repo settings',
   // every day
-  cronTime: cronGenerator.every(1).days(),
+  cronTime: IS_DEV_ENV ? cronGenerator.every(5).minutes() : cronGenerator.every(1).days(),
   onTrigger: async () => {
     await refreshGithubRepoSettings()
   },

backend/src/database/migrations/V1762508771__autobackup-integrations.sql

@@ -0,0 +1,29 @@
+create table "integrationsHistory" as
+select *
+from "integrations"
+where 1 = 2;
+
+alter table "integrationsHistory"
+    add column "historyCreatedAt" timestamptz not null default now();
+
+create index if not exists ix_integration_history_integration_id on "integrationsHistory" ("id");
+
+create function log_integrations_changes()
+    returns trigger as
+$$
+begin
+    insert into "integrationsHistory" select old.*;
+
+    if (tg_op = 'DELETE') then
+        return old;
+    else
+        return new;
+    end if;
+end;
+$$ language plpgsql;
+
+create trigger integrations_audit_trigger
+    after update or delete
+    on integrations
+    for each row
+execute function log_integrations_changes();

backend/src/segment/track.ts

@@ -1,7 +1,7 @@
 import { getServiceChildLogger } from '@crowd/logging'
 import { Edition } from '@crowd/types'
 
-import { API_CONFIG, IS_TEST_ENV, SEGMENT_CONFIG } from '../conf'
+import { API_CONFIG, IS_DEV_ENV, IS_TEST_ENV, SEGMENT_CONFIG } from '../conf'
 import SequelizeRepository from '../database/repositories/sequelizeRepository'
 
 import { CROWD_ANALYTICS_PLATORM_NAME } from './addProductDataToCrowdTenant'
@@ -21,6 +21,7 @@ export default async function identify(
   }).email
   if (
     !IS_TEST_ENV &&
+    !IS_DEV_ENV &&
     SEGMENT_CONFIG.writeKey &&
     // This is only for events in the hosted version. Self-hosted has less telemetry.
     (API_CONFIG.edition === Edition.CROWD_HOSTED || API_CONFIG.edition === Edition.LFX) &&
@@ -41,6 +42,10 @@ export default async function identify(
 
     const { userIdOut, tenantIdOut } = getTenatUser(userId, options)
 
+    if (!userIdOut) {
+      return
+    }
+
     const payload = {
       userId: userIdOut,
       event,

backend/src/services/collectionService.ts

@@ -1,6 +1,7 @@
 import { uniq } from 'lodash'
 
 import { getCleanString } from '@crowd/common'
+import GithubIntegrationService from '@crowd/common_services/src/services/github.integration.service'
 import { OrganizationField, QueryExecutor, findOrgById, queryOrgs } from '@crowd/data-access-layer'
 import { listCategoriesByIds } from '@crowd/data-access-layer/src/categories'
 import {
@@ -49,7 +50,6 @@ import SequelizeRepository from '@/database/repositories/sequelizeRepository'
 import { IGithubInsights } from '@/types/githubTypes'
 
 import { IServiceOptions } from './IServiceOptions'
-import GithubIntegrationService from './githubIntegrationService'
 
 export class CollectionService extends LoggerBase {
   options: IServiceOptions

backend/src/services/integrationService.ts

@@ -4,16 +4,17 @@ import { request } from '@octokit/request'
 import axios, { AxiosRequestConfig, AxiosResponse } from 'axios'
 import lodash from 'lodash'
 import moment from 'moment'
-import { QueryTypes, Transaction } from 'sequelize'
-import { v4 as uuidv4 } from 'uuid'
+import { Transaction } from 'sequelize'
 
 import { EDITION, Error400, Error404, Error542 } from '@crowd/common'
+import { getGithubInstallationToken } from '@crowd/common_services'
 import {
   ICreateInsightsProject,
   deleteMissingSegmentRepositories,
   deleteSegmentRepositories,
   upsertSegmentRepositories,
 } from '@crowd/data-access-layer/src/collections'
+import { syncRepositoriesToGitV2 } from '@crowd/data-access-layer/src/integrations'
 import {
   NangoIntegration,
   connectNangoIntegration,
@@ -68,7 +69,6 @@ import { encryptData } from '../utils/crypto'
 
 import { IServiceOptions } from './IServiceOptions'
 import { CollectionService } from './collectionService'
-import { getGithubInstallationToken } from './helpers/githubToken'
 
 const discordToken = DISCORD_CONFIG.token || DISCORD_CONFIG.token2
 
@@ -1340,11 +1340,15 @@ export default class IntegrationService {
       )
 
       // upsert repositories to git.repositories in order to be processed by git-integration V2
-      await this.syncRepositoriesToGitV2(
-        remotes,
-        options || this.options,
+      const qx = SequelizeRepository.getQueryExecutor({
+        ...(options || this.options),
         transaction,
+      })
+      await syncRepositoriesToGitV2(
+        qx,
+        remotes,
         integration.id,
+        (options || this.options).currentSegments[0].id,
       )
 
       // Only commit if we created the transaction ourselves
@@ -1362,95 +1366,6 @@ export default class IntegrationService {
     return integration
   }
 
-  /**
-   * Syncs repositories to git.repositories table (git-integration V2)
-   * @param remotes Array of repository objects with url and optional forkedFrom
-   * @param options Repository options
-   * @param transaction Database transaction
-   * @param integrationId The integration ID from the git integration
-   * @param inheritFromExistingRepos If true, queries githubRepos and gitlabRepos for IDs; if false, generates new UUIDs
-   *
-   * TODO: @Mouad After migration is complete, simplify this function by:
-   * 1. Using an object parameter instead of multiple parameters for better maintainability
-   * 2. Removing the inheritFromExistingRepos parameter since git.repositories will be the source of truth
-   * 3. Simplifying the logic to only handle git.repositories operations
-   */
-  private async syncRepositoriesToGitV2(
-    remotes: Array<{ url: string; forkedFrom?: string | null }>,
-    options: IRepositoryOptions,
-    transaction: Transaction,
-    integrationId: string,
-  ) {
-    const seq = SequelizeRepository.getSequelize(options)
-
-    let repositoriesToSync: Array<{
-      id: string
-      url: string
-      integrationId: string
-      segmentId: string
-      forkedFrom?: string | null
-    }> = []
-    // check GitHub repos first, fallback to GitLab repos if none found
-    const existingRepos: Array<{
-      id: string
-      url: string
-    }> = await seq.query(
-      `
-          WITH github_repos AS (
-            SELECT id, url FROM "githubRepos" 
-            WHERE url IN (:urls) AND "deletedAt" IS NULL
-          ),
-          gitlab_repos AS (
-            SELECT id, url FROM "gitlabRepos" 
-            WHERE url IN (:urls) AND "deletedAt" IS NULL
-          )
-          SELECT id, url FROM github_repos
-          UNION ALL
-          SELECT id, url FROM gitlab_repos
-          WHERE NOT EXISTS (SELECT 1 FROM github_repos)
-        `,
-      {
-        replacements: {
-          urls: remotes.map((r) => r.url),
-        },
-        type: QueryTypes.SELECT,
-        transaction,
-      },
-    )
-
-    // Create a map of url to forkedFrom for quick lookup
-    const forkedFromMap = new Map(remotes.map((r) => [r.url, r.forkedFrom]))
-
-    repositoriesToSync = existingRepos.map((repo) => ({
-      id: repo.id,
-      url: repo.url,
-      integrationId,
-      segmentId: options.currentSegments[0].id,
-      forkedFrom: forkedFromMap.get(repo.url) || null,
-    }))
-
-    if (repositoriesToSync.length === 0) {
-      this.options.log.warn(
-        'No existing repos found in githubRepos or gitlabRepos - inserting new to git.repositories with new uuid',
-      )
-      repositoriesToSync = remotes.map((remote) => ({
-        id: uuidv4(), // Generate new UUID
-        url: remote.url,
-        integrationId,
-        segmentId: options.currentSegments[0].id,
-        forkedFrom: remote.forkedFrom || null,
-      }))
-    }
-
-    // Sync to git.repositories v2
-    await GitReposRepository.upsert(repositoriesToSync, {
-      ...options,
-      transaction,
-    })
-
-    this.options.log.info(`Synced ${repositoriesToSync.length} repos to git v2`)
-  }
-
   async atlassianAdminConnect(adminApi: string, organizationId: string) {
     const nangoPayload = {
       params: {

backend/src/services/memberService.ts

@@ -6,7 +6,7 @@ import validator from 'validator'
 
 import { captureApiChange, memberUnmergeAction } from '@crowd/audit-logs'
 import { Error400, calculateReach, getProperDisplayName, isDomainExcluded } from '@crowd/common'
-import { CommonMemberService } from '@crowd/common_services'
+import { CommonMemberService, getGithubInstallationToken } from '@crowd/common_services'
 import { findMemberAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations'
 import {
   MemberField,
@@ -58,7 +58,6 @@ import {
 import telemetryTrack from '../segment/telemetryTrack'
 
 import { IServiceOptions } from './IServiceOptions'
-import { getGithubInstallationToken } from './helpers/githubToken'
 import MemberAttributeSettingsService from './memberAttributeSettingsService'
 import MemberOrganizationService from './memberOrganizationService'
 import OrganizationService from './organizationService'