chore: github integration sync (CM-710) (#3572)

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

backend/src/api/integration/helpers/githubOrgRepos.ts

@@ -1,5 +1,6 @@
+import { GithubIntegrationService } from '@crowd/common_services'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {

backend/src/api/integration/helpers/githubSearchOrgs.ts

@@ -1,5 +1,6 @@
+import { GithubIntegrationService } from '@crowd/common_services'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {

backend/src/api/integration/helpers/githubSearchRepos.ts

@@ -1,11 +1,12 @@
+import { GithubIntegrationService } from '@crowd/common_services'
+
 import Permissions from '@/security/permissions'
-import GithubIntegrationService from '@/services/githubIntegrationService'
 import PermissionChecker from '@/services/user/permissionChecker'
 
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.integrationEdit)
 
-  const payload = await new GithubIntegrationService(req).findGithubRepos(
+  const payload = await new GithubIntegrationService(req.log).findGithubRepos(
     req.query.query,
     req.query.limit,
     req.query.offset,

backend/src/bin/jobs/index.ts

@@ -4,7 +4,7 @@ import autoImportGroups from './autoImportGroupsioGroups'
 import checkStuckIntegrationRuns from './checkStuckIntegrationRuns'
 import cleanUp from './cleanUp'
 import integrationTicks from './integrationTicks'
-// import refreshGithubRepoSettingsJob from './refreshGithubRepoSettings'
+import refreshGithubRepoSettingsJob from './refreshGithubRepoSettings'
 import refreshGitlabToken from './refreshGitlabToken'
 import refreshGroupsioToken from './refreshGroupsioToken'
 import refreshMaterializedViews from './refreshMaterializedViews'
@@ -16,7 +16,7 @@ const jobs: CrowdJob[] = [
   checkStuckIntegrationRuns,
   refreshGroupsioToken,
   refreshGitlabToken,
-  // refreshGithubRepoSettingsJob,
+  refreshGithubRepoSettingsJob,
   autoImportGroups,
 ]
 

backend/src/bin/jobs/refreshGithubRepoSettings.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-continue */
 import cronGenerator from 'cron-time-generator'
 
-import { timeout } from '@crowd/common'
+import { IS_DEV_ENV, timeout } from '@crowd/common'
 import { getServiceChildLogger } from '@crowd/logging'
 
 import SequelizeRepository from '../../database/repositories/sequelizeRepository'
@@ -59,7 +59,7 @@ export const refreshGithubRepoSettings = async () => {
 const job: CrowdJob = {
   name: 'Refresh Github repo settings',
   // every day
-  cronTime: cronGenerator.every(1).days(),
+  cronTime: IS_DEV_ENV ? cronGenerator.every(5).minutes() : cronGenerator.every(1).days(),
   onTrigger: async () => {
     await refreshGithubRepoSettings()
   },

backend/src/database/migrations/U1762508771__autobackup-integrations.sql

@@ -0,0 +1,29 @@
+create table "integrationsHistory" as
+select *
+from "integrations"
+where 1 = 2;
+
+alter table "integrationsHistory"
+    add column "historyCreatedAt" timestamptz not null default now();
+
+create index if not exists ix_integration_history_integration_id on "integrationsHistory" ("id");
+
+create function log_integrations_changes()
+    returns trigger as
+$$
+begin
+    insert into "integrationsHistory" select old.*;
+
+    if (tg_op = 'DELETE') then
+        return old;
+    else
+        return new;
+    end if;
+end;
+$$ language plpgsql;
+
+create trigger integrations_audit_trigger
+    after update or delete
+    on integrations
+    for each row
+execute function log_integrations_changes();

backend/src/database/migrations/V1762508771__autobackup-integrations.sql

@@ -1,7 +1,7 @@
 import { getServiceChildLogger } from '@crowd/logging'
 import { Edition } from '@crowd/types'
 
-import { API_CONFIG, IS_TEST_ENV, SEGMENT_CONFIG } from '../conf'
+import { API_CONFIG, IS_DEV_ENV, IS_TEST_ENV, SEGMENT_CONFIG } from '../conf'
 import SequelizeRepository from '../database/repositories/sequelizeRepository'
 
 import { CROWD_ANALYTICS_PLATORM_NAME } from './addProductDataToCrowdTenant'
@@ -21,6 +21,7 @@ export default async function identify(
   }).email
   if (
     !IS_TEST_ENV &&
+    !IS_DEV_ENV &&
     SEGMENT_CONFIG.writeKey &&
     // This is only for events in the hosted version. Self-hosted has less telemetry.
     (API_CONFIG.edition === Edition.CROWD_HOSTED || API_CONFIG.edition === Edition.LFX) &&
@@ -41,6 +42,10 @@ export default async function identify(
 
     const { userIdOut, tenantIdOut } = getTenatUser(userId, options)
 
+    if (!userIdOut) {
+      return
+    }
+
     const payload = {
       userId: userIdOut,
       event,

backend/src/segment/track.ts

@@ -1,6 +1,7 @@
 import { uniq } from 'lodash'
 
 import { getCleanString } from '@crowd/common'
+import { GithubIntegrationService } from '@crowd/common_services'
 import { OrganizationField, QueryExecutor, findOrgById, queryOrgs } from '@crowd/data-access-layer'
 import { listCategoriesByIds } from '@crowd/data-access-layer/src/categories'
 import {
@@ -49,7 +50,6 @@ import SequelizeRepository from '@/database/repositories/sequelizeRepository'
 import { IGithubInsights } from '@/types/githubTypes'
 
 import { IServiceOptions } from './IServiceOptions'
-import GithubIntegrationService from './githubIntegrationService'
 
 export class CollectionService extends LoggerBase {
   options: IServiceOptions