Skip to content

Commit e2fa9b5

Browse files
jankucajankuca
authored
feat: Use integration listing from project, not from blocks (#109)
1 parent 5c3d0f1 commit e2fa9b5

19 files changed

+672
-112
lines changed

‎cspell.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"dntk",
3030
"dont",
3131
"DONT",
32+
"duckdb",
3233
"ename",
3334
"evalue",
3435
"findstr",
@@ -42,6 +43,7 @@
4243
"millis",
4344
"nbformat",
4445
"numpy",
46+
"pgsql",
4547
"pids",
4648
"Pids",
4749
"PYTHONHOME",

‎src/messageTypes.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ export type LocalizedMessages = {
179179
integrationsConfigureTitle: string;
180180
integrationsCancel: string;
181181
integrationsSave: string;
182+
// Integration type labels
183+
integrationsPostgresTypeLabel: string;
184+
integrationsBigQueryTypeLabel: string;
182185
// PostgreSQL form strings
183186
integrationsPostgresNameLabel: string;
184187
integrationsPostgresNamePlaceholder: string;

‎src/notebooks/deepnote/deepnoteNotebookManager.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { injectable } from 'inversify';
22

3-
import { IDeepnoteNotebookManager } from '../types';
3+
import { IDeepnoteNotebookManager, ProjectIntegration } from '../types';
44
import type { DeepnoteProject } from '../../platform/deepnote/deepnoteTypes';
55

66
/**
@@ -75,6 +75,35 @@ export class DeepnoteNotebookManager implements IDeepnoteNotebookManager {
7575
this.currentNotebookId.set(projectId, notebookId);
7676
}
7777

78+
/**
79+
* Updates the integrations list in the project data.
80+
* This modifies the stored project to reflect changes in configured integrations.
81+
*
82+
* @param projectId - Project identifier
83+
* @param integrations - Array of integration metadata to store in the project
84+
* @returns `true` if the project was found and updated successfully, `false` if the project does not exist
85+
*/
86+
updateProjectIntegrations(projectId: string, integrations: ProjectIntegration[]): boolean {
87+
const project = this.originalProjects.get(projectId);
88+
89+
if (!project) {
90+
return false;
91+
}
92+
93+
const updatedProject = JSON.parse(JSON.stringify(project)) as DeepnoteProject;
94+
updatedProject.project.integrations = integrations;
95+
96+
const currentNotebookId = this.currentNotebookId.get(projectId);
97+
98+
if (currentNotebookId) {
99+
this.storeOriginalProject(projectId, updatedProject, currentNotebookId);
100+
} else {
101+
this.originalProjects.set(projectId, updatedProject);
102+
}
103+
104+
return true;
105+
}
106+
78107
/**
79108
* Checks if the init notebook has already been run for a project.
80109
* @param projectId Project identifier

‎src/notebooks/deepnote/deepnoteNotebookManager.unit.test.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,117 @@ suite('DeepnoteNotebookManager', () => {
183183
});
184184
});
185185

186+
suite('updateProjectIntegrations', () => {
187+
test('should update integrations list for existing project and return true', () => {
188+
manager.storeOriginalProject('project-123', mockProject, 'notebook-456');
189+
190+
const integrations = [
191+
{ id: 'int-1', name: 'PostgreSQL', type: 'pgsql' },
192+
{ id: 'int-2', name: 'BigQuery', type: 'big-query' }
193+
];
194+
195+
const result = manager.updateProjectIntegrations('project-123', integrations);
196+
197+
assert.strictEqual(result, true);
198+
199+
const updatedProject = manager.getOriginalProject('project-123');
200+
assert.deepStrictEqual(updatedProject?.project.integrations, integrations);
201+
});
202+
203+
test('should replace existing integrations list and return true', () => {
204+
const projectWithIntegrations: DeepnoteProject = {
205+
...mockProject,
206+
project: {
207+
...mockProject.project,
208+
integrations: [{ id: 'old-int', name: 'Old Integration', type: 'pgsql' }]
209+
}
210+
};
211+
212+
manager.storeOriginalProject('project-123', projectWithIntegrations, 'notebook-456');
213+
214+
const newIntegrations = [
215+
{ id: 'new-int-1', name: 'New Integration 1', type: 'pgsql' },
216+
{ id: 'new-int-2', name: 'New Integration 2', type: 'big-query' }
217+
];
218+
219+
const result = manager.updateProjectIntegrations('project-123', newIntegrations);
220+
221+
assert.strictEqual(result, true);
222+
223+
const updatedProject = manager.getOriginalProject('project-123');
224+
assert.deepStrictEqual(updatedProject?.project.integrations, newIntegrations);
225+
});
226+
227+
test('should handle empty integrations array and return true', () => {
228+
const projectWithIntegrations: DeepnoteProject = {
229+
...mockProject,
230+
project: {
231+
...mockProject.project,
232+
integrations: [{ id: 'int-1', name: 'Integration 1', type: 'pgsql' }]
233+
}
234+
};
235+
236+
manager.storeOriginalProject('project-123', projectWithIntegrations, 'notebook-456');
237+
238+
const result = manager.updateProjectIntegrations('project-123', []);
239+
240+
assert.strictEqual(result, true);
241+
242+
const updatedProject = manager.getOriginalProject('project-123');
243+
assert.deepStrictEqual(updatedProject?.project.integrations, []);
244+
});
245+
246+
test('should return false for unknown project', () => {
247+
const result = manager.updateProjectIntegrations('unknown-project', [
248+
{ id: 'int-1', name: 'Integration', type: 'pgsql' }
249+
]);
250+
251+
assert.strictEqual(result, false);
252+
253+
const project = manager.getOriginalProject('unknown-project');
254+
assert.strictEqual(project, undefined);
255+
});
256+
257+
test('should preserve other project properties and return true', () => {
258+
manager.storeOriginalProject('project-123', mockProject, 'notebook-456');
259+
260+
const integrations = [{ id: 'int-1', name: 'PostgreSQL', type: 'pgsql' }];
261+
262+
const result = manager.updateProjectIntegrations('project-123', integrations);
263+
264+
assert.strictEqual(result, true);
265+
266+
const updatedProject = manager.getOriginalProject('project-123');
267+
assert.strictEqual(updatedProject?.project.id, mockProject.project.id);
268+
assert.strictEqual(updatedProject?.project.name, mockProject.project.name);
269+
assert.strictEqual(updatedProject?.version, mockProject.version);
270+
assert.deepStrictEqual(updatedProject?.metadata, mockProject.metadata);
271+
});
272+
273+
test('should update integrations when currentNotebookId is undefined and return true', () => {
274+
// Store project with a notebook ID, then clear it to simulate the edge case
275+
manager.storeOriginalProject('project-123', mockProject, 'notebook-456');
276+
manager.updateCurrentNotebookId('project-123', undefined as any);
277+
278+
const integrations = [
279+
{ id: 'int-1', name: 'PostgreSQL', type: 'pgsql' },
280+
{ id: 'int-2', name: 'BigQuery', type: 'big-query' }
281+
];
282+
283+
const result = manager.updateProjectIntegrations('project-123', integrations);
284+
285+
assert.strictEqual(result, true);
286+
287+
const updatedProject = manager.getOriginalProject('project-123');
288+
assert.deepStrictEqual(updatedProject?.project.integrations, integrations);
289+
// Verify other properties remain unchanged
290+
assert.strictEqual(updatedProject?.project.id, mockProject.project.id);
291+
assert.strictEqual(updatedProject?.project.name, mockProject.project.name);
292+
assert.strictEqual(updatedProject?.version, mockProject.version);
293+
assert.deepStrictEqual(updatedProject?.metadata, mockProject.metadata);
294+
});
295+
});
296+
186297
suite('integration scenarios', () => {
187298
test('should handle complete workflow for multiple projects', () => {
188299
manager.storeOriginalProject('project-1', mockProject, 'notebook-1');

‎src/notebooks/deepnote/integrations/integrationDetector.ts‎

Lines changed: 52 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@ import { inject, injectable } from 'inversify';
22

33
import { logger } from '../../../platform/logging';
44
import { IDeepnoteNotebookManager } from '../../types';
5-
import { IntegrationStatus, IntegrationWithStatus } from '../../../platform/notebooks/deepnote/integrationTypes';
5+
import {
6+
DATAFRAME_SQL_INTEGRATION_ID,
7+
DEEPNOTE_TO_INTEGRATION_TYPE,
8+
IntegrationStatus,
9+
IntegrationWithStatus,
10+
RawIntegrationType
11+
} from '../../../platform/notebooks/deepnote/integrationTypes';
612
import { IIntegrationDetector, IIntegrationStorage } from './types';
7-
import { BlockWithIntegration, scanBlocksForIntegrations } from './integrationUtils';
813

914
/**
1015
* Service for detecting integrations used in Deepnote notebooks
@@ -17,7 +22,8 @@ export class IntegrationDetector implements IIntegrationDetector {
1722
) {}
1823

1924
/**
20-
* Detect all integrations used in the given project
25+
* Detect all integrations used in the given project.
26+
* Uses the project's integrations field as the source of truth.
2127
*/
2228
async detectIntegrations(projectId: string): Promise<Map<string, IntegrationWithStatus>> {
2329
// Get the project
@@ -29,33 +35,52 @@ export class IntegrationDetector implements IIntegrationDetector {
2935
return new Map();
3036
}
3137

32-
logger.debug(
33-
`IntegrationDetector: Scanning project ${projectId} with ${project.project.notebooks.length} notebooks`
34-
);
35-
36-
// Collect all blocks with SQL integration metadata from all notebooks
37-
const blocksWithIntegrations: BlockWithIntegration[] = [];
38-
for (const notebook of project.project.notebooks) {
39-
logger.trace(`IntegrationDetector: Scanning notebook ${notebook.id} with ${notebook.blocks.length} blocks`);
40-
41-
for (const block of notebook.blocks) {
42-
// Check if this is a code block with SQL integration metadata
43-
if (block.type === 'code' && block.metadata?.sql_integration_id) {
44-
blocksWithIntegrations.push({
45-
id: block.id,
46-
sql_integration_id: block.metadata.sql_integration_id
47-
});
48-
} else if (block.type === 'code') {
49-
logger.trace(
50-
`IntegrationDetector: Block ${block.id} has no sql_integration_id. Metadata:`,
51-
block.metadata
52-
);
53-
}
38+
logger.debug(`IntegrationDetector: Scanning project ${projectId} for integrations`);
39+
40+
const integrations = new Map<string, IntegrationWithStatus>();
41+
42+
// Use the project's integrations field as the source of truth
43+
const projectIntegrations = project.project.integrations || [];
44+
logger.debug(`IntegrationDetector: Found ${projectIntegrations.length} integrations in project.integrations`);
45+
46+
for (const projectIntegration of projectIntegrations) {
47+
const integrationId = projectIntegration.id;
48+
49+
// Skip the internal DuckDB integration
50+
if (integrationId === DATAFRAME_SQL_INTEGRATION_ID) {
51+
continue;
52+
}
53+
54+
logger.debug(`IntegrationDetector: Found integration: ${integrationId} (${projectIntegration.type})`);
55+
56+
// Map the Deepnote integration type to our IntegrationType
57+
const integrationType = DEEPNOTE_TO_INTEGRATION_TYPE[projectIntegration.type as RawIntegrationType];
58+
59+
// Skip unknown integration types
60+
if (!integrationType) {
61+
logger.warn(
62+
`IntegrationDetector: Unknown integration type '${projectIntegration.type}' for integration ID '${integrationId}'. Skipping.`
63+
);
64+
continue;
5465
}
66+
67+
// Check if the integration is configured
68+
const config = await this.integrationStorage.getIntegrationConfig(integrationId);
69+
70+
const status: IntegrationWithStatus = {
71+
config: config || null,
72+
status: config ? IntegrationStatus.Connected : IntegrationStatus.Disconnected,
73+
// Include integration metadata from project for prefilling when config is null
74+
integrationName: projectIntegration.name,
75+
integrationType: integrationType
76+
};
77+
78+
integrations.set(integrationId, status);
5579
}
5680

57-
// Use the shared utility to scan blocks and build the status map
58-
return scanBlocksForIntegrations(blocksWithIntegrations, this.integrationStorage, 'IntegrationDetector');
81+
logger.debug(`IntegrationDetector: Found ${integrations.size} integrations`);
82+
83+
return integrations;
5984
}
6085

6186
/**

‎src/notebooks/deepnote/integrations/integrationManager.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ export class IntegrationManager implements IIntegrationManager {
162162
}
163163

164164
// Show the webview with optional selected integration
165-
await this.webviewProvider.show(integrations, selectedIntegrationId);
165+
await this.webviewProvider.show(projectId, integrations, selectedIntegrationId);
166166
}
167167

168168
/**

0 commit comments

Comments
 (0)