diff --git a/nodes/MarkdownifyDescription.ts b/nodes/MarkdownifyDescription.ts index 3dbbfd7..d9aa90e 100644 --- a/nodes/MarkdownifyDescription.ts +++ b/nodes/MarkdownifyDescription.ts @@ -18,12 +18,34 @@ export const markdownifyOperations: INodeProperties[] = [ action: 'Convert a webpage or article to clean markdown useful for blogs dev docs and more', description: 'Convert a webpage or article to clean markdown – useful for blogs, dev docs and more', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a conversion', + description: 'Retrieve the status and results of a markdown conversion by request ID', + }, ], default: 'convert', }, ]; export const markdownifyFields: INodeProperties[] = [ + // Fields for getStatus operation + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a convert operation', + displayOptions: { + show: { + resource: ['markdownify'], + operation: ['getStatus'], + }, + }, + }, + // Fields for convert operation { displayName: 'Website URL', name: 'websiteUrl', diff --git a/nodes/ScrapegraphAI/ScrapegraphAi.node.ts b/nodes/ScrapegraphAI/ScrapegraphAi.node.ts index 51291e6..f6b1a8b 100644 --- a/nodes/ScrapegraphAI/ScrapegraphAi.node.ts +++ b/nodes/ScrapegraphAI/ScrapegraphAi.node.ts @@ -148,12 +148,28 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/smartscraper/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'searchscraper') { if (operation === 'search') { const userPrompt = this.getNodeParameter('userPrompt', i) as string; const numResults = this.getNodeParameter('numResults', i) as number; + const extractionMode = this.getNodeParameter('extractionMode', i, true) as boolean; const enableScrolling = this.getNodeParameter('enableScrolling', i, false) as boolean; const enablePagination = this.getNodeParameter('enablePagination', i, false) as boolean; const useOutputSchema = this.getNodeParameter('useOutputSchema', i, false) as boolean; @@ -161,6 +177,7 @@ export class ScrapegraphAi implements INodeType { const requestBody: any = { user_prompt: userPrompt, num_results: numResults, + extraction_mode: extractionMode, }; // Add number_of_scrolls if scrolling is enabled @@ -202,6 +219,21 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/searchscraper/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'smartcrawler') { @@ -286,6 +318,21 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/markdownify/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'scrape') { diff --git a/nodes/SearchscraperDescription.ts b/nodes/SearchscraperDescription.ts index 1a67aac..61f4a9a 100644 --- a/nodes/SearchscraperDescription.ts +++ b/nodes/SearchscraperDescription.ts @@ -18,12 +18,34 @@ export const searchscraperOperations: INodeProperties[] = [ action: 'Perform ai powered site wide search and structured data extraction ideal for knowledge retrieval', description: 'Perform AI-powered site-wide search and structured data extraction – ideal for knowledge retrieval', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a search operation', + description: 'Retrieve the status and results of a search scraping operation by request ID', + }, ], default: 'search', }, ]; export const searchscraperFields: INodeProperties[] = [ + // Fields for getStatus operation + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a search operation', + displayOptions: { + show: { + resource: ['searchscraper'], + operation: ['getStatus'], + }, + }, + }, + // Fields for search operation { displayName: 'User Prompt', name: 'userPrompt', @@ -97,6 +119,19 @@ export const searchscraperFields: INodeProperties[] = [ }, ], }, + { + displayName: 'Extraction Mode', + name: 'extractionMode', + type: 'boolean', + default: true, + description: 'Whether to use AI extraction (true) for structured data or markdown conversion (false) for raw content', + displayOptions: { + show: { + resource: ['searchscraper'], + operation: ['search'], + }, + }, + }, { displayName: 'Enable Infinite Scrolling', name: 'enableScrolling', diff --git a/nodes/SmartscraperDescription.ts b/nodes/SmartscraperDescription.ts index 86d6ff2..bea7a06 100644 --- a/nodes/SmartscraperDescription.ts +++ b/nodes/SmartscraperDescription.ts @@ -18,12 +18,34 @@ export const smartscraperOperations: INodeProperties[] = [ action: 'Autonomously extract live data from any website perfect for e commerce job boards lead capture and more', description: 'Autonomously extract live data from any website – perfect for e-commerce, job boards, lead capture and more', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a scraping operation', + description: 'Retrieve the status and results of a scraping operation by request ID', + }, ], default: 'scrape', }, ]; export const smartscraperFields: INodeProperties[] = [ + // Fields for getStatus operation + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a scrape operation', + displayOptions: { + show: { + resource: ['smartscraper'], + operation: ['getStatus'], + }, + }, + }, + // Fields for scrape operation { displayName: 'Website URL', name: 'websiteUrl',