diff --git a/api/api.py b/api/api.py index d40e73f96..690d0056e 100644 --- a/api/api.py +++ b/api/api.py @@ -1,4 +1,5 @@ import os +import shutil import logging from fastapi import FastAPI, HTTPException, Query, Request, WebSocket from fastapi.middleware.cors import CORSMiddleware @@ -400,6 +401,23 @@ def generate_json_export(repo_url: str, pages: List[WikiPage]) -> str: # Add the WebSocket endpoint app.add_websocket_route("/ws/chat", handle_websocket_chat) +# --- Wiki Repo Helper Functions --- +WIKI_REPOS_DIR = os.path.join(get_adalflow_default_root_path(), "repos") +os.makedirs(WIKI_REPOS_DIR, exist_ok=True) + +def get_wiki_repo_path(owner: str, repo: str) -> str: + filename = f"{owner}_{repo}" + return os.path.join(WIKI_REPOS_DIR, filename) + + +# --- Wiki Databases Helper Functions --- +WIKI_DATABASE_DIR = os.path.join(get_adalflow_default_root_path(), "databases") +os.makedirs(WIKI_DATABASE_DIR, exist_ok=True) + +def get_wiki_database_path(owner: str, repo: str) -> str: + filename = f"{owner}_{repo}.pkl" + return os.path.join(WIKI_DATABASE_DIR, filename) + # --- Wiki Cache Helper Functions --- WIKI_CACHE_DIR = os.path.join(get_adalflow_default_root_path(), "wikicache") @@ -507,7 +525,8 @@ async def delete_wiki_cache( repo: str = Query(..., description="Repository name"), repo_type: str = Query(..., description="Repository type (e.g., github, gitlab)"), language: str = Query(..., description="Language of the wiki content"), - authorization_code: Optional[str] = Query(None, description="Authorization code") + authorization_code: Optional[str] = Query(None, description="Authorization code"), + force_refetch: bool = Query(False, description="Force re-clone the repository") ): """ Deletes a specific wiki cache from the file system. @@ -527,16 +546,29 @@ async def delete_wiki_cache( if os.path.exists(cache_path): try: + if force_refetch: + repo_dir = get_wiki_repo_path(owner, repo) + database_file = get_wiki_database_path(owner, repo) + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + if os.path.exists(database_file): + os.remove(database_file) + logger.info(f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully") os.remove(cache_path) logger.info(f"Successfully deleted wiki cache: {cache_path}") return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} except Exception as e: - logger.error(f"Error deleting wiki cache {cache_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache: {str(e)}") + logger.error(f"Error during deletion for {owner}/{repo} ({language}): {e}") + detail = "Failed to delete wiki cache" + if force_refetch: + detail += ", repository and database" + raise HTTPException(status_code=500, detail=f"{detail}: {str(e)}") else: logger.warning(f"Wiki cache not found, cannot delete: {cache_path}") raise HTTPException(status_code=404, detail="Wiki cache not found") + + @app.get("/health") async def health_check(): """Health check endpoint for Docker and monitoring""" diff --git a/api/data_pipeline.py b/api/data_pipeline.py index fcea34ced..a234fea3f 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -170,8 +170,12 @@ def read_all_documents(path: str, embedder_type: str = None, is_ollama_embedder: embedder_type = 'ollama' if is_ollama_embedder else None documents = [] # File extensions to look for, prioritizing code files - code_extensions = [".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs", - ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs"] + code_extensions = [ + ".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs", + ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs", ".ino", ".class", ".kts", + ".sh", ".bat", ".ps1", ".rb", ".pl", ".lua", ".m", ".mm", ".vb", ".dart", + ".sql", ".r", ".scala", ".erl", ".ex", ".exs", ".clj", ".groovy" + ] doc_extensions = [".md", ".txt", ".rst", ".json", ".yaml", ".yml"] # Determine filtering mode: inclusion or exclusion diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 80e403b8f..838a0d73e 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -245,7 +245,7 @@ export default function RepoWikiPage() { const includedFiles = searchParams.get('included_files') || ''; const [modelIncludedDirs, setModelIncludedDirs] = useState(includedDirs); const [modelIncludedFiles, setModelIncludedFiles] = useState(includedFiles); - + const [forceRefetch, setForceRefetch] = useState(false); // Wiki type state - default to comprehensive view const isComprehensiveParam = searchParams.get('comprehensive') !== 'false'; @@ -1562,6 +1562,7 @@ IMPORTANT: is_custom_model: isCustomSelectedModelState.toString(), custom_model: customSelectedModelState, comprehensive: isComprehensiveView.toString(), + force_refetch: forceRefetch.toString(), authorization_code: authCode, }); @@ -1658,7 +1659,7 @@ IMPORTANT: // For now, we rely on the standard loadData flow initiated by resetting effectRan and dependencies. // This will re-trigger the main data loading useEffect. // No direct call to fetchRepositoryStructure here, let the useEffect handle it based on effectRan.current = false. - }, [effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]); + }, [forceRefetch, effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]); // Start wiki generation when component mounts useEffect(() => { @@ -2248,6 +2249,8 @@ IMPORTANT: authCode={authCode} setAuthCode={setAuthCode} isAuthLoading={isAuthLoading} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> ); diff --git a/src/app/page.tsx b/src/app/page.tsx index 9e05a2ef9..c615a1b34 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -135,6 +135,7 @@ export default function Home() { const [includedDirs, setIncludedDirs] = useState(''); const [includedFiles, setIncludedFiles] = useState(''); const [selectedPlatform, setSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket'>('github'); + const [forceRefetch, setForceRefetch] = useState(false); const [accessToken, setAccessToken] = useState(''); const [error, setError] = useState(null); const [isSubmitting, setIsSubmitting] = useState(false); @@ -476,6 +477,8 @@ export default function Home() { authCode={authCode} setAuthCode={setAuthCode} isAuthLoading={isAuthLoading} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> diff --git a/src/components/Ask.tsx b/src/components/Ask.tsx index 056afcd23..e2e371fec 100644 --- a/src/components/Ask.tsx +++ b/src/components/Ask.tsx @@ -920,6 +920,8 @@ const Ask: React.FC = ({ showWikiType={false} authRequired={false} isAuthLoading={false} + forceRefetch={false} + setForceRefetch={() => {}} /> ); diff --git a/src/components/ConfigurationModal.tsx b/src/components/ConfigurationModal.tsx index 7a1dae697..567bcf66c 100644 --- a/src/components/ConfigurationModal.tsx +++ b/src/components/ConfigurationModal.tsx @@ -49,6 +49,10 @@ interface ConfigurationModalProps { includedFiles: string; setIncludedFiles: (value: string) => void; + // Repository refetch options + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; + // Form submission onSubmit: () => void; isSubmitting: boolean; @@ -94,7 +98,9 @@ export default function ConfigurationModal({ authRequired, authCode, setAuthCode, - isAuthLoading + isAuthLoading, + forceRefetch, + setForceRefetch, }: ConfigurationModalProps) { const { messages: t } = useLanguage(); @@ -211,23 +217,25 @@ export default function ConfigurationModal({ {/* Model Selector */}
diff --git a/src/components/ModelSelectionModal.tsx b/src/components/ModelSelectionModal.tsx index 5a8ed3fe4..4ea934cd9 100644 --- a/src/components/ModelSelectionModal.tsx +++ b/src/components/ModelSelectionModal.tsx @@ -18,6 +18,8 @@ interface ModelSelectionModalProps { customModel: string; setCustomModel: (value: string) => void; onApply: (token?: string) => void; + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; // Wiki type options isComprehensiveView: boolean; @@ -75,6 +77,8 @@ export default function ModelSelectionModal({ showWikiType = true, showTokenInput = false, repositoryType = 'github', + forceRefetch, + setForceRefetch, }: ModelSelectionModalProps) { const { messages: t } = useLanguage(); @@ -187,6 +191,8 @@ export default function ModelSelectionModal({ setIncludedDirs={showFileFilters ? (value: string) => setLocalIncludedDirs(value) : undefined} includedFiles={localIncludedFiles} setIncludedFiles={showFileFilters ? (value: string) => setLocalIncludedFiles(value) : undefined} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> {/* Token Input Section for refresh */} diff --git a/src/components/UserSelector.tsx b/src/components/UserSelector.tsx index a959f2bd6..c15d4779c 100644 --- a/src/components/UserSelector.tsx +++ b/src/components/UserSelector.tsx @@ -1,25 +1,12 @@ 'use client'; -import React, { useState, useEffect } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { useLanguage } from '@/contexts/LanguageContext'; -// Define the interfaces for our model configuration -interface Model { - id: string; - name: string; -} - -interface Provider { - id: string; - name: string; - models: Model[]; - supportsCustomModel?: boolean; -} - -interface ModelConfig { - providers: Provider[]; - defaultProvider: string; -} +// ===== Types ===== +interface Model { id: string; name: string } +interface Provider { id: string; name: string; models: Model[]; supportsCustomModel?: boolean } +interface ModelConfig { providers: Provider[]; defaultProvider: string } interface ModelSelectorProps { provider: string; @@ -30,6 +17,8 @@ interface ModelSelectorProps { setIsCustomModel: (value: boolean) => void; customModel: string; setCustomModel: (value: string) => void; + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; // File filter configuration showFileFilters?: boolean; @@ -43,99 +32,8 @@ interface ModelSelectorProps { setIncludedFiles?: (value: string) => void; } -export default function UserSelector({ - provider, - setProvider, - model, - setModel, - isCustomModel, - setIsCustomModel, - customModel, - setCustomModel, - - // File filter configuration - showFileFilters = false, - excludedDirs = '', - setExcludedDirs, - excludedFiles = '', - setExcludedFiles, - includedDirs = '', - setIncludedDirs, - includedFiles = '', - setIncludedFiles -}: ModelSelectorProps) { - // State to manage the visibility of the filters modal and filter section - const [isFilterSectionOpen, setIsFilterSectionOpen] = useState(false); - // State to manage filter mode: 'exclude' or 'include' - const [filterMode, setFilterMode] = useState<'exclude' | 'include'>('exclude'); - const { messages: t } = useLanguage(); - - // State for model configurations from backend - const [modelConfig, setModelConfig] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); - - // State for viewing default values - const [showDefaultDirs, setShowDefaultDirs] = useState(false); - const [showDefaultFiles, setShowDefaultFiles] = useState(false); - - // Fetch model configurations from the backend - useEffect(() => { - const fetchModelConfig = async () => { - try { - setIsLoading(true); - setError(null); - - const response = await fetch('/api/models/config'); - - if (!response.ok) { - throw new Error(`Error fetching model configurations: ${response.status}`); - } - - const data = await response.json(); - setModelConfig(data); - - // Initialize provider and model with defaults from API if not already set - if (!provider && data.defaultProvider) { - setProvider(data.defaultProvider); - - // Find the default provider and set its default model - const selectedProvider = data.providers.find((p: Provider) => p.id === data.defaultProvider); - if (selectedProvider && selectedProvider.models.length > 0) { - setModel(selectedProvider.models[0].id); - } - } - } catch (err) { - console.error('Failed to fetch model configurations:', err); - setError('Failed to load model configurations. Using default options.'); - } finally { - setIsLoading(false); - } - }; - - fetchModelConfig(); - }, [provider, setModel, setProvider]); - - // Handler for changing provider - const handleProviderChange = (newProvider: string) => { - setProvider(newProvider); - setTimeout(() => { - // Reset custom model state when changing providers - setIsCustomModel(false); - - // Set default model for the selected provider - if (modelConfig) { - const selectedProvider = modelConfig.providers.find((p: Provider) => p.id === newProvider); - if (selectedProvider && selectedProvider.models.length > 0) { - setModel(selectedProvider.models[0].id); - } - } - }, 10); - }; - - // Default excluded directories from config.py - const defaultExcludedDirs = -`./.venv/ +// ===== Constants (kept outside component so they're not re-created) ===== +const DEFAULT_EXCLUDED_DIRS = `./.venv/ ./venv/ ./env/ ./virtualenv/ @@ -172,9 +70,7 @@ export default function UserSelector({ ./temp/ ./.eng`; - // Default excluded files from config.py - const defaultExcludedFiles = -`package-lock.json +const DEFAULT_EXCLUDED_FILES = `package-lock.json yarn.lock pnpm-lock.yaml npm-shrinkwrap.json @@ -260,263 +156,371 @@ next.config.js *.docx *.pptx`; - // Display loading state +// ===== Small UI primitives ===== +const Label: React.FC> = ({ children, htmlFor }) => ( + +); + +const InputShell: React.FC> = ({ children }) => ( +
+ {children} +
+); + +const Select = ( + props: React.DetailedHTMLProps, HTMLSelectElement> +) => ( +