From fb866b5a59561f9c7aaaeb9fd893edbb1eacf6f7 Mon Sep 17 00:00:00 2001 From: Daniil <8039921+DaniilSmirnov@users.noreply.github.com> Date: Tue, 14 Oct 2025 20:17:33 +0300 Subject: [PATCH 1/9] add force refresh toggle --- api/api.py | 44 ++++++++++++++++++++++---- api/data_pipeline.py | 8 +++-- src/app/[owner]/[repo]/page.tsx | 5 ++- src/app/page.tsx | 3 ++ src/components/ConfigurationModal.tsx | 44 +++++++++++++++----------- src/components/ModelSelectionModal.tsx | 8 +++++ src/components/UserSelector.tsx | 43 +++++++++++++++++++++++++ 7 files changed, 128 insertions(+), 27 deletions(-) diff --git a/api/api.py b/api/api.py index d40e73f96..c2bd46b1f 100644 --- a/api/api.py +++ b/api/api.py @@ -400,6 +400,23 @@ def generate_json_export(repo_url: str, pages: List[WikiPage]) -> str: # Add the WebSocket endpoint app.add_websocket_route("/ws/chat", handle_websocket_chat) +# --- Wiki Repo Helper Functions --- +WIKI_REPOS_DIR = os.path.join(get_adalflow_default_root_path(), "repos") +os.makedirs(WIKI_REPOS_DIR, exist_ok=True) + +def get_wiki_repo_path(owner: str, repo: str) -> str: + filename = f"{owner}_{repo}" + return os.path.join(WIKI_REPOS_DIR, filename) + + +# --- Wiki Databases Helper Functions --- +WIKI_DATABASE_DIR = os.path.join(get_adalflow_default_root_path(), "databases") +os.makedirs(WIKI_REPOS_DIR, exist_ok=True) + +def get_wiki_database_path(owner: str, repo: str) -> str: + filename = f"{owner}_{repo}.pkl" + return os.path.join(WIKI_DATABASE_DIR, filename) + # --- Wiki Cache Helper Functions --- WIKI_CACHE_DIR = os.path.join(get_adalflow_default_root_path(), "wikicache") @@ -507,7 +524,8 @@ async def delete_wiki_cache( repo: str = Query(..., description="Repository name"), repo_type: str = Query(..., description="Repository type (e.g., github, gitlab)"), language: str = Query(..., description="Language of the wiki content"), - authorization_code: Optional[str] = Query(None, description="Authorization code") + authorization_code: Optional[str] = Query(None, description="Authorization code"), + force_refetch: bool = Query(False, description="Force re-clone the repository") ): """ Deletes a specific wiki cache from the file system. @@ -527,16 +545,30 @@ async def delete_wiki_cache( if os.path.exists(cache_path): try: - os.remove(cache_path) - logger.info(f"Successfully deleted wiki cache: {cache_path}") - return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} + if not force_refetch: + os.remove(cache_path) + logger.info(f"Successfully deleted wiki cache: {cache_path}") + return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} + else: + repo_dir = get_wiki_repo_path(owner, repo) + database_file = get_wiki_database_path(owner, repo) + os.remove(repo_dir) + os.remove(database_file) + logger.info(f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully") + return {"message": f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully"} except Exception as e: - logger.error(f"Error deleting wiki cache {cache_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache: {str(e)}") + if not force_refetch: + logger.error(f"Error deleting wiki cache {cache_path}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache: {str(e)}") + else: + logger.error(f"Error deleting wiki cache, repository and database {cache_path}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache with force refresh: {str(e)}") else: logger.warning(f"Wiki cache not found, cannot delete: {cache_path}") raise HTTPException(status_code=404, detail="Wiki cache not found") + + @app.get("/health") async def health_check(): """Health check endpoint for Docker and monitoring""" diff --git a/api/data_pipeline.py b/api/data_pipeline.py index fcea34ced..a234fea3f 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -170,8 +170,12 @@ def read_all_documents(path: str, embedder_type: str = None, is_ollama_embedder: embedder_type = 'ollama' if is_ollama_embedder else None documents = [] # File extensions to look for, prioritizing code files - code_extensions = [".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs", - ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs"] + code_extensions = [ + ".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs", + ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs", ".ino", ".class", ".kts", + ".sh", ".bat", ".ps1", ".rb", ".pl", ".lua", ".m", ".mm", ".vb", ".dart", + ".sql", ".r", ".scala", ".erl", ".ex", ".exs", ".clj", ".groovy" + ] doc_extensions = [".md", ".txt", ".rst", ".json", ".yaml", ".yml"] # Determine filtering mode: inclusion or exclusion diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 80e403b8f..39651deb4 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -245,7 +245,7 @@ export default function RepoWikiPage() { const includedFiles = searchParams.get('included_files') || ''; const [modelIncludedDirs, setModelIncludedDirs] = useState(includedDirs); const [modelIncludedFiles, setModelIncludedFiles] = useState(includedFiles); - + const [forceRefetch, setForceRefetch] = useState(false); // Wiki type state - default to comprehensive view const isComprehensiveParam = searchParams.get('comprehensive') !== 'false'; @@ -1562,6 +1562,7 @@ IMPORTANT: is_custom_model: isCustomSelectedModelState.toString(), custom_model: customSelectedModelState, comprehensive: isComprehensiveView.toString(), + force_refetch: forceRefetch.toString(), authorization_code: authCode, }); @@ -2248,6 +2249,8 @@ IMPORTANT: authCode={authCode} setAuthCode={setAuthCode} isAuthLoading={isAuthLoading} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> ); diff --git a/src/app/page.tsx b/src/app/page.tsx index 9e05a2ef9..c615a1b34 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -135,6 +135,7 @@ export default function Home() { const [includedDirs, setIncludedDirs] = useState(''); const [includedFiles, setIncludedFiles] = useState(''); const [selectedPlatform, setSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket'>('github'); + const [forceRefetch, setForceRefetch] = useState(false); const [accessToken, setAccessToken] = useState(''); const [error, setError] = useState(null); const [isSubmitting, setIsSubmitting] = useState(false); @@ -476,6 +477,8 @@ export default function Home() { authCode={authCode} setAuthCode={setAuthCode} isAuthLoading={isAuthLoading} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> diff --git a/src/components/ConfigurationModal.tsx b/src/components/ConfigurationModal.tsx index 7a1dae697..567bcf66c 100644 --- a/src/components/ConfigurationModal.tsx +++ b/src/components/ConfigurationModal.tsx @@ -49,6 +49,10 @@ interface ConfigurationModalProps { includedFiles: string; setIncludedFiles: (value: string) => void; + // Repository refetch options + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; + // Form submission onSubmit: () => void; isSubmitting: boolean; @@ -94,7 +98,9 @@ export default function ConfigurationModal({ authRequired, authCode, setAuthCode, - isAuthLoading + isAuthLoading, + forceRefetch, + setForceRefetch, }: ConfigurationModalProps) { const { messages: t } = useLanguage(); @@ -211,23 +217,25 @@ export default function ConfigurationModal({ {/* Model Selector */}
diff --git a/src/components/ModelSelectionModal.tsx b/src/components/ModelSelectionModal.tsx index 5a8ed3fe4..bda5165f9 100644 --- a/src/components/ModelSelectionModal.tsx +++ b/src/components/ModelSelectionModal.tsx @@ -18,6 +18,8 @@ interface ModelSelectionModalProps { customModel: string; setCustomModel: (value: string) => void; onApply: (token?: string) => void; + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; // Wiki type options isComprehensiveView: boolean; @@ -75,6 +77,8 @@ export default function ModelSelectionModal({ showWikiType = true, showTokenInput = false, repositoryType = 'github', + forceRefetch, + setForceRefetch, }: ModelSelectionModalProps) { const { messages: t } = useLanguage(); @@ -88,6 +92,7 @@ export default function ModelSelectionModal({ const [localExcludedFiles, setLocalExcludedFiles] = useState(excludedFiles); const [localIncludedDirs, setLocalIncludedDirs] = useState(includedDirs); const [localIncludedFiles, setLocalIncludedFiles] = useState(includedFiles); + const [localForceRefetch, setLocalForceRefetch] = useState(forceRefetch); // Token input state const [localAccessToken, setLocalAccessToken] = useState(''); @@ -123,6 +128,7 @@ export default function ModelSelectionModal({ if (setExcludedFiles) setExcludedFiles(localExcludedFiles); if (setIncludedDirs) setIncludedDirs(localIncludedDirs); if (setIncludedFiles) setIncludedFiles(localIncludedFiles); + if (setForceRefetch) setForceRefetch(localForceRefetch); // Pass token to onApply if needed if (showTokenInput) { @@ -187,6 +193,8 @@ export default function ModelSelectionModal({ setIncludedDirs={showFileFilters ? (value: string) => setLocalIncludedDirs(value) : undefined} includedFiles={localIncludedFiles} setIncludedFiles={showFileFilters ? (value: string) => setLocalIncludedFiles(value) : undefined} + forceRefetch={localForceRefetch} + setForceRefetch={setLocalForceRefetch} /> {/* Token Input Section for refresh */} diff --git a/src/components/UserSelector.tsx b/src/components/UserSelector.tsx index a959f2bd6..4488d6f0e 100644 --- a/src/components/UserSelector.tsx +++ b/src/components/UserSelector.tsx @@ -30,6 +30,8 @@ interface ModelSelectorProps { setIsCustomModel: (value: boolean) => void; customModel: string; setCustomModel: (value: string) => void; + forceRefetch: boolean; + setForceRefetch: (forceRefetch: boolean) => void; // File filter configuration showFileFilters?: boolean; @@ -52,6 +54,8 @@ export default function UserSelector({ setIsCustomModel, customModel, setCustomModel, + forceRefetch, + setForceRefetch, // File filter configuration showFileFilters = false, @@ -331,6 +335,45 @@ next.config.js )} +
+
+
{ + const newValue = !forceRefetch; + setForceRefetch(newValue); + if (newValue) { + setForceRefetch(true); + } + }} + > + {}} + className="sr-only" + /> +
+
+
+ +
+
+ {/* Custom model toggle - only when provider supports it */} {modelConfig?.providers.find((p: Provider) => p.id === provider)?.supportsCustomModel && (
From 4996f3579c9e8e729ddbefe0eb0ca7d1b6dd0c9f Mon Sep 17 00:00:00 2001 From: Daniil <8039921+DaniilSmirnov@users.noreply.github.com> Date: Tue, 14 Oct 2025 21:11:33 +0300 Subject: [PATCH 2/9] fix behavior --- src/app/[owner]/[repo]/page.tsx | 4 ++-- src/components/ModelSelectionModal.tsx | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 39651deb4..b2d08124d 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -306,7 +306,7 @@ export default function RepoWikiPage() { // Fallback to just the file path return filePath; - }, [effectiveRepoInfo, defaultBranch]); + }, [effectiveRepoInfo, defaultBranch, forceRefetch]); // Memoize repo info to avoid triggering updates in callbacks @@ -1659,7 +1659,7 @@ IMPORTANT: // For now, we rely on the standard loadData flow initiated by resetting effectRan and dependencies. // This will re-trigger the main data loading useEffect. // No direct call to fetchRepositoryStructure here, let the useEffect handle it based on effectRan.current = false. - }, [effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]); + }, [forceRefetch, effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]); // Start wiki generation when component mounts useEffect(() => { diff --git a/src/components/ModelSelectionModal.tsx b/src/components/ModelSelectionModal.tsx index bda5165f9..4ea934cd9 100644 --- a/src/components/ModelSelectionModal.tsx +++ b/src/components/ModelSelectionModal.tsx @@ -92,7 +92,6 @@ export default function ModelSelectionModal({ const [localExcludedFiles, setLocalExcludedFiles] = useState(excludedFiles); const [localIncludedDirs, setLocalIncludedDirs] = useState(includedDirs); const [localIncludedFiles, setLocalIncludedFiles] = useState(includedFiles); - const [localForceRefetch, setLocalForceRefetch] = useState(forceRefetch); // Token input state const [localAccessToken, setLocalAccessToken] = useState(''); @@ -128,7 +127,6 @@ export default function ModelSelectionModal({ if (setExcludedFiles) setExcludedFiles(localExcludedFiles); if (setIncludedDirs) setIncludedDirs(localIncludedDirs); if (setIncludedFiles) setIncludedFiles(localIncludedFiles); - if (setForceRefetch) setForceRefetch(localForceRefetch); // Pass token to onApply if needed if (showTokenInput) { @@ -193,8 +191,8 @@ export default function ModelSelectionModal({ setIncludedDirs={showFileFilters ? (value: string) => setLocalIncludedDirs(value) : undefined} includedFiles={localIncludedFiles} setIncludedFiles={showFileFilters ? (value: string) => setLocalIncludedFiles(value) : undefined} - forceRefetch={localForceRefetch} - setForceRefetch={setLocalForceRefetch} + forceRefetch={forceRefetch} + setForceRefetch={setForceRefetch} /> {/* Token Input Section for refresh */} From a575fd6f38028d07d954e01a77693b6c634ef020 Mon Sep 17 00:00:00 2001 From: Daniil <8039921+DaniilSmirnov@users.noreply.github.com> Date: Tue, 14 Oct 2025 22:05:40 +0300 Subject: [PATCH 3/9] fix api behaviour --- api/api.py | 25 +- src/components/UserSelector.tsx | 734 +++++++++++++++----------------- 2 files changed, 350 insertions(+), 409 deletions(-) diff --git a/api/api.py b/api/api.py index c2bd46b1f..f994ec85d 100644 --- a/api/api.py +++ b/api/api.py @@ -1,4 +1,5 @@ import os +import shutil import logging from fastapi import FastAPI, HTTPException, Query, Request, WebSocket from fastapi.middleware.cors import CORSMiddleware @@ -545,24 +546,24 @@ async def delete_wiki_cache( if os.path.exists(cache_path): try: - if not force_refetch: - os.remove(cache_path) - logger.info(f"Successfully deleted wiki cache: {cache_path}") - return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} - else: + if force_refetch: repo_dir = get_wiki_repo_path(owner, repo) database_file = get_wiki_database_path(owner, repo) - os.remove(repo_dir) - os.remove(database_file) + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + if os.path.exists(database_file): + os.remove(database_file) logger.info(f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully") - return {"message": f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully"} + os.remove(cache_path) + logger.info(f"Successfully deleted wiki cache: {cache_path}") + return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} except Exception as e: - if not force_refetch: + if force_refetch: + logger.error(f"Error deleting repository and database {cache_path}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache, repository and database: {str(e)}") + else: logger.error(f"Error deleting wiki cache {cache_path}: {e}") raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache: {str(e)}") - else: - logger.error(f"Error deleting wiki cache, repository and database {cache_path}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache with force refresh: {str(e)}") else: logger.warning(f"Wiki cache not found, cannot delete: {cache_path}") raise HTTPException(status_code=404, detail="Wiki cache not found") diff --git a/src/components/UserSelector.tsx b/src/components/UserSelector.tsx index 4488d6f0e..0a427d9cd 100644 --- a/src/components/UserSelector.tsx +++ b/src/components/UserSelector.tsx @@ -1,25 +1,12 @@ 'use client'; -import React, { useState, useEffect } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { useLanguage } from '@/contexts/LanguageContext'; -// Define the interfaces for our model configuration -interface Model { - id: string; - name: string; -} - -interface Provider { - id: string; - name: string; - models: Model[]; - supportsCustomModel?: boolean; -} - -interface ModelConfig { - providers: Provider[]; - defaultProvider: string; -} +// ===== Types ===== +interface Model { id: string; name: string } +interface Provider { id: string; name: string; models: Model[]; supportsCustomModel?: boolean } +interface ModelConfig { providers: Provider[]; defaultProvider: string } interface ModelSelectorProps { provider: string; @@ -45,101 +32,8 @@ interface ModelSelectorProps { setIncludedFiles?: (value: string) => void; } -export default function UserSelector({ - provider, - setProvider, - model, - setModel, - isCustomModel, - setIsCustomModel, - customModel, - setCustomModel, - forceRefetch, - setForceRefetch, - - // File filter configuration - showFileFilters = false, - excludedDirs = '', - setExcludedDirs, - excludedFiles = '', - setExcludedFiles, - includedDirs = '', - setIncludedDirs, - includedFiles = '', - setIncludedFiles -}: ModelSelectorProps) { - // State to manage the visibility of the filters modal and filter section - const [isFilterSectionOpen, setIsFilterSectionOpen] = useState(false); - // State to manage filter mode: 'exclude' or 'include' - const [filterMode, setFilterMode] = useState<'exclude' | 'include'>('exclude'); - const { messages: t } = useLanguage(); - - // State for model configurations from backend - const [modelConfig, setModelConfig] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); - - // State for viewing default values - const [showDefaultDirs, setShowDefaultDirs] = useState(false); - const [showDefaultFiles, setShowDefaultFiles] = useState(false); - - // Fetch model configurations from the backend - useEffect(() => { - const fetchModelConfig = async () => { - try { - setIsLoading(true); - setError(null); - - const response = await fetch('/api/models/config'); - - if (!response.ok) { - throw new Error(`Error fetching model configurations: ${response.status}`); - } - - const data = await response.json(); - setModelConfig(data); - - // Initialize provider and model with defaults from API if not already set - if (!provider && data.defaultProvider) { - setProvider(data.defaultProvider); - - // Find the default provider and set its default model - const selectedProvider = data.providers.find((p: Provider) => p.id === data.defaultProvider); - if (selectedProvider && selectedProvider.models.length > 0) { - setModel(selectedProvider.models[0].id); - } - } - } catch (err) { - console.error('Failed to fetch model configurations:', err); - setError('Failed to load model configurations. Using default options.'); - } finally { - setIsLoading(false); - } - }; - - fetchModelConfig(); - }, [provider, setModel, setProvider]); - - // Handler for changing provider - const handleProviderChange = (newProvider: string) => { - setProvider(newProvider); - setTimeout(() => { - // Reset custom model state when changing providers - setIsCustomModel(false); - - // Set default model for the selected provider - if (modelConfig) { - const selectedProvider = modelConfig.providers.find((p: Provider) => p.id === newProvider); - if (selectedProvider && selectedProvider.models.length > 0) { - setModel(selectedProvider.models[0].id); - } - } - }, 10); - }; - - // Default excluded directories from config.py - const defaultExcludedDirs = -`./.venv/ +// ===== Constants (kept outside component so they're not re-created) ===== +const DEFAULT_EXCLUDED_DIRS = `./.venv/ ./venv/ ./env/ ./virtualenv/ @@ -176,9 +70,7 @@ export default function UserSelector({ ./temp/ ./.eng`; - // Default excluded files from config.py - const defaultExcludedFiles = -`package-lock.json +const DEFAULT_EXCLUDED_FILES = `package-lock.json yarn.lock pnpm-lock.yaml npm-shrinkwrap.json @@ -264,302 +156,350 @@ next.config.js *.docx *.pptx`; - // Display loading state +// ===== Small UI primitives ===== +const Label: React.FC> = ({ children, htmlFor }) => ( + +); + +const InputShell: React.FC> = ({ children }) => ( +
+ {children} +
+); + +const Select = ( + props: React.DetailedHTMLProps, HTMLSelectElement> +) => ( +