diff --git a/.claude/commands/implement-feture.md b/.claude/commands/implement-feture.md new file mode 100644 index 00000000..33302a4f --- /dev/null +++ b/.claude/commands/implement-feture.md @@ -0,0 +1,7 @@ +You will be implementing a new feature in this codebase + +$ARGUMENTS + +IMPORTANT: Only do this for front-end features. +Once this feature is built, make sure to write the changes you made to file called frontend-changes.md +Do not ask for permissions to modify this file, assume you can always do it. \ No newline at end of file diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 00000000..4caf96a2 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,54 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + # Optional: Filter by PR author + # if: | + # github.event.pull_request.user.login == 'external-contributor' || + # github.event.pull_request.user.login == 'new-developer' || + # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + prompt: | + Please review this pull request and provide feedback on: + - Code quality and best practices + - Potential bugs or issues + - Performance considerations + - Security concerns + - Test coverage + + Use the repository's CLAUDE.md for guidance on style and conventions. Be constructive and helpful in your feedback. + + Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. + + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options + claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' + diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 00000000..ae36c007 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options + # claude_args: '--model claude-opus-4-1-20250805 --allowed-tools Bash(gh pr:*)' + diff --git a/.playwright-mcp/page-2025-09-05T12-05-05-819Z.png b/.playwright-mcp/page-2025-09-05T12-05-05-819Z.png new file mode 100644 index 00000000..5953fe44 Binary files /dev/null and b/.playwright-mcp/page-2025-09-05T12-05-05-819Z.png differ diff --git a/.playwright-mcp/page-2025-09-05T12-06-19-531Z.png b/.playwright-mcp/page-2025-09-05T12-06-19-531Z.png new file mode 100644 index 00000000..cb63a64e Binary files /dev/null and b/.playwright-mcp/page-2025-09-05T12-06-19-531Z.png differ diff --git a/.python-version b/.python-version index 24ee5b1b..92536a9e 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.13 +3.12.0 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..dab889de --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,175 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is a **Course Materials RAG System** - a full-stack Retrieval-Augmented Generation application that allows users to query course materials and receive AI-powered responses with proper source attribution. + +## Architecture + +The system uses a **modular, three-tier architecture**: + +### Backend (`/backend/`) +- **FastAPI** web framework with CORS and proxy middleware +- **RAG System Core**: Main orchestrator (`rag_system.py`) +- **Vector Storage**: ChromaDB with SentenceTransformers embeddings (`vector_store.py`) +- **AI Generation**: Anthropic Claude integration with tool calling (`ai_generator.py`) +- **Document Processing**: Handles PDF/DOCX/TXT files (`document_processor.py`) +- **Tool-Based Search**: Semantic search with course/lesson filtering (`search_tools.py`) +- **Session Management**: Conversation history tracking (`session_manager.py`) + +### Frontend (`/frontend/`) +- **Vanilla JavaScript** SPA with marked.js for markdown rendering +- **Real-time chat interface** with loading states and source attribution +- **Course statistics sidebar** with collapsible sections +- **Suggested questions** for user guidance + +### Data Models (`/backend/models.py`) +- **Course**: Title, description, lessons, instructor, URL +- **Lesson**: Number, title, content, URL +- **CourseChunk**: Processed text chunks for vector storage + +## Development Commands + +### Quick Start +```bash +chmod +x run.sh +./run.sh +``` + +### Manual Development +```bash +# Install dependencies (first time) +uv sync + +# Start backend server +cd backend && uv run uvicorn app:app --reload --port 8000 + +# Run tests +cd backend && uv run pytest tests/ -v + +# Application runs at: +# - Web Interface: http://localhost:8000 +# - API Docs: http://localhost:8000/docs +``` + +### Code Quality Commands + +```bash +# Format code with Black and Ruff +uv run black backend/ main.py +uv run ruff format backend/ main.py +uv run ruff check --fix backend/ main.py + +# Run type checking +uv run mypy backend/ main.py + +# Run linting checks only +uv run ruff check backend/ main.py + +# Complete quality check (format + lint + test) +./scripts/quality-check.sh + +# Individual scripts +./scripts/format.sh # Format code only +./scripts/lint.sh # Lint and type check only +``` + +### Environment Setup +Create `.env` file in root: +``` +ANTHROPIC_API_KEY=your_key_here +``` + +## Key Technical Patterns + +### RAG Query Flow +1. User query → FastAPI endpoint (`/api/query`) +2. RAG system creates AI prompt with tool definitions +3. Claude uses `search_course_content` tool with semantic matching +4. Vector store searches ChromaDB with course/lesson filtering +5. Search results formatted with source attribution +6. Claude synthesizes response using retrieved content +7. Response returned with clickable source links + +### Tool-Based Search Architecture +- **CourseSearchTool**: Handles semantic search with course name fuzzy matching +- **ToolManager**: Registers and executes tools for AI agent +- **Source Tracking**: Last search sources stored for UI display +- **Flexible Filtering**: Supports course title and lesson number filters + +### Vector Storage Strategy +- **SentenceTransformers**: `all-MiniLM-L6-v2` for embeddings +- **ChromaDB Collections**: Separate storage for course metadata vs content chunks +- **Smart Deduplication**: Avoids re-processing existing courses +- **Metadata Enrichment**: Course titles, lesson numbers, URLs stored as metadata + +### Session Management +- **Conversation History**: Tracks user-assistant exchanges per session +- **Context Limits**: Configurable max history (default: 2 messages) +- **Session Creation**: Auto-generated UUIDs for frontend sessions + +## Configuration (`/backend/config.py`) + +Key settings: +- **ANTHROPIC_MODEL**: `claude-sonnet-4-20250514` +- **EMBEDDING_MODEL**: `all-MiniLM-L6-v2` +- **CHUNK_SIZE**: 800 characters +- **CHUNK_OVERLAP**: 100 characters +- **MAX_RESULTS**: 5 search results +- **MAX_HISTORY**: 2 conversation turns + +## Document Processing + +Supports: **PDF, DOCX, TXT** files +- Course documents placed in `/docs/` folder +- Auto-loaded on server startup +- Structured parsing extracts course metadata and lessons +- Text chunking with overlap for semantic search +- Duplicate detection prevents re-processing + +## API Endpoints + +- **POST** `/api/query` - Process user questions +- **GET** `/api/courses` - Get course statistics +- **Static files** served at `/` (frontend) + +## Testing and Development + +### Testing + +The project includes comprehensive test coverage: + +- **Unit Tests**: Individual component testing (CourseSearchTool, VectorStore, AIGenerator) +- **Integration Tests**: RAG system end-to-end testing +- **API Tests**: FastAPI endpoint testing + +```bash +# Run all tests +cd backend && uv run pytest tests/ -v + +# Run specific test file +cd backend && uv run pytest tests/test_course_search_tool.py -v + +# Run with coverage (requires pytest-cov: uv add pytest-cov) +cd backend && uv run pytest tests/ --cov=. --cov-report=html +``` + +### Development Guidelines + +Since this is a RAG system with AI components: +- Test with sample course documents in `/docs/` +- Verify ChromaDB storage at `./backend/chroma_db/` +- Monitor API logs for tool usage and search results +- Test different question types (general vs course-specific) +- Validate source attribution and clickable links +- **Always use `uv` for dependency management and running commands** + +#### Code Quality Standards +- **Black**: Automatic code formatting (line length: 88) +- **Ruff**: Fast linting and import organization +- **MyPy**: Static type checking with strict settings +- **Run quality checks before committing**: Use `./scripts/quality-check.sh` +- **Consistent formatting**: All code is formatted with Black and Ruff +- **Type hints required**: MyPy enforces type annotations \ No newline at end of file diff --git a/backend-tool-refactor.md b/backend-tool-refactor.md new file mode 100644 index 00000000..de23ae5c --- /dev/null +++ b/backend-tool-refactor.md @@ -0,0 +1,28 @@ +Refactor @backend/ai_generator.py to support sequential tool calling where Claude can make up to 2 tool calls in separate API rounds. + +Current behavior: +- Claude makes 1 tool call → tools are removed from API params → final response +- If Claude wants another tool call after seeing results, it can't (gets empty response) + +Desired behavior: +- Each tool call should be a separate API request where Claude can reason about previous results +- Support for complex queries requiring multiple searches for comparisons, multi-part questions, or when information from different courses/lessons is needed + +Example flow: +1. User: "Search for a course that discusses the same topic as lesson 4 of course X" +2. Claude: get course outline for course X → gets title of lesson 4 +3. Claude: uses the title to search for a course that discusses the same topic → returns course information +4. Claude: provides complete answer + +Requirements: +- Maximum 2 sequential rounds per user query +- Terminate when: (a) 2 rounds completed, (b) Claude's response has no tool_use blocks, or (c) tool call fails +- Preserve conversation context between rounds +- Handle tool execution errors gracefully + +Notes: +- Update the system prompt in @backend/ai_generator.py +- Update the test @backend/tests/test_ai_generator.py +- Write tests that verify the external behavior (API calls made, tools executed, results returned) rather than internal state details. + +Use two parallel subagents to brainstorm possible plans. Do not implement any code. diff --git a/backend/ai_generator.py b/backend/ai_generator.py index 0363ca90..98eb31c9 100644 --- a/backend/ai_generator.py +++ b/backend/ai_generator.py @@ -1,135 +1,366 @@ +from typing import Any + import anthropic -from typing import List, Optional, Dict, Any + class AIGenerator: """Handles interactions with Anthropic's Claude API for generating responses""" - + # Static system prompt to avoid rebuilding on each call - SYSTEM_PROMPT = """ You are an AI assistant specialized in course materials and educational content with access to a comprehensive search tool for course information. + SYSTEM_PROMPT = """You are an AI assistant specialized in course materials and educational content with access to specialized tools for course information. -Search Tool Usage: -- Use the search tool **only** for questions about specific course content or detailed educational materials -- **One search per query maximum** -- Synthesize search results into accurate, fact-based responses -- If search yields no results, state this clearly without offering alternatives +Tool Usage Guidelines: +- **Multiple tool calls allowed**: You can use tools sequentially to gather comprehensive information +- **Tool call strategy**: Start with broader searches, then narrow down with specific filters if needed +- **Course outline queries**: Use get_course_outline tool for course structure questions +- **Content queries**: Use search_course_content tool for topic-specific questions +- **Sequential refinement**: If initial results are insufficient, you may make additional tool calls with different parameters +- **Maximum efficiency**: Use tools thoughtfully - each call should add value to your response Response Protocol: -- **General knowledge questions**: Answer using existing knowledge without searching -- **Course-specific questions**: Search first, then answer -- **No meta-commentary**: - - Provide direct answers only — no reasoning process, search explanations, or question-type analysis - - Do not mention "based on the search results" - +- **Comprehensive answers**: Use multiple tool calls when necessary to provide complete responses +- **Source synthesis**: Combine information from multiple searches coherently +- **Clear attribution**: Reference sources appropriately +- **Direct answers**: Provide the information requested without meta-commentary about your process All responses must be: -1. **Brief, Concise and focused** - Get to the point quickly +1. **Brief and focused** - Get to the point quickly 2. **Educational** - Maintain instructional value 3. **Clear** - Use accessible language -4. **Example-supported** - Include relevant examples when they aid understanding -Provide only the direct answer to what was asked. +4. **Well-sourced** - Include relevant examples and references +5. **Complete** - Address all aspects of the question + +Provide comprehensive, well-researched answers using available tools as needed. """ - + def __init__(self, api_key: str, model: str): self.client = anthropic.Anthropic(api_key=api_key) self.model = model - + # Pre-build base API parameters - self.base_params = { - "model": self.model, - "temperature": 0, - "max_tokens": 800 - } - - def generate_response(self, query: str, - conversation_history: Optional[str] = None, - tools: Optional[List] = None, - tool_manager=None) -> str: + self.base_params = {"model": self.model, "temperature": 0, "max_tokens": 800} + + def generate_response( + self, + query: str, + conversation_history: str | None = None, + tools: list | None = None, + tool_manager=None, + max_rounds: int = 2, + ) -> str: """ - Generate AI response with optional tool usage and conversation context. - + Generate AI response with sequential tool calling support up to max_rounds. + Args: query: The user's question or request conversation_history: Previous messages for context tools: Available tools the AI can use tool_manager: Manager to execute tools - + max_rounds: Maximum number of sequential tool calling rounds (default: 2) + Returns: Generated response as string """ - - # Build system content efficiently - avoid string ops when possible + + # Build system content efficiently system_content = ( f"{self.SYSTEM_PROMPT}\n\nPrevious conversation:\n{conversation_history}" - if conversation_history + if conversation_history else self.SYSTEM_PROMPT ) - - # Prepare API call parameters efficiently - api_params = { - **self.base_params, - "messages": [{"role": "user", "content": query}], - "system": system_content - } - - # Add tools if available - if tools: - api_params["tools"] = tools - api_params["tool_choice"] = {"type": "auto"} - - # Get response from Claude - response = self.client.messages.create(**api_params) - - # Handle tool execution if needed - if response.stop_reason == "tool_use" and tool_manager: - return self._handle_tool_execution(response, api_params, tool_manager) - - # Return direct response - return response.content[0].text - - def _handle_tool_execution(self, initial_response, base_params: Dict[str, Any], tool_manager): + + # Initialize conversation context for sequential rounds + messages = [{"role": "user", "content": query}] + + # Execute sequential rounds + try: + final_response = self._execute_sequential_rounds( + messages=messages, + system_content=system_content, + tools=tools, + tool_manager=tool_manager, + max_rounds=max_rounds, + ) + return final_response + + except Exception as e: + # Graceful fallback on any error + print(f"Error in sequential tool calling: {e}") + return self._fallback_response(query, system_content) + + def _execute_sequential_rounds( + self, + messages: list[dict], + system_content: str, + tools: list | None, + tool_manager, + max_rounds: int, + ) -> str: + """ + Execute up to max_rounds of sequential tool calling. + + Args: + messages: Conversation messages + system_content: System prompt with context + tools: Available tools + tool_manager: Tool execution manager + max_rounds: Maximum rounds allowed + + Returns: + Final response text + + Raises: + Exception: On unrecoverable errors + """ + + current_round = 0 + + while current_round < max_rounds: + current_round += 1 + + # Prepare API parameters for this round + api_params = { + **self.base_params, + "messages": messages, + "system": system_content, + } + + # Add tools if available and tool manager exists + if tools and tool_manager: + api_params["tools"] = tools + api_params["tool_choice"] = {"type": "auto"} + + # Make API call + try: + response = self.client.messages.create(**api_params) + except Exception as e: + raise Exception(f"API call failed in round {current_round}: {str(e)}") + + # Add Claude's response to conversation + messages.append({"role": "assistant", "content": response.content}) + + # Check termination conditions + termination_result = self._check_termination_conditions( + response, current_round, max_rounds + ) + + if termination_result["should_terminate"]: + return termination_result["response"] + + # Execute tools and continue to next round + try: + tool_results = self._execute_tools_for_round(response, tool_manager) + if tool_results: + messages.append({"role": "user", "content": tool_results}) + else: + # No tools executed - this shouldn't happen if stop_reason is tool_use + return self._extract_text_response(response) + + except Exception as e: + # Tool execution failed - terminate gracefully + print(f"Tool execution failed in round {current_round}: {e}") + return f"I encountered an error while using tools to answer your question. {self._extract_text_response(response)}" + + # If we reach here, we've exhausted max_rounds + # Make final call without tools to get conclusion + return self._make_final_call_without_tools(messages, system_content) + + def _check_termination_conditions( + self, response, current_round: int, max_rounds: int + ) -> dict[str, Any]: + """ + Check if we should terminate the sequential tool calling. + + Termination occurs when: + 1. Claude's response has no tool_use blocks + 2. Maximum rounds completed + + Args: + response: Claude's response + current_round: Current round number + max_rounds: Maximum allowed rounds + + Returns: + Dict with 'should_terminate' boolean and 'response' text if terminating + """ + + # Condition 1: No tool use - Claude provided final answer + if response.stop_reason != "tool_use": + return { + "should_terminate": True, + "response": self._extract_text_response(response), + } + + # Condition 2: Max rounds completed - only terminate if we exceed max rounds + # Note: We should allow max_rounds to complete, so only terminate if current_round > max_rounds + if current_round > max_rounds: + return { + "should_terminate": True, + "response": self._extract_text_response(response), + } + + # Continue to next round + return {"should_terminate": False, "response": None} + + def _execute_tools_for_round(self, response, tool_manager) -> list[dict] | None: + """ + Execute all tool calls in the current response. + + Args: + response: Claude's response containing tool_use blocks + tool_manager: Tool execution manager + + Returns: + List of tool results or None if no tools executed + + Raises: + Exception: On tool execution failures + """ + + tool_results = [] + + for content_block in response.content: + if content_block.type == "tool_use": + try: + # Execute the tool + tool_result = tool_manager.execute_tool( + content_block.name, **content_block.input + ) + + # Handle tool execution errors + if isinstance(tool_result, str) and tool_result.startswith( + "Error:" + ): + # Tool returned an error - we can continue but should handle gracefully + print(f"Tool execution error: {tool_result}") + + tool_results.append( + { + "type": "tool_result", + "tool_use_id": content_block.id, + "content": tool_result, + } + ) + + except Exception as e: + # Critical tool execution failure + error_msg = ( + f"Failed to execute tool '{content_block.name}': {str(e)}" + ) + print(f"Critical tool error: {error_msg}") + + # Re-raise the exception to terminate the sequential tool calling + raise e + + return tool_results if tool_results else None + + def _handle_tool_execution( + self, initial_response, base_params: dict[str, Any], tool_manager + ): """ Handle execution of tool calls and get follow-up response. - + Args: initial_response: The response containing tool use requests base_params: Base API parameters tool_manager: Manager to execute tools - + Returns: Final response text after tool execution """ # Start with existing messages messages = base_params["messages"].copy() - + # Add AI's tool use response messages.append({"role": "assistant", "content": initial_response.content}) - + # Execute all tool calls and collect results tool_results = [] for content_block in initial_response.content: if content_block.type == "tool_use": tool_result = tool_manager.execute_tool( - content_block.name, - **content_block.input + content_block.name, **content_block.input + ) + + tool_results.append( + { + "type": "tool_result", + "tool_use_id": content_block.id, + "content": tool_result, + } ) - - tool_results.append({ - "type": "tool_result", - "tool_use_id": content_block.id, - "content": tool_result - }) - + # Add tool results as single message if tool_results: messages.append({"role": "user", "content": tool_results}) - + # Prepare final API call without tools final_params = { **self.base_params, "messages": messages, - "system": base_params["system"] + "system": base_params["system"], } - + # Get final response final_response = self.client.messages.create(**final_params) - return final_response.content[0].text \ No newline at end of file + return final_response.content[0].text + + def _extract_text_response(self, response) -> str: + """Extract text content from Claude's response, handling mixed content.""" + text_parts = [] + for content_block in response.content: + if hasattr(content_block, "type") and str(content_block.type) == "text": + # Real API response or Mock with type="text" + text_parts.append(str(content_block.text)) + elif hasattr(content_block, "text"): + # Mock object for testing - convert to string + text_parts.append(str(content_block.text)) + + return ( + "".join(text_parts) + if text_parts + else "I don't have a clear answer to provide." + ) + + def _make_final_call_without_tools( + self, messages: list[dict], system_content: str + ) -> str: + """Make final API call without tools to get conclusion.""" + + # Add instruction for Claude to provide final answer + messages.append( + { + "role": "user", + "content": "Please provide your final answer based on the information gathered.", + } + ) + + api_params = { + **self.base_params, + "messages": messages, + "system": system_content, + # Explicitly no tools parameter + } + + try: + final_response = self.client.messages.create(**api_params) + return self._extract_text_response(final_response) + except Exception as e: + print(f"Final call failed: {e}") + return "I apologize, but I encountered an error while formulating my final response." + + def _fallback_response(self, query: str, system_content: str) -> str: + """Fallback to single API call without tools on error.""" + try: + api_params = { + **self.base_params, + "messages": [{"role": "user", "content": query}], + "system": system_content, + } + + response = self.client.messages.create(**api_params) + return self._extract_text_response(response) + + except Exception as e: + print(f"Fallback response failed: {e}") + return "I'm sorry, I'm unable to process your request at this time. Please try again later." diff --git a/backend/app.py b/backend/app.py index 5a69d741..d01c84ab 100644 --- a/backend/app.py +++ b/backend/app.py @@ -1,25 +1,22 @@ import warnings + warnings.filterwarnings("ignore", message="resource_tracker: There appear to be.*") +import os + +from config import config from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles from fastapi.middleware.trustedhost import TrustedHostMiddleware +from fastapi.staticfiles import StaticFiles from pydantic import BaseModel -from typing import List, Optional -import os - -from config import config from rag_system import RAGSystem # Initialize FastAPI app app = FastAPI(title="Course Materials RAG System", root_path="") # Add trusted host middleware for proxy -app.add_middleware( - TrustedHostMiddleware, - allowed_hosts=["*"] -) +app.add_middleware(TrustedHostMiddleware, allowed_hosts=["*"]) # Enable CORS with proper settings for proxy app.add_middleware( @@ -34,25 +31,35 @@ # Initialize RAG system rag_system = RAGSystem(config) + # Pydantic models for request/response class QueryRequest(BaseModel): """Request model for course queries""" + query: str - session_id: Optional[str] = None + session_id: str | None = None + class QueryResponse(BaseModel): """Response model for course queries""" + answer: str - sources: List[str] + sources: list[ + str | dict[str, str] + ] # Support both string and {"text": "...", "url": "..."} formats session_id: str + class CourseStats(BaseModel): """Response model for course statistics""" + total_courses: int - course_titles: List[str] + course_titles: list[str] + # API Endpoints + @app.post("/api/query", response_model=QueryResponse) async def query_documents(request: QueryRequest): """Process a query and return response with sources""" @@ -61,18 +68,15 @@ async def query_documents(request: QueryRequest): session_id = request.session_id if not session_id: session_id = rag_system.session_manager.create_session() - + # Process query using RAG system answer, sources = rag_system.query(request.query, session_id) - - return QueryResponse( - answer=answer, - sources=sources, - session_id=session_id - ) + + return QueryResponse(answer=answer, sources=sources, session_id=session_id) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + @app.get("/api/courses", response_model=CourseStats) async def get_course_stats(): """Get course analytics and statistics""" @@ -80,11 +84,12 @@ async def get_course_stats(): analytics = rag_system.get_course_analytics() return CourseStats( total_courses=analytics["total_courses"], - course_titles=analytics["course_titles"] + course_titles=analytics["course_titles"], ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + @app.on_event("startup") async def startup_event(): """Load initial documents on startup""" @@ -92,16 +97,16 @@ async def startup_event(): if os.path.exists(docs_path): print("Loading initial documents...") try: - courses, chunks = rag_system.add_course_folder(docs_path, clear_existing=False) + courses, chunks = rag_system.add_course_folder( + docs_path, clear_existing=False + ) print(f"Loaded {courses} courses with {chunks} chunks") except Exception as e: print(f"Error loading documents: {e}") + # Custom static file handler with no-cache headers for development -from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse -import os -from pathlib import Path class DevStaticFiles(StaticFiles): @@ -113,7 +118,7 @@ async def get_response(self, path: str, scope): response.headers["Pragma"] = "no-cache" response.headers["Expires"] = "0" return response - - + + # Serve static files for the frontend -app.mount("/", StaticFiles(directory="../frontend", html=True), name="static") \ No newline at end of file +app.mount("/", StaticFiles(directory="../frontend", html=True), name="static") diff --git a/backend/config.py b/backend/config.py index d9f6392e..cab6dccc 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,29 +1,31 @@ import os from dataclasses import dataclass + from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() + @dataclass class Config: """Configuration settings for the RAG system""" + # Anthropic API settings ANTHROPIC_API_KEY: str = os.getenv("ANTHROPIC_API_KEY", "") ANTHROPIC_MODEL: str = "claude-sonnet-4-20250514" - + # Embedding model settings EMBEDDING_MODEL: str = "all-MiniLM-L6-v2" - + # Document processing settings - CHUNK_SIZE: int = 800 # Size of text chunks for vector storage - CHUNK_OVERLAP: int = 100 # Characters to overlap between chunks - MAX_RESULTS: int = 5 # Maximum search results to return - MAX_HISTORY: int = 2 # Number of conversation messages to remember - + CHUNK_SIZE: int = 800 # Size of text chunks for vector storage + CHUNK_OVERLAP: int = 100 # Characters to overlap between chunks + MAX_RESULTS: int = 5 # Maximum search results to return + MAX_HISTORY: int = 2 # Number of conversation messages to remember + # Database paths CHROMA_PATH: str = "./chroma_db" # ChromaDB storage location -config = Config() - +config = Config() diff --git a/backend/document_processor.py b/backend/document_processor.py index 266e8590..230d1d4c 100644 --- a/backend/document_processor.py +++ b/backend/document_processor.py @@ -1,83 +1,86 @@ import os import re -from typing import List, Tuple -from models import Course, Lesson, CourseChunk + +from models import Course, CourseChunk, Lesson + class DocumentProcessor: """Processes course documents and extracts structured information""" - + def __init__(self, chunk_size: int, chunk_overlap: int): self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap - + def read_file(self, file_path: str) -> str: """Read content from file with UTF-8 encoding""" try: - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, encoding="utf-8") as file: return file.read() except UnicodeDecodeError: # If UTF-8 fails, try with error handling - with open(file_path, 'r', encoding='utf-8', errors='ignore') as file: + with open(file_path, encoding="utf-8", errors="ignore") as file: return file.read() - - - def chunk_text(self, text: str) -> List[str]: + def chunk_text(self, text: str) -> list[str]: """Split text into sentence-based chunks with overlap using config settings""" - + # Clean up the text - text = re.sub(r'\s+', ' ', text.strip()) # Normalize whitespace - + text = re.sub(r"\s+", " ", text.strip()) # Normalize whitespace + # Better sentence splitting that handles abbreviations # This regex looks for periods followed by whitespace and capital letters # but ignores common abbreviations - sentence_endings = re.compile(r'(? self.chunk_size and current_chunk: break - + current_chunk.append(sentence) current_size += total_addition - + # Add chunk if we have content if current_chunk: - chunks.append(' '.join(current_chunk)) - + chunks.append(" ".join(current_chunk)) + # Calculate overlap for next chunk - if hasattr(self, 'chunk_overlap') and self.chunk_overlap > 0: + if hasattr(self, "chunk_overlap") and self.chunk_overlap > 0: # Find how many sentences to overlap overlap_size = 0 overlap_sentences = 0 - + # Count backwards from end of current chunk for k in range(len(current_chunk) - 1, -1, -1): - sentence_len = len(current_chunk[k]) + (1 if k < len(current_chunk) - 1 else 0) + sentence_len = len(current_chunk[k]) + ( + 1 if k < len(current_chunk) - 1 else 0 + ) if overlap_size + sentence_len <= self.chunk_overlap: overlap_size += sentence_len overlap_sentences += 1 else: break - + # Move start position considering overlap next_start = i + len(current_chunk) - overlap_sentences i = max(next_start, i + 1) # Ensure we make progress @@ -87,14 +90,12 @@ def chunk_text(self, text: str) -> List[str]: else: # No sentences fit, move to next i += 1 - - return chunks - - + return chunks - - def process_course_document(self, file_path: str) -> Tuple[Course, List[CourseChunk]]: + def process_course_document( + self, file_path: str + ) -> tuple[Course, list[CourseChunk]]: """ Process a course document with expected format: Line 1: Course Title: [title] @@ -104,47 +105,51 @@ def process_course_document(self, file_path: str) -> Tuple[Course, List[CourseCh """ content = self.read_file(file_path) filename = os.path.basename(file_path) - - lines = content.strip().split('\n') - + + lines = content.strip().split("\n") + # Extract course metadata from first three lines course_title = filename # Default fallback course_link = None instructor_name = "Unknown" - + # Parse course title from first line if len(lines) >= 1 and lines[0].strip(): - title_match = re.match(r'^Course Title:\s*(.+)$', lines[0].strip(), re.IGNORECASE) + title_match = re.match( + r"^Course Title:\s*(.+)$", lines[0].strip(), re.IGNORECASE + ) if title_match: course_title = title_match.group(1).strip() else: course_title = lines[0].strip() - + # Parse remaining lines for course metadata for i in range(1, min(len(lines), 4)): # Check first 4 lines for metadata line = lines[i].strip() if not line: continue - + # Try to match course link - link_match = re.match(r'^Course Link:\s*(.+)$', line, re.IGNORECASE) + link_match = re.match(r"^Course Link:\s*(.+)$", line, re.IGNORECASE) if link_match: course_link = link_match.group(1).strip() continue - + # Try to match instructor - instructor_match = re.match(r'^Course Instructor:\s*(.+)$', line, re.IGNORECASE) + instructor_match = re.match( + r"^Course Instructor:\s*(.+)$", line, re.IGNORECASE + ) if instructor_match: instructor_name = instructor_match.group(1).strip() continue - + # Create course object with title as ID course = Course( title=course_title, course_link=course_link, - instructor=instructor_name if instructor_name != "Unknown" else None + instructor=instructor_name if instructor_name != "Unknown" else None, ) - + # Process lessons and create chunks course_chunks = [] current_lesson = None @@ -152,108 +157,114 @@ def process_course_document(self, file_path: str) -> Tuple[Course, List[CourseCh lesson_link = None lesson_content = [] chunk_counter = 0 - + # Start processing from line 4 (after metadata) start_index = 3 if len(lines) > 3 and not lines[3].strip(): start_index = 4 # Skip empty line after instructor - + i = start_index while i < len(lines): line = lines[i] - + # Check for lesson markers (e.g., "Lesson 0: Introduction") - lesson_match = re.match(r'^Lesson\s+(\d+):\s*(.+)$', line.strip(), re.IGNORECASE) - + lesson_match = re.match( + r"^Lesson\s+(\d+):\s*(.+)$", line.strip(), re.IGNORECASE + ) + if lesson_match: # Process previous lesson if it exists if current_lesson is not None and lesson_content: - lesson_text = '\n'.join(lesson_content).strip() + lesson_text = "\n".join(lesson_content).strip() if lesson_text: # Add lesson to course lesson = Lesson( lesson_number=current_lesson, title=lesson_title, - lesson_link=lesson_link + lesson_link=lesson_link, ) course.lessons.append(lesson) - + # Create chunks for this lesson chunks = self.chunk_text(lesson_text) for idx, chunk in enumerate(chunks): # For the first chunk of each lesson, add lesson context if idx == 0: - chunk_with_context = f"Lesson {current_lesson} content: {chunk}" + chunk_with_context = ( + f"Lesson {current_lesson} content: {chunk}" + ) else: chunk_with_context = chunk - + course_chunk = CourseChunk( content=chunk_with_context, course_title=course.title, lesson_number=current_lesson, - chunk_index=chunk_counter + chunk_index=chunk_counter, ) course_chunks.append(course_chunk) chunk_counter += 1 - + # Start new lesson current_lesson = int(lesson_match.group(1)) lesson_title = lesson_match.group(2).strip() lesson_link = None - + # Check if next line is a lesson link if i + 1 < len(lines): next_line = lines[i + 1].strip() - link_match = re.match(r'^Lesson Link:\s*(.+)$', next_line, re.IGNORECASE) + link_match = re.match( + r"^Lesson Link:\s*(.+)$", next_line, re.IGNORECASE + ) if link_match: lesson_link = link_match.group(1).strip() i += 1 # Skip the link line so it's not added to content - + lesson_content = [] else: # Add line to current lesson content lesson_content.append(line) - + i += 1 - + # Process the last lesson if current_lesson is not None and lesson_content: - lesson_text = '\n'.join(lesson_content).strip() + lesson_text = "\n".join(lesson_content).strip() if lesson_text: lesson = Lesson( lesson_number=current_lesson, title=lesson_title, - lesson_link=lesson_link + lesson_link=lesson_link, ) course.lessons.append(lesson) - + chunks = self.chunk_text(lesson_text) for idx, chunk in enumerate(chunks): # For any chunk of each lesson, add lesson context & course title - + chunk_with_context = f"Course {course_title} Lesson {current_lesson} content: {chunk}" - + course_chunk = CourseChunk( content=chunk_with_context, course_title=course.title, lesson_number=current_lesson, - chunk_index=chunk_counter + chunk_index=chunk_counter, ) course_chunks.append(course_chunk) chunk_counter += 1 - + # If no lessons found, treat entire content as one document if not course_chunks and len(lines) > 2: - remaining_content = '\n'.join(lines[start_index:]).strip() + remaining_content = "\n".join(lines[start_index:]).strip() if remaining_content: chunks = self.chunk_text(remaining_content) for chunk in chunks: course_chunk = CourseChunk( content=chunk, course_title=course.title, - chunk_index=chunk_counter + chunk_index=chunk_counter, ) course_chunks.append(course_chunk) chunk_counter += 1 - + return course, course_chunks diff --git a/backend/models.py b/backend/models.py index 7f7126fa..24a9652a 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,22 +1,27 @@ -from typing import List, Dict, Optional from pydantic import BaseModel + class Lesson(BaseModel): """Represents a lesson within a course""" + lesson_number: int # Sequential lesson number (1, 2, 3, etc.) - title: str # Lesson title - lesson_link: Optional[str] = None # URL link to the lesson + title: str # Lesson title + lesson_link: str | None = None # URL link to the lesson + class Course(BaseModel): """Represents a complete course with its lessons""" - title: str # Full course title (used as unique identifier) - course_link: Optional[str] = None # URL link to the course - instructor: Optional[str] = None # Course instructor name (optional metadata) - lessons: List[Lesson] = [] # List of lessons in this course + + title: str # Full course title (used as unique identifier) + course_link: str | None = None # URL link to the course + instructor: str | None = None # Course instructor name (optional metadata) + lessons: list[Lesson] = [] # List of lessons in this course + class CourseChunk(BaseModel): """Represents a text chunk from a course for vector storage""" - content: str # The actual text content - course_title: str # Which course this chunk belongs to - lesson_number: Optional[int] = None # Which lesson this chunk is from - chunk_index: int # Position of this chunk in the document \ No newline at end of file + + content: str # The actual text content + course_title: str # Which course this chunk belongs to + lesson_number: int | None = None # Which lesson this chunk is from + chunk_index: int # Position of this chunk in the document diff --git a/backend/rag_system.py b/backend/rag_system.py index 50d848c8..6dba30fa 100644 --- a/backend/rag_system.py +++ b/backend/rag_system.py @@ -1,147 +1,188 @@ -from typing import List, Tuple, Optional, Dict import os -from document_processor import DocumentProcessor -from vector_store import VectorStore + from ai_generator import AIGenerator +from document_processor import DocumentProcessor +from models import Course +from search_tools import CourseOutlineTool, CourseSearchTool, ToolManager from session_manager import SessionManager -from search_tools import ToolManager, CourseSearchTool -from models import Course, Lesson, CourseChunk +from vector_store import VectorStore + class RAGSystem: """Main orchestrator for the Retrieval-Augmented Generation system""" - + def __init__(self, config): self.config = config - + # Initialize core components - self.document_processor = DocumentProcessor(config.CHUNK_SIZE, config.CHUNK_OVERLAP) - self.vector_store = VectorStore(config.CHROMA_PATH, config.EMBEDDING_MODEL, config.MAX_RESULTS) - self.ai_generator = AIGenerator(config.ANTHROPIC_API_KEY, config.ANTHROPIC_MODEL) + self.document_processor = DocumentProcessor( + config.CHUNK_SIZE, config.CHUNK_OVERLAP + ) + self.vector_store = VectorStore( + config.CHROMA_PATH, config.EMBEDDING_MODEL, config.MAX_RESULTS + ) + self.ai_generator = AIGenerator( + config.ANTHROPIC_API_KEY, config.ANTHROPIC_MODEL + ) self.session_manager = SessionManager(config.MAX_HISTORY) - + # Initialize search tools self.tool_manager = ToolManager() self.search_tool = CourseSearchTool(self.vector_store) + self.outline_tool = CourseOutlineTool(self.vector_store) self.tool_manager.register_tool(self.search_tool) - - def add_course_document(self, file_path: str) -> Tuple[Course, int]: + self.tool_manager.register_tool(self.outline_tool) + + def add_course_document(self, file_path: str) -> tuple[Course, int]: """ Add a single course document to the knowledge base. - + Args: file_path: Path to the course document - + Returns: Tuple of (Course object, number of chunks created) """ try: # Process the document - course, course_chunks = self.document_processor.process_course_document(file_path) - + course, course_chunks = self.document_processor.process_course_document( + file_path + ) + # Add course metadata to vector store for semantic search self.vector_store.add_course_metadata(course) - + # Add course content chunks to vector store self.vector_store.add_course_content(course_chunks) - + return course, len(course_chunks) except Exception as e: print(f"Error processing course document {file_path}: {e}") return None, 0 - - def add_course_folder(self, folder_path: str, clear_existing: bool = False) -> Tuple[int, int]: + + def add_course_folder( + self, folder_path: str, clear_existing: bool = False + ) -> tuple[int, int]: """ Add all course documents from a folder. - + Args: folder_path: Path to folder containing course documents clear_existing: Whether to clear existing data first - + Returns: Tuple of (total courses added, total chunks created) """ total_courses = 0 total_chunks = 0 - + # Clear existing data if requested if clear_existing: print("Clearing existing data for fresh rebuild...") self.vector_store.clear_all_data() - + if not os.path.exists(folder_path): print(f"Folder {folder_path} does not exist") return 0, 0 - + # Get existing course titles to avoid re-processing existing_course_titles = set(self.vector_store.get_existing_course_titles()) - + # Process each file in the folder for file_name in os.listdir(folder_path): file_path = os.path.join(folder_path, file_name) - if os.path.isfile(file_path) and file_name.lower().endswith(('.pdf', '.docx', '.txt')): + if os.path.isfile(file_path) and file_name.lower().endswith( + (".pdf", ".docx", ".txt") + ): try: # Check if this course might already exist # We'll process the document to get the course ID, but only add if new - course, course_chunks = self.document_processor.process_course_document(file_path) - + course, course_chunks = ( + self.document_processor.process_course_document(file_path) + ) + if course and course.title not in existing_course_titles: # This is a new course - add it to the vector store self.vector_store.add_course_metadata(course) self.vector_store.add_course_content(course_chunks) total_courses += 1 total_chunks += len(course_chunks) - print(f"Added new course: {course.title} ({len(course_chunks)} chunks)") + print( + f"Added new course: {course.title} ({len(course_chunks)} chunks)" + ) existing_course_titles.add(course.title) elif course: print(f"Course already exists: {course.title} - skipping") except Exception as e: print(f"Error processing {file_name}: {e}") - + return total_courses, total_chunks - - def query(self, query: str, session_id: Optional[str] = None) -> Tuple[str, List[str]]: + + def query( + self, query: str, session_id: str | None = None + ) -> tuple[str, list[str]]: """ Process a user query using the RAG system with tool-based search. - + Args: query: User's question session_id: Optional session ID for conversation context - + Returns: Tuple of (response, sources list - empty for tool-based approach) """ - # Create prompt for the AI with clear instructions - prompt = f"""Answer this question about course materials: {query}""" - - # Get conversation history if session exists - history = None - if session_id: - history = self.session_manager.get_conversation_history(session_id) - - # Generate response using AI with tools - response = self.ai_generator.generate_response( - query=prompt, - conversation_history=history, - tools=self.tool_manager.get_tool_definitions(), - tool_manager=self.tool_manager - ) - - # Get sources from the search tool - sources = self.tool_manager.get_last_sources() - - # Reset sources after retrieving them - self.tool_manager.reset_sources() - - # Update conversation history - if session_id: - self.session_manager.add_exchange(session_id, query, response) - - # Return response with sources from tool searches - return response, sources - - def get_course_analytics(self) -> Dict: + try: + # Input validation + if not query or not query.strip(): + return "Please provide a valid question.", [] + + # Create prompt for the AI with clear instructions + prompt = f"""Answer this question about course materials: {query}""" + + # Get conversation history if session exists + history = None + if session_id: + try: + history = self.session_manager.get_conversation_history(session_id) + except Exception as e: + print(f"Warning: Failed to get conversation history: {e}") + # Continue without history rather than failing entirely + + # Generate response using AI with tools + response = self.ai_generator.generate_response( + query=prompt, + conversation_history=history, + tools=self.tool_manager.get_tool_definitions(), + tool_manager=self.tool_manager, + ) + + # Get sources from the search tool + sources = self.tool_manager.get_last_sources() + + # Reset sources after retrieving them + self.tool_manager.reset_sources() + + # Update conversation history (don't fail if this fails) + if session_id: + try: + self.session_manager.add_exchange(session_id, query, response) + except Exception as e: + print(f"Warning: Failed to update conversation history: {e}") + + # Return response with sources from tool searches + return response, sources + + except Exception as e: + # Log the full error for debugging + print(f"RAG system error: {e}") + + # Return user-friendly error message + error_msg = "I'm sorry, I encountered an error while processing your question. Please try again or rephrase your question." + return error_msg, [] + + def get_course_analytics(self) -> dict: """Get analytics about the course catalog""" return { "total_courses": self.vector_store.get_course_count(), - "course_titles": self.vector_store.get_existing_course_titles() - } \ No newline at end of file + "course_titles": self.vector_store.get_existing_course_titles(), + } diff --git a/backend/search_tools.py b/backend/search_tools.py index adfe8235..7f575785 100644 --- a/backend/search_tools.py +++ b/backend/search_tools.py @@ -1,16 +1,17 @@ -from typing import Dict, Any, Optional, Protocol from abc import ABC, abstractmethod -from vector_store import VectorStore, SearchResults +from typing import Any + +from vector_store import SearchResults, VectorStore class Tool(ABC): """Abstract base class for all tools""" - + @abstractmethod - def get_tool_definition(self) -> Dict[str, Any]: + def get_tool_definition(self) -> dict[str, Any]: """Return Anthropic tool definition for this tool""" pass - + @abstractmethod def execute(self, **kwargs) -> str: """Execute the tool with given parameters""" @@ -19,12 +20,12 @@ def execute(self, **kwargs) -> str: class CourseSearchTool(Tool): """Tool for searching course content with semantic course name matching""" - + def __init__(self, vector_store: VectorStore): self.store = vector_store self.last_sources = [] # Track sources from last search - - def get_tool_definition(self) -> Dict[str, Any]: + + def get_tool_definition(self) -> dict[str, Any]: """Return Anthropic tool definition for this tool""" return { "name": "search_course_content", @@ -33,92 +34,196 @@ def get_tool_definition(self) -> Dict[str, Any]: "type": "object", "properties": { "query": { - "type": "string", - "description": "What to search for in the course content" + "type": "string", + "description": "What to search for in the course content", }, "course_name": { "type": "string", - "description": "Course title (partial matches work, e.g. 'MCP', 'Introduction')" + "description": "Course title (partial matches work, e.g. 'MCP', 'Introduction')", }, "lesson_number": { "type": "integer", - "description": "Specific lesson number to search within (e.g. 1, 2, 3)" - } + "description": "Specific lesson number to search within (e.g. 1, 2, 3)", + }, }, - "required": ["query"] - } + "required": ["query"], + }, } - - def execute(self, query: str, course_name: Optional[str] = None, lesson_number: Optional[int] = None) -> str: + + def execute( + self, + query: str, + course_name: str | None = None, + lesson_number: int | None = None, + ) -> str: """ Execute the search tool with given parameters. - + Args: query: What to search for course_name: Optional course filter lesson_number: Optional lesson filter - + Returns: Formatted search results or error message """ - - # Use the vector store's unified search interface - results = self.store.search( - query=query, - course_name=course_name, - lesson_number=lesson_number - ) - - # Handle errors - if results.error: - return results.error - - # Handle empty results - if results.is_empty(): - filter_info = "" - if course_name: - filter_info += f" in course '{course_name}'" - if lesson_number: - filter_info += f" in lesson {lesson_number}" - return f"No relevant content found{filter_info}." - - # Format and return results - return self._format_results(results) - + + # Input validation + if query is None: + return "Error: Search query cannot be None." + + try: + # Use the vector store's unified search interface + results = self.store.search( + query=query, course_name=course_name, lesson_number=lesson_number + ) + + # Handle errors from vector store + if results.error: + return results.error + + # Handle empty results + if results.is_empty(): + filter_info = "" + if course_name: + filter_info += f" in course '{course_name}'" + if lesson_number: + filter_info += f" in lesson {lesson_number}" + return f"No relevant content found{filter_info}." + + # Format and return results + return self._format_results(results) + + except Exception as e: + # Handle any unexpected errors gracefully + error_msg = f"Search failed due to an internal error: {str(e)}" + print(f"CourseSearchTool error: {e}") # Log for debugging + return error_msg + def _format_results(self, results: SearchResults) -> str: """Format search results with course and lesson context""" formatted = [] sources = [] # Track sources for the UI - - for doc, meta in zip(results.documents, results.metadata): - course_title = meta.get('course_title', 'unknown') - lesson_num = meta.get('lesson_number') - + + for doc, meta in zip(results.documents, results.metadata, strict=False): + course_title = meta.get("course_title", "unknown") + lesson_num = meta.get("lesson_number") + # Build context header header = f"[{course_title}" if lesson_num is not None: header += f" - Lesson {lesson_num}" header += "]" - - # Track source for the UI - source = course_title + + # Track source for the UI with link if available + source_text = course_title if lesson_num is not None: - source += f" - Lesson {lesson_num}" - sources.append(source) - + source_text += f" - Lesson {lesson_num}" + # Try to get lesson link + lesson_link = self.store.get_lesson_link(course_title, lesson_num) + if lesson_link: + # Create source object with link + sources.append({"text": source_text, "url": lesson_link}) + else: + # Fallback to plain text + sources.append(source_text) + else: + # For course-level content, try to get course link + course_link = self.store.get_course_link(course_title) + if course_link: + sources.append({"text": source_text, "url": course_link}) + else: + sources.append(source_text) + formatted.append(f"{header}\n{doc}") - + # Store sources for retrieval self.last_sources = sources - + return "\n\n".join(formatted) + +class CourseOutlineTool(Tool): + """Tool for getting complete course outlines with lesson structure""" + + def __init__(self, vector_store: VectorStore): + self.store = vector_store + self.last_sources = [] # Track sources from last search + + def get_tool_definition(self) -> dict[str, Any]: + """Return Anthropic tool definition for this tool""" + return { + "name": "get_course_outline", + "description": "Get the complete outline/structure of a course including all lessons with numbers and titles", + "input_schema": { + "type": "object", + "properties": { + "course_name": { + "type": "string", + "description": "Course title or partial course name (e.g. 'MCP', 'Introduction', 'RAG')", + } + }, + "required": ["course_name"], + }, + } + + def execute(self, course_name: str) -> str: + """ + Execute the course outline tool with given course name. + + Args: + course_name: Course title to get outline for + + Returns: + Formatted course outline with lessons or error message + """ + + # Get course outline from vector store + outline = self.store.get_course_outline(course_name) + + # Handle course not found + if not outline: + return f"No course found matching '{course_name}'. Please check the course name or try a partial match." + + # Format the outline response + return self._format_outline(outline) + + def _format_outline(self, outline: dict[str, Any]) -> str: + """Format course outline for AI response""" + course_title = outline.get("course_title", "Unknown Course") + course_link = outline.get("course_link") + lessons = outline.get("lessons", []) + + # Build formatted response + formatted = [f"Course: {course_title}"] + + if lessons: + formatted.append(f"\nLessons ({len(lessons)} total):") + for lesson in lessons: + lesson_num = lesson.get("lesson_number", "?") + lesson_title = lesson.get("lesson_title", "Untitled Lesson") + formatted.append(f" {lesson_num}. {lesson_title}") + else: + formatted.append("\nNo lesson structure available for this course.") + + # Track sources for the UI + sources = [] + if course_link: + sources.append({"text": course_title, "url": course_link}) + else: + sources.append(course_title) + + self.last_sources = sources + + return "\n".join(formatted) + + class ToolManager: """Manages available tools for the AI""" - + def __init__(self): self.tools = {} - + def register_tool(self, tool: Tool): """Register any tool that implements the Tool interface""" tool_def = tool.get_tool_definition() @@ -127,28 +232,27 @@ def register_tool(self, tool: Tool): raise ValueError("Tool must have a 'name' in its definition") self.tools[tool_name] = tool - def get_tool_definitions(self) -> list: """Get all tool definitions for Anthropic tool calling""" return [tool.get_tool_definition() for tool in self.tools.values()] - + def execute_tool(self, tool_name: str, **kwargs) -> str: """Execute a tool by name with given parameters""" if tool_name not in self.tools: return f"Tool '{tool_name}' not found" - + return self.tools[tool_name].execute(**kwargs) - + def get_last_sources(self) -> list: """Get sources from the last search operation""" # Check all tools for last_sources attribute for tool in self.tools.values(): - if hasattr(tool, 'last_sources') and tool.last_sources: + if hasattr(tool, "last_sources") and tool.last_sources: return tool.last_sources return [] def reset_sources(self): """Reset sources from all tools that track sources""" for tool in self.tools.values(): - if hasattr(tool, 'last_sources'): - tool.last_sources = [] \ No newline at end of file + if hasattr(tool, "last_sources"): + tool.last_sources = [] diff --git a/backend/session_manager.py b/backend/session_manager.py index a5a96b1a..cae2f3ca 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -1,61 +1,65 @@ -from typing import Dict, List, Optional from dataclasses import dataclass + @dataclass class Message: """Represents a single message in a conversation""" - role: str # "user" or "assistant" + + role: str # "user" or "assistant" content: str # The message content + class SessionManager: """Manages conversation sessions and message history""" - + def __init__(self, max_history: int = 5): self.max_history = max_history - self.sessions: Dict[str, List[Message]] = {} + self.sessions: dict[str, list[Message]] = {} self.session_counter = 0 - + def create_session(self) -> str: """Create a new conversation session""" self.session_counter += 1 session_id = f"session_{self.session_counter}" self.sessions[session_id] = [] return session_id - + def add_message(self, session_id: str, role: str, content: str): """Add a message to the conversation history""" if session_id not in self.sessions: self.sessions[session_id] = [] - + message = Message(role=role, content=content) self.sessions[session_id].append(message) - + # Keep conversation history within limits if len(self.sessions[session_id]) > self.max_history * 2: - self.sessions[session_id] = self.sessions[session_id][-self.max_history * 2:] - + self.sessions[session_id] = self.sessions[session_id][ + -self.max_history * 2 : + ] + def add_exchange(self, session_id: str, user_message: str, assistant_message: str): """Add a complete question-answer exchange""" self.add_message(session_id, "user", user_message) self.add_message(session_id, "assistant", assistant_message) - - def get_conversation_history(self, session_id: Optional[str]) -> Optional[str]: + + def get_conversation_history(self, session_id: str | None) -> str | None: """Get formatted conversation history for a session""" if not session_id or session_id not in self.sessions: return None - + messages = self.sessions[session_id] if not messages: return None - + # Format messages for context formatted_messages = [] for msg in messages: formatted_messages.append(f"{msg.role.title()}: {msg.content}") - + return "\n".join(formatted_messages) - + def clear_session(self, session_id: str): """Clear all messages from a session""" if session_id in self.sessions: - self.sessions[session_id] = [] \ No newline at end of file + self.sessions[session_id] = [] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 00000000..82ecf1ba --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,321 @@ +"""Pytest configuration and shared fixtures for RAG system tests""" + +import os +import shutil +import sys +import tempfile +from unittest.mock import Mock + +import pytest + +# Add backend directory to Python path for imports +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from config import Config +from models import Course, CourseChunk, Lesson +from search_tools import CourseOutlineTool, CourseSearchTool, ToolManager +from vector_store import SearchResults, VectorStore + + +@pytest.fixture +def temp_chroma_path(): + """Create temporary ChromaDB path for testing""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + +@pytest.fixture +def test_config(temp_chroma_path): + """Test configuration with temporary paths""" + config = Config() + config.CHROMA_PATH = temp_chroma_path + config.ANTHROPIC_API_KEY = "test-key" + config.MAX_RESULTS = 3 # Smaller for testing + return config + + +@pytest.fixture +def sample_course(): + """Sample course for testing""" + return Course( + title="Introduction to Machine Learning", + course_link="https://example.com/ml-course", + instructor="Dr. Smith", + lessons=[ + Lesson( + lesson_number=1, + title="What is ML?", + lesson_link="https://example.com/lesson1", + ), + Lesson( + lesson_number=2, + title="Types of ML", + lesson_link="https://example.com/lesson2", + ), + Lesson( + lesson_number=3, + title="ML Algorithms", + lesson_link="https://example.com/lesson3", + ), + ], + ) + + +@pytest.fixture +def sample_course_chunks(sample_course): + """Sample course chunks for testing""" + return [ + CourseChunk( + content="Machine learning is a subset of artificial intelligence that focuses on algorithms.", + course_title=sample_course.title, + lesson_number=1, + chunk_index=0, + ), + CourseChunk( + content="There are three main types of machine learning: supervised, unsupervised, and reinforcement learning.", + course_title=sample_course.title, + lesson_number=2, + chunk_index=1, + ), + CourseChunk( + content="Popular ML algorithms include linear regression, decision trees, and neural networks.", + course_title=sample_course.title, + lesson_number=3, + chunk_index=2, + ), + ] + + +@pytest.fixture +def mock_vector_store(): + """Mock VectorStore for testing search tools""" + mock_store = Mock(spec=VectorStore) + + # Configure default successful search response + mock_store.search.return_value = SearchResults( + documents=["Machine learning is a subset of artificial intelligence."], + metadata=[ + {"course_title": "Introduction to Machine Learning", "lesson_number": 1} + ], + distances=[0.2], + error=None, + ) + + # Configure course outline response + mock_store.get_course_outline.return_value = { + "course_title": "Introduction to Machine Learning", + "course_link": "https://example.com/ml-course", + "lessons": [ + {"lesson_number": 1, "lesson_title": "What is ML?"}, + {"lesson_number": 2, "lesson_title": "Types of ML?"}, + {"lesson_number": 3, "lesson_title": "ML Algorithms"}, + ], + } + + # Configure link methods + mock_store.get_lesson_link.return_value = "https://example.com/lesson1" + mock_store.get_course_link.return_value = "https://example.com/ml-course" + + return mock_store + + +@pytest.fixture +def mock_anthropic_client(): + """Mock Anthropic client for testing AI generator""" + mock_client = Mock() + + # Mock response without tool use + mock_response = Mock() + mock_response.content = [Mock(text="This is a test response")] + mock_response.stop_reason = "end_turn" + + mock_client.messages.create.return_value = mock_response + return mock_client + + +@pytest.fixture +def mock_anthropic_tool_response(): + """Mock Anthropic response with tool usage""" + mock_response = Mock() + mock_response.stop_reason = "tool_use" + + # Mock tool use content block + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "search_course_content" + tool_block.id = "tool_123" + tool_block.input = {"query": "machine learning", "course_name": "ML"} + + mock_response.content = [tool_block] + return mock_response + + +@pytest.fixture +def mock_final_anthropic_response(): + """Mock final Anthropic response after tool execution""" + mock_response = Mock() + mock_response.content = [ + Mock(text="Based on the search results, machine learning is...") + ] + return mock_response + + +@pytest.fixture +def real_vector_store(test_config): + """Real VectorStore instance for integration tests""" + return VectorStore( + chroma_path=test_config.CHROMA_PATH, + embedding_model=test_config.EMBEDDING_MODEL, + max_results=test_config.MAX_RESULTS, + ) + + +@pytest.fixture +def populated_vector_store(real_vector_store, sample_course, sample_course_chunks): + """Vector store populated with test data""" + real_vector_store.add_course_metadata(sample_course) + real_vector_store.add_course_content(sample_course_chunks) + return real_vector_store + + +@pytest.fixture +def course_search_tool(mock_vector_store): + """CourseSearchTool with mocked dependencies""" + return CourseSearchTool(mock_vector_store) + + +@pytest.fixture +def course_outline_tool(mock_vector_store): + """CourseOutlineTool with mocked dependencies""" + return CourseOutlineTool(mock_vector_store) + + +@pytest.fixture +def tool_manager(course_search_tool, course_outline_tool): + """ToolManager with registered tools""" + manager = ToolManager() + manager.register_tool(course_search_tool) + manager.register_tool(course_outline_tool) + return manager + + +@pytest.fixture +def mock_rag_system(): + """Mock RAG system for API testing""" + mock_rag = Mock() + + # Default successful query response + mock_rag.query.return_value = ( + "This is a test response about machine learning.", + [{"text": "Introduction to ML - Lesson 1", "url": "https://example.com/ml/lesson1"}] + ) + + # Default session creation + mock_rag.session_manager.create_session.return_value = "test-session-123" + + # Default course analytics + mock_rag.get_course_analytics.return_value = { + "total_courses": 2, + "course_titles": ["Introduction to Machine Learning", "Advanced Python Programming"] + } + + # Default course folder loading + mock_rag.add_course_folder.return_value = (2, 15) # 2 courses, 15 chunks + + return mock_rag + + +@pytest.fixture +def test_app_factory(): + """Factory for creating test apps with mocked dependencies""" + def create_test_app(mock_rag=None): + """Create FastAPI test app with mocked RAG system""" + from fastapi import FastAPI, HTTPException + from fastapi.middleware.cors import CORSMiddleware + from fastapi.middleware.trustedhost import TrustedHostMiddleware + from pydantic import BaseModel + from typing import List, Optional, Union, Dict + + # Create test app without static file mounting to avoid frontend dependency + app = FastAPI(title="Course Materials RAG System - Test", root_path="") + + # Add middleware + app.add_middleware(TrustedHostMiddleware, allowed_hosts=["*"]) + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + expose_headers=["*"], + ) + + # Use provided mock or create default + if mock_rag is None: + mock_rag = Mock() + mock_rag.query.return_value = ("Test response", []) + mock_rag.session_manager.create_session.return_value = "test-session" + mock_rag.get_course_analytics.return_value = {"total_courses": 0, "course_titles": []} + + # Pydantic models (duplicated to avoid import issues in tests) + class QueryRequest(BaseModel): + query: str + session_id: Optional[str] = None + + class QueryResponse(BaseModel): + answer: str + sources: List[Union[str, Dict[str, str]]] + session_id: str + + class CourseStats(BaseModel): + total_courses: int + course_titles: List[str] + + # API endpoints + @app.post("/api/query", response_model=QueryResponse) + async def query_documents(request: QueryRequest): + try: + session_id = request.session_id + if not session_id: + session_id = mock_rag.session_manager.create_session() + + answer, sources = mock_rag.query(request.query, session_id) + + return QueryResponse( + answer=answer, + sources=sources, + session_id=session_id + ) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @app.get("/api/courses", response_model=CourseStats) + async def get_course_stats(): + try: + analytics = mock_rag.get_course_analytics() + return CourseStats( + total_courses=analytics["total_courses"], + course_titles=analytics["course_titles"] + ) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @app.get("/health") + async def health_check(): + return {"status": "healthy"} + + return app, mock_rag + + return create_test_app + + +@pytest.fixture +def test_client(test_app_factory, mock_rag_system): + """Test client with default mocked RAG system""" + from fastapi.testclient import TestClient + + app, rag_mock = test_app_factory(mock_rag_system) + client = TestClient(app) + + return client, rag_mock diff --git a/backend/tests/test_ai_generator.py b/backend/tests/test_ai_generator.py new file mode 100644 index 00000000..bdf52439 --- /dev/null +++ b/backend/tests/test_ai_generator.py @@ -0,0 +1,733 @@ +"""Unit tests for AIGenerator""" + +from unittest.mock import Mock, patch + +from ai_generator import AIGenerator + + +class TestAIGeneratorBasic: + """Test basic AIGenerator functionality""" + + def test_init(self): + """Test AIGenerator initialization""" + generator = AIGenerator("test-api-key", "claude-sonnet-4") + + assert generator.model == "claude-sonnet-4" + assert generator.base_params["model"] == "claude-sonnet-4" + assert generator.base_params["temperature"] == 0 + assert generator.base_params["max_tokens"] == 800 + + @patch("anthropic.Anthropic") + def test_generate_response_simple_query(self, mock_anthropic): + """Test simple response generation without tools""" + # Mock client and response + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_response.content = [Mock(text="This is a simple response")] + mock_response.stop_reason = "end_turn" + mock_client.messages.create.return_value = mock_response + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response("What is machine learning?") + + assert result == "This is a simple response" + mock_client.messages.create.assert_called_once() + + @patch("anthropic.Anthropic") + def test_generate_response_with_conversation_history(self, mock_anthropic): + """Test response generation with conversation history""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_response.content = [Mock(text="Response with context")] + mock_response.stop_reason = "end_turn" + mock_client.messages.create.return_value = mock_response + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "Follow up question", conversation_history="Previous conversation context" + ) + + assert result == "Response with context" + + # Verify system prompt includes history + call_args = mock_client.messages.create.call_args + system_content = call_args[1]["system"] + assert "Previous conversation context" in system_content + + +class TestAIGeneratorToolIntegration: + """Test AIGenerator tool calling functionality""" + + @patch("anthropic.Anthropic") + def test_generate_response_with_tools_no_tool_use(self, mock_anthropic): + """Test response generation with tools available but not used""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_response.content = [Mock(text="Direct response without tools")] + mock_response.stop_reason = "end_turn" + mock_client.messages.create.return_value = mock_response + + # Create mock tools and tool manager + mock_tools = [{"name": "search_tool", "description": "Search content"}] + mock_tool_manager = Mock() + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "General question", tools=mock_tools, tool_manager=mock_tool_manager + ) + + assert result == "Direct response without tools" + + # Verify tools were included in API call + call_args = mock_client.messages.create.call_args + assert call_args[1]["tools"] == mock_tools + assert call_args[1]["tool_choice"] == {"type": "auto"} + + @patch("anthropic.Anthropic") + def test_generate_response_with_tool_use_single_tool(self, mock_anthropic): + """Test response generation with single tool call""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock initial response with tool use + mock_tool_response = Mock() + mock_tool_response.stop_reason = "tool_use" + + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "search_course_content" + tool_block.id = "tool_123" + tool_block.input = {"query": "machine learning", "course_name": "ML"} + mock_tool_response.content = [tool_block] + + # Mock final response after tool execution + mock_final_response = Mock() + mock_final_response.content = [Mock(text="Based on search results: ML is...")] + + # Configure client to return different responses for each call + mock_client.messages.create.side_effect = [ + mock_tool_response, + mock_final_response, + ] + + # Mock tool manager + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.return_value = ( + "Search results: Machine learning is AI subset" + ) + + mock_tools = [ + {"name": "search_course_content", "description": "Search content"} + ] + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "What is machine learning?", + tools=mock_tools, + tool_manager=mock_tool_manager, + ) + + assert result == "Based on search results: ML is..." + + # Verify tool was executed + mock_tool_manager.execute_tool.assert_called_once_with( + "search_course_content", query="machine learning", course_name="ML" + ) + + # Verify two API calls were made + assert mock_client.messages.create.call_count == 2 + + @patch("anthropic.Anthropic") + def test_generate_response_with_multiple_tool_calls(self, mock_anthropic): + """Test response generation with multiple tool calls""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock initial response with multiple tool uses + mock_tool_response = Mock() + mock_tool_response.stop_reason = "tool_use" + + tool_block1 = Mock() + tool_block1.type = "tool_use" + tool_block1.name = "search_course_content" + tool_block1.id = "tool_1" + tool_block1.input = {"query": "machine learning"} + + tool_block2 = Mock() + tool_block2.type = "tool_use" + tool_block2.name = "get_course_outline" + tool_block2.id = "tool_2" + tool_block2.input = {"course_name": "ML Course"} + + mock_tool_response.content = [tool_block1, tool_block2] + + # Mock final response + mock_final_response = Mock() + mock_final_response.content = [Mock(text="Combined results from both tools")] + + mock_client.messages.create.side_effect = [ + mock_tool_response, + mock_final_response, + ] + + # Mock tool manager with multiple tool results + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.side_effect = [ + "Search result 1", + "Course outline result", + ] + + mock_tools = [ + {"name": "search_course_content", "description": "Search content"}, + {"name": "get_course_outline", "description": "Get outline"}, + ] + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "Tell me about ML course", tools=mock_tools, tool_manager=mock_tool_manager + ) + + assert result == "Combined results from both tools" + + # Verify both tools were executed + assert mock_tool_manager.execute_tool.call_count == 2 + mock_tool_manager.execute_tool.assert_any_call( + "search_course_content", query="machine learning" + ) + mock_tool_manager.execute_tool.assert_any_call( + "get_course_outline", course_name="ML Course" + ) + + +class TestAIGeneratorToolExecutionHandling: + """Test _handle_tool_execution method specifically""" + + def test_handle_tool_execution_message_construction(self): + """Test correct message construction during tool execution""" + # Create a real AIGenerator (without mocking Anthropic for this test) + with patch("anthropic.Anthropic") as mock_anthropic: + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock final response + mock_final_response = Mock() + mock_final_response.content = [Mock(text="Final response after tool use")] + mock_client.messages.create.return_value = mock_final_response + + generator = AIGenerator("test-key", "claude-sonnet-4") + + # Create mock initial response + initial_response = Mock() + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "search_tool" + tool_block.id = "tool_123" + tool_block.input = {"query": "test"} + initial_response.content = [tool_block] + + # Create base params + base_params = { + "messages": [{"role": "user", "content": "test query"}], + "system": "test system prompt", + } + + # Mock tool manager + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.return_value = "Tool execution result" + + # Call the method + result = generator._handle_tool_execution( + initial_response, base_params, mock_tool_manager + ) + + assert result == "Final response after tool use" + + # Verify tool was executed + mock_tool_manager.execute_tool.assert_called_once_with( + "search_tool", query="test" + ) + + # Verify final API call structure + call_args = mock_client.messages.create.call_args + messages = call_args[1]["messages"] + + # Should have original user message, AI tool use message, and tool result message + assert len(messages) == 3 + assert messages[0]["role"] == "user" + assert messages[1]["role"] == "assistant" + assert messages[2]["role"] == "user" + + # Tool result should be in proper format + tool_result = messages[2]["content"][0] + assert tool_result["type"] == "tool_result" + assert tool_result["tool_use_id"] == "tool_123" + assert tool_result["content"] == "Tool execution result" + + def test_handle_tool_execution_no_tool_blocks(self): + """Test handling when response contains no tool use blocks""" + with patch("anthropic.Anthropic") as mock_anthropic: + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_final_response = Mock() + mock_final_response.content = [Mock(text="No tools used")] + mock_client.messages.create.return_value = mock_final_response + + generator = AIGenerator("test-key", "claude-sonnet-4") + + # Create mock initial response with no tool blocks + initial_response = Mock() + text_block = Mock() + text_block.type = "text" + initial_response.content = [text_block] + + base_params = { + "messages": [{"role": "user", "content": "test query"}], + "system": "test system prompt", + } + + mock_tool_manager = Mock() + + result = generator._handle_tool_execution( + initial_response, base_params, mock_tool_manager + ) + + assert result == "No tools used" + # Tool manager should not be called + mock_tool_manager.execute_tool.assert_not_called() + + +class TestAIGeneratorErrorHandling: + """Test error handling in AIGenerator""" + + @patch("anthropic.Anthropic") + def test_anthropic_api_error(self, mock_anthropic): + """Test handling of Anthropic API errors""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock API to raise exception + mock_client.messages.create.side_effect = Exception("API rate limit exceeded") + + generator = AIGenerator("test-key", "claude-sonnet-4") + + result = generator.generate_response("test query") + + # Should fallback gracefully + assert "I'm sorry, I'm unable to process your request at this time" in result + + @patch("anthropic.Anthropic") + def test_tool_execution_error(self, mock_anthropic): + """Test handling of tool execution errors""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock initial tool use response + mock_tool_response = Mock() + mock_tool_response.stop_reason = "tool_use" + + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "failing_tool" + tool_block.id = "tool_123" + tool_block.input = {"param": "value"} + mock_tool_response.content = [tool_block] + + # Mock final response + mock_final_response = Mock() + mock_final_response.content = [Mock(text="Handled tool error")] + + mock_client.messages.create.side_effect = [ + mock_tool_response, + mock_final_response, + ] + + # Mock tool manager to raise exception + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.side_effect = Exception("Tool execution failed") + + mock_tools = [{"name": "failing_tool", "description": "A failing tool"}] + + generator = AIGenerator("test-key", "claude-sonnet-4") + + # Should handle the tool execution error gracefully + result = generator.generate_response( + "test query", tools=mock_tools, tool_manager=mock_tool_manager + ) + + # Should return error message + assert "I encountered an error while using tools" in result + + @patch("anthropic.Anthropic") + def test_malformed_tool_response(self, mock_anthropic): + """Test handling of malformed tool response""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Mock response with malformed content + mock_tool_response = Mock() + mock_tool_response.stop_reason = "tool_use" + mock_tool_response.content = [] # Empty content + + mock_final_response = Mock() + mock_final_response.content = [Mock(text="Handled malformed response")] + + mock_client.messages.create.side_effect = [ + mock_tool_response, + mock_final_response, + ] + + mock_tool_manager = Mock() + mock_tools = [{"name": "test_tool", "description": "Test tool"}] + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "test query", tools=mock_tools, tool_manager=mock_tool_manager + ) + + # Should handle gracefully and not call any tools + # With empty content, it should extract no text and return default message + assert result == "I don't have a clear answer to provide." + mock_tool_manager.execute_tool.assert_not_called() + + +class TestAIGeneratorSystemPrompt: + """Test system prompt construction and usage""" + + def test_system_prompt_content(self): + """Test that system prompt contains expected content""" + generator = AIGenerator("test-key", "claude-sonnet-4") + + assert "specialized in course materials" in generator.SYSTEM_PROMPT + assert "search_course_content tool" in generator.SYSTEM_PROMPT + assert "get_course_outline tool" in generator.SYSTEM_PROMPT + assert "Multiple tool calls allowed" in generator.SYSTEM_PROMPT + + @patch("anthropic.Anthropic") + def test_system_prompt_with_history(self, mock_anthropic): + """Test system prompt construction with conversation history""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_response.content = [Mock(text="Response with history")] + mock_response.stop_reason = "end_turn" + mock_client.messages.create.return_value = mock_response + + generator = AIGenerator("test-key", "claude-sonnet-4") + generator.generate_response( + "Current question", + conversation_history="User: Previous question\nAssistant: Previous answer", + ) + + # Verify system prompt includes history + call_args = mock_client.messages.create.call_args + system_content = call_args[1]["system"] + + assert generator.SYSTEM_PROMPT in system_content + assert "Previous conversation:" in system_content + assert "User: Previous question" in system_content + assert "Assistant: Previous answer" in system_content + + +class TestAIGeneratorSequentialToolCalling: + """Test sequential tool calling functionality (up to 2 rounds)""" + + @patch("anthropic.Anthropic") + def test_sequential_tool_calling_two_rounds_success(self, mock_anthropic): + """Test successful 2-round sequential tool calling""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Round 1: Tool use response + round1_response = Mock() + round1_response.stop_reason = "tool_use" + tool_block1 = Mock() + tool_block1.type = "tool_use" + tool_block1.name = "get_course_outline" + tool_block1.id = "tool_1" + tool_block1.input = {"course_name": "ML Course"} + round1_response.content = [tool_block1] + + # Round 2: Another tool use response + round2_response = Mock() + round2_response.stop_reason = "tool_use" + tool_block2 = Mock() + tool_block2.type = "tool_use" + tool_block2.name = "search_course_content" + tool_block2.id = "tool_2" + tool_block2.input = {"query": "neural networks", "course_name": "Advanced AI"} + round2_response.content = [tool_block2] + + # Final response + final_response = Mock() + final_response.content = [Mock(text="Comparison of neural network concepts")] + final_response.stop_reason = "end_turn" + + # Configure mock to return responses in sequence + mock_client.messages.create.side_effect = [ + round1_response, + round2_response, + final_response, + ] + + # Mock tool manager + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.side_effect = [ + "Course outline with lesson on neural networks", + "Detailed neural network content from Advanced AI course", + ] + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "Find courses that discuss similar topics to lesson 3 of ML Course", + tools=[{"name": "get_course_outline"}, {"name": "search_course_content"}], + tool_manager=mock_tool_manager, + ) + + # Assertions + assert result == "Comparison of neural network concepts" + assert mock_client.messages.create.call_count == 3 # 2 tool rounds + final + assert mock_tool_manager.execute_tool.call_count == 2 + + # Verify tool calls + mock_tool_manager.execute_tool.assert_any_call( + "get_course_outline", course_name="ML Course" + ) + mock_tool_manager.execute_tool.assert_any_call( + "search_course_content", query="neural networks", course_name="Advanced AI" + ) + + @patch("anthropic.Anthropic") + def test_sequential_tool_calling_single_round_sufficient(self, mock_anthropic): + """Test when first round provides sufficient answer""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # Round 1: Tool use response + round1_response = Mock() + round1_response.stop_reason = "tool_use" + tool_block1 = Mock() + tool_block1.type = "tool_use" + tool_block1.name = "search_course_content" + tool_block1.id = "tool_1" + tool_block1.input = {"query": "machine learning"} + round1_response.content = [tool_block1] + + # Round 2: Final text response (no tools) + final_response = Mock() + final_response.content = [ + Mock(text="Machine learning is covered in these courses...", type="text") + ] + final_response.stop_reason = "end_turn" + + mock_client.messages.create.side_effect = [round1_response, final_response] + + # Mock tool manager + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.return_value = "ML content from multiple courses" + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "What courses cover machine learning?", + tools=[{"name": "search_course_content"}], + tool_manager=mock_tool_manager, + ) + + # Should terminate after 2 API calls (1 tool round + 1 final) + assert result == "Machine learning is covered in these courses..." + assert mock_client.messages.create.call_count == 2 + assert mock_tool_manager.execute_tool.call_count == 1 + + +class TestAIGeneratorTerminationConditions: + """Test various termination conditions for sequential tool calling""" + + @patch("anthropic.Anthropic") + def test_termination_after_two_rounds_max(self, mock_anthropic): + """Test termination after exactly 2 rounds even if Claude wants more""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + def create_tool_use_response(tool_name, **kwargs): + response = Mock() + response.stop_reason = "tool_use" + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = tool_name + tool_block.id = f"tool_{tool_name}" + tool_block.input = kwargs + response.content = [tool_block] + return response + + def create_text_response(text): + response = Mock() + response.stop_reason = "end_turn" + response.content = [Mock(text=text, type="text")] + return response + + # Configure responses for 3 potential rounds, but only 2 should execute + mock_responses = [ + # Round 1: tool use + create_tool_use_response("get_course_outline", course_name="ML Course"), + # Round 2: tool use + create_tool_use_response("search_course_content", query="deep learning"), + # Round 3: final response after max rounds reached + create_text_response("Final answer based on two tool calls"), + ] + + mock_client.messages.create.side_effect = mock_responses + + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.side_effect = [ + "Course outline result", + "Search result", + ] + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "Complex query requiring multiple searches", + tools=[{"name": "get_course_outline"}, {"name": "search_course_content"}], + tool_manager=mock_tool_manager, + ) + + # Should only make 3 API calls total (2 tool rounds + 1 final) + assert mock_client.messages.create.call_count == 3 + assert mock_tool_manager.execute_tool.call_count == 2 + assert "Final answer based on two tool calls" in result + + @patch("anthropic.Anthropic") + def test_termination_no_tool_use_in_response(self, mock_anthropic): + """Test termination when Claude doesn't request tools""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + def create_tool_use_response(tool_name, **kwargs): + response = Mock() + response.stop_reason = "tool_use" + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = tool_name + tool_block.id = f"tool_{tool_name}" + tool_block.input = kwargs + response.content = [tool_block] + return response + + def create_text_response(text): + response = Mock() + response.stop_reason = "end_turn" + response.content = [Mock(text=text, type="text")] + return response + + # First response: tool use + # Second response: text only (no tool use) + mock_responses = [ + create_tool_use_response("search_course_content", query="machine learning"), + create_text_response("Based on search results, here's the answer"), + ] + + mock_client.messages.create.side_effect = mock_responses + + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.return_value = "Search results" + + generator = AIGenerator("test-key", "claude-sonnet-4") + result = generator.generate_response( + "Simple query", + tools=[{"name": "search_course_content"}], + tool_manager=mock_tool_manager, + ) + + # Should terminate after 2 API calls (1 tool round + 1 final) + assert mock_client.messages.create.call_count == 2 + assert mock_tool_manager.execute_tool.call_count == 1 + assert result == "Based on search results, here's the answer" + + +class TestAIGeneratorSequentialErrorHandling: + """Test error handling in sequential tool calling scenarios""" + + @patch("anthropic.Anthropic") + def test_error_recovery_tool_failure_round_two(self, mock_anthropic): + """Test error handling when second round tool fails""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + def create_tool_use_response(tool_name, **kwargs): + response = Mock() + response.stop_reason = "tool_use" + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = tool_name + tool_block.id = f"tool_{tool_name}" + tool_block.input = kwargs + response.content = [tool_block] + return response + + # First round succeeds, second round tool fails + mock_client.messages.create.side_effect = [ + create_tool_use_response("get_course_outline", course_name="ML Course"), + create_tool_use_response("search_course_content", query="neural networks"), + ] + + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.side_effect = [ + "Successful course outline result", + Exception("Tool execution failed in round 2"), + ] + + generator = AIGenerator("test-key", "claude-sonnet-4") + + # Should handle error gracefully + result = generator.generate_response( + "Complex query", + tools=[{"name": "get_course_outline"}, {"name": "search_course_content"}], + tool_manager=mock_tool_manager, + ) + + assert mock_client.messages.create.call_count == 2 # Both rounds attempted + assert mock_tool_manager.execute_tool.call_count == 2 # Both tools attempted + assert "I encountered an error while using tools" in result + + @patch("anthropic.Anthropic") + def test_api_error_during_sequential_calls(self, mock_anthropic): + """Test API error handling during sequential calls""" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + # First call succeeds, second call fails + def create_tool_use_response(): + response = Mock() + response.stop_reason = "tool_use" + tool_block = Mock() + tool_block.type = "tool_use" + tool_block.name = "search_tool" + tool_block.id = "tool_1" + tool_block.input = {"query": "test"} + response.content = [tool_block] + return response + + mock_client.messages.create.side_effect = [ + create_tool_use_response(), + Exception("API error in round 2"), + ] + + mock_tool_manager = Mock() + mock_tool_manager.execute_tool.return_value = "Tool result" + + generator = AIGenerator("test-key", "claude-sonnet-4") + + # Should fall back gracefully + result = generator.generate_response( + "Test query", + tools=[{"name": "search_tool"}], + tool_manager=mock_tool_manager, + ) + + assert "I'm sorry, I'm unable to process your request" in result diff --git a/backend/tests/test_api_endpoints.py b/backend/tests/test_api_endpoints.py new file mode 100644 index 00000000..3ea8d1e0 --- /dev/null +++ b/backend/tests/test_api_endpoints.py @@ -0,0 +1,385 @@ +"""Comprehensive API endpoint tests using new fixtures and test app factory""" +import pytest +import json +from unittest.mock import Mock +from fastapi.testclient import TestClient + + +@pytest.mark.api +class TestQueryEndpointEnhanced: + """Enhanced API tests for /api/query endpoint using new fixtures""" + + def test_query_with_new_fixtures(self, test_client): + """Test basic query functionality with new fixtures""" + client, mock_rag = test_client + + response = client.post("/api/query", json={ + "query": "What is machine learning?" + }) + + assert response.status_code == 200 + data = response.json() + + assert data["answer"] == "This is a test response about machine learning." + assert len(data["sources"]) == 1 + assert data["sources"][0]["text"] == "Introduction to ML - Lesson 1" + assert data["sources"][0]["url"] == "https://example.com/ml/lesson1" + assert data["session_id"] == "test-session-123" + + def test_query_with_custom_mock(self, test_app_factory): + """Test query with custom mock configuration""" + custom_mock = Mock() + custom_mock.query.return_value = ( + "Custom response about neural networks.", + [ + {"text": "Neural Networks - Chapter 1", "url": "https://example.com/nn/ch1"}, + {"text": "Deep Learning Basics", "url": "https://example.com/dl/basics"} + ] + ) + custom_mock.session_manager.create_session.return_value = "custom-session-456" + + app, rag_mock = test_app_factory(custom_mock) + client = TestClient(app) + + response = client.post("/api/query", json={ + "query": "How do neural networks work?" + }) + + assert response.status_code == 200 + data = response.json() + + assert data["answer"] == "Custom response about neural networks." + assert len(data["sources"]) == 2 + assert data["session_id"] == "custom-session-456" + + # Verify mock was called correctly + custom_mock.query.assert_called_once_with("How do neural networks work?", "custom-session-456") + + def test_query_large_response(self, test_app_factory): + """Test query with large response data""" + large_mock = Mock() + + # Create large response + large_answer = "A" * 5000 # 5KB response + large_sources = [ + {"text": f"Large Source {i}", "url": f"https://example.com/large/{i}"} + for i in range(20) + ] + + large_mock.query.return_value = (large_answer, large_sources) + large_mock.session_manager.create_session.return_value = "large-session" + + app, _ = test_app_factory(large_mock) + client = TestClient(app) + + response = client.post("/api/query", json={ + "query": "Tell me everything about machine learning" + }) + + assert response.status_code == 200 + data = response.json() + + assert len(data["answer"]) == 5000 + assert len(data["sources"]) == 20 + assert all(source["text"].startswith("Large Source") for source in data["sources"]) + + def test_query_unicode_content(self, test_app_factory): + """Test query with Unicode and special characters""" + unicode_mock = Mock() + unicode_mock.query.return_value = ( + "这是关于机器学习的回答。Machine learning is 机械学習。", + [{"text": "多语言课程 🚀", "url": "https://example.com/unicode/课程"}] + ) + unicode_mock.session_manager.create_session.return_value = "unicode-session" + + app, _ = test_app_factory(unicode_mock) + client = TestClient(app) + + response = client.post("/api/query", json={ + "query": "什么是机器学习?" + }) + + assert response.status_code == 200 + data = response.json() + + assert "机器学习" in data["answer"] + assert "多语言课程 🚀" in data["sources"][0]["text"] + + def test_query_concurrent_sessions(self, test_app_factory): + """Test handling multiple concurrent sessions""" + session_mock = Mock() + session_mock.session_manager.create_session.side_effect = [ + "session-1", "session-2", "session-3" + ] + session_mock.query.return_value = ("Response", []) + + app, _ = test_app_factory(session_mock) + client = TestClient(app) + + # Simulate concurrent requests + responses = [] + for i in range(3): + response = client.post("/api/query", json={ + "query": f"Query {i+1}" + }) + responses.append(response) + + # All should succeed with different session IDs + session_ids = set() + for response in responses: + assert response.status_code == 200 + session_ids.add(response.json()["session_id"]) + + assert len(session_ids) == 3 # All different sessions + + +@pytest.mark.api +class TestCoursesEndpointEnhanced: + """Enhanced API tests for /api/courses endpoint""" + + def test_courses_with_new_fixtures(self, test_client): + """Test courses endpoint with new fixtures""" + client, mock_rag = test_client + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + + assert data["total_courses"] == 2 + assert len(data["course_titles"]) == 2 + assert "Introduction to Machine Learning" in data["course_titles"] + assert "Advanced Python Programming" in data["course_titles"] + + def test_courses_large_dataset(self, test_app_factory): + """Test courses endpoint with large dataset""" + large_courses_mock = Mock() + + # Create large course list + large_titles = [f"Course {i:03d}: Advanced Topic {i}" for i in range(100)] + large_courses_mock.get_course_analytics.return_value = { + "total_courses": 100, + "course_titles": large_titles + } + + app, _ = test_app_factory(large_courses_mock) + client = TestClient(app) + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + + assert data["total_courses"] == 100 + assert len(data["course_titles"]) == 100 + assert "Course 050: Advanced Topic 50" in data["course_titles"] + + def test_courses_unicode_titles(self, test_app_factory): + """Test courses endpoint with Unicode course titles""" + unicode_courses_mock = Mock() + unicode_courses_mock.get_course_analytics.return_value = { + "total_courses": 3, + "course_titles": [ + "机器学习入门", + "Aprendizaje Automático 🤖", + "Машинное обучение" + ] + } + + app, _ = test_app_factory(unicode_courses_mock) + client = TestClient(app) + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + + assert data["total_courses"] == 3 + assert "机器学习入门" in data["course_titles"] + assert "Aprendizaje Automático 🤖" in data["course_titles"] + assert "Машинное обучение" in data["course_titles"] + + +@pytest.mark.api +class TestHealthEndpoint: + """Test health check endpoint""" + + def test_health_check(self, test_client): + """Test health check endpoint""" + client, _ = test_client + + response = client.get("/health") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + + def test_health_check_no_dependencies(self, test_app_factory): + """Test health check works even if RAG system is broken""" + broken_mock = Mock() + broken_mock.query.side_effect = Exception("RAG system broken") + broken_mock.get_course_analytics.side_effect = Exception("Analytics broken") + + app, _ = test_app_factory(broken_mock) + client = TestClient(app) + + # Health check should still work + response = client.get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "healthy" + + +@pytest.mark.api +class TestErrorHandlingEnhanced: + """Enhanced error handling tests""" + + def test_query_timeout_simulation(self, test_app_factory): + """Test query timeout handling""" + import time + + timeout_mock = Mock() + def slow_query(*args, **kwargs): + time.sleep(0.1) # Simulate slow operation + raise TimeoutError("Query timeout") + + timeout_mock.query.side_effect = slow_query + timeout_mock.session_manager.create_session.return_value = "timeout-session" + + app, _ = test_app_factory(timeout_mock) + client = TestClient(app) + + response = client.post("/api/query", json={"query": "slow query"}) + + assert response.status_code == 500 + assert "Query timeout" in response.json()["detail"] + + def test_malformed_json_request(self, test_client): + """Test handling of malformed JSON requests""" + client, _ = test_client + + # Send malformed JSON + response = client.post( + "/api/query", + data='{"query": "test", "invalid": json}', # Invalid JSON + headers={"content-type": "application/json"} + ) + + assert response.status_code == 422 + + def test_extremely_long_query(self, test_client): + """Test handling of extremely long query strings""" + client, mock_rag = test_client + + # Create very long query (100KB) + long_query = "A" * 100000 + + response = client.post("/api/query", json={ + "query": long_query + }) + + # Should handle gracefully + assert response.status_code == 200 + + # Verify the long query was passed to RAG system + args, kwargs = mock_rag.query.call_args + assert args[0] == long_query + + def test_empty_course_analytics(self, test_app_factory): + """Test courses endpoint with empty analytics""" + empty_mock = Mock() + empty_mock.get_course_analytics.return_value = { + "total_courses": 0, + "course_titles": [] + } + + app, _ = test_app_factory(empty_mock) + client = TestClient(app) + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + assert data["total_courses"] == 0 + assert data["course_titles"] == [] + + +@pytest.mark.api +class TestMiddlewareConfiguration: + """Test middleware and CORS configuration""" + + def test_cors_headers_query(self, test_client): + """Test CORS headers on query endpoint""" + client, _ = test_client + + # Make request with custom origin + response = client.post( + "/api/query", + json={"query": "test"}, + headers={"Origin": "https://example.com"} + ) + + assert response.status_code == 200 + # TestClient doesn't process CORS middleware the same way, + # but we can verify the endpoint works + + def test_cors_headers_courses(self, test_client): + """Test CORS headers on courses endpoint""" + client, _ = test_client + + response = client.get( + "/api/courses", + headers={"Origin": "https://localhost:3000"} + ) + + assert response.status_code == 200 + + def test_options_request(self, test_client): + """Test OPTIONS request handling""" + client, _ = test_client + + response = client.options("/api/query") + + # FastAPI should handle OPTIONS requests + # Status could be 405 (method not allowed) or 200 depending on CORS setup + assert response.status_code in [200, 405] + + +@pytest.mark.integration +class TestAppFactory: + """Test the app factory fixture itself""" + + def test_app_factory_creates_different_apps(self, test_app_factory): + """Test that app factory creates independent app instances""" + mock1 = Mock() + mock1.query.return_value = ("Response 1", []) + mock1.session_manager.create_session.return_value = "session-1" + + mock2 = Mock() + mock2.query.return_value = ("Response 2", []) + mock2.session_manager.create_session.return_value = "session-2" + + app1, _ = test_app_factory(mock1) + app2, _ = test_app_factory(mock2) + + client1 = TestClient(app1) + client2 = TestClient(app2) + + response1 = client1.post("/api/query", json={"query": "test"}) + response2 = client2.post("/api/query", json={"query": "test"}) + + assert response1.json()["answer"] == "Response 1" + assert response1.json()["session_id"] == "session-1" + + assert response2.json()["answer"] == "Response 2" + assert response2.json()["session_id"] == "session-2" + + def test_app_factory_with_none_mock(self, test_app_factory): + """Test app factory creates default mock when None provided""" + app, rag_mock = test_app_factory(None) + client = TestClient(app) + + response = client.post("/api/query", json={"query": "test"}) + + assert response.status_code == 200 + assert response.json()["answer"] == "Test response" + assert response.json()["session_id"] == "test-session" \ No newline at end of file diff --git a/backend/tests/test_app.py b/backend/tests/test_app.py new file mode 100644 index 00000000..cab1806c --- /dev/null +++ b/backend/tests/test_app.py @@ -0,0 +1,412 @@ +"""API layer tests for FastAPI endpoints""" + +import os +import sys +from unittest.mock import Mock, patch + +import pytest +from fastapi.testclient import TestClient + +# Add backend directory to Python path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + + +class TestQueryEndpoint: + """Test /api/query endpoint""" + + @pytest.fixture + def mock_app(self): + """Create test client with mocked RAG system""" + with patch("app.RAGSystem") as mock_rag_class: + # Import app after patching + from app import app + + # Configure mock RAG system + mock_rag = Mock() + mock_rag.query.return_value = ( + "Test response", + [{"text": "Test Course", "url": "https://example.com"}], + ) + mock_rag.session_manager.create_session.return_value = "test-session-123" + mock_rag_class.return_value = mock_rag + + client = TestClient(app) + return client, mock_rag + + def test_query_without_session(self, mock_app): + """Test query endpoint without session ID""" + client, mock_rag = mock_app + + response = client.post( + "/api/query", json={"query": "What is machine learning?"} + ) + + assert response.status_code == 200 + data = response.json() + + assert data["answer"] == "Test response" + assert len(data["sources"]) == 1 + assert data["sources"][0]["text"] == "Test Course" + assert data["session_id"] == "test-session-123" + + # Verify RAG system was called correctly + mock_rag.query.assert_called_once_with( + "What is machine learning?", "test-session-123" + ) + + def test_query_with_session(self, mock_app): + """Test query endpoint with existing session ID""" + client, mock_rag = mock_app + + response = client.post( + "/api/query", + json={"query": "Follow up question", "session_id": "existing-session-456"}, + ) + + assert response.status_code == 200 + data = response.json() + + assert data["session_id"] == "existing-session-456" + + # Verify RAG system was called with existing session + mock_rag.query.assert_called_once_with( + "Follow up question", "existing-session-456" + ) + # Session creation should not be called + mock_rag.session_manager.create_session.assert_not_called() + + def test_query_with_string_sources(self, mock_app): + """Test query endpoint with string sources (backward compatibility)""" + client, mock_rag = mock_app + + # Configure RAG to return string sources + mock_rag.query.return_value = ("Test response", ["Course 1", "Course 2"]) + + response = client.post("/api/query", json={"query": "Test query"}) + + assert response.status_code == 200 + data = response.json() + + assert data["sources"] == ["Course 1", "Course 2"] + + def test_query_with_mixed_sources(self, mock_app): + """Test query endpoint with mixed source types""" + client, mock_rag = mock_app + + # Configure RAG to return mixed sources + mixed_sources = [ + {"text": "Course with link", "url": "https://example.com/course"}, + "Plain text source", + ] + mock_rag.query.return_value = ("Test response", mixed_sources) + + response = client.post("/api/query", json={"query": "Test query"}) + + assert response.status_code == 200 + data = response.json() + + assert len(data["sources"]) == 2 + assert data["sources"][0]["text"] == "Course with link" + assert data["sources"][0]["url"] == "https://example.com/course" + assert data["sources"][1] == "Plain text source" + + def test_query_empty_query(self, mock_app): + """Test query endpoint with empty query""" + client, mock_rag = mock_app + + response = client.post("/api/query", json={"query": ""}) + + assert response.status_code == 200 + # Should still process empty query + mock_rag.query.assert_called_once() + + def test_query_missing_query_field(self, mock_app): + """Test query endpoint with missing query field""" + client, mock_rag = mock_app + + response = client.post("/api/query", json={"session_id": "test-session"}) + + assert response.status_code == 422 # Validation error + + def test_query_rag_system_exception(self, mock_app): + """Test query endpoint when RAG system raises exception""" + client, mock_rag = mock_app + + # Configure RAG to raise exception + mock_rag.query.side_effect = Exception("RAG system error") + + response = client.post("/api/query", json={"query": "Test query"}) + + assert response.status_code == 500 + data = response.json() + assert "RAG system error" in data["detail"] + + def test_query_session_creation_exception(self, mock_app): + """Test query endpoint when session creation fails""" + client, mock_rag = mock_app + + # Configure session manager to raise exception + mock_rag.session_manager.create_session.side_effect = Exception( + "Session creation failed" + ) + + response = client.post("/api/query", json={"query": "Test query"}) + + assert response.status_code == 500 + data = response.json() + assert "Session creation failed" in data["detail"] + + +class TestCoursesEndpoint: + """Test /api/courses endpoint""" + + @pytest.fixture + def mock_app_courses(self): + """Create test client with mocked RAG system for courses endpoint""" + with patch("app.RAGSystem") as mock_rag_class: + from app import app + + mock_rag = Mock() + mock_rag.get_course_analytics.return_value = { + "total_courses": 3, + "course_titles": ["Course 1", "Course 2", "Course 3"], + } + mock_rag_class.return_value = mock_rag + + client = TestClient(app) + return client, mock_rag + + def test_get_course_stats_success(self, mock_app_courses): + """Test successful course statistics retrieval""" + client, mock_rag = mock_app_courses + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + + assert data["total_courses"] == 3 + assert len(data["course_titles"]) == 3 + assert "Course 1" in data["course_titles"] + + # Verify RAG system was called + mock_rag.get_course_analytics.assert_called_once() + + def test_get_course_stats_empty(self, mock_app_courses): + """Test course statistics when no courses exist""" + client, mock_rag = mock_app_courses + + # Configure RAG to return empty analytics + mock_rag.get_course_analytics.return_value = { + "total_courses": 0, + "course_titles": [], + } + + response = client.get("/api/courses") + + assert response.status_code == 200 + data = response.json() + + assert data["total_courses"] == 0 + assert data["course_titles"] == [] + + def test_get_course_stats_exception(self, mock_app_courses): + """Test course statistics endpoint when RAG system raises exception""" + client, mock_rag = mock_app_courses + + # Configure RAG to raise exception + mock_rag.get_course_analytics.side_effect = Exception("Analytics error") + + response = client.get("/api/courses") + + assert response.status_code == 500 + data = response.json() + assert "Analytics error" in data["detail"] + + +class TestAppStartup: + """Test application startup behavior""" + + @patch("app.os.path.exists") + @patch("app.RAGSystem") + def test_startup_with_docs_folder(self, mock_rag_class, mock_exists): + """Test startup behavior when docs folder exists""" + mock_exists.return_value = True + mock_rag = Mock() + mock_rag.add_course_folder.return_value = (2, 10) # 2 courses, 10 chunks + mock_rag_class.return_value = mock_rag + + from app import app + + with TestClient(app): + # Trigger startup event + pass + + # Verify docs folder was processed + mock_rag.add_course_folder.assert_called_once_with( + "../docs", clear_existing=False + ) + + @patch("app.os.path.exists") + @patch("app.RAGSystem") + def test_startup_without_docs_folder(self, mock_rag_class, mock_exists): + """Test startup behavior when docs folder doesn't exist""" + mock_exists.return_value = False + mock_rag = Mock() + mock_rag_class.return_value = mock_rag + + from app import app + + with TestClient(app): + pass + + # Verify docs processing was not attempted + mock_rag.add_course_folder.assert_not_called() + + @patch("app.os.path.exists") + @patch("app.RAGSystem") + def test_startup_docs_processing_error(self, mock_rag_class, mock_exists): + """Test startup behavior when docs processing fails""" + mock_exists.return_value = True + mock_rag = Mock() + mock_rag.add_course_folder.side_effect = Exception("Processing error") + mock_rag_class.return_value = mock_rag + + from app import app + + # Should not crash despite processing error + with TestClient(app): + pass + + +class TestAppConfiguration: + """Test app configuration and middleware""" + + def test_cors_configuration(self): + """Test CORS middleware configuration""" + from app import app + + client = TestClient(app) + + # Test CORS headers on OPTIONS request + response = client.options("/api/query") + + # Should handle CORS properly + assert ( + response.status_code == 405 + ) # Method not allowed, but CORS headers should be present + + def test_trusted_host_middleware(self): + """Test trusted host middleware allows requests""" + from app import app + + client = TestClient(app) + + # Should accept requests from any host (configured with "*") + with patch("app.RAGSystem"): + response = client.get("/api/courses") + # Should not be blocked by trusted host middleware + assert response.status_code != 400 + + +class TestErrorHandling: + """Test error handling across the application""" + + @pytest.fixture + def error_app(self): + """App configured to test error scenarios""" + with patch("app.RAGSystem") as mock_rag_class: + from app import app + + mock_rag = Mock() + mock_rag_class.return_value = mock_rag + + return TestClient(app), mock_rag + + def test_query_validation_error(self, error_app): + """Test validation error handling""" + client, mock_rag = error_app + + # Send invalid JSON + response = client.post("/api/query", json={"invalid_field": "value"}) + + assert response.status_code == 422 + data = response.json() + assert "detail" in data + + def test_query_unexpected_error(self, error_app): + """Test unexpected error handling in query endpoint""" + client, mock_rag = error_app + + # Configure RAG to raise unexpected error + mock_rag.query.side_effect = RuntimeError("Unexpected system error") + + response = client.post("/api/query", json={"query": "test"}) + + assert response.status_code == 500 + data = response.json() + assert "Unexpected system error" in data["detail"] + + def test_courses_unexpected_error(self, error_app): + """Test unexpected error handling in courses endpoint""" + client, mock_rag = error_app + + # Configure RAG to raise unexpected error + mock_rag.get_course_analytics.side_effect = ValueError( + "Analytics computation error" + ) + + response = client.get("/api/courses") + + assert response.status_code == 500 + data = response.json() + assert "Analytics computation error" in data["detail"] + + +class TestRequestResponseModels: + """Test Pydantic request/response models""" + + def test_query_request_model_validation(self): + """Test QueryRequest model validation""" + from app import QueryRequest + + # Valid request + request = QueryRequest(query="test query", session_id="test-session") + assert request.query == "test query" + assert request.session_id == "test-session" + + # Request without session_id + request = QueryRequest(query="test query") + assert request.query == "test query" + assert request.session_id is None + + # Invalid request (missing query) + with pytest.raises(ValueError): + QueryRequest(session_id="test-session") + + def test_query_response_model_validation(self): + """Test QueryResponse model validation""" + from app import QueryResponse + + # Valid response with dict sources + response = QueryResponse( + answer="test answer", + sources=[{"text": "source", "url": "https://example.com"}], + session_id="test-session", + ) + assert response.answer == "test answer" + assert len(response.sources) == 1 + + # Valid response with string sources + response = QueryResponse( + answer="test answer", sources=["string source"], session_id="test-session" + ) + assert response.sources == ["string source"] + + def test_course_stats_model_validation(self): + """Test CourseStats model validation""" + from app import CourseStats + + stats = CourseStats(total_courses=5, course_titles=["Course 1", "Course 2"]) + assert stats.total_courses == 5 + assert len(stats.course_titles) == 2 diff --git a/backend/tests/test_course_search_tool.py b/backend/tests/test_course_search_tool.py new file mode 100644 index 00000000..9626d2d6 --- /dev/null +++ b/backend/tests/test_course_search_tool.py @@ -0,0 +1,294 @@ +"""Unit tests for CourseSearchTool""" + +from unittest.mock import Mock + +from search_tools import CourseSearchTool +from vector_store import SearchResults + + +class TestCourseSearchTool: + """Test cases for CourseSearchTool functionality""" + + def test_get_tool_definition(self, course_search_tool): + """Test that tool definition is correctly structured""" + definition = course_search_tool.get_tool_definition() + + assert definition["name"] == "search_course_content" + assert "description" in definition + assert "input_schema" in definition + assert definition["input_schema"]["required"] == ["query"] + + # Check properties structure + properties = definition["input_schema"]["properties"] + assert "query" in properties + assert "course_name" in properties + assert "lesson_number" in properties + + def test_execute_successful_search_basic_query(self, course_search_tool): + """Test successful search with basic query only""" + result = course_search_tool.execute("machine learning") + + # Should call vector store search with correct parameters + course_search_tool.store.search.assert_called_once_with( + query="machine learning", course_name=None, lesson_number=None + ) + + # Should return formatted results + assert "[Introduction to Machine Learning" in result + assert "Machine learning is a subset" in result + + # Should track sources + assert len(course_search_tool.last_sources) > 0 + + def test_execute_successful_search_with_course_filter(self, course_search_tool): + """Test successful search with course name filter""" + result = course_search_tool.execute( + "machine learning", course_name="Introduction to Machine Learning" + ) + + course_search_tool.store.search.assert_called_once_with( + query="machine learning", + course_name="Introduction to Machine Learning", + lesson_number=None, + ) + + assert "[Introduction to Machine Learning" in result + + def test_execute_successful_search_with_lesson_filter(self, course_search_tool): + """Test successful search with lesson number filter""" + result = course_search_tool.execute( + "machine learning", + course_name="Introduction to Machine Learning", + lesson_number=1, + ) + + course_search_tool.store.search.assert_called_once_with( + query="machine learning", + course_name="Introduction to Machine Learning", + lesson_number=1, + ) + + assert "[Introduction to Machine Learning - Lesson 1]" in result + + def test_execute_vector_store_error(self, course_search_tool): + """Test handling of vector store errors""" + # Mock vector store to return error + course_search_tool.store.search.return_value = SearchResults.empty( + "Database connection failed" + ) + + result = course_search_tool.execute("machine learning") + + assert result == "Database connection failed" + assert course_search_tool.last_sources == [] + + def test_execute_no_results_found_basic_query(self, course_search_tool): + """Test handling when no results are found""" + # Mock vector store to return empty results + course_search_tool.store.search.return_value = SearchResults( + documents=[], metadata=[], distances=[] + ) + + result = course_search_tool.execute("nonexistent topic") + + assert result == "No relevant content found." + assert course_search_tool.last_sources == [] + + def test_execute_no_results_found_with_filters(self, course_search_tool): + """Test handling when no results are found with filters""" + course_search_tool.store.search.return_value = SearchResults( + documents=[], metadata=[], distances=[] + ) + + result = course_search_tool.execute( + "nonexistent topic", course_name="ML Course", lesson_number=5 + ) + + assert result == "No relevant content found in course 'ML Course' in lesson 5." + assert course_search_tool.last_sources == [] + + def test_execute_partial_course_filter_message(self, course_search_tool): + """Test error message construction with partial filters""" + course_search_tool.store.search.return_value = SearchResults( + documents=[], metadata=[], distances=[] + ) + + # Test with only course name + result = course_search_tool.execute("test query", course_name="Some Course") + assert result == "No relevant content found in course 'Some Course'." + + # Test with only lesson number + result = course_search_tool.execute("test query", lesson_number=3) + assert result == "No relevant content found in lesson 3." + + def test_format_results_with_links(self, course_search_tool): + """Test result formatting with lesson and course links""" + # Configure mock to return links + course_search_tool.store.get_lesson_link.return_value = ( + "https://example.com/lesson1" + ) + course_search_tool.store.get_course_link.return_value = ( + "https://example.com/course" + ) + + # Create search results with multiple documents + search_results = SearchResults( + documents=[ + "Content about machine learning algorithms", + "More content about neural networks", + ], + metadata=[ + {"course_title": "ML Course", "lesson_number": 1}, + {"course_title": "ML Course", "lesson_number": 2}, + ], + distances=[0.1, 0.2], + ) + + course_search_tool.store.search.return_value = search_results + + result = course_search_tool.execute("algorithms") + + # Should format with lesson headers + assert "[ML Course - Lesson 1]" in result + assert "[ML Course - Lesson 2]" in result + assert "Content about machine learning algorithms" in result + assert "More content about neural networks" in result + + # Should track sources with links + expected_sources = [ + {"text": "ML Course - Lesson 1", "url": "https://example.com/lesson1"}, + { + "text": "ML Course - Lesson 2", + "url": "https://example.com/lesson1", + }, # Mock returns same link + ] + assert len(course_search_tool.last_sources) == 2 + + def test_format_results_without_links(self, course_search_tool): + """Test result formatting when links are not available""" + # Configure mock to return no links + course_search_tool.store.get_lesson_link.return_value = None + course_search_tool.store.get_course_link.return_value = None + + search_results = SearchResults( + documents=["Content without links"], + metadata=[{"course_title": "No Link Course", "lesson_number": 1}], + distances=[0.1], + ) + + course_search_tool.store.search.return_value = search_results + + result = course_search_tool.execute("test") + + # Should still format properly but sources should be plain text + assert "[No Link Course - Lesson 1]" in result + assert course_search_tool.last_sources == ["No Link Course - Lesson 1"] + + def test_format_results_course_level_content(self, course_search_tool): + """Test result formatting for course-level content (no lesson number)""" + course_search_tool.store.get_course_link.return_value = ( + "https://example.com/course" + ) + + search_results = SearchResults( + documents=["Course overview content"], + metadata=[{"course_title": "Overview Course"}], # No lesson_number + distances=[0.1], + ) + + course_search_tool.store.search.return_value = search_results + + result = course_search_tool.execute("overview") + + # Should format without lesson number + assert "[Overview Course]" in result + assert "Course overview content" in result + + # Should track course-level source with link + expected_source = { + "text": "Overview Course", + "url": "https://example.com/course", + } + assert course_search_tool.last_sources == [expected_source] + + def test_format_results_malformed_metadata(self, course_search_tool): + """Test handling of malformed metadata""" + search_results = SearchResults( + documents=["Some content"], + metadata=[{}], + distances=[0.1], # Empty metadata + ) + + course_search_tool.store.search.return_value = search_results + + result = course_search_tool.execute("test") + + # Should handle gracefully with unknown course + assert "[unknown]" in result + assert "Some content" in result + + def test_sources_reset_between_searches(self, course_search_tool): + """Test that sources are properly managed between searches""" + # First search + course_search_tool.execute("first query") + first_sources = course_search_tool.last_sources.copy() + assert len(first_sources) > 0 + + # Second search with different results + course_search_tool.store.search.return_value = SearchResults( + documents=["Different content"], + metadata=[{"course_title": "Different Course", "lesson_number": 2}], + distances=[0.3], + ) + + course_search_tool.execute("second query") + second_sources = course_search_tool.last_sources + + # Sources should be different and reflect the new search + assert second_sources != first_sources + assert any("Different Course" in str(source) for source in second_sources) + + +class TestCourseSearchToolEdgeCases: + """Test edge cases and error conditions""" + + def test_execute_with_none_query(self): + """Test behavior with None query - should return error message""" + mock_store = Mock() + tool = CourseSearchTool(mock_store) + + result = tool.execute(None) + assert result == "Error: Search query cannot be None." + + def test_execute_with_empty_string_query(self, course_search_tool): + """Test behavior with empty string query""" + result = course_search_tool.execute("") + + course_search_tool.store.search.assert_called_once_with( + query="", course_name=None, lesson_number=None + ) + + def test_execute_with_invalid_lesson_number(self, course_search_tool): + """Test behavior with invalid lesson number types""" + # Should handle string lesson numbers + result = course_search_tool.execute("test", lesson_number="not_a_number") + + course_search_tool.store.search.assert_called_once_with( + query="test", + course_name=None, + lesson_number="not_a_number", # Vector store should handle validation + ) + + def test_vector_store_exception_handling(self): + """Test handling when vector store raises unexpected exceptions""" + mock_store = Mock() + mock_store.search.side_effect = Exception("Unexpected database error") + + tool = CourseSearchTool(mock_store) + + result = tool.execute("test query") + + # Should handle gracefully and return error message + assert isinstance(result, str) + assert "Search failed due to an internal error" in result + assert "Unexpected database error" in result diff --git a/backend/tests/test_rag_system.py b/backend/tests/test_rag_system.py new file mode 100644 index 00000000..639a1a90 --- /dev/null +++ b/backend/tests/test_rag_system.py @@ -0,0 +1,419 @@ +"""Integration tests for RAGSystem""" + +from unittest.mock import Mock, patch + +import pytest +from models import Course, CourseChunk, Lesson +from rag_system import RAGSystem + + +class TestRAGSystemIntegration: + """Test RAG system integration and complete query flow""" + + @pytest.fixture + def mock_rag_system(self, test_config): + """Create RAG system with mocked dependencies""" + with ( + patch("rag_system.DocumentProcessor"), + patch("rag_system.VectorStore") as mock_vector_store, + patch("rag_system.AIGenerator") as mock_ai_generator, + patch("rag_system.SessionManager"), + ): + rag = RAGSystem(test_config) + + # Configure mocks + rag.vector_store = mock_vector_store.return_value + rag.ai_generator = mock_ai_generator.return_value + + # Mock AI generator to return simple response + rag.ai_generator.generate_response.return_value = "Mocked AI response" + + return rag + + def test_rag_system_initialization(self, test_config): + """Test that RAG system initializes all components correctly""" + with ( + patch("rag_system.DocumentProcessor"), + patch("rag_system.VectorStore"), + patch("rag_system.AIGenerator"), + patch("rag_system.SessionManager"), + ): + rag = RAGSystem(test_config) + + # Verify tools are registered + assert "search_course_content" in rag.tool_manager.tools + assert "get_course_outline" in rag.tool_manager.tools + assert len(rag.tool_manager.tools) == 2 + + def test_query_without_session(self, mock_rag_system): + """Test query processing without session ID""" + response, sources = mock_rag_system.query("What is machine learning?") + + assert response == "Mocked AI response" + assert isinstance(sources, list) + + # Verify AI generator was called with correct parameters + mock_rag_system.ai_generator.generate_response.assert_called_once() + call_args = mock_rag_system.ai_generator.generate_response.call_args + + # Check query format + assert ( + "Answer this question about course materials: What is machine learning?" + in call_args[0][0] + ) + + # Check tools are provided + assert call_args[1]["tools"] is not None + assert call_args[1]["tool_manager"] is not None + + def test_query_with_session(self, mock_rag_system): + """Test query processing with session ID""" + session_id = "test-session-123" + + # Mock session manager + mock_rag_system.session_manager.get_conversation_history.return_value = ( + "Previous context" + ) + + response, sources = mock_rag_system.query("Follow up question", session_id) + + assert response == "Mocked AI response" + + # Verify session history was retrieved + mock_rag_system.session_manager.get_conversation_history.assert_called_once_with( + session_id + ) + + # Verify conversation history was passed to AI generator + call_args = mock_rag_system.ai_generator.generate_response.call_args + assert call_args[1]["conversation_history"] == "Previous context" + + # Verify session was updated + mock_rag_system.session_manager.add_exchange.assert_called_once_with( + session_id, "Follow up question", "Mocked AI response" + ) + + def test_query_with_tool_execution(self, mock_rag_system): + """Test query that triggers tool execution""" + # Configure search tool to return results + mock_rag_system.search_tool.last_sources = [ + {"text": "ML Course - Lesson 1", "url": "https://example.com/lesson1"} + ] + + # Configure tool manager to return sources + mock_rag_system.tool_manager.get_last_sources.return_value = ( + mock_rag_system.search_tool.last_sources + ) + + response, sources = mock_rag_system.query("What is machine learning?") + + assert response == "Mocked AI response" + assert len(sources) == 1 + assert sources[0]["text"] == "ML Course - Lesson 1" + assert sources[0]["url"] == "https://example.com/lesson1" + + # Verify sources were reset after retrieval + mock_rag_system.tool_manager.reset_sources.assert_called_once() + + def test_query_ai_generator_exception(self, mock_rag_system): + """Test handling when AI generator raises exception""" + mock_rag_system.ai_generator.generate_response.side_effect = Exception( + "API error" + ) + + with pytest.raises(Exception) as exc_info: + mock_rag_system.query("test query") + + assert "API error" in str(exc_info.value) + + +class TestRAGSystemDocumentProcessing: + """Test document processing functionality""" + + @pytest.fixture + def mock_rag_with_docs(self, test_config): + """RAG system with document processing mocks""" + with ( + patch("rag_system.DocumentProcessor") as mock_doc_processor, + patch("rag_system.VectorStore") as mock_vector_store, + patch("rag_system.AIGenerator"), + patch("rag_system.SessionManager"), + ): + rag = RAGSystem(test_config) + + # Configure document processor mock + sample_course = Course( + title="Test Course", + course_link="https://example.com/course", + lessons=[Lesson(lesson_number=1, title="Test Lesson")], + ) + sample_chunks = [ + CourseChunk( + content="Test content", + course_title="Test Course", + lesson_number=1, + chunk_index=0, + ) + ] + + mock_doc_processor.return_value.process_course_document.return_value = ( + sample_course, + sample_chunks, + ) + rag.document_processor = mock_doc_processor.return_value + rag.vector_store = mock_vector_store.return_value + + return rag + + def test_add_course_document_success(self, mock_rag_with_docs): + """Test successful course document addition""" + course, chunk_count = mock_rag_with_docs.add_course_document( + "/path/to/course.pdf" + ) + + assert course.title == "Test Course" + assert chunk_count == 1 + + # Verify document was processed + mock_rag_with_docs.document_processor.process_course_document.assert_called_once_with( + "/path/to/course.pdf" + ) + + # Verify data was added to vector store + mock_rag_with_docs.vector_store.add_course_metadata.assert_called_once() + mock_rag_with_docs.vector_store.add_course_content.assert_called_once() + + def test_add_course_document_processing_error(self, mock_rag_with_docs): + """Test handling of document processing errors""" + mock_rag_with_docs.document_processor.process_course_document.side_effect = ( + Exception("Processing failed") + ) + + course, chunk_count = mock_rag_with_docs.add_course_document( + "/path/to/invalid.pdf" + ) + + assert course is None + assert chunk_count == 0 + + # Vector store should not be called + mock_rag_with_docs.vector_store.add_course_metadata.assert_not_called() + mock_rag_with_docs.vector_store.add_course_content.assert_not_called() + + @patch("os.path.exists") + @patch("os.listdir") + @patch("os.path.isfile") + def test_add_course_folder_success( + self, mock_isfile, mock_listdir, mock_exists, mock_rag_with_docs + ): + """Test successful course folder processing""" + # Mock file system + mock_exists.return_value = True + mock_listdir.return_value = ["course1.pdf", "course2.docx", "readme.txt"] + mock_isfile.return_value = True + + # Mock existing courses (empty) + mock_rag_with_docs.vector_store.get_existing_course_titles.return_value = [] + + courses, chunks = mock_rag_with_docs.add_course_folder("/docs") + + assert courses == 3 # All three files processed + assert chunks == 3 # One chunk per file + + # Verify all files were processed + assert ( + mock_rag_with_docs.document_processor.process_course_document.call_count + == 3 + ) + + @patch("os.path.exists") + def test_add_course_folder_missing_folder(self, mock_exists, mock_rag_with_docs): + """Test handling of missing course folder""" + mock_exists.return_value = False + + courses, chunks = mock_rag_with_docs.add_course_folder("/nonexistent") + + assert courses == 0 + assert chunks == 0 + + @patch("os.path.exists") + @patch("os.listdir") + @patch("os.path.isfile") + def test_add_course_folder_skip_existing( + self, mock_isfile, mock_listdir, mock_exists, mock_rag_with_docs + ): + """Test skipping existing courses when adding folder""" + mock_exists.return_value = True + mock_listdir.return_value = ["course1.pdf"] + mock_isfile.return_value = True + + # Mock existing courses to include the course we're trying to add + mock_rag_with_docs.vector_store.get_existing_course_titles.return_value = [ + "Test Course" + ] + + courses, chunks = mock_rag_with_docs.add_course_folder("/docs") + + assert courses == 0 # Should skip existing course + assert chunks == 0 + + # Document should still be processed to check if it's duplicate + mock_rag_with_docs.document_processor.process_course_document.assert_called_once() + + # But vector store should not be updated + mock_rag_with_docs.vector_store.add_course_metadata.assert_not_called() + mock_rag_with_docs.vector_store.add_course_content.assert_not_called() + + @patch("os.path.exists") + @patch("os.listdir") + @patch("os.path.isfile") + def test_add_course_folder_clear_existing( + self, mock_isfile, mock_listdir, mock_exists, mock_rag_with_docs + ): + """Test clearing existing data before adding folder""" + mock_exists.return_value = True + mock_listdir.return_value = ["course1.pdf"] + mock_isfile.return_value = True + + mock_rag_with_docs.vector_store.get_existing_course_titles.return_value = [] + + courses, chunks = mock_rag_with_docs.add_course_folder( + "/docs", clear_existing=True + ) + + # Verify data was cleared + mock_rag_with_docs.vector_store.clear_all_data.assert_called_once() + + assert courses == 1 + assert chunks == 1 + + +class TestRAGSystemAnalytics: + """Test RAG system analytics functionality""" + + def test_get_course_analytics(self, mock_rag_system): + """Test course analytics retrieval""" + # Configure vector store mock + mock_rag_system.vector_store.get_course_count.return_value = 5 + mock_rag_system.vector_store.get_existing_course_titles.return_value = [ + "Course 1", + "Course 2", + "Course 3", + "Course 4", + "Course 5", + ] + + analytics = mock_rag_system.get_course_analytics() + + assert analytics["total_courses"] == 5 + assert len(analytics["course_titles"]) == 5 + assert "Course 1" in analytics["course_titles"] + + +class TestRAGSystemRealIntegration: + """Test RAG system with real components (integration test)""" + + @pytest.fixture + def real_rag_system(self, test_config): + """RAG system with real components""" + # Only mock the Anthropic client to avoid real API calls + with patch("ai_generator.anthropic.Anthropic") as mock_anthropic: + mock_client = Mock() + mock_response = Mock() + mock_response.content = [Mock(text="Real integration test response")] + mock_response.stop_reason = "end_turn" + mock_client.messages.create.return_value = mock_response + mock_anthropic.return_value = mock_client + + return RAGSystem(test_config) + + def test_real_integration_query_flow( + self, real_rag_system, sample_course, sample_course_chunks + ): + """Test complete query flow with real components""" + # Add test data + real_rag_system.vector_store.add_course_metadata(sample_course) + real_rag_system.vector_store.add_course_content(sample_course_chunks) + + # Execute query + response, sources = real_rag_system.query("What is machine learning?") + + assert "Real integration test response" in response + assert isinstance(sources, list) + + # Verify vector store has the data + assert real_rag_system.vector_store.get_course_count() == 1 + assert ( + "Introduction to Machine Learning" + in real_rag_system.vector_store.get_existing_course_titles() + ) + + def test_real_tool_registration(self, real_rag_system): + """Test that tools are properly registered in real system""" + tool_definitions = real_rag_system.tool_manager.get_tool_definitions() + + assert len(tool_definitions) == 2 + + # Check search tool + search_tool = next( + ( + tool + for tool in tool_definitions + if tool["name"] == "search_course_content" + ), + None, + ) + assert search_tool is not None + assert "course materials" in search_tool["description"].lower() + + # Check outline tool + outline_tool = next( + (tool for tool in tool_definitions if tool["name"] == "get_course_outline"), + None, + ) + assert outline_tool is not None + assert "outline" in outline_tool["description"].lower() + + def test_real_search_tool_execution( + self, real_rag_system, sample_course, sample_course_chunks + ): + """Test that search tool actually works with real vector store""" + # Add test data + real_rag_system.vector_store.add_course_metadata(sample_course) + real_rag_system.vector_store.add_course_content(sample_course_chunks) + + # Execute search tool directly + result = real_rag_system.tool_manager.execute_tool( + "search_course_content", + query="machine learning", + course_name="Introduction to Machine Learning", + ) + + assert isinstance(result, str) + assert len(result) > 0 + assert "Introduction to Machine Learning" in result + + # Check that sources are tracked + sources = real_rag_system.tool_manager.get_last_sources() + assert len(sources) > 0 + + def test_real_outline_tool_execution( + self, real_rag_system, sample_course, sample_course_chunks + ): + """Test that outline tool actually works with real vector store""" + # Add test data + real_rag_system.vector_store.add_course_metadata(sample_course) + real_rag_system.vector_store.add_course_content(sample_course_chunks) + + # Execute outline tool directly + result = real_rag_system.tool_manager.execute_tool( + "get_course_outline", + course_name="Machine Learning", # Partial name to test fuzzy matching + ) + + assert isinstance(result, str) + assert "Introduction to Machine Learning" in result + assert "Lessons (3 total):" in result + assert "1. What is ML?" in result + assert "2. Types of ML" in result + assert "3. ML Algorithms" in result diff --git a/backend/tests/test_static_files.py b/backend/tests/test_static_files.py new file mode 100644 index 00000000..f74706f2 --- /dev/null +++ b/backend/tests/test_static_files.py @@ -0,0 +1,364 @@ +"""Tests for static file serving and frontend integration""" +import pytest +import tempfile +import os +from pathlib import Path +from unittest.mock import Mock, patch +from fastapi.testclient import TestClient + + +@pytest.mark.api +class TestStaticFileHandling: + """Test static file serving without requiring actual frontend files""" + + @pytest.fixture + def temp_frontend_dir(self): + """Create temporary frontend directory with test files""" + with tempfile.TemporaryDirectory() as temp_dir: + frontend_path = Path(temp_dir) / "frontend" + frontend_path.mkdir() + + # Create test HTML file + index_html = frontend_path / "index.html" + index_html.write_text(""" + + +
Ask questions about courses, instructors, and content
+Ask questions about courses, instructors, and content
+