From d18b55142db35ea4b300385049fa021c5efda08d Mon Sep 17 00:00:00 2001 From: omsherikar Date: Wed, 29 Oct 2025 02:51:27 +0530 Subject: [PATCH 1/6] Successfully created the essential workflows , CI, security, and code quality, etc --- .github/ISSUE_TEMPLATE/config.yml | 17 + .github/ISSUE_TEMPLATE/feature_request.yml | 196 ++++++-- .github/workflows/ci.yml | 414 +++++++++++++++++ .github/workflows/code-quality.yml | 90 ++++ .github/workflows/dependencies.yml | 495 +++++++++++++++++++++ .github/workflows/docker.yml | 370 +++++++++++++++ .github/workflows/release.yml | 408 +++++++++++++++++ .github/workflows/security.yml | 274 ++++++++++++ .pre-commit-config.yaml | 186 ++++++++ .secrets.baseline | 125 ++++++ .yamllint.yml | 37 ++ frontend/.prettierignore | 26 ++ frontend/.prettierrc | 16 + frontend/package.json | 17 +- frontend/src/test-setup.ts | 30 ++ frontend/vitest.config.ts | 31 ++ pyproject.toml | 147 +++++- 17 files changed, 2846 insertions(+), 33 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/dependencies.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/security.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets.baseline create mode 100644 .yamllint.yml create mode 100644 frontend/.prettierignore create mode 100644 frontend/.prettierrc create mode 100644 frontend/src/test-setup.ts create mode 100644 frontend/vitest.config.ts diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7f67b3e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,17 @@ +blank_issues_enabled: false +contact_links: + - name: ๐Ÿ’ฌ GitHub Discussions + url: https://github.com/AOSSIE-Org/Devr.AI/discussions + about: Ask questions, share ideas, and discuss with the community + + - name: ๐Ÿ“š Documentation + url: https://github.com/AOSSIE-Org/Devr.AI/blob/main/README.md + about: Check the documentation for setup guides and API references + + - name: ๐Ÿ†˜ Discord Support + url: https://discord.gg/BjaG8DJx2G + about: Get help from the community on our Discord server + + - name: ๐Ÿ”’ Security Issues + url: https://github.com/AOSSIE-Org/Devr.AI/security/advisories/new + about: Report security vulnerabilities privately diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 380b21d..af4db14 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,50 +1,184 @@ -name: "โœจ Feature Request" -description: "Suggest a new feature or enhancement" -title: "FEATURE REQUEST:" -labels: ["Feature Request"] +name: ๐Ÿš€ Feature Request +description: Suggest a new feature or enhancement for Devr.AI +title: "[Feature]: " +labels: ["enhancement", "needs-triage"] +assignees: [] + body: + - type: markdown + attributes: + value: | + Thanks for suggesting a new feature! ๐Ÿš€ + + Please search existing issues and discussions before creating a new request. + - type: checkboxes + id: terms attributes: - label: "Is your feature request related to a problem?" - description: "Please check if your feature request is related to an existing problem." + label: Pre-flight Checklist + description: Please confirm the following before proceeding options: - - label: "Yes, it is related to a problem" + - label: I have searched for existing feature requests required: true - - type: markdown + - label: I have checked the roadmap and project discussions + required: true + - label: This feature aligns with Devr.AI's goals + required: true + + - type: dropdown + id: category attributes: - value: | - > [!IMPORTANT] - > Before submitting, please ensure your feature request aligns with the project's goals and scope. + label: Feature Category + description: What type of feature is this? + options: + - AI/LLM Enhancement + - Discord Integration + - GitHub Integration + - Slack Integration + - User Interface + - Backend API + - Database/Storage + - Performance + - Security + - Documentation + - Developer Experience + - Other + validations: + required: true + + - type: dropdown + id: priority + attributes: + label: Priority + description: How important is this feature to you? + options: + - Critical (Blocking my use case) + - High (Would significantly improve my workflow) + - Medium (Nice to have improvement) + - Low (Minor enhancement) + validations: + required: true + - type: textarea + id: problem attributes: - label: "Describe the feature you'd like" - description: "Provide a clear and concise description of the feature or enhancement you'd like to see." - value: | - ## ๐ŸŒŸ Feature Description - + label: Problem Statement + description: What problem does this feature solve? + placeholder: | + Describe the problem you're trying to solve... + + "I'm frustrated when..." + "It's difficult to..." + "Users need to..." + validations: + required: true - ## ๐Ÿ” Problem Statement - + - type: textarea + id: solution + attributes: + label: Proposed Solution + description: Describe the feature you'd like to see + placeholder: | + A clear and concise description of what you want to happen... + + "I would like to be able to..." + "The system should..." + "Add a feature that..." + validations: + required: true - ## ๐ŸŽฏ Expected Outcome - + - type: textarea + id: alternatives + attributes: + label: Alternative Solutions + description: Have you considered any alternative approaches? + placeholder: | + Describe alternatives you've considered... + + "Another approach could be..." + "We could also..." - ## ๐Ÿ“ท Screenshots and Design Ideas - + - type: textarea + id: use-cases + attributes: + label: Use Cases + description: Who would benefit from this feature and how? + placeholder: | + Describe specific use cases... + + 1. As a [user type], I want to [action] so that [benefit] + 2. When [scenario], users could [action] + 3. This would help [audience] by [benefit] + validations: + required: true - ## ๐Ÿ“‹ Additional Context - + - type: textarea + id: mockups + attributes: + label: Mockups/Examples + description: If applicable, add mockups, wireframes, or examples + placeholder: | + Add links to mockups, screenshots, or examples of similar features... + + You can drag and drop images here or use markdown: + ![Description](URL) + + - type: textarea + id: acceptance-criteria + attributes: + label: Acceptance Criteria + description: What defines "done" for this feature? + placeholder: | + Define clear acceptance criteria... + + - [ ] User can perform [action] + - [ ] System responds with [behavior] + - [ ] Feature works on [platforms] + - [ ] Performance meets [requirements] + + - type: dropdown + id: breaking-changes + attributes: + label: Breaking Changes + description: Would this feature require breaking changes? + options: + - "No breaking changes" + - "Minor breaking changes (with migration path)" + - "Major breaking changes" + - "Unknown/Need analysis" validations: required: true + + - type: textarea + id: technical-considerations + attributes: + label: Technical Considerations + description: Any technical aspects to consider? + placeholder: | + Consider technical implications... + + - Database changes needed + - API modifications required + - Performance impact + - Security implications + - Third-party integrations + - type: checkboxes + id: contribution attributes: - label: "Record" + label: Contribution + description: Are you willing to contribute to implementing this feature? options: - - label: "I agree to follow this project's Code of Conduct" - required: true - - label: "I want to work on implementing this feature" + - label: I would like to work on implementing this feature required: false - - type: markdown + - label: I can help with design/specification + required: false + - label: I can help with testing + required: false + + - type: textarea + id: additional attributes: - value: | - Thanks for suggesting a new feature! ๐Ÿš€ We appreciate your input and will review it soon. + label: Additional Context + description: Add any other context or information about the feature request + placeholder: Any additional information that might be helpful... \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f04a541 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,414 @@ +name: ๐Ÿš€ CI/CD Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + workflow_dispatch: + +env: + POETRY_VERSION: "2.2.1" + NODE_VERSION: "22.x" + PYTHON_VERSION: "3.13" + +jobs: + # ============================================================================ + # SETUP & VALIDATION + # ============================================================================ + setup: + name: ๐Ÿ”ง Setup & Validate Environment + runs-on: ubuntu-latest + outputs: + backend-changed: ${{ steps.changes.outputs.backend }} + frontend-changed: ${{ steps.changes.outputs.frontend }} + docker-changed: ${{ steps.changes.outputs.docker }} + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐Ÿ” Detect Changes + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + backend: + - 'backend/**' + - 'pyproject.toml' + - 'poetry.lock' + - 'tests/**' + frontend: + - 'frontend/**' + - 'landing/**' + - 'package*.json' + docker: + - 'backend/docker-compose.yml' + - '**/Dockerfile*' + + - name: ๐Ÿ“Š Repository Stats + run: | + echo "### ๐Ÿ“Š Repository Overview" >> $GITHUB_STEP_SUMMARY + echo "- **Python files**: $(find . -name '*.py' | wc -l)" >> $GITHUB_STEP_SUMMARY + echo "- **TypeScript/JavaScript files**: $(find . -name '*.ts' -o -name '*.tsx' -o -name '*.js' -o -name '*.jsx' | wc -l)" >> $GITHUB_STEP_SUMMARY + echo "- **Test files**: $(find . -name 'test_*.py' -o -name '*.test.ts' -o -name '*.test.js' | wc -l)" >> $GITHUB_STEP_SUMMARY + echo "- **Changed paths**: Backend: ${{ steps.changes.outputs.backend }}, Frontend: ${{ steps.changes.outputs.frontend }}" >> $GITHUB_STEP_SUMMARY + + # ============================================================================ + # BACKEND TESTING & QUALITY + # ============================================================================ + backend-test: + name: ๐Ÿ Backend Tests + runs-on: ubuntu-latest + needs: setup + if: needs.setup.outputs.backend-changed == 'true' || github.event_name == 'workflow_dispatch' + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + os: [ubuntu-latest, macos-latest] + fail-fast: false + + services: + weaviate: + image: cr.weaviate.io/semitechnologies/weaviate:1.31.0 + ports: + - 8080:8080 + env: + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + options: >- + --health-cmd "wget --no-verbose --tries=3 --spider http://localhost:8080/v1/meta || exit 1" + --health-interval 30s + --health-timeout 10s + --health-retries 5 + + rabbitmq: + image: rabbitmq:3-management + ports: + - 5672:5672 + env: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + options: >- + --health-cmd "rabbitmq-diagnostics status" + --health-interval 30s + --health-timeout 10s + --health-retries 5 + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + + - name: ๐Ÿ”„ Load Cached Dependencies + uses: actions/cache@v4 + id: cached-poetry-dependencies + with: + path: .venv + key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: ๐Ÿ“ฅ Install Dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: | + cd backend + poetry install --with dev + + - name: ๐Ÿ”ง Setup Test Environment + run: | + # Create test .env file + cat > .env << EOF + GEMINI_API_KEY=test_key_placeholder + TAVILY_API_KEY=test_key_placeholder + DISCORD_BOT_TOKEN=test_token_placeholder + GITHUB_TOKEN=test_token_placeholder + SUPABASE_URL=https://test.supabase.co + SUPABASE_KEY=test_key_placeholder + BACKEND_URL=http://localhost:8001 + RABBITMQ_URL=amqp://localhost:5672/ + LANGSMITH_TRACING=false + EOF + + - name: ๐Ÿงช Run Backend Tests + working-directory: backend + run: | + poetry run pytest tests/ \ + --cov=app \ + --cov-report=xml \ + --cov-report=html \ + --cov-report=term-missing \ + --cov-fail-under=20 \ + --junit-xml=test-results.xml \ + -v + + - name: ๐Ÿ“Š Upload Coverage to Codecov + if: matrix.python-version == env.PYTHON_VERSION && matrix.os == 'ubuntu-latest' + uses: codecov/codecov-action@v4 + with: + file: backend/coverage.xml + flags: backend + name: backend-coverage + + - name: ๐Ÿ“‹ Upload Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: backend-test-results-${{ matrix.python-version }}-${{ matrix.os }} + path: | + backend/test-results.xml + backend/htmlcov/ + + # ============================================================================ + # FRONTEND TESTING & QUALITY + # ============================================================================ + frontend-test: + name: โš›๏ธ Frontend Tests + runs-on: ubuntu-latest + needs: setup + if: needs.setup.outputs.frontend-changed == 'true' || github.event_name == 'workflow_dispatch' + strategy: + matrix: + node-version: ["18.x", "20.x", "22.x"] + fail-fast: false + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐ŸŸข Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: 'frontend/package-lock.json' + + - name: ๐Ÿ“ฆ Install Frontend Dependencies + working-directory: frontend + run: npm ci + + - name: ๐Ÿ” TypeScript Check + working-directory: frontend + run: npx tsc --noEmit + + - name: ๐Ÿงช Run Frontend Tests + working-directory: frontend + run: | + npm run test -- --coverage --watchAll=false + env: + CI: true + + - name: ๐Ÿ—๏ธ Build Frontend + working-directory: frontend + run: npm run build + + - name: ๐Ÿ“Š Upload Coverage + if: matrix.node-version == env.NODE_VERSION + uses: codecov/codecov-action@v4 + with: + file: frontend/coverage/lcov.info + flags: frontend + name: frontend-coverage + + - name: ๐Ÿ“ฆ Upload Build Artifacts + if: matrix.node-version == env.NODE_VERSION + uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/dist/ + + # ============================================================================ + # CODE QUALITY & LINTING + # ============================================================================ + code-quality: + name: ๐Ÿ” Code Quality & Linting + runs-on: ubuntu-latest + needs: setup + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: 'frontend/package-lock.json' + + - name: ๐Ÿ“ฅ Install Dependencies + run: | + cd backend && poetry install --with dev + cd ../frontend && npm ci + + # Backend Quality Checks + - name: ๐Ÿ Python Code Formatting (Black) + working-directory: backend + run: poetry run black --check --diff . + + - name: ๐Ÿ”ข Import Sorting (isort) + working-directory: backend + run: poetry run isort --check-only --diff . + + - name: ๐Ÿ” Python Linting (flake8) + working-directory: backend + run: poetry run flake8 . + + - name: ๐Ÿท๏ธ Type Checking (mypy) + working-directory: backend + run: poetry run mypy . || true # Allow to continue for now + + # Frontend Quality Checks + - name: โš›๏ธ Frontend Linting (ESLint) + working-directory: frontend + run: npx eslint . --ext .ts,.tsx,.js,.jsx --max-warnings 0 + + - name: ๐Ÿ’… Frontend Formatting (Prettier) + working-directory: frontend + run: npx prettier --check . + + - name: ๐Ÿ” TypeScript Strict Check + working-directory: frontend + run: npx tsc --noEmit --strict + + # ============================================================================ + # SECURITY SCANNING + # ============================================================================ + security-scan: + name: ๐Ÿ”’ Security Scanning + runs-on: ubuntu-latest + needs: setup + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + - name: ๐Ÿ” Python Security Scan (Safety) + working-directory: backend + run: | + poetry run pip install safety + poetry export -f requirements.txt --output requirements.txt + poetry run safety check -r requirements.txt || true + + - name: ๐Ÿ”’ Code Security Analysis (Bandit) + working-directory: backend + run: | + poetry run pip install bandit[toml] + poetry run bandit -r . -f json -o bandit-report.json || true + + - name: ๐Ÿ” Frontend Security Audit + working-directory: frontend + run: npm audit --audit-level high || true + + - name: ๐Ÿ“‹ Upload Security Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports + path: | + backend/bandit-report.json + backend/requirements.txt + + # ============================================================================ + # DOCKER & INTEGRATION TESTS + # ============================================================================ + docker-test: + name: ๐Ÿณ Docker & Integration Tests + runs-on: ubuntu-latest + needs: [setup, backend-test] + if: needs.setup.outputs.docker-changed == 'true' || github.event_name == 'workflow_dispatch' + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: ๐Ÿ—๏ธ Test Docker Compose Build + working-directory: backend + run: | + docker-compose -f docker-compose.yml build --no-cache + docker-compose -f docker-compose.yml up -d + sleep 30 # Wait for services to be ready + + - name: ๐Ÿงช Integration Health Checks + run: | + echo "Testing service endpoints..." + + # Test Weaviate + curl -f http://localhost:8080/v1/meta || echo "Weaviate check failed" + + # Test RabbitMQ + curl -f http://localhost:15672 || echo "RabbitMQ check failed" + + # Test FalkorDB + curl -f http://localhost:3000 || echo "FalkorDB check failed" + + - name: ๐Ÿงน Cleanup Docker + if: always() + working-directory: backend + run: | + docker-compose -f docker-compose.yml down -v + docker system prune -f + + # ============================================================================ + # FINAL STATUS CHECK + # ============================================================================ + ci-success: + name: โœ… CI Pipeline Success + runs-on: ubuntu-latest + needs: [backend-test, frontend-test, code-quality, security-scan, docker-test] + if: always() + + steps: + - name: ๐ŸŽ‰ All Checks Passed + if: ${{ needs.backend-test.result == 'success' && needs.frontend-test.result == 'success' && needs.code-quality.result == 'success' && needs.security-scan.result == 'success' && (needs.docker-test.result == 'success' || needs.docker-test.result == 'skipped') }} + run: | + echo "๐ŸŽ‰ All CI checks passed successfully!" + echo "### โœ… CI Pipeline Results" >> $GITHUB_STEP_SUMMARY + echo "- Backend Tests: โœ… Passed" >> $GITHUB_STEP_SUMMARY + echo "- Frontend Tests: โœ… Passed" >> $GITHUB_STEP_SUMMARY + echo "- Code Quality: โœ… Passed" >> $GITHUB_STEP_SUMMARY + echo "- Security Scan: โœ… Passed" >> $GITHUB_STEP_SUMMARY + echo "- Docker Tests: โœ… Passed" >> $GITHUB_STEP_SUMMARY + + - name: โŒ Some Checks Failed + if: ${{ needs.backend-test.result != 'success' || needs.frontend-test.result != 'success' || needs.code-quality.result != 'success' || needs.security-scan.result != 'success' || (needs.docker-test.result != 'success' && needs.docker-test.result != 'skipped') }} + run: | + echo "โŒ Some CI checks failed!" + echo "### โŒ CI Pipeline Results" >> $GITHUB_STEP_SUMMARY + echo "- Backend Tests: ${{ needs.backend-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Frontend Tests: ${{ needs.frontend-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Code Quality: ${{ needs.code-quality.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Security Scan: ${{ needs.security-scan.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Docker Tests: ${{ needs.docker-test.result }}" >> $GITHUB_STEP_SUMMARY + exit 1 diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..aa87d42 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,90 @@ +name: ๐Ÿ” Code Quality & Formatting + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + pre-commit-checks: + name: ๐Ÿ”ง Pre-commit Hooks & Formatting + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22.x" + + - name: ๐Ÿ“ฅ Install Dependencies + run: | + cd backend && poetry install --with dev + cd ../frontend && npm ci + + # Python Code Quality + - name: ๐Ÿ Check Python Formatting (Black) + working-directory: backend + run: | + poetry run black --check --diff . || { + echo "โŒ Python code formatting issues found!" + echo "Run: cd backend && poetry run black ." + exit 1 + } + + - name: ๐Ÿ“Š Check Import Sorting (isort) + working-directory: backend + run: | + poetry run isort --check-only --diff . || { + echo "โŒ Import sorting issues found!" + echo "Run: cd backend && poetry run isort ." + exit 1 + } + + - name: ๐Ÿ” Python Linting (flake8) + working-directory: backend + run: | + poetry run flake8 --statistics --tee --output-file=flake8-report.txt . || { + echo "โŒ Python linting issues found!" + echo "Check: backend/flake8-report.txt" + exit 1 + } + + # Frontend Code Quality + - name: โš›๏ธ Frontend Linting (ESLint) + working-directory: frontend + run: | + npx eslint . --ext .ts,.tsx,.js,.jsx --format=unix --max-warnings 0 || { + echo "โŒ Frontend linting issues found!" + echo "Run: cd frontend && npx eslint . --ext .ts,.tsx,.js,.jsx --fix" + exit 1 + } + + - name: ๐Ÿ’… Frontend Formatting (Prettier) + working-directory: frontend + run: | + npx prettier --check . || { + echo "โŒ Frontend formatting issues found!" + echo "Run: cd frontend && npx prettier --write ." + exit 1 + } + + - name: ๐Ÿ“‹ Upload Reports + uses: actions/upload-artifact@v4 + if: failure() + with: + name: code-quality-reports + path: | + backend/flake8-report.txt + frontend/eslint-report.json diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 0000000..eada852 --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,495 @@ +name: ๐Ÿ“ฆ Dependency Management & Updates + +on: + schedule: + # Run weekly on Mondays at 9 AM UTC + - cron: '0 9 * * 1' + workflow_dispatch: + inputs: + update_type: + description: 'Type of updates to check' + required: true + type: choice + options: + - 'all' + - 'security' + - 'minor' + - 'patch' + default: 'security' + auto_merge: + description: 'Auto-merge security updates' + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.13" + NODE_VERSION: "22.x" + +jobs: + # ============================================================================ + # ANALYZE CURRENT DEPENDENCIES + # ============================================================================ + dependency-audit: + name: ๐Ÿ” Dependency Audit & Analysis + runs-on: ubuntu-latest + + outputs: + python-updates: ${{ steps.python-check.outputs.updates }} + node-updates: ${{ steps.node-check.outputs.updates }} + security-updates: ${{ steps.security-check.outputs.updates }} + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + # Python Dependency Analysis + - name: ๐Ÿ Analyze Python Dependencies + id: python-check + working-directory: backend + run: | + # Install poetry-plugin-up for updates + poetry self add poetry-plugin-up + + # Check for updates + poetry up --dry-run --latest > poetry-updates.txt || true + + # Check for security updates + poetry install --only=main + poetry export -f requirements.txt --output requirements.txt + pip install safety + safety check -r requirements.txt --json --output safety-report.json || true + + # Count updates + UPDATES_COUNT=$(grep -c "โ†’" poetry-updates.txt || echo "0") + echo "updates=$UPDATES_COUNT" >> $GITHUB_OUTPUT + + echo "๐Ÿ“Š Python Dependencies Analysis:" >> $GITHUB_STEP_SUMMARY + echo "- Available updates: $UPDATES_COUNT" >> $GITHUB_STEP_SUMMARY + + # Node.js Dependency Analysis + - name: โš›๏ธ Analyze Frontend Dependencies + id: node-check + working-directory: frontend + run: | + npm ci + + # Check for updates using npm-check-updates + npx npm-check-updates --format group --color > npm-updates.txt || true + + # Security audit + npm audit --audit-level moderate --json > npm-audit.json || true + + # Count updates + UPDATES_COUNT=$(grep -c "โ†’" npm-updates.txt || echo "0") + echo "updates=$UPDATES_COUNT" >> $GITHUB_OUTPUT + + echo "๐Ÿ“Š Frontend Dependencies Analysis:" >> $GITHUB_STEP_SUMMARY + echo "- Available updates: $UPDATES_COUNT" >> $GITHUB_STEP_SUMMARY + + # Security Updates Analysis + - name: ๐Ÿ”’ Analyze Security Updates + id: security-check + run: | + # Combine security reports + PYTHON_SECURITY=$(jq '.vulnerabilities | length' backend/safety-report.json 2>/dev/null || echo "0") + NODE_SECURITY=$(jq '.metadata.vulnerabilities.total' frontend/npm-audit.json 2>/dev/null || echo "0") + + TOTAL_SECURITY=$((PYTHON_SECURITY + NODE_SECURITY)) + echo "updates=$TOTAL_SECURITY" >> $GITHUB_OUTPUT + + echo "๐Ÿ”’ Security Updates:" >> $GITHUB_STEP_SUMMARY + echo "- Python vulnerabilities: $PYTHON_SECURITY" >> $GITHUB_STEP_SUMMARY + echo "- Node.js vulnerabilities: $NODE_SECURITY" >> $GITHUB_STEP_SUMMARY + echo "- **Total security issues: $TOTAL_SECURITY**" >> $GITHUB_STEP_SUMMARY + + - name: ๐Ÿ“‹ Upload Dependency Reports + uses: actions/upload-artifact@v4 + with: + name: dependency-reports + path: | + backend/poetry-updates.txt + backend/safety-report.json + frontend/npm-updates.txt + frontend/npm-audit.json + + # ============================================================================ + # CREATE SECURITY UPDATE PR + # ============================================================================ + security-updates: + name: ๐Ÿšจ Apply Security Updates + runs-on: ubuntu-latest + needs: dependency-audit + if: needs.dependency-audit.outputs.security-updates > 0 + + permissions: + contents: write + pull-requests: write + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + - name: ๐Ÿ”ง Configure Git + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: ๐ŸŒฟ Create Security Update Branch + run: | + BRANCH_NAME="security-updates/$(date +%Y-%m-%d)" + git checkout -b "$BRANCH_NAME" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + + # Apply Python Security Updates + - name: ๐Ÿ Apply Python Security Updates + working-directory: backend + run: | + poetry self add poetry-plugin-up + + # Update only packages with known vulnerabilities + poetry export -f requirements.txt --output requirements.txt + pip install safety + + # Get vulnerable packages + VULNERABLE_PACKAGES=$(safety check -r requirements.txt --json | jq -r '.vulnerabilities[].package_name' | sort -u || true) + + if [ -n "$VULNERABLE_PACKAGES" ]; then + for package in $VULNERABLE_PACKAGES; do + echo "Updating vulnerable package: $package" + poetry add "$package@latest" || true + done + fi + + # Apply Node.js Security Updates + - name: โš›๏ธ Apply Frontend Security Updates + working-directory: frontend + run: | + # Fix security vulnerabilities + npm audit fix --audit-level moderate || true + + # Update packages with security issues + npm update --save + + - name: ๐Ÿงช Test After Security Updates + run: | + # Quick validation that updates don't break basic functionality + cd backend && poetry install --with dev + cd ../frontend && npm ci + + # Basic import test + cd ../backend && poetry run python -c "import app; print('Backend imports OK')" || echo "Backend import issues detected" + + # Basic build test + cd ../frontend && npm run build || echo "Frontend build issues detected" + + - name: ๐Ÿ“ Generate Security Update Summary + run: | + # Create commit message and PR description + cat > security-update-summary.md << EOF + # ๐Ÿ”’ Security Updates $(date +%Y-%m-%d) + + This automated PR applies security updates to resolve known vulnerabilities. + + ## ๐Ÿ” Security Issues Addressed + + ### Python Dependencies + $(cd backend && poetry export -f requirements.txt --output requirements.txt && pip install safety && safety check -r requirements.txt --short 2>/dev/null || echo "No Python security issues found") + + ### Node.js Dependencies + $(cd frontend && npm audit --audit-level moderate 2>/dev/null | grep -E "(High|Critical)" || echo "No Node.js security issues found") + + ## ๐Ÿงช Testing Status + + - โœ… Backend imports validated + - โœ… Frontend build validated + - โœ… Dependencies resolved + + ## ๐Ÿš€ Auto-merge Policy + + This PR can be auto-merged if: + - All CI checks pass + - Only security patches (no major version changes) + - No breaking changes detected + + --- + *This PR was automatically created by GitHub Actions* + EOF + + - name: ๐Ÿ’พ Commit Security Updates + run: | + git add . + if ! git diff --cached --quiet; then + git commit -m "๐Ÿ”’ Security updates $(date +%Y-%m-%d) + + - Apply security patches for vulnerable dependencies + - Update Python packages with known CVEs + - Update Node.js packages with security issues + - Maintain compatibility with existing functionality + + Auto-generated by: .github/workflows/dependencies.yml" + + git push origin "$BRANCH_NAME" + echo "CHANGES_MADE=true" >> $GITHUB_ENV + else + echo "No changes to commit" + echo "CHANGES_MADE=false" >> $GITHUB_ENV + fi + + - name: ๐Ÿ”„ Create Security Update PR + if: env.CHANGES_MADE == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const summary = fs.readFileSync('security-update-summary.md', 'utf8'); + + const { data: pr } = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `๐Ÿ”’ Security Updates ${new Date().toISOString().split('T')[0]}`, + head: process.env.BRANCH_NAME, + base: 'main', + body: summary + }); + + // Add labels + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + labels: ['security', 'dependencies', 'automated'] + }); + + console.log(`Created PR #${pr.number}`); + + # ============================================================================ + # CREATE REGULAR UPDATE PR + # ============================================================================ + regular-updates: + name: ๐Ÿ“ˆ Regular Dependency Updates + runs-on: ubuntu-latest + needs: dependency-audit + if: github.event.inputs.update_type != 'security' && (needs.dependency-audit.outputs.python-updates > 0 || needs.dependency-audit.outputs.node-updates > 0) + + permissions: + contents: write + pull-requests: write + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + - name: ๐Ÿ”ง Configure Git + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: ๐ŸŒฟ Create Update Branch + run: | + UPDATE_TYPE="${{ github.event.inputs.update_type || 'minor' }}" + BRANCH_NAME="dependency-updates/$UPDATE_TYPE/$(date +%Y-%m-%d)" + git checkout -b "$BRANCH_NAME" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + echo "UPDATE_TYPE=$UPDATE_TYPE" >> $GITHUB_ENV + + - name: ๐Ÿ Update Python Dependencies + working-directory: backend + run: | + poetry self add poetry-plugin-up + + case "${{ env.UPDATE_TYPE }}" in + "patch") + poetry up --patch + ;; + "minor") + poetry up --minor + ;; + "all") + poetry up --latest + ;; + esac + + - name: โš›๏ธ Update Frontend Dependencies + working-directory: frontend + run: | + case "${{ env.UPDATE_TYPE }}" in + "patch") + npx npm-check-updates --upgrade --target patch + ;; + "minor") + npx npm-check-updates --upgrade --target minor + ;; + "all") + npx npm-check-updates --upgrade + ;; + esac + + npm install + + - name: ๐Ÿงช Test After Updates + run: | + echo "Testing updated dependencies..." + + # Test backend + cd backend + poetry install --with dev + poetry run python -c "import app; print('โœ… Backend imports successful')" + + # Test frontend + cd ../frontend + npm ci + npm run build + echo "โœ… Frontend build successful" + + - name: ๐Ÿ’พ Commit Updates + run: | + git add . + if ! git diff --cached --quiet; then + git commit -m "๐Ÿ“ฆ Dependency updates (${{ env.UPDATE_TYPE }}) $(date +%Y-%m-%d) + + - Update Python dependencies (${{ env.UPDATE_TYPE }} versions) + - Update Node.js dependencies (${{ env.UPDATE_TYPE }} versions) + - Maintain backward compatibility + - All tests passing + + Update type: ${{ env.UPDATE_TYPE }} + Auto-generated by: .github/workflows/dependencies.yml" + + git push origin "$BRANCH_NAME" + echo "CHANGES_MADE=true" >> $GITHUB_ENV + else + echo "No dependency updates available" + echo "CHANGES_MADE=false" >> $GITHUB_ENV + fi + + - name: ๐Ÿ”„ Create Update PR + if: env.CHANGES_MADE == 'true' + uses: actions/github-script@v7 + with: + script: | + const updateType = process.env.UPDATE_TYPE; + const title = `๐Ÿ“ฆ ${updateType.charAt(0).toUpperCase() + updateType.slice(1)} Dependency Updates ${new Date().toISOString().split('T')[0]}`; + + const body = `## ๐Ÿ“ฆ Dependency Updates (${updateType}) + + This PR updates dependencies to their latest ${updateType} versions. + + ### ๐Ÿ”„ What's Updated + + - **Python**: Updated to latest ${updateType} versions + - **Node.js**: Updated to latest ${updateType} versions + + ### โœ… Validation + + - Backend imports tested + - Frontend build tested + - No breaking changes expected for ${updateType} updates + + ### ๐Ÿš€ Merge Guidelines + + - Review changelog for any notable changes + - Run full test suite before merging + - Monitor for any runtime issues after deployment + + --- + *This PR was automatically created by GitHub Actions*`; + + const { data: pr } = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + head: process.env.BRANCH_NAME, + base: 'main', + body: body + }); + + // Add labels + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + labels: ['dependencies', `${updateType}-update`, 'automated'] + }); + + console.log(`Created PR #${pr.number}`); + + # ============================================================================ + # SUMMARY & NOTIFICATIONS + # ============================================================================ + dependency-summary: + name: ๐Ÿ“Š Dependency Update Summary + runs-on: ubuntu-latest + needs: [dependency-audit, security-updates, regular-updates] + if: always() + + steps: + - name: ๐Ÿ“Š Generate Summary Report + run: | + echo "### ๐Ÿ“ฆ Dependency Management Report" >> $GITHUB_STEP_SUMMARY + echo "**Date**: $(date)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "#### ๐Ÿ” Audit Results" >> $GITHUB_STEP_SUMMARY + echo "- Python updates available: ${{ needs.dependency-audit.outputs.python-updates }}" >> $GITHUB_STEP_SUMMARY + echo "- Node.js updates available: ${{ needs.dependency-audit.outputs.node-updates }}" >> $GITHUB_STEP_SUMMARY + echo "- Security vulnerabilities: ${{ needs.dependency-audit.outputs.security-updates }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "#### ๐Ÿš€ Actions Taken" >> $GITHUB_STEP_SUMMARY + if [ "${{ needs.security-updates.result }}" = "success" ]; then + echo "- โœ… Security updates applied" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.security-updates.result }}" = "skipped" ]; then + echo "- โญ๏ธ No security updates needed" >> $GITHUB_STEP_SUMMARY + else + echo "- โŒ Security updates failed" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.regular-updates.result }}" = "success" ]; then + echo "- โœ… Regular updates applied" >> $GITHUB_STEP_SUMMARY + elif [ "${{ needs.regular-updates.result }}" = "skipped" ]; then + echo "- โญ๏ธ No regular updates requested" >> $GITHUB_STEP_SUMMARY + else + echo "- โŒ Regular updates failed" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..ae56526 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,370 @@ +name: ๐Ÿณ Docker Build & Deploy + +on: + push: + branches: [ main, develop ] + tags: [ 'v*' ] + pull_request: + branches: [ main ] + paths: + - 'backend/**' + - 'frontend/**' + - '**/Dockerfile*' + - 'docker-compose*.yml' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + # ============================================================================ + # BUILD MATRIX FOR MULTI-COMPONENT DOCKER IMAGES + # ============================================================================ + docker-build: + name: ๐Ÿ—๏ธ Build Docker Images + runs-on: ubuntu-latest + + strategy: + matrix: + component: [backend, frontend] + fail-fast: false + + permissions: + contents: read + packages: write + + outputs: + backend-digest: ${{ steps.backend-build.outputs.digest }} + frontend-digest: ${{ steps.frontend-build.outputs.digest }} + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: ๐Ÿ” Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Backend Docker Build + - name: ๐Ÿ Build Backend Dockerfile + if: matrix.component == 'backend' + working-directory: backend + run: | + cat > Dockerfile << 'EOF' + # Multi-stage build for Python backend + FROM python:3.13-slim as builder + + # Install system dependencies + RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + + # Install Poetry + RUN pip install poetry==2.2.1 + RUN poetry config virtualenvs.create false + + # Copy dependency files + WORKDIR /app + COPY pyproject.toml poetry.lock ./ + + # Install dependencies + RUN poetry install --only=main --no-dev + + # Production stage + FROM python:3.13-slim as production + + # Create app user + RUN groupadd -r appuser && useradd -r -g appuser appuser + + # Install runtime dependencies + RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + + # Copy installed packages from builder + COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages + COPY --from=builder /usr/local/bin /usr/local/bin + + # Set working directory + WORKDIR /app + + # Copy application code + COPY --chown=appuser:appuser . . + + # Switch to non-root user + USER appuser + + # Health check + HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000/api/v1/health || exit 1 + + # Expose port + EXPOSE 8000 + + # Start application + CMD ["uvicorn", "main:api", "--host", "0.0.0.0", "--port", "8000"] + EOF + + # Frontend Docker Build + - name: โš›๏ธ Build Frontend Dockerfile + if: matrix.component == 'frontend' + working-directory: frontend + run: | + cat > Dockerfile << 'EOF' + # Multi-stage build for React frontend + FROM node:22-alpine as builder + + # Set working directory + WORKDIR /app + + # Copy package files + COPY package*.json ./ + + # Install dependencies + RUN npm ci --only=production + + # Copy source code + COPY . . + + # Build application + RUN npm run build + + # Production stage with nginx + FROM nginx:alpine as production + + # Copy custom nginx config + COPY --from=builder /app/dist /usr/share/nginx/html + + # Copy nginx configuration + RUN echo 'server { + listen 80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + try_files $uri $uri/ /index.html; + } + + location /api { + proxy_pass http://backend:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "no-referrer-when-downgrade" always; + add_header Content-Security-Policy "default-src '\''self'\'' http: https: data: blob: '\''unsafe-inline'\''" always; + }' > /etc/nginx/conf.d/default.conf + + # Health check + HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost/ || exit 1 + + # Expose port + EXPOSE 80 + + # Start nginx + CMD ["nginx", "-g", "daemon off;"] + EOF + + - name: ๐Ÿท๏ธ Extract Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.component }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: ๐Ÿ—๏ธ Build and Push Docker Image + id: build + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.component }} + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Store build outputs + - name: ๐Ÿ“‹ Output Build Info + id: backend-build + if: matrix.component == 'backend' + run: echo "digest=${{ steps.build.outputs.digest }}" >> $GITHUB_OUTPUT + + - name: ๐Ÿ“‹ Output Build Info + id: frontend-build + if: matrix.component == 'frontend' + run: echo "digest=${{ steps.build.outputs.digest }}" >> $GITHUB_OUTPUT + + # ============================================================================ + # DOCKER COMPOSE INTEGRATION TEST + # ============================================================================ + integration-test: + name: ๐Ÿงช Integration Test with Docker Compose + runs-on: ubuntu-latest + needs: docker-build + if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Create Integration Docker Compose + working-directory: backend + run: | + # Create integration test compose file + cat > docker-compose.integration.yml << 'EOF' + version: '3.8' + + services: + weaviate: + image: cr.weaviate.io/semitechnologies/weaviate:1.31.0 + ports: + - "8080:8080" + environment: + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:8080/v1/meta"] + interval: 30s + timeout: 10s + retries: 5 + + rabbitmq: + image: rabbitmq:3-management + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "status"] + interval: 30s + timeout: 10s + retries: 5 + + falkordb: + image: falkordb/falkordb:latest + ports: + - "6379:6379" + - "3000:3000" + healthcheck: + test: ["CMD", "redis-cli", "-p", "6379", "ping"] + interval: 30s + timeout: 10s + retries: 5 + + networks: + default: + name: devr-ai-integration + EOF + + - name: ๐Ÿš€ Start Integration Environment + working-directory: backend + run: | + docker-compose -f docker-compose.integration.yml up -d + + # Wait for services to be healthy + echo "Waiting for services to be ready..." + sleep 60 + + # Health check all services + docker-compose -f docker-compose.integration.yml ps + + - name: ๐Ÿงช Run Integration Tests + run: | + echo "Testing service endpoints..." + + # Test Weaviate + curl -f http://localhost:8080/v1/meta || exit 1 + echo "โœ… Weaviate is healthy" + + # Test RabbitMQ Management + curl -f http://localhost:15672 || exit 1 + echo "โœ… RabbitMQ is healthy" + + # Test FalkorDB + curl -f http://localhost:3000 || exit 1 + echo "โœ… FalkorDB is healthy" + + - name: ๐Ÿงน Cleanup Integration Environment + if: always() + working-directory: backend + run: | + docker-compose -f docker-compose.integration.yml down -v + docker system prune -f + + # ============================================================================ + # VULNERABILITY SCANNING FOR BUILT IMAGES + # ============================================================================ + image-security-scan: + name: ๐Ÿ” Scan Built Images for Vulnerabilities + runs-on: ubuntu-latest + needs: docker-build + if: github.event_name != 'pull_request' + + strategy: + matrix: + component: [backend, frontend] + + steps: + - name: ๐Ÿ” Scan Image with Trivy + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.component }}:${{ github.ref_name }} + format: 'sarif' + output: 'trivy-results-${{ matrix.component }}.sarif' + + - name: ๐Ÿ“‹ Upload Trivy Scan Results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results-${{ matrix.component }}.sarif' + + # ============================================================================ + # SUCCESS SUMMARY + # ============================================================================ + docker-success: + name: โœ… Docker Build Success + runs-on: ubuntu-latest + needs: [docker-build, integration-test] + if: always() + + steps: + - name: ๐ŸŽ‰ Build Summary + run: | + echo "### ๐Ÿณ Docker Build Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Docker Build**: ${{ needs.docker-build.result }}" >> $GITHUB_STEP_SUMMARY + echo "- **Integration Test**: ${{ needs.integration-test.result }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.docker-build.result }}" = "success" ] && [ "${{ needs.integration-test.result }}" = "success" ]; then + echo "๐ŸŽ‰ All Docker workflows completed successfully!" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ Some Docker workflows failed. Check the logs above." >> $GITHUB_STEP_SUMMARY + exit 1 + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..210ce1e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,408 @@ +name: ๐Ÿš€ Release & Deployment + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + version: + description: 'Version to release (e.g., v1.0.0)' + required: true + type: string + prerelease: + description: 'Mark as pre-release' + required: false + type: boolean + default: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + # ============================================================================ + # VALIDATE RELEASE + # ============================================================================ + validate-release: + name: โœ… Validate Release + runs-on: ubuntu-latest + + outputs: + version: ${{ steps.version.outputs.version }} + is-prerelease: ${{ steps.version.outputs.is-prerelease }} + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐Ÿท๏ธ Extract Version Info + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.version }}" + IS_PRERELEASE="${{ github.event.inputs.prerelease }}" + else + VERSION="${{ github.ref_name }}" + # Check if version contains alpha, beta, rc + if [[ "$VERSION" =~ (alpha|beta|rc) ]]; then + IS_PRERELEASE="true" + else + IS_PRERELEASE="false" + fi + fi + + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "is-prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT + + echo "### ๐Ÿท๏ธ Release Information" >> $GITHUB_STEP_SUMMARY + echo "- **Version**: $VERSION" >> $GITHUB_STEP_SUMMARY + echo "- **Pre-release**: $IS_PRERELEASE" >> $GITHUB_STEP_SUMMARY + + - name: ๐Ÿ” Validate Version Format + run: | + VERSION="${{ steps.version.outputs.version }}" + if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then + echo "โŒ Invalid version format: $VERSION" + echo "Expected format: v1.0.0 or v1.0.0-beta1" + exit 1 + fi + echo "โœ… Version format is valid: $VERSION" + + - name: ๐Ÿ” Check if Tag Exists + if: github.event_name == 'workflow_dispatch' + run: | + VERSION="${{ steps.version.outputs.version }}" + if git rev-parse "$VERSION" >/dev/null 2>&1; then + echo "โŒ Tag $VERSION already exists" + exit 1 + fi + echo "โœ… Tag $VERSION is available" + + # ============================================================================ + # BUILD RELEASE ARTIFACTS + # ============================================================================ + build-artifacts: + name: ๐Ÿ—๏ธ Build Release Artifacts + runs-on: ubuntu-latest + needs: validate-release + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: ๐ŸŸข Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22.x" + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + # Build Backend Package + - name: ๐Ÿ Build Python Package + working-directory: backend + run: | + poetry version ${{ needs.validate-release.outputs.version }} + poetry build + + # Create backend archive + tar -czf ../devr-ai-backend-${{ needs.validate-release.outputs.version }}.tar.gz . + + # Build Frontend Package + - name: โš›๏ธ Build Frontend + working-directory: frontend + run: | + # Update version in package.json + npm version ${{ needs.validate-release.outputs.version }} --no-git-tag-version + npm ci + npm run build + + # Create frontend archive + tar -czf ../devr-ai-frontend-${{ needs.validate-release.outputs.version }}.tar.gz dist/ + + # Create Complete Release Archive + - name: ๐Ÿ“ฆ Create Release Archive + run: | + mkdir -p release-package + + # Copy important files + cp README.md release-package/ + cp LICENSE release-package/ || echo "No LICENSE file found" + cp env.example release-package/ + cp -r docs/ release-package/ || echo "No docs directory found" + + # Copy built artifacts + cp backend/dist/* release-package/ || true + cp devr-ai-backend-${{ needs.validate-release.outputs.version }}.tar.gz release-package/ + cp devr-ai-frontend-${{ needs.validate-release.outputs.version }}.tar.gz release-package/ + + # Create installation script + cat > release-package/install.sh << 'EOF' + #!/bin/bash + set -e + + echo "๐Ÿš€ Installing Devr.AI..." + + # Check prerequisites + command -v python3 >/dev/null 2>&1 || { echo "โŒ Python 3 is required"; exit 1; } + command -v node >/dev/null 2>&1 || { echo "โŒ Node.js is required"; exit 1; } + command -v docker >/dev/null 2>&1 || { echo "โŒ Docker is required"; exit 1; } + + # Extract backend + echo "๐Ÿ“ฆ Extracting backend..." + tar -xzf devr-ai-backend-*.tar.gz -C backend/ + + # Extract frontend + echo "๐Ÿ“ฆ Extracting frontend..." + mkdir -p frontend/ + tar -xzf devr-ai-frontend-*.tar.gz -C frontend/ + + # Setup environment + echo "๐Ÿ”ง Setting up environment..." + cp env.example .env + + echo "โœ… Devr.AI installed successfully!" + echo "๐Ÿ“– Please read README.md for configuration instructions." + EOF + + chmod +x release-package/install.sh + + # Create final archive + tar -czf devr-ai-${{ needs.validate-release.outputs.version }}.tar.gz release-package/ + + - name: ๐Ÿ“‹ Upload Release Artifacts + uses: actions/upload-artifact@v4 + with: + name: release-artifacts + path: | + devr-ai-${{ needs.validate-release.outputs.version }}.tar.gz + devr-ai-backend-${{ needs.validate-release.outputs.version }}.tar.gz + devr-ai-frontend-${{ needs.validate-release.outputs.version }}.tar.gz + + # ============================================================================ + # BUILD & PUSH RELEASE DOCKER IMAGES + # ============================================================================ + build-release-images: + name: ๐Ÿณ Build Release Docker Images + runs-on: ubuntu-latest + needs: validate-release + + strategy: + matrix: + component: [backend, frontend] + + permissions: + contents: read + packages: write + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: ๐Ÿ” Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: ๐Ÿท๏ธ Extract Docker Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.component }} + tags: | + type=semver,pattern={{version}},value=${{ needs.validate-release.outputs.version }} + type=semver,pattern={{major}}.{{minor}},value=${{ needs.validate-release.outputs.version }} + type=semver,pattern={{major}},value=${{ needs.validate-release.outputs.version }} + + # Use the Dockerfiles created in docker.yml workflow + - name: ๐Ÿ Create Backend Dockerfile + if: matrix.component == 'backend' + working-directory: backend + run: | + cat > Dockerfile << 'EOF' + FROM python:3.13-slim as builder + RUN apt-get update && apt-get install -y build-essential curl && rm -rf /var/lib/apt/lists/* + RUN pip install poetry==2.2.1 + RUN poetry config virtualenvs.create false + WORKDIR /app + COPY pyproject.toml poetry.lock ./ + RUN poetry install --only=main --no-dev + + FROM python:3.13-slim as production + RUN groupadd -r appuser && useradd -r -g appuser appuser + RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages + COPY --from=builder /usr/local/bin /usr/local/bin + WORKDIR /app + COPY --chown=appuser:appuser . . + USER appuser + HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost:8000/api/v1/health || exit 1 + EXPOSE 8000 + CMD ["uvicorn", "main:api", "--host", "0.0.0.0", "--port", "8000"] + EOF + + - name: โš›๏ธ Create Frontend Dockerfile + if: matrix.component == 'frontend' + working-directory: frontend + run: | + cat > Dockerfile << 'EOF' + FROM node:22-alpine as builder + WORKDIR /app + COPY package*.json ./ + RUN npm ci --only=production + COPY . . + RUN npm run build + + FROM nginx:alpine as production + COPY --from=builder /app/dist /usr/share/nginx/html + RUN echo 'server { listen 80; location / { root /usr/share/nginx/html; try_files $uri /index.html; } }' > /etc/nginx/conf.d/default.conf + HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 CMD curl -f http://localhost/ || exit 1 + EXPOSE 80 + CMD ["nginx", "-g", "daemon off;"] + EOF + + - name: ๐Ÿ—๏ธ Build and Push Release Image + uses: docker/build-push-action@v5 + with: + context: ./${{ matrix.component }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # ============================================================================ + # GENERATE CHANGELOG + # ============================================================================ + generate-changelog: + name: ๐Ÿ“ Generate Changelog + runs-on: ubuntu-latest + needs: validate-release + + outputs: + changelog: ${{ steps.changelog.outputs.changelog }} + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐Ÿ“ Generate Changelog + id: changelog + run: | + # Get the previous tag + PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + CURRENT_TAG="${{ needs.validate-release.outputs.version }}" + + echo "Generating changelog from $PREVIOUS_TAG to $CURRENT_TAG" + + # Create changelog + CHANGELOG="## What's Changed\n\n" + + if [ -n "$PREVIOUS_TAG" ]; then + # Get commits since last tag + COMMITS=$(git log --pretty=format:"- %s (%h)" $PREVIOUS_TAG..HEAD) + else + # Get all commits if no previous tag + COMMITS=$(git log --pretty=format:"- %s (%h)") + fi + + # Categorize commits + FEATURES=$(echo "$COMMITS" | grep -i "feat\|feature\|add" || true) + FIXES=$(echo "$COMMITS" | grep -i "fix\|bug" || true) + IMPROVEMENTS=$(echo "$COMMITS" | grep -i "improve\|enhance\|update" || true) + OTHERS=$(echo "$COMMITS" | grep -v -i "feat\|feature\|add\|fix\|bug\|improve\|enhance\|update" || true) + + if [ -n "$FEATURES" ]; then + CHANGELOG="${CHANGELOG}### ๐Ÿš€ New Features\n$FEATURES\n\n" + fi + + if [ -n "$FIXES" ]; then + CHANGELOG="${CHANGELOG}### ๐Ÿ› Bug Fixes\n$FIXES\n\n" + fi + + if [ -n "$IMPROVEMENTS" ]; then + CHANGELOG="${CHANGELOG}### ๐Ÿ”ง Improvements\n$IMPROVEMENTS\n\n" + fi + + if [ -n "$OTHERS" ]; then + CHANGELOG="${CHANGELOG}### ๐Ÿ“‹ Other Changes\n$OTHERS\n\n" + fi + + # Add installation instructions + CHANGELOG="${CHANGELOG}### ๐Ÿ“ฆ Installation\n\n" + CHANGELOG="${CHANGELOG}**Docker:**\n" + CHANGELOG="${CHANGELOG}\`\`\`bash\n" + CHANGELOG="${CHANGELOG}docker pull ghcr.io/${{ github.repository }}-backend:$CURRENT_TAG\n" + CHANGELOG="${CHANGELOG}docker pull ghcr.io/${{ github.repository }}-frontend:$CURRENT_TAG\n" + CHANGELOG="${CHANGELOG}\`\`\`\n\n" + CHANGELOG="${CHANGELOG}**Manual:**\n" + CHANGELOG="${CHANGELOG}Download and extract \`devr-ai-$CURRENT_TAG.tar.gz\` from the release assets.\n\n" + + # Save changelog + echo -e "$CHANGELOG" > CHANGELOG.md + + # Output for GitHub release + echo "changelog<> $GITHUB_OUTPUT + echo -e "$CHANGELOG" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: ๐Ÿ“‹ Upload Changelog + uses: actions/upload-artifact@v4 + with: + name: changelog + path: CHANGELOG.md + + # ============================================================================ + # CREATE GITHUB RELEASE + # ============================================================================ + create-release: + name: ๐ŸŽ‰ Create GitHub Release + runs-on: ubuntu-latest + needs: [validate-release, build-artifacts, build-release-images, generate-changelog] + + permissions: + contents: write + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ“‹ Download Release Artifacts + uses: actions/download-artifact@v4 + with: + name: release-artifacts + + - name: ๐ŸŽ‰ Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.validate-release.outputs.version }} + name: ${{ needs.validate-release.outputs.version }} + body: ${{ needs.generate-changelog.outputs.changelog }} + prerelease: ${{ needs.validate-release.outputs.is-prerelease }} + files: | + devr-ai-${{ needs.validate-release.outputs.version }}.tar.gz + devr-ai-backend-${{ needs.validate-release.outputs.version }}.tar.gz + devr-ai-frontend-${{ needs.validate-release.outputs.version }}.tar.gz + generate_release_notes: true + + - name: ๐Ÿ“Š Release Summary + run: | + echo "### ๐ŸŽ‰ Release Created Successfully" >> $GITHUB_STEP_SUMMARY + echo "- **Version**: ${{ needs.validate-release.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "- **Pre-release**: ${{ needs.validate-release.outputs.is-prerelease }}" >> $GITHUB_STEP_SUMMARY + echo "- **Docker Images**: Available on GitHub Container Registry" >> $GITHUB_STEP_SUMMARY + echo "- **Release Assets**: Complete installation packages attached" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..02ebe06 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,274 @@ +name: ๐Ÿ”’ Security Scanning & Vulnerability Assessment + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + schedule: + # Run security scans daily at 2 AM UTC + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + # ============================================================================ + # DEPENDENCY VULNERABILITY SCANNING + # ============================================================================ + dependency-scan: + name: ๐Ÿ” Dependency Vulnerability Scan + runs-on: ubuntu-latest + + strategy: + matrix: + component: [backend, frontend] + fail-fast: false + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + # Backend Dependency Scanning + - name: ๐Ÿ Setup Python (Backend) + if: matrix.component == 'backend' + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: ๐Ÿ“ฆ Install Poetry (Backend) + if: matrix.component == 'backend' + uses: snok/install-poetry@v1 + + - name: ๐Ÿ” Python Dependency Security Check + if: matrix.component == 'backend' + working-directory: backend + run: | + poetry install --only=main + poetry export -f requirements.txt --output requirements.txt + pip install safety + safety check -r requirements.txt --json --output safety-report.json || true + safety check -r requirements.txt || true + + # Frontend Dependency Scanning + - name: ๐ŸŸข Setup Node.js (Frontend) + if: matrix.component == 'frontend' + uses: actions/setup-node@v4 + with: + node-version: "22.x" + + - name: ๐Ÿ” Frontend Security Audit + if: matrix.component == 'frontend' + working-directory: frontend + run: | + npm ci + npm audit --audit-level high --json > npm-audit-report.json || true + npm audit --audit-level high || true + + - name: ๐Ÿ“‹ Upload Dependency Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: dependency-scan-${{ matrix.component }} + path: | + backend/safety-report.json + backend/requirements.txt + frontend/npm-audit-report.json + + # ============================================================================ + # CODE SECURITY ANALYSIS + # ============================================================================ + code-security: + name: ๐Ÿ›ก๏ธ Static Code Security Analysis + runs-on: ubuntu-latest + + permissions: + security-events: write + contents: read + actions: read + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: ๐Ÿ“ฆ Install Poetry + uses: snok/install-poetry@v1 + + - name: ๐Ÿ“ฅ Install Dependencies + working-directory: backend + run: poetry install --with dev + + # Bandit Security Linting + - name: ๐Ÿ” Python Security Linting (Bandit) + working-directory: backend + run: | + poetry run bandit -r . \ + -f json \ + -o bandit-report.json \ + --exclude ./tests,./venv,./.venv || true + + poetry run bandit -r . \ + --exclude ./tests,./venv,./.venv || true + + # CodeQL Analysis + - name: ๐Ÿ”ฌ Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python, javascript + queries: security-extended,security-and-quality + + - name: ๐Ÿ—๏ธ Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: ๐Ÿ” Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:python" + + - name: ๐Ÿ“‹ Upload Security Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: code-security-reports + path: | + backend/bandit-report.json + + # ============================================================================ + # SECRET SCANNING + # ============================================================================ + secret-scan: + name: ๐Ÿ” Secret & Credential Scanning + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for comprehensive scanning + + - name: ๐Ÿ” TruffleHog Secret Scanning + uses: trufflesecurity/trufflehog@main + with: + path: ./ + base: main + head: HEAD + extra_args: --debug --only-verified + + # ============================================================================ + # DOCKER IMAGE SECURITY + # ============================================================================ + container-security: + name: ๐Ÿณ Container Security Scan + runs-on: ubuntu-latest + if: contains(github.event.head_commit.message, 'docker') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + + steps: + - name: ๐Ÿ“ฅ Checkout Code + uses: actions/checkout@v4 + + - name: ๐Ÿณ Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: ๐Ÿ—๏ธ Build Test Image + working-directory: backend + run: | + # Create a simple Dockerfile for testing if it doesn't exist + if [ ! -f Dockerfile ]; then + cat > Dockerfile << EOF + FROM python:3.13-slim + WORKDIR /app + COPY pyproject.toml poetry.lock ./ + RUN pip install poetry && poetry config virtualenvs.create false + RUN poetry install --only=main + COPY . . + CMD ["python", "main.py"] + EOF + fi + docker build -t devr-ai:test . + + - name: ๐Ÿ” Trivy Container Scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'devr-ai:test' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: ๐Ÿ“‹ Upload Trivy Scan Results + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' + + # ============================================================================ + # SECURITY SUMMARY & NOTIFICATION + # ============================================================================ + security-summary: + name: ๐Ÿ“Š Security Summary + runs-on: ubuntu-latest + needs: [dependency-scan, code-security, secret-scan] + if: always() + + steps: + - name: ๐Ÿ“ฅ Download Security Reports + uses: actions/download-artifact@v4 + with: + path: security-reports/ + + - name: ๐Ÿ“Š Generate Security Summary + run: | + echo "### ๐Ÿ”’ Security Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Check job statuses + if [ "${{ needs.dependency-scan.result }}" = "success" ]; then + echo "โœ… **Dependency Scan**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Dependency Scan**: Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.code-security.result }}" = "success" ]; then + echo "โœ… **Code Security**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Code Security**: Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.secret-scan.result }}" = "success" ]; then + echo "โœ… **Secret Scan**: Passed" >> $GITHUB_STEP_SUMMARY + else + echo "โŒ **Secret Scan**: Failed" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "๐Ÿ“‹ **Detailed reports available in workflow artifacts**" >> $GITHUB_STEP_SUMMARY + + - name: ๐Ÿ’ฌ Comment Security Results on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + let comment = '## ๐Ÿ”’ Security Scan Results\n\n'; + + const results = { + 'Dependency Scan': '${{ needs.dependency-scan.result }}', + 'Code Security': '${{ needs.code-security.result }}', + 'Secret Scan': '${{ needs.secret-scan.result }}' + }; + + for (const [check, result] of Object.entries(results)) { + const icon = result === 'success' ? 'โœ…' : 'โŒ'; + comment += `${icon} **${check}**: ${result}\n`; + } + + comment += '\n๐Ÿ“‹ Check the workflow run for detailed reports.'; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7b42e6a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,186 @@ +# Pre-commit hooks configuration for Devr.AI +# Install with: pip install pre-commit && pre-commit install + +repos: + # ============================================================================ + # GENERAL CODE QUALITY + # ============================================================================ + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + exclude: '\.md$' + - id: end-of-file-fixer + - id: check-yaml + args: ['--unsafe'] # Allow %%GITHUB_TOKEN%% in workflows + - id: check-toml + - id: check-json + - id: check-xml + - id: check-merge-conflict + - id: check-case-conflict + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: debug-statements + - id: detect-private-key + - id: fix-byte-order-marker + - id: mixed-line-ending + args: ['--fix=lf'] + + # ============================================================================ + # PYTHON CODE QUALITY + # ============================================================================ + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + name: Format Python code (black) + files: ^backend/ + language_version: python3 + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: Sort Python imports (isort) + files: ^backend/ + args: ["--profile", "black", "--line-length", "100"] + + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + name: Lint Python code (flake8) + files: ^backend/ + args: ["--max-line-length=100", "--extend-ignore=E203,W503"] + additional_dependencies: + - flake8-docstrings + - flake8-bugbear + - flake8-comprehensions + + - repo: https://github.com/pycqa/autoflake + rev: v2.3.1 + hooks: + - id: autoflake + name: Remove unused imports (autoflake) + files: ^backend/ + args: + - --remove-all-unused-imports + - --remove-unused-variables + - --in-place + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.15.0 + hooks: + - id: mypy + name: Type checking (mypy) + files: ^backend/ + args: ["--ignore-missing-imports", "--check-untyped-defs"] + additional_dependencies: + - types-requests + - types-PyYAML + + # ============================================================================ + # PYTHON SECURITY + # ============================================================================ + - repo: https://github.com/PyCQA/bandit + rev: 1.8.0 + hooks: + - id: bandit + name: Security linting (bandit) + files: ^backend/ + args: ["-r", "-x", "tests/", "-f", "json"] + exclude: ^backend/tests/ + + - repo: https://github.com/Lucas-C/pre-commit-hooks-safety + rev: v1.3.2 + hooks: + - id: python-safety-dependencies-check + name: Check for known security vulnerabilities (safety) + files: ^backend/ + args: ["--json"] + + # ============================================================================ + # FRONTEND CODE QUALITY + # ============================================================================ + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v9.17.0 + hooks: + - id: eslint + name: Lint TypeScript/JavaScript (ESLint) + files: ^frontend/.*\.(js|jsx|ts|tsx)$ + args: ["--fix", "--max-warnings=0"] + additional_dependencies: + - eslint@^9.9.1 + - "@typescript-eslint/parser@^8.3.0" + - "@typescript-eslint/eslint-plugin@^8.3.0" + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + name: Format frontend code (Prettier) + files: ^frontend/.*\.(js|jsx|ts|tsx|json|css|md)$ + exclude: ^frontend/package-lock\.json$ + + # ============================================================================ + # DOCKER & YAML + # ============================================================================ + - repo: https://github.com/adrienverge/yamllint + rev: v1.37.0 + hooks: + - id: yamllint + name: Lint YAML files + args: [-c=.yamllint.yml] + + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker + name: Lint Dockerfiles + files: .*Dockerfile.* + + # ============================================================================ + # DOCUMENTATION & MARKDOWN + # ============================================================================ + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.42.0 + hooks: + - id: markdownlint + name: Lint Markdown files + args: ["--fix"] + exclude: ^CHANGELOG\.md$ + + # ============================================================================ + # SECRETS & SECURITY + # ============================================================================ + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + name: Detect secrets and credentials + args: ['--baseline', '.secrets.baseline'] + exclude: package-lock\.json|poetry\.lock + + # ============================================================================ + # COMMIT MESSAGE FORMAT + # ============================================================================ + - repo: https://github.com/compilerla/conventional-pre-commit + rev: v3.6.0 + hooks: + - id: conventional-pre-commit + name: Check commit message format + stages: [commit-msg] + args: [optional-scope, feat, fix, docs, style, refactor, perf, test, chore, ci, build] + +# CI-specific configurations +ci: + autofix_commit_msg: | + ๐Ÿ”ง [pre-commit.ci] auto fixes from pre-commit hooks + + Applied automatic fixes for code quality issues. + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: 'โฌ†๏ธ [pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..af5bb6e --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,125 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": { + "env.example": [ + { + "type": "Generic High Entropy String", + "filename": "env.example", + "hashed_secret": "placeholder_hash_will_be_generated", + "is_verified": false, + "line_number": 2 + } + ] + }, + "generated_at": "2024-10-28T20:00:00Z" +} diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..aee7bb5 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,37 @@ +# YAML linting configuration for yamllint + +extends: default + +rules: + # Allow long lines in YAML (common in GitHub Actions) + line-length: + max: 120 + level: warning + + # Allow truthy values like 'yes', 'no', 'on', 'off' + truthy: + allowed-values: ['true', 'false', 'yes', 'no', 'on', 'off'] + check-keys: false + + # Allow empty values (common in GitHub Actions) + empty-values: + forbid-in-block-mappings: false + forbid-in-flow-mappings: false + + # Allow duplicate keys (sometimes needed in workflows) + key-duplicates: disable + + # Be more lenient with indentation + indentation: + spaces: 2 + indent-sequences: true + check-multi-line-strings: false + + # Allow comments with less than 2 spaces + comments: + min-spaces-from-content: 1 + +ignore: | + /.github/workflows/* + /docker-compose*.yml + /*.yml diff --git a/frontend/.prettierignore b/frontend/.prettierignore new file mode 100644 index 0000000..f46ad30 --- /dev/null +++ b/frontend/.prettierignore @@ -0,0 +1,26 @@ +# Build outputs +dist/ +build/ +coverage/ + +# Dependencies +node_modules/ + +# Logs +*.log + +# Environment files +.env* + +# Package managers +package-lock.json +yarn.lock +pnpm-lock.yaml + +# Cache +.cache/ +.parcel-cache/ + +# OS generated files +.DS_Store +Thumbs.db diff --git a/frontend/.prettierrc b/frontend/.prettierrc new file mode 100644 index 0000000..d8765f3 --- /dev/null +++ b/frontend/.prettierrc @@ -0,0 +1,16 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2, + "useTabs": false, + "bracketSpacing": true, + "arrowParens": "avoid", + "endOfLine": "lf", + "quoteProps": "as-needed", + "jsxSingleQuote": true, + "bracketSameLine": false, + "proseWrap": "preserve", + "htmlWhitespaceSensitivity": "css" +} diff --git a/frontend/package.json b/frontend/package.json index d5f8e48..e5a688e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,6 +7,13 @@ "dev": "vite", "build": "vite build", "lint": "eslint .", + "lint:fix": "eslint . --fix", + "format": "prettier --write .", + "format:check": "prettier --check .", + "test": "vitest", + "test:coverage": "vitest --coverage", + "test:ui": "vitest --ui", + "type-check": "tsc --noEmit", "preview": "vite preview" }, "dependencies": { @@ -25,18 +32,26 @@ }, "devDependencies": { "@eslint/js": "^9.9.1", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.1.0", + "@testing-library/user-event": "^14.5.4", "@types/react": "^18.3.5", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", + "@vitest/coverage-v8": "^2.1.8", + "@vitest/ui": "^2.1.8", "autoprefixer": "^10.4.18", "eslint": "^9.9.1", "eslint-plugin-react-hooks": "^5.1.0-rc.0", "eslint-plugin-react-refresh": "^0.4.11", "globals": "^15.9.0", + "jsdom": "^25.0.1", "postcss": "^8.4.35", + "prettier": "^3.4.2", "tailwindcss": "^3.4.1", "typescript": "^5.5.3", "typescript-eslint": "^8.3.0", - "vite": "^5.4.2" + "vite": "^5.4.2", + "vitest": "^2.1.8" } } diff --git a/frontend/src/test-setup.ts b/frontend/src/test-setup.ts new file mode 100644 index 0000000..3a83420 --- /dev/null +++ b/frontend/src/test-setup.ts @@ -0,0 +1,30 @@ +import '@testing-library/jest-dom' + +// Mock environment variables +Object.defineProperty(window, 'matchMedia', { + writable: true, + value: vi.fn().mockImplementation(query => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), // deprecated + removeListener: vi.fn(), // deprecated + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), +}) + +// Mock ResizeObserver +global.ResizeObserver = vi.fn().mockImplementation(() => ({ + observe: vi.fn(), + unobserve: vi.fn(), + disconnect: vi.fn(), +})) + +// Mock IntersectionObserver +global.IntersectionObserver = vi.fn().mockImplementation(() => ({ + observe: vi.fn(), + unobserve: vi.fn(), + disconnect: vi.fn(), +})) diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts new file mode 100644 index 0000000..221033e --- /dev/null +++ b/frontend/vitest.config.ts @@ -0,0 +1,31 @@ +/// +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' + +export default defineConfig({ + plugins: [react()], + test: { + globals: true, + environment: 'jsdom', + setupFiles: ['./src/test-setup.ts'], + css: true, + reporters: ['verbose'], + coverage: { + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'src/test-setup.ts', + '**/*.d.ts', + '**/*.config.*', + '**/coverage/**', + '**/dist/**', + '**/.{idea,git,cache,output,temp}/**', + ], + }, + }, + resolve: { + alias: { + '@': '/src', + }, + }, +}) diff --git a/pyproject.toml b/pyproject.toml index 3d22557..43d2c08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,11 +48,156 @@ package-mode = false [tool.poetry.group.dev.dependencies] pytest = "^8.3.5" +pytest-cov = "^6.0.0" +pytest-asyncio = "^0.25.0" +pytest-mock = "^3.15.0" flake8 = "^7.1.2" +black = "^24.0.0" +isort = "^5.13.2" +mypy = "^1.15.0" +safety = "^4.0.0" +bandit = "^1.8.0" autopep8 = "^2.3.2" autoflake = "^2.3.1" -isort = "^6.0.1" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" + +# ============================================================================= +# DEVELOPMENT TOOLS CONFIGURATION +# ============================================================================= + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--verbose", + "--tb=short", + "--cov-report=term-missing:skip-covered", + "--cov-report=html:htmlcov", + "--cov-report=xml", +] +filterwarnings = [ + "error", + "ignore::UserWarning", + "ignore::DeprecationWarning", +] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["backend/app"] +branch = true +omit = [ + "*/tests/*", + "*/venv/*", + "*/.venv/*", + "*/migrations/*", + "*/test_*.py", + "*/__pycache__/*" +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:" +] +show_missing = true +skip_covered = false +precision = 2 + +[tool.black] +line-length = 100 +target-version = ["py310", "py311", "py312", "py313"] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 100 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +skip_glob = ["*/migrations/*"] + +[tool.flake8] +max-line-length = 100 +select = ["E", "W", "F", "C", "N"] +ignore = [ + "E203", # whitespace before ':' + "E501", # line too long (handled by black) + "W503", # line break before binary operator + "C901", # too complex (we'll handle this separately) +] +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + ".venv", + "venv", + ".tox", + ".mypy_cache", + "migrations" +] +per-file-ignores = [ + "__init__.py:F401", # imported but unused + "test_*.py:S101", # use of assert +] + +[tool.mypy] +python_version = "3.10" +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +strict_equality = true +show_error_codes = true + +[[tool.mypy.overrides]] +module = [ + "supabase.*", + "weaviate.*", + "discord.*", + "aio_pika.*", + "langgraph.*", + "langchain_google_genai.*", + "tavily_python.*" +] +ignore_missing_imports = true + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101"] # Skip assert_used test From 7cda2c3e2d0a9f8448db790534becc000121ae34 Mon Sep 17 00:00:00 2001 From: Om Sherikar Date: Wed, 29 Oct 2025 03:16:33 +0530 Subject: [PATCH 2/6] Update .github/workflows/security.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/security.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 02ebe06..7ac22d2 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -45,8 +45,8 @@ jobs: poetry install --only=main poetry export -f requirements.txt --output requirements.txt pip install safety - safety check -r requirements.txt --json --output safety-report.json || true - safety check -r requirements.txt || true + safety check -r requirements.txt --json --output safety-report.json + safety check -r requirements.txt # Frontend Dependency Scanning - name: ๐ŸŸข Setup Node.js (Frontend) From 06b79ac16a5a2e0b31aecf73a85fcfdd43fe042b Mon Sep 17 00:00:00 2001 From: Om Sherikar Date: Wed, 29 Oct 2025 03:21:32 +0530 Subject: [PATCH 3/6] Update .github/workflows/dependencies.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index eada852..39d3620 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -133,7 +133,7 @@ jobs: name: ๐Ÿšจ Apply Security Updates runs-on: ubuntu-latest needs: dependency-audit - if: needs.dependency-audit.outputs.security-updates > 0 + if: needs.dependency-audit.outputs.security-updates != '0' && needs.dependency-audit.outputs.security-updates != '' permissions: contents: write From 4ecc0b29c0c1991c05ef27cec9418fa120c412a7 Mon Sep 17 00:00:00 2001 From: Om Sherikar Date: Wed, 29 Oct 2025 03:23:46 +0530 Subject: [PATCH 4/6] Refactor npm audit commands in security workflow --- .github/workflows/security.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 7ac22d2..5a5c851 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -60,8 +60,8 @@ jobs: working-directory: frontend run: | npm ci - npm audit --audit-level high --json > npm-audit-report.json || true - npm audit --audit-level high || true + npm audit --audit-level high --json > npm-audit-report.json + npm audit --audit-level high - name: ๐Ÿ“‹ Upload Dependency Reports uses: actions/upload-artifact@v4 From 230dbfd76d4ea323ec28d28380885d28a821df05 Mon Sep 17 00:00:00 2001 From: Om Sherikar Date: Wed, 29 Oct 2025 03:24:34 +0530 Subject: [PATCH 5/6] Update .github/workflows/security.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/security.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 5a5c851..abb5c7e 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -108,10 +108,7 @@ jobs: poetry run bandit -r . \ -f json \ -o bandit-report.json \ - --exclude ./tests,./venv,./.venv || true - - poetry run bandit -r . \ - --exclude ./tests,./venv,./.venv || true + --exclude ./tests,./venv,./.venv # CodeQL Analysis - name: ๐Ÿ”ฌ Initialize CodeQL From 3db9d0d585f55dc007ae72899da7fbe978519382 Mon Sep 17 00:00:00 2001 From: Om Sherikar Date: Wed, 29 Oct 2025 03:25:06 +0530 Subject: [PATCH 6/6] Update .yamllint.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .yamllint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.yamllint.yml b/.yamllint.yml index aee7bb5..8db0b59 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -18,8 +18,6 @@ rules: forbid-in-block-mappings: false forbid-in-flow-mappings: false - # Allow duplicate keys (sometimes needed in workflows) - key-duplicates: disable # Be more lenient with indentation indentation: