diff --git a/app/aimodels/bertopic/crud/crud_bertopic_embedding_pretrained.py b/app/aimodels/bertopic/crud/crud_bertopic_embedding_pretrained.py index b0f2e6b..0536618 100644 --- a/app/aimodels/bertopic/crud/crud_bertopic_embedding_pretrained.py +++ b/app/aimodels/bertopic/crud/crud_bertopic_embedding_pretrained.py @@ -2,7 +2,7 @@ from typing import Union from sqlalchemy.orm import Session from app.crud.base import CRUDBase -from app.core.config import get_originated_from, OriginationEnum +from app.core.config import OriginationEnum from ..models.bertopic_embedding_pretrained import BertopicEmbeddingPretrainedModel from app.ppg_common.schemas.bertopic.bertopic_embedding_pretrained import BertopicEmbeddingPretrainedCreate, BertopicEmbeddingPretrainedUpdate diff --git a/app/aimodels/bertopic/models/document.py b/app/aimodels/bertopic/models/document.py index 3366bb9..87fe676 100644 --- a/app/aimodels/bertopic/models/document.py +++ b/app/aimodels/bertopic/models/document.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING from sqlalchemy import Column, DateTime, UUID, String, Enum from sqlalchemy.orm import relationship -from sqlalchemy.sql import func from app.db.base_class import Base from app.core.config import get_originated_from, OriginationEnum import uuid diff --git a/app/aimodels/bertopic/models/document_bertopic_trained_model.py b/app/aimodels/bertopic/models/document_bertopic_trained_model.py index cb09625..8e8dce4 100644 --- a/app/aimodels/bertopic/models/document_bertopic_trained_model.py +++ b/app/aimodels/bertopic/models/document_bertopic_trained_model.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING -from sqlalchemy import Column, ForeignKey, Table, Enum +from sqlalchemy import Column, ForeignKey, Table from app.db.base_class import Base if TYPE_CHECKING: diff --git a/app/aimodels/bertopic/models/topic.py b/app/aimodels/bertopic/models/topic.py index 6cf32d9..202ac04 100644 --- a/app/aimodels/bertopic/models/topic.py +++ b/app/aimodels/bertopic/models/topic.py @@ -1,8 +1,5 @@ -from typing import TYPE_CHECKING from sqlalchemy import Column, UUID, String, Enum, JSON, Integer, Boolean, ForeignKey from sqlalchemy.ext.mutable import MutableDict -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func from app.db.base_class import Base from app.core.config import get_originated_from, OriginationEnum import uuid diff --git a/app/aimodels/bertopic/routers/bertopic_embedding_pretrained.py b/app/aimodels/bertopic/routers/bertopic_embedding_pretrained.py index 213b2ae..016e717 100644 --- a/app/aimodels/bertopic/routers/bertopic_embedding_pretrained.py +++ b/app/aimodels/bertopic/routers/bertopic_embedding_pretrained.py @@ -2,7 +2,7 @@ import pickle import pandas as pd from typing import Union -from io import TextIOWrapper, StringIO, BytesIO, BufferedReader +from io import StringIO, BytesIO from fastapi import Depends, APIRouter, UploadFile from fastapi import HTTPException diff --git a/app/aimodels/bertopic/routers/documents.py b/app/aimodels/bertopic/routers/documents.py index 500699d..8c80480 100644 --- a/app/aimodels/bertopic/routers/documents.py +++ b/app/aimodels/bertopic/routers/documents.py @@ -1,7 +1,5 @@ -import os from typing import Union from fastapi import Depends, APIRouter -from pydantic import UUID4 from sqlalchemy.orm import Session from app.dependencies import get_db from app.core.errors import HTTPValidationError diff --git a/app/aimodels/bertopic/routers/summarization.py b/app/aimodels/bertopic/routers/summarization.py index ba46b29..d713d35 100644 --- a/app/aimodels/bertopic/routers/summarization.py +++ b/app/aimodels/bertopic/routers/summarization.py @@ -1,8 +1,6 @@ -import json from typing import Union, Any -from pydantic import BaseModel, UUID4 +from pydantic import UUID4 from fastapi import Depends, APIRouter, HTTPException -from fastapi.encoders import jsonable_encoder from fastapi.responses import HTMLResponse from sqlalchemy.orm import Session from app.dependencies import get_db diff --git a/app/aimodels/bertopic/routers/train.py b/app/aimodels/bertopic/routers/train.py index 46b49da..88c010a 100644 --- a/app/aimodels/bertopic/routers/train.py +++ b/app/aimodels/bertopic/routers/train.py @@ -6,7 +6,7 @@ import pandas as pd from app.core.logging import logger from app.core.s3 import pickle_and_upload_object_to_s3 -from app.core.errors import ValidationError, HTTPValidationError +from app.core.errors import HTTPValidationError from app.core.config import settings, get_acronym_dictionary, get_icao_dictionary from app.dependencies import get_db, get_s3 from app.ppg_common.schemas.bertopic.document_embedding_computation import DocumentEmbeddingComputationCreate diff --git a/app/aimodels/gpt4all/crud/crud_llm_pretrained.py b/app/aimodels/gpt4all/crud/crud_llm_pretrained.py index d62abc1..68a51e9 100644 --- a/app/aimodels/gpt4all/crud/crud_llm_pretrained.py +++ b/app/aimodels/gpt4all/crud/crud_llm_pretrained.py @@ -1,8 +1,7 @@ from typing import Union from sqlalchemy.orm import Session -from sqlalchemy import func -from app.core.config import get_originated_from, OriginationEnum +from app.core.config import OriginationEnum from app.crud.base import CRUDBase from app.ppg_common.schemas.gpt4all.llm_pretrained import LlmPretrainedCreate, LlmPretrainedUpdate from ..models import LlmPretrainedModel diff --git a/app/aimodels/gpt4all/models/llm_pretrained.py b/app/aimodels/gpt4all/models/llm_pretrained.py index 88fe5b0..11c745d 100644 --- a/app/aimodels/gpt4all/models/llm_pretrained.py +++ b/app/aimodels/gpt4all/models/llm_pretrained.py @@ -1,7 +1,5 @@ import uuid -from typing import TYPE_CHECKING from sqlalchemy import Column, Enum, Integer, UUID, String, Boolean, Sequence -from sqlalchemy.orm import relationship from app.db.base_class import Base from app.core.config import get_originated_from, OriginationEnum from app.ppg_common.schemas.gpt4all.llm_pretrained import LlmFilenameEnum diff --git a/app/aimodels/gpt4all/routers/pretrained.py b/app/aimodels/gpt4all/routers/pretrained.py index d848621..0e0de25 100644 --- a/app/aimodels/gpt4all/routers/pretrained.py +++ b/app/aimodels/gpt4all/routers/pretrained.py @@ -1,7 +1,6 @@ import hashlib from typing import Union -from aiofiles import open as open_aio from fastapi import Depends, APIRouter, UploadFile, HTTPException from pydantic import UUID4 from sqlalchemy.orm import Session diff --git a/app/chat_search/ai_services/service.py b/app/chat_search/ai_services/service.py index 1530802..806610d 100644 --- a/app/chat_search/ai_services/service.py +++ b/app/chat_search/ai_services/service.py @@ -18,7 +18,6 @@ from app.chat_search.ai_services.marco_rerank_retriever import MarcoRerankRetriever from app.core.errors import ValidationError from app.core.s3 import download_pickled_object_from_s3 -from app.core.model_cache import MODEL_CACHE_BASEDIR from app.core.config import settings from app.aimodels.bertopic.crud import ( bertopic_embedding_pretrained as bertopic_embedding_pretrained_crud, diff --git a/app/chat_search/routers/retrieval.py b/app/chat_search/routers/retrieval.py index bc4247d..b632ce2 100644 --- a/app/chat_search/routers/retrieval.py +++ b/app/chat_search/routers/retrieval.py @@ -1,25 +1,17 @@ import os -from fastapi import Depends, APIRouter, HTTPException +from fastapi import Depends, APIRouter from fastapi.responses import HTMLResponse from jinja2 import Environment, FileSystemLoader -from pydantic import BaseModel from sqlalchemy.orm import Session from mypy_boto3_s3.client import S3Client -from app.aimodels.gpt4all.models.llm_pretrained import ( - LlmFilenameEnum, - LlmPretrainedModel, -) from app.aimodels.gpt4all.routers.completions import ( validate_inputs_and_generate_service, ) from app.dependencies import get_db, get_s3 -from app.core.config import settings -from ... import crud from app.aimodels.gpt4all.ai_services.completion_inference import ( CompletionInference, CompletionInferenceInputs, - CompletionInferenceOutputs, ) from ..ai_services.service import RetrievalService diff --git a/app/core/config.py b/app/core/config.py index 0505415..29e3369 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -1,7 +1,7 @@ import os import enum from typing import Optional, Any -from pydantic import PostgresDsn, validator, field_validator, ValidationInfo +from pydantic import PostgresDsn, field_validator, ValidationInfo from pydantic_settings import BaseSettings class OriginationEnum(str, enum.Enum): @@ -16,7 +16,7 @@ class EnvironmentSettings(BaseSettings): class Settings(BaseSettings): # general settings docs_ui_root_path: str = "" - log_level: str = "INFO" + log_level: str = "WARNING" originated_from: OriginationEnum = OriginationEnum.ORIGINATED_FROM_APP acronym_dictionary: dict = {} icao_dictionary: dict = {} diff --git a/app/initial_data.py b/app/initial_data.py index 947daae..1a5d250 100644 --- a/app/initial_data.py +++ b/app/initial_data.py @@ -11,7 +11,6 @@ from tqdm import tqdm from typing import Union from fastapi import UploadFile -from fastapi.encoders import jsonable_encoder from mypy_boto3_s3.client import S3Client import app.ppg_common.services.mattermost_utils as mattermost_utils @@ -21,7 +20,6 @@ from app.ppg_common.schemas.mattermost.mattermost_documents import ThreadTypeEnum from app.aimodels.bertopic.models.bertopic_embedding_pretrained import BertopicEmbeddingPretrainedModel, EmbeddingModelTypeEnum -from app.aimodels.bertopic.models.document import DocumentModel from app.aimodels.gpt4all.models.llm_pretrained import LlmPretrainedModel, LlmFilenameEnum from app.db.init_db import init_db, wipe_db, drop_constraints diff --git a/app/mattermost/crud/crud_mattermost.py b/app/mattermost/crud/crud_mattermost.py index 1565a29..7a8ee0d 100644 --- a/app/mattermost/crud/crud_mattermost.py +++ b/app/mattermost/crud/crud_mattermost.py @@ -1,7 +1,6 @@ from typing import Union from datetime import datetime, timedelta import pandas as pd -import numpy as np from fastapi import HTTPException from sqlalchemy.orm import Session from app.core.config import settings diff --git a/app/mattermost/models/mattermost_channels.py b/app/mattermost/models/mattermost_channels.py index 0db8206..6fcd3a5 100644 --- a/app/mattermost/models/mattermost_channels.py +++ b/app/mattermost/models/mattermost_channels.py @@ -6,7 +6,7 @@ from app.core.config import get_originated_from, OriginationEnum if TYPE_CHECKING: - from .mattermost_documents import MattermostDocumentModel + pass class MattermostChannelModel(Base): diff --git a/app/mattermost/models/mattermost_documents.py b/app/mattermost/models/mattermost_documents.py index 709b9be..10be7ef 100644 --- a/app/mattermost/models/mattermost_documents.py +++ b/app/mattermost/models/mattermost_documents.py @@ -7,9 +7,7 @@ from app.ppg_common.schemas.mattermost.mattermost_documents import InfoTypeEnum, ThreadTypeEnum if TYPE_CHECKING: - from .mattermost_channels import MattermostChannelModel - from .mattermost_users import MattermostUserModel - from app.aimodels.bertopic.models.document import DocumentModel + pass class MattermostDocumentModel(Base): diff --git a/app/nitmre_nlp_utils/preprocess.py b/app/nitmre_nlp_utils/preprocess.py index 5fa78e5..b0866f8 100644 --- a/app/nitmre_nlp_utils/preprocess.py +++ b/app/nitmre_nlp_utils/preprocess.py @@ -13,7 +13,7 @@ def _tokenize(msg: str) -> set[str]: / often used between acronyms in an explicit/implied coordination line (e.g., DO/LL) """ - p = re.compile('[^\w/#@]+') + p = re.compile(r'[^\w/#@]+') return set(p.split(msg)) # only keep unique tokens to avoid revisiting during preprocessing diff --git a/tests/aimodels/bertopic/routers/test_bertopic_embedding_pretrained.py b/tests/aimodels/bertopic/routers/test_bertopic_embedding_pretrained.py index 31b10cd..386ae8d 100644 --- a/tests/aimodels/bertopic/routers/test_bertopic_embedding_pretrained.py +++ b/tests/aimodels/bertopic/routers/test_bertopic_embedding_pretrained.py @@ -11,8 +11,7 @@ from fastapi.testclient import TestClient from app.core.config import OriginationEnum from app.ppg_common.schemas.bertopic.bertopic_embedding_pretrained import BertopicEmbeddingPretrainedCreate -from app.aimodels.bertopic.models.bertopic_embedding_pretrained import BertopicEmbeddingPretrainedModel -from app.aimodels.bertopic.crud.crud_bertopic_embedding_pretrained import CRUDBertopicEmbeddingPretrained, bertopic_embedding_pretrained +from app.aimodels.bertopic.crud.crud_bertopic_embedding_pretrained import bertopic_embedding_pretrained from app.aimodels.bertopic.ai_services.weak_learning import WeakLearner from fastapi.encoders import jsonable_encoder diff --git a/tests/aimodels/gpt4all/ai_services/conftest.py b/tests/aimodels/gpt4all/ai_services/conftest.py index 383fb28..aaa9756 100644 --- a/tests/aimodels/gpt4all/ai_services/conftest.py +++ b/tests/aimodels/gpt4all/ai_services/conftest.py @@ -1,8 +1,6 @@ from unittest.mock import create_autospec from sqlalchemy.orm import Session -from app.ppg_common.schemas.gpt4all.llm_pretrained import LlmPretrainedCreate -from app.aimodels.gpt4all import crud from app.aimodels.gpt4all.models.llm_pretrained import LlmPretrainedModel import pytest