Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Individual connector page #640

Merged
merged 8 commits into from
Oct 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions backend/danswer/db/connector_credential_pair.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,16 @@ def get_connector_credential_pair(
return result.scalar_one_or_none()


def get_connector_credential_pair_from_id(
cc_pair_id: int,
db_session: Session,
) -> ConnectorCredentialPair | None:
stmt = select(ConnectorCredentialPair)
stmt = stmt.where(ConnectorCredentialPair.id == cc_pair_id)
result = db_session.execute(stmt)
return result.scalar_one_or_none()


def get_last_successful_attempt_time(
connector_id: int,
credential_id: int,
Expand Down
18 changes: 18 additions & 0 deletions backend/danswer/db/index_attempt.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,24 @@ def get_latest_index_attempts(
return db_session.execute(stmt).scalars().all()


def get_index_attempts_for_cc_pair(
db_session: Session, cc_pair_identifier: ConnectorCredentialPairIdentifier
) -> Sequence[IndexAttempt]:
stmt = (
select(IndexAttempt)
.where(
and_(
IndexAttempt.connector_id == cc_pair_identifier.connector_id,
IndexAttempt.credential_id == cc_pair_identifier.credential_id,
)
)
.order_by(
IndexAttempt.time_created.desc(),
)
)
return db_session.execute(stmt).scalars().all()


def delete_index_attempts(
connector_id: int,
credential_id: int,
Expand Down
4 changes: 4 additions & 0 deletions backend/danswer/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
from danswer.datastores.document_index import get_default_document_index
from danswer.db.credentials import create_initial_public_credential
from danswer.direct_qa.llm_utils import get_default_qa_model
from danswer.server.cc_pair.api import router as cc_pair_router
from danswer.server.chat_backend import router as chat_router
from danswer.server.connector import router as connector_router
from danswer.server.credential import router as credential_router
from danswer.server.document_set import router as document_set_router
from danswer.server.event_loading import router as event_processing_router
Expand Down Expand Up @@ -77,7 +79,9 @@ def get_application() -> FastAPI:
application.include_router(event_processing_router)
application.include_router(admin_router)
application.include_router(user_router)
application.include_router(connector_router)
application.include_router(credential_router)
application.include_router(cc_pair_router)
application.include_router(document_set_router)
application.include_router(slack_bot_management_router)
application.include_router(state_router)
Expand Down
67 changes: 67 additions & 0 deletions backend/danswer/server/cc_pair/api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
from fastapi import APIRouter
from fastapi import Depends
from fastapi import HTTPException
from sqlalchemy.orm import Session

from danswer.auth.users import current_admin_user
from danswer.background.celery.celery_utils import get_deletion_status
from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
from danswer.db.document import get_document_cnts_for_cc_pairs
from danswer.db.engine import get_session
from danswer.db.index_attempt import get_index_attempts_for_cc_pair
from danswer.db.models import User
from danswer.server.cc_pair.models import CCPairFullInfo
from danswer.server.models import ConnectorCredentialPairIdentifier


router = APIRouter(prefix="/manage")


@router.get("/admin/cc-pair/{cc_pair_id}")
def get_cc_pair_full_info(
cc_pair_id: int,
_: User | None = Depends(current_admin_user),
db_session: Session = Depends(get_session),
) -> CCPairFullInfo:
cc_pair = get_connector_credential_pair_from_id(
cc_pair_id=cc_pair_id,
db_session=db_session,
)
if cc_pair is None:
raise HTTPException(
status_code=400,
detail=f"Connector Credential Pair with id {cc_pair_id} not found.",
)

cc_pair_identifier = ConnectorCredentialPairIdentifier(
connector_id=cc_pair.connector_id,
credential_id=cc_pair.credential_id,
)

index_attempts = get_index_attempts_for_cc_pair(
db_session=db_session,
cc_pair_identifier=cc_pair_identifier,
)

document_count_info_list = list(
get_document_cnts_for_cc_pairs(
db_session=db_session,
cc_pair_identifiers=[cc_pair_identifier],
)
)
documents_indexed = (
document_count_info_list[0][-1] if document_count_info_list else 0
)

latest_deletion_attempt = get_deletion_status(
connector_id=cc_pair.connector.id,
credential_id=cc_pair.credential.id,
db_session=db_session,
)

return CCPairFullInfo.from_models(
cc_pair_model=cc_pair,
index_attempt_models=list(index_attempts),
latest_deletion_attempt=latest_deletion_attempt,
num_docs_indexed=documents_indexed,
)
43 changes: 43 additions & 0 deletions backend/danswer/server/cc_pair/models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
from pydantic import BaseModel

from danswer.db.models import ConnectorCredentialPair
from danswer.db.models import IndexAttempt
from danswer.server.models import ConnectorSnapshot
from danswer.server.models import CredentialSnapshot
from danswer.server.models import DeletionAttemptSnapshot
from danswer.server.models import IndexAttemptSnapshot


class CCPairFullInfo(BaseModel):
id: int
name: str
num_docs_indexed: int
connector: ConnectorSnapshot
credential: CredentialSnapshot
index_attempts: list[IndexAttemptSnapshot]
latest_deletion_attempt: DeletionAttemptSnapshot | None

@classmethod
def from_models(
cls,
cc_pair_model: ConnectorCredentialPair,
index_attempt_models: list[IndexAttempt],
latest_deletion_attempt: DeletionAttemptSnapshot | None,
num_docs_indexed: int, # not ideal, but this must be computed seperately
) -> "CCPairFullInfo":
return cls(
id=cc_pair_model.id,
name=cc_pair_model.name,
num_docs_indexed=num_docs_indexed,
connector=ConnectorSnapshot.from_connector_db_model(
cc_pair_model.connector
),
credential=CredentialSnapshot.from_credential_db_model(
cc_pair_model.credential
),
index_attempts=[
IndexAttemptSnapshot.from_index_attempt_db_model(index_attempt_model)
for index_attempt_model in index_attempt_models
],
latest_deletion_attempt=latest_deletion_attempt,
)
Loading