diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 817811f10..6b54f97e5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye +FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bookworm # Avoid warnings by switching to noninteractive ENV DEBIAN_FRONTEND=noninteractive @@ -22,7 +22,7 @@ RUN sudo apt-get update \ && sudo apt-get -y install --no-install-recommends apt-utils dialog nano bash-completion sudo bsdmainutils cmake \ # # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && sudo apt-get -y install git iproute2 procps lsb-release figlet build-essential + && sudo apt-get -y install git iproute2 procps lsb-release figlet build-essential dnsutils openvpn # Save command line history RUN echo "export HISTFILE=/home/$USERNAME/commandhistory/.bash_history" >> "/home/$USERNAME/.bashrc" \ @@ -34,11 +34,11 @@ RUN echo "export HISTFILE=/home/$USERNAME/commandhistory/.bash_history" >> "/hom # docker-client RUN sudo apt-get update \ # libgl dependencies and install Docker CE CLI - && sudo apt-get install ffmpeg libsm6 libxext6 apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release -y \ - && curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | sudo apt-key add - 2>/dev/null \ - && sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" \ - && sudo apt-get update \ - && sudo apt-get install -y docker-ce-cli + && sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release -y \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && sudo apt-get update -y \ + && sudo apt-get install -y docker-ce-cli ARG DOCKER_GROUP_ID COPY ./scripts/docker-client.sh /tmp/ @@ -49,7 +49,7 @@ RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prom && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/home/$USERNAME/.bashrc" # terraform + tflint -ARG TERRAFORM_VERSION=1.3.7 +ARG TERRAFORM_VERSION=1.8.5 ARG TFLINT_VERSION=0.44.1 RUN mkdir -p /tmp/docker-downloads \ && curl -sSL -o /tmp/docker-downloads/terraform.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \ @@ -75,12 +75,6 @@ RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian && apt-get update \ && apt-get install -y azure-functions-core-tools-4 -# Install Powershell -RUN apt-get update \ - && apt-get install -y powershell \ - # Alias Powershell - && echo "alias powershell=pwsh" >> "/home/$USERNAME/.bashrc" - #Install npm and nodejs in the Docker dev container COPY ./scripts/nodejs.sh /tmp/ RUN /tmp/nodejs.sh @@ -100,5 +94,13 @@ RUN pip install azure-cosmos RUN pip install pyfiglet RUN pip install azure-identity RUN pip install azure-keyvault-secrets + # Python Package for Backend Testing -RUN pip install pytest \ No newline at end of file +RUN pip install pytest + +# Install skopeo - This must be at the end due to pulling from trixie +RUN sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install usrmerge + +RUN echo 'deb http://ftp.debian.org/debian trixie main' | sudo tee /etc/apt/sources.list.d/debian.trixie.list \ + && sudo apt-get update \ + && sudo apt-get install skopeo \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8276a6457..e8e15be35 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ }, "forwardPorts": [ 7071 ], "runArgs": [ - "--network", "host" // use host networking so that the dev container can access the API when running the container locally + "--network", "host", "--cap-add", "NET_ADMIN" // use host networking so that the dev container can access the API when running the container locally ], "mounts": [ diff --git a/.devcontainer/scripts/nodejs.sh b/.devcontainer/scripts/nodejs.sh index 6f983a946..62b59ff5a 100755 --- a/.devcontainer/scripts/nodejs.sh +++ b/.devcontainer/scripts/nodejs.sh @@ -3,4 +3,4 @@ set -e curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash source $NVM_DIR/nvm.sh -nvm install v18.12.1 \ No newline at end of file +nvm install v20.13.0 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 19b523a6a..4327c5237 100644 --- a/.gitignore +++ b/.gitignore @@ -400,4 +400,8 @@ infra/sp_config/config.json #Upgrade & Migrate Support scripts/upgrade_repoint.config.json azcopy.tar.gz -azcopy_dir \ No newline at end of file +azcopy_dir + +#Openvpn setting +openvpn/* +!openvpn/info.txt \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index be3394284..84a61a14f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -86,8 +86,8 @@ "request": "launch", "program": "debug_tests.py", "args": [ - "--storage_account_connection_str", - "${env:STORAGE_ACCOUNT_CONNECTION_STR},", + "--storage_account_url", + "${env:AZURE_BLOB_STORAGE_ENDPOINT},", "--search_service_endpoint", "${env:SEARCH_SERVICE_ENDPOINT}", "--search_index", @@ -98,10 +98,9 @@ "60" ], "env": { - "STORAGE_ACCOUNT_CONNECTION_STR": "${env:BLOB_CONNECTION_STRING}", + "storage_account_url": "${env:AZURE_BLOB_STORAGE_ENDPOINT}", "SEARCH_SERVICE_ENDPOINT": "${env:AZURE_SEARCH_SERVICE_ENDPOINT}", - "SEARCH_INDEX": "${env:AZURE_SEARCH_INDEX}", - "SEARCH_KEY": "${env:AZURE_SEARCH_SERVICE_KEY}" + "SEARCH_INDEX": "${env:AZURE_SEARCH_INDEX}" }, "cwd": "${workspaceFolder}/tests", "envFile": "${workspaceFolder}/scripts/environments/infrastructure.debug.env", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a6a4e27a..0ed46971e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to Information Assistant +# Contributing to Information Assistant copilot template This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us diff --git a/Makefile b/Makefile index 998ef5985..db2542260 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ help: ## Show this help | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%s\033[0m|%s\n", $$1, $$2}' \ | column -t -s '|' -deploy: build infrastructure extract-env deploy-enrichments deploy-search-indexes deploy-webapp deploy-functions ## Deploy infrastructure and application code +deploy: build infrastructure extract-env deploy-search-indexes deploy-functions deploy-webapp deploy-enrichments ## Deploy infrastructure and application code build-deploy-webapp: build extract-env deploy-webapp ##Build and Deploy the Webapp build-deploy-enrichments: build extract-env deploy-enrichments ##Build and Deploy the Enrichment Webapp @@ -18,15 +18,15 @@ build-deploy-functions: build extract-env deploy-functions ##Build and Deploy th build: ## Build application code @./scripts/build.sh -build-containers: extract-env - @./app/enrichment/docker-build.sh - infrastructure: check-subscription ## Deploy infrastructure @./scripts/inf-create.sh -extract-env: extract-env-debug-webapp extract-env-debug-functions ## Extract infrastructure.env file from Terraform output +extract-env: check-secure-mode-connectivity extract-env-debug-webapp extract-env-debug-functions ## Extract infrastructure.env file from Terraform output @./scripts/json-to-env.sh < inf_output.json > ./scripts/environments/infrastructure.env +check-secure-mode-connectivity: ## Check secure mode connectivity + @./scripts/check-secure-mode-connectivity.sh + deploy-webapp: extract-env ## Deploys the web app code to Azure App Service @./scripts/deploy-webapp.sh diff --git a/NOTICE.md b/NOTICE.md index beb584eea..5a668cb97 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -113,10 +113,6 @@ From the following locations: -### @azure/storage-blob - - - ### classnames diff --git a/README.md b/README.md index dbec14e0b..2fb3d51a0 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,35 @@ -# Information Assistant Accelerator +# Information Assistant (IA) copilot template > [!IMPORTANT] > As of November 15, 2023, Azure Cognitive Search has been renamed to Azure AI Search. Azure Cognitive Services have also been renamed to Azure AI Services. -## Table of Contents +## Table of Contents -- [Response Generation Approaches](#response-generation-approaches) +- [Response generation approaches](#response-generation-approaches) - [Features](#features) - [Azure account requirements](#azure-account-requirements) -- [Azure Deployment](./docs/deployment/deployment.md) - - [GitHub Codespaces Setup](./docs/deployment/deployment.md#development-environment-configuration) - - [Cost Estimation](./docs/deployment/deployment.md#sizing-estimator) +- [Azure deployment](./docs/deployment/deployment.md) + - [GitHub Codespaces setup](./docs/deployment/deployment.md#development-environment-configuration) + - [Cost estimation](./docs/deployment/deployment.md#sizing-estimator) - [Configuring ENV parameters](./docs/deployment/deployment.md#configure-env-files) - [Authenticating to Azure](./docs/deployment/deployment.md#log-into-azure-using-the-azure-cli) - [Deploying to Azure](./docs/deployment/deployment.md#deploy-and-configure-azure-resources) - - [Troubleshooting Common Issues](./docs/deployment/troubleshooting.md) - - [Considerations for Production Adoption](./docs/deployment/considerations_production.md) + - [Troubleshooting common issues](./docs/deployment/troubleshooting.md) + - [Considerations for production adoption](./docs/deployment/considerations_production.md) +- [Secure mode deployment](./docs/secure_deployment/secure_deployment.md) + - [Cost estimation](./docs/secure_deployment/secure_costestimator.md) - [Enabling optional features](./docs/features/optional_features.md) - [Using the app](/docs/deployment/using_ia_first_time.md) - [Responsible AI](#responsible-ai) - [Transparency Note](#transparency-note) - [Content Safety](#content-safety) - [Data Collection Notice](#data-collection-notice) +- [Shared responsibility and customer responsibilities](#shared-responsibility-and-customer-responsibilities) - [Resources](#resources) - - [Known Issues](./docs/knownissues.md) - - [Functional Tests](./tests/README.md) + - [Known issues](./docs/knownissues.md) + - [Functional tests](./tests/README.md) - [Navigating the source code](#navigating-the-source-code) - - [Architectural Decisions](/docs/features/architectural_decisions.md) + - [Architectural decisions](/docs/features/architectural_decisions.md) - [References](#references) - [Trademarks](#trademarks) - [Code of Conduct](#code-of-conduct) @@ -35,23 +38,27 @@ [![Open in GitHub Codespaces](https://img.shields.io/static/v1?style=for-the-badge&label=GitHub+Codespaces&message=Open&color=brightgreen&logo=github)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=601652366&machine=basicLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=eastus) -This industry accelerator showcases integration between Azure and OpenAI's large language models. It leverages Azure AI Search for data retrieval and ChatGPT-style Q&A interactions. Using the Retrieval Augmented Generation (RAG) design pattern with Azure OpenAI's GPT models, it provides a natural language interaction to discover relevant responses to user queries. Azure AI Search simplifies data ingestion, transformation, indexing, and multilingual translation. +Information Assistant (IA) copilot template provides a starting point for organizations to build their own custom generative AI capability to extend the power of Azure OpenAI. It showcases a common scenario using large language models (LLMs) to “chat with your own data” through the [Retrieval Augmented Generation (RAG) pattern](https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview). This pattern lets you use the reasoning abilities of LLMs to generate responses based on your domain data without fine-tuning the model. -The accelerator adapts prompts based on the model type for enhanced performance. Users can customize settings like temperature and persona for personalized AI interactions. It offers features like explainable thought processes, referenceable citations, and direct content for verification. +Information Assistant copilot template is an end-to-end solution which is a comprehensive reference sample including documentation, source code, and deployment to allow you to take and extend for your own purposes. -Please [see this video](https://aka.ms/InfoAssist/video) for use cases that may be achievable with this accelerator. +This copilot template showcases integration between Azure and OpenAI's LLMs. It leverages Azure AI Search for data retrieval and ChatGPT-style Q&A interactions. Using the RAG design pattern with Azure OpenAI's GPT models, it provides a natural language interaction to discover relevant responses to user queries. Azure AI Search simplifies data ingestion, transformation, indexing, and multilingual translation. -# Response Generation Approaches +The copilot adapts prompts based on the model type for enhanced performance. Users can customize settings like temperature and persona for personalized AI interactions. It offers features like explainable thought processes, referenceable citations, and direct content for verification. -## Work(Grounded) -It utilizes a retrieval-augmented generation (RAG) pattern to generate responses grounded in specific data sourced from your own dataset. By combining retrieval of relevant information with generative capabilities, It can produce responses that are not only contextually relevant but also grounded in verified data. The RAG pipeline accesses your dataset to retrieve relevant information before generating responses, ensuring accuracy and reliability. Additionally, each response includes a citation to the document chunk from which the answer is derived, providing transparency and allowing users to verify the source. This approach is particularly advantageous in domains where precision and factuality are paramount. Users can trust that the responses generated are based on reliable data sources, enhancing the credibility and usefulness of the application. Specific information on our Grounded (RAG) can be found in [RAG](docs/features/cognitive_search.md#azure-ai-search-integration) +Please [see this video](https://aka.ms/InfoAssist/video) for use cases that may be achievable with Information Assistant copilot template. + +# Response generation approaches + +## Work (Grounded) +It utilizes a Retrieval Augmented Generation (RAG) pattern to generate responses grounded in specific data sourced from your own dataset. By combining retrieval of relevant information with generative capabilities, it can produce responses that are not only contextually relevant but also grounded in verified data. The RAG pipeline accesses your dataset to retrieve relevant information before generating responses, ensuring accuracy and reliability. Additionally, each response includes a citation to the document chunk from which the answer is derived, providing transparency and allowing users to verify the source. This approach is particularly advantageous in domains where precision and factuality are paramount. Users can trust that the responses generated are based on reliable data sources, enhancing the credibility and usefulness of the application. Specific information on our Grounded (RAG) can be found in [RAG](docs/features/cognitive_search.md#azure-ai-search-integration). ## Ungrounded It leverages the capabilities of a large language model (LLM) to generate responses in an ungrounded manner, without relying on external data sources or retrieval-augmented generation techniques. The LLM has been trained on a vast corpus of text data, enabling it to generate coherent and contextually relevant responses solely based on the input provided. This approach allows for open-ended and creative generation, making it suitable for tasks such as ideation, brainstorming, and exploring hypothetical scenarios. It's important to note that the generated responses are not grounded in specific factual data and should be evaluated critically, especially in domains where accuracy and verifiability are paramount. ## Work and Web -It offers 3 response options: one generated through our retrieval-augmented generation (RAG) pipeline, and the other grounded in content directly from the web. When users opt for the RAG response, they receive a grounded answer sourced from your data, complete with citations to document chunks for transparency and verification. Conversely, selecting the web response provides access to a broader range of sources, potentially offering more diverse perspectives. Each web response is grounded in content from the web accompanied by citations of web links, allowing users to explore the original sources for further context and validation. Upon request, It can also generate a final response that compares and contrasts both responses. This comparative analysis allows users to make informed decisions based on the reliability, relevance, and context of the information provided. -Specific information about our Grounded and Web can be found in [Web](/docs/features/features.md#bing-search-and-compare) +It offers 2 response options: one generated through our Retrieval Augmented Generation (RAG) pipeline, and the other grounded in content directly from the web. When users opt for the RAG response, they receive a grounded answer sourced from their data, complete with citations to document chunks for transparency and verification. Conversely, selecting the web response provides access to a broader range of sources, potentially offering more diverse perspectives. Each web response is grounded in content from the web accompanied by citations of web links, allowing users to explore the original sources for further context and validation. Upon request, It can also generate a final response that compares and contrasts both responses. This comparative analysis allows users to make informed decisions based on the reliability, relevance, and context of the information provided. +Specific information about our Work and Web can be found in [Web](/docs/features/features.md#bing-search-and-compare). ## Assistants It generates response by using LLM as a reasoning engine. The key strength lies in agent's ability to autonomously reason about tasks, decompose them into steps, and determine the appropriate tools and data sources to leverage, all without the need for predefined task definitions or rigid workflows. This approach allows for a dynamic and adaptive response generation process without predefining set of tasks. It harnesses the capabilities of LLM to understand natural language queries and generate responses tailored to specific tasks. These Agents are being released in preview mode as we continue to evaluate and mitigate the potential risks associated with autonomous reasoning, such as misuse of external tools, lack of transparency, biased outputs, privacy concerns, and remote code execution vulnerabilities. With future releases, we plan to work to enhance the safety and robustness of these autonomous reasoning capabilities. Specific information on our preview agents can be found in [Assistants](/docs/features/features.md#autonomous-reasoning-with-assistants-agents). @@ -59,7 +66,7 @@ It generates response by using LLM as a reasoning engine. The key strength lies ## Features -The IA Accelerator contains several features, many of which have their own documentation. +The Information Assistant copilot template contains several features, many of which have their own documentation. - Examples of custom Retrieval Augmented Generation (RAG), Prompt Engineering, and Document Pre-Processing - Azure AI Search Integration to include text search of both text documents and images @@ -68,11 +75,11 @@ The IA Accelerator contains several features, many of which have their own docum For a detailed review see our [Features](./docs/features/features.md) page. -### Process Flow for Work(Grounded), Ungrounded, and Work and Web +### Process flow for Work (Grounded), Ungrounded, and Work and Web ![Process Flow for Chat](/docs/process_flow_chat.png) -### Process Flow for Assistants +### Process flow for Assistants ![Process Flow for Assistants](/docs/process_flow_agent.png) @@ -81,17 +88,18 @@ For a detailed review see our [Features](./docs/features/features.md) page. **IMPORTANT:** In order to deploy and run this example, you'll need: * **Azure account**. If you're new to Azure, [get an Azure account for free](https://azure.microsoft.com/free/cognitive-search/) and you'll get some free Azure credits to get started. -* **Azure subscription with access enabled for the Azure OpenAI service**. You can request access with [this form](https://aka.ms/oaiapply). +* **Azure subscription with Azure OpenAI service**. Learn more about [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) * **Access to one of the following Azure OpenAI models**: Model Name | Supported Versions ---|--- gpt-35-turbo | current version - **gpt-35-turbo-16k** | current version - **gpt-4** | current version + gpt-35-turbo-16k | current version + gpt-4 | current version gpt-4-32k | current version + **gpt-4o** | current version - **Important:** Gpt-35-turbo-16k (0613) is recommended. GPT 4 models may achieve better results from the IA Accelerator. + **Important:** Gpt-4o (2024-05-13) is recommended. The gpt-4 models may achieve better results but slower performance than gpt-35 models when used with Information Assistant. * (Optional) **Access to the following Azure OpenAI model for embeddings**. Some open source embedding models may perform better for your specific data or use case. For the use case and data Information Assistant was tested for we recommend using the following Azure OpenAI embedding model. Model Name | Supported Versions @@ -100,31 +108,32 @@ For a detailed review see our [Features](./docs/features/features.md) page. * **Azure account permissions**: * Your Azure account must have `Microsoft.Authorization/roleAssignments/write` permissions, such as [Role Based Access Control Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview), [User Access Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator), or [Owner](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#owner) on the subscription. * Your Azure account also needs `Microsoft.Resources/deployments/write` permissions on the subscription level. - * Your Azure account also needs `microsoft.directory/applications/create` and `microsoft.directory/servicePrincipals/create`, such as [Application Administrator](https://learn.microsoft.com/en-us/entra/identity/role-based-access-control/permissions-reference#application-administrator) Entra built-in role. + * Your Azure account also needs `microsoft.directory/applications/create` and `microsoft.directory/servicePrincipals/create`, such as [Application Administrator](https://learn.microsoft.com/entra/identity/role-based-access-control/permissions-reference#application-administrator) Entra built-in role. * **To have accepted the Azure AI Services Responsible AI Notice** for your subscription. If you have not manually accepted this notice please follow our guide at [Accepting Azure AI Service Responsible AI Notice](./docs/deployment/accepting_responsible_ai_notice.md). +* **To have accepted the Azure AI Services Multi-service Account Responsible AI Notice** for your subscription. If you have not manually accepted this notice please follow our guide at [Accepting Azure AI Services Multi-service Account Responsible AI Notice](./docs/deployment/accepting_responsible_ai_notice_multi_service.md). * (Optional) Have [Visual Studio Code](https://code.visualstudio.com/) installed on your development machine. If your Azure tenant and subscription have conditional access policies or device policies required, you may need to open your GitHub Codespaces in VS Code to satisfy the required polices. ## Deployment -Please follow the instructions in [the deployment guide](/docs/deployment/deployment.md) to install the IA Accelerator in your Azure subscription. +Please follow the instructions in [the deployment guide](/docs/deployment/deployment.md) to install the Information Assistant copilot template in your Azure subscription. -Once completed, follow the [instructions for using IA Accelerator for the first time](/docs/deployment/using_ia_first_time.md). +Once completed, follow the [instructions for using Information Assistant copilot template for the first time](/docs/deployment/using_ia_first_time.md). You may choose to **[view the deployment and usage click-through guides](https://aka.ms/InfoAssist/deploy)** to see the steps in action. These videos may be useful to help clarify specific steps or actions in the instructions. ## Responsible AI -The Information Assistant (IA) Accelerator and Microsoft are committed to the advancement of AI driven by ethical principles that put people first. +The Information Assistant (IA) copilot template and Microsoft are committed to the advancement of AI driven by ethical principles that put people first. ### Transparency Note -**Read our [Transparency Note](/docs/transparency.md)** +**Read our [Transparency Note](/docs/transparency.md).** -Find out more with Microsoft's [Responsible AI resources](https://www.microsoft.com/en-us/ai/responsible-ai) +Find out more with Microsoft's [Responsible AI resources](https://www.microsoft.com/ai/responsible-ai). ### Content Safety -Content safety is provided through Azure OpenAI service. The Azure OpenAI Service includes a content filtering system that runs alongside the core AI models. This system uses an ensemble of classification models to detect four categories of potentially harmful content (violence, hate, sexual, and self-harm) at four severity levels (safe, low, medium, high).These 4 categories may not be sufficient for all use cases, especially for minors. Please read our [Transaparncy Note](/docs/transparency.md) +Content safety is provided through Azure OpenAI service. The Azure OpenAI Service includes a [content filtering](https://learn.microsoft.com/azure/ai-services/openai/concepts/content-filter) system that runs alongside the core AI models. This system uses an ensemble of classification models to detect four categories of potentially harmful content (violence, hate, sexual, and self-harm) at four severity levels (safe, low, medium, high). These 4 categories may not be sufficient for all use cases, especially for minors. Please read our [Transparency Note](/docs/transparency.md). By default, the content filters are set to filter out prompts and completions that are detected as medium or high severity for those four harm categories. Content labeled as low or safe severity is not filtered. @@ -134,7 +143,7 @@ The filtering configuration can be customized at the resource level, allowing cu This provides controls for Azure customers to tailor the content filtering behavior to their needs while aiming to prevent potentially harmful generated content and any copyright violations from public content. -Instructions on how to confiure content filters via Azure OpenAI Studio can be found here +Learn how to [configure content filters via Azure OpenAI Studio (preview)](https://learn.microsoft.com/azure/ai-services/openai/how-to/content-filters#configuring-content-filters-via-azure-openai-studio-preview). ## Data Collection Notice @@ -142,7 +151,7 @@ The software may collect information about you and your use of the software and ### About Data Collection -Data collection by the software in this repository is used by Microsoft solely to help justify the efforts of the teams who build and maintain this accelerator for our customers. It is your choice to leave this enabled, or to disable data collection. +Data collection by the software in this repository is used by Microsoft solely to help justify the efforts of the teams who build and maintain this copilot template for our customers. It is your choice to leave this enabled, or to disable data collection. Data collection is implemented by the presence of a tracking GUID in the environment variables at deployment time. The GUID is associated with each Azure resource deployed by the installation scripts. This GUID is used by Microsoft to track the Azure consumption this open source solution generates. @@ -152,7 +161,7 @@ To disable data collection, follow the instructions in the [Configure ENV files] ## Resources -### Navigating the Source Code +### Navigating the source code This project has the following structure: @@ -168,13 +177,13 @@ docs/deployment/ | Detailed documentation on how to deploy and start using Infor docs/features/ | Detailed documentation of specific features and development level configuration for Information Assistant. docs/ | Other supporting documentation that is primarily linked to from the other markdown files. functions/ | The pipeline of Azure Functions that handle the document extraction and chunking as well as the custom CosmosDB logging. -infra/ | The Terraform scripts that deploy the entire IA Accelerator. The overall accelerator is orchestrated via the `main.tf` file but most of the resource deployments are modularized under the **core** folder. -pipelines/ | Azure DevOps pipelines that can be used to enable CI/CD deployments of the accelerator. +infra/ | The Terraform scripts that deploy the entire IA copilot template. The overall copilot template is orchestrated via the `main.tf` file but most of the resource deployments are modularized under the **core** folder. +pipelines/ | Azure DevOps pipelines that can be used to enable CI/CD deployments of the copilot template. scripts/environments/ | Deployment configuration files. This is where all external configuration values will be set. scripts/ | Supporting scripts that perform the various deployment tasks such as infrastructure deployment, Azure WebApp and Function deployments, building of the webapp and functions source code, etc. These scripts align to the available commands in the `Makefile`. tests/ | Functional Test scripts that are used to validate a deployed Information Assistant's document processing pipelines are working as expected. Makefile | Deployment command definitions and configurations. You can use `make help` to get more details on available commands. -README.md | Starting point for this repo. It covers overviews of the Accelerator, Responsible AI, Environment, Deployment, and Usage of the Accelerator. +README.md | Starting point for this repo. It covers overviews of the copilot template, Responsible AI, Environment, Deployment, and Usage of the copilot template. ### References @@ -182,15 +191,27 @@ README.md | Starting point for this repo. It covers overviews of the Accelerator - [Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search) - [Azure OpenAI Service](https://learn.microsoft.com/azure/cognitive-services/openai/overview) -### Trademarks +## Shared responsibility and customer responsibilities -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft’s Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party’s policies. +To ensure your data is secure and your privacy controls are addressed, we recommend that you follow a set of best practices when deploying into Azure: -### Code of Conduct +- [Azure security best practices and patterns](https://learn.microsoft.com/azure/security/fundamentals/best-practices-and-patterns) +- [Microsoft Services in Cybersecurity](https://learn.microsoft.com/azure/security/fundamentals/cyber-services) -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +Protecting your data also requires that all aspects of your security and compliance program include your cloud infrastructure and data. The following guidance can help you to secure your deployment. + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft’s Trademark & Brand Guidelines](https://www.microsoft.com/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party’s policies. -### Reporting Security Issues +## Microsoft Legal Notice + +**Notice**. The Information Assistant copilot template (the "IA") is PROVIDED "AS-IS," "WITH ALL FAULTS," AND "AS AVAILABLE," AND ARE EXCLUDED FROM THE SERVICE LEVEL AGREEMENTS AND LIMITED WARRANTY. The IA may employ lesser or different privacy and security measures than those typically present in Azure Services. Unless otherwise noted, The IA should not be used to process Personal Data or other data that is subject to legal or regulatory compliance requirements. The following terms in the DPA do not apply to the IA: Processing of Personal Data, GDPR, Data Security, and HIPAA Business Associate. We may change or discontinue the IA at any time without notice. The IA (1) is not designed, intended, or made available as legal services, (2) is not intended to substitute for professional legal counsel or judgment, and (3) should not be used in place of consulting with a qualified professional legal professional for your specific needs. Microsoft makes no warranty that the IA is accurate, up-to-date, or complete. You are wholly responsible for ensuring your own compliance with all applicable laws and regulations. + +## Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -For security concerns, please see [Security Guidelines](./SECURITY.md) +## Reporting security issues +For security concerns, please see [Security Guidelines](./SECURITY.md). \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md index 4b9bef5ee..cbbd35b30 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,7 @@ Microsoft takes the security of our software products and services seriously, wh If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. -## Reporting Security Issues +## Reporting security issues **Please do not report security vulnerabilities through public GitHub issues.** @@ -30,10 +30,11 @@ This information will help us triage your report more quickly. If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. -## Providing Feedback +## Providing feedback Please refer to the [Contributing](./CONTRIBUTING.md) guidelines for acceptable methods to provide feedback for issues which are not security related. -## Preferred Languages + +## Preferred languages We prefer all communications to be in English. diff --git a/SUPPORT.md b/SUPPORT.md index ca4b6073a..9234bb520 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,19 +1,21 @@ -# Support - -## How to file issues and get help - -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or -feature request as a new Issue. - -For help and questions about using this project, please use the [Discussion](https://github.com/microsoft/PubSec-Info-Assistant/discussions) forums on our GitHub Repo page. - -For customer support deploying this accelerator, please reach out to your local Microsoft representative or email the [Industry Solutions Accelerator Team](mailto:isat-support@microsoft.com). - -## Providing Feedback - -Please refer to the [Contributing](./CONTRIBUTING.md) guidelines for acceptable methods to provide feedback which are not security related. - -## Microsoft Support Policy - -Support for this **PROJECT** is limited to the resources listed above. +# Support + +## How to file issues and get help + +This project uses [GitHub Issues](https://github.com/microsoft/PubSec-Info-Assistant/issues) to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +Please provide as much information as possible when filing an issue (please redact any sensitive information). + +For help and questions about using this project, please use the [Discussion](https://github.com/microsoft/PubSec-Info-Assistant/discussions) forums on our GitHub repo page. + +For customer support deploying Information Assistant copilot template, please reach out to your local Microsoft representative or email the [Public Sector Industries & Regulated Products Accelerators Team](mailto:isat-support@microsoft.com). + +## Providing feedback + +Please refer to the [Contributing](./CONTRIBUTING.md) guidelines for acceptable methods to provide feedback which are not security related. + +## Microsoft Support Policy + +Support for this **PROJECT** is limited to the resources listed above. diff --git a/app/backend/app.py b/app/backend/app.py index 77ecfff85..ea424c00d 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -1,19 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. + from io import StringIO from typing import Optional +from datetime import datetime import asyncio -#from sse_starlette.sse import EventSourceResponse -#from starlette.responses import StreamingResponse -from starlette.responses import Response import logging import os import json import urllib.parse import pandas as pd -from datetime import datetime, time, timedelta +import pydantic from fastapi.staticfiles import StaticFiles -from fastapi import FastAPI, File, HTTPException, Request, UploadFile +from fastapi import FastAPI, File, HTTPException, Request, UploadFile, Form from fastapi.responses import RedirectResponse, StreamingResponse import openai from approaches.comparewebwithwork import CompareWebWithWork @@ -22,19 +21,12 @@ from approaches.chatwebretrieveread import ChatWebRetrieveRead from approaches.gpt_direct_approach import GPTDirectApproach from approaches.approach import Approaches -from azure.core.credentials import AzureKeyCredential -from azure.identity import DefaultAzureCredential, AzureAuthorityHosts +from azure.identity import ManagedIdentityCredential, AzureAuthorityHosts, DefaultAzureCredential, get_bearer_token_provider from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient from azure.search.documents import SearchClient -from azure.storage.blob import ( - AccountSasPermissions, - BlobServiceClient, - ResourceTypes, - generate_account_sas, -) +from azure.storage.blob import BlobServiceClient, ContentSettings from approaches.mathassistant import( generate_response, - process_agent_scratch_pad, process_agent_response, stream_agent_responses ) @@ -44,9 +36,8 @@ process_agent_response as td_agent_response, process_agent_scratch_pad as td_agent_scratch_pad, get_images_in_temp - ) -from shared_code.status_log import State, StatusClassification, StatusLog, StatusQueryLevel +from shared_code.status_log import State, StatusClassification, StatusLog from azure.cosmos import CosmosClient @@ -55,13 +46,12 @@ ENV = { "AZURE_BLOB_STORAGE_ACCOUNT": None, "AZURE_BLOB_STORAGE_ENDPOINT": None, - "AZURE_BLOB_STORAGE_KEY": None, "AZURE_BLOB_STORAGE_CONTAINER": "content", "AZURE_BLOB_STORAGE_UPLOAD_CONTAINER": "upload", "AZURE_SEARCH_SERVICE": "gptkb", "AZURE_SEARCH_SERVICE_ENDPOINT": None, - "AZURE_SEARCH_SERVICE_KEY": None, "AZURE_SEARCH_INDEX": "gptkbindex", + "AZURE_SEARCH_AUDIENCE": None, "USE_SEMANTIC_RERANKER": "true", "AZURE_OPENAI_SERVICE": "myopenai", "AZURE_OPENAI_RESOURCE_GROUP": "", @@ -74,25 +64,23 @@ "EMBEDDING_DEPLOYMENT_NAME": "", "AZURE_OPENAI_EMBEDDINGS_MODEL_NAME": "", "AZURE_OPENAI_EMBEDDINGS_VERSION": "", - "AZURE_OPENAI_SERVICE_KEY": None, "AZURE_SUBSCRIPTION_ID": None, "AZURE_ARM_MANAGEMENT_API": "https://management.azure.com", "CHAT_WARNING_BANNER_TEXT": "", "APPLICATION_TITLE": "Information Assistant, built with Azure OpenAI", "KB_FIELDS_CONTENT": "content", "KB_FIELDS_PAGENUMBER": "pages", - "KB_FIELDS_SOURCEFILE": "file_uri", + "KB_FIELDS_SOURCEFILE": "file_name", "KB_FIELDS_CHUNKFILE": "chunk_file", "COSMOSDB_URL": None, - "COSMOSDB_KEY": None, "COSMOSDB_LOG_DATABASE_NAME": "statusdb", "COSMOSDB_LOG_CONTAINER_NAME": "statuscontainer", "QUERY_TERM_LANGUAGE": "English", "TARGET_EMBEDDINGS_MODEL": "BAAI/bge-small-en-v1.5", "ENRICHMENT_APPSERVICE_URL": "enrichment", "TARGET_TRANSLATION_LANGUAGE": "en", - "ENRICHMENT_ENDPOINT": None, - "ENRICHMENT_KEY": None, + "AZURE_AI_ENDPOINT": None, + "AZURE_AI_LOCATION": "", "BING_SEARCH_ENDPOINT": "https://api.bing.microsoft.com/", "BING_SEARCH_KEY": "", "ENABLE_BING_SAFE_SEARCH": "true", @@ -100,8 +88,9 @@ "ENABLE_UNGROUNDED_CHAT": "false", "ENABLE_MATH_ASSISTANT": "false", "ENABLE_TABULAR_DATA_ASSISTANT": "false", - "ENABLE_MULTIMEDIA": "false", - "MAX_CSV_FILE_SIZE": "7" + "MAX_CSV_FILE_SIZE": "7", + "LOCAL_DEBUG": "false", + "AZURE_AI_CREDENTIAL_DOMAIN": "cognitiveservices.azure.com" } for key, value in ENV.items(): @@ -117,7 +106,17 @@ log.setLevel('DEBUG') log.propagate = True -dffinal = None +class StatusResponse(pydantic.BaseModel): + """The response model for the health check endpoint""" + status: str + uptime_seconds: float + version: str + +start_time = datetime.now() + +IS_READY = False + +DF_FINAL = None # Used by the OpenAI SDK openai.api_type = "azure" openai.api_base = ENV["AZURE_OPENAI_ENDPOINT"] @@ -126,39 +125,49 @@ else: AUTHORITY = AzureAuthorityHosts.AZURE_PUBLIC_CLOUD openai.api_version = "2024-02-01" -# Use the current user identity to authenticate with Azure OpenAI, Cognitive Search and Blob Storage (no secrets needed, -# just use 'az login' locally, and managed identity when deployed on Azure). If you need to use keys, use separate AzureKeyCredential instances with the -# keys for each service -# If you encounter a blocking error during a DefaultAzureCredntial resolution, you can exclude the problematic credential by using a parameter (ex. exclude_shared_token_cache_credential=True) -azure_credential = DefaultAzureCredential(authority=AUTHORITY) -# Comment these two lines out if using keys, set your API key in the OPENAI_API_KEY environment variable instead -# openai.api_type = "azure_ad" -# openai_token = azure_credential.get_token("https://cognitiveservices.azure.com/.default") -openai.api_key = ENV["AZURE_OPENAI_SERVICE_KEY"] +# When debugging in VSCode, use the current user identity to authenticate with Azure OpenAI, +# Cognitive Search and Blob Storage (no secrets needed, just use 'az login' locally) +# Use managed identity when deployed on Azure. +# If you encounter a blocking error during a DefaultAzureCredntial resolution, you can exclude +# the problematic credential by using a parameter (ex. exclude_shared_token_cache_credential=True) +if ENV["LOCAL_DEBUG"] == "true": + azure_credential = DefaultAzureCredential(authority=AUTHORITY) +else: + azure_credential = ManagedIdentityCredential(authority=AUTHORITY) +# Comment these two lines out if using keys, set your API key in the OPENAI_API_KEY +# environment variable instead +openai.api_type = "azure_ad" +token_provider = get_bearer_token_provider(azure_credential, + f'https://{ENV["AZURE_AI_CREDENTIAL_DOMAIN"]}/.default') +openai.azure_ad_token_provider = token_provider +#openai.api_key = ENV["AZURE_OPENAI_SERVICE_KEY"] # Setup StatusLog to allow access to CosmosDB for logging statusLog = StatusLog( ENV["COSMOSDB_URL"], - ENV["COSMOSDB_KEY"], + azure_credential, ENV["COSMOSDB_LOG_DATABASE_NAME"], ENV["COSMOSDB_LOG_CONTAINER_NAME"] ) -azure_search_key_credential = AzureKeyCredential(ENV["AZURE_SEARCH_SERVICE_KEY"]) # Set up clients for Cognitive Search and Storage search_client = SearchClient( endpoint=ENV["AZURE_SEARCH_SERVICE_ENDPOINT"], index_name=ENV["AZURE_SEARCH_INDEX"], - credential=azure_search_key_credential, + credential=azure_credential, + audience=ENV["AZURE_SEARCH_AUDIENCE"] ) + blob_client = BlobServiceClient( account_url=ENV["AZURE_BLOB_STORAGE_ENDPOINT"], - credential=ENV["AZURE_BLOB_STORAGE_KEY"], + credential=azure_credential, ) blob_container = blob_client.get_container_client(ENV["AZURE_BLOB_STORAGE_CONTAINER"]) +blob_upload_container_client = blob_client.get_container_client( + os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"]) -model_name = '' -model_version = '' +MODEL_NAME = '' +MODEL_VERSION = '' # Set up OpenAI management client openai_mgmt_client = CognitiveServicesManagementClient( @@ -172,26 +181,25 @@ account_name=ENV["AZURE_OPENAI_SERVICE"], deployment_name=ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"]) -model_name = deployment.properties.model.name -model_version = deployment.properties.model.version +MODEL_NAME = deployment.properties.model.name +MODEL_VERSION = deployment.properties.model.version -if (str_to_bool.get(ENV["USE_AZURE_OPENAI_EMBEDDINGS"])): +if str_to_bool.get(ENV["USE_AZURE_OPENAI_EMBEDDINGS"]): embedding_deployment = openai_mgmt_client.deployments.get( resource_group_name=ENV["AZURE_OPENAI_RESOURCE_GROUP"], account_name=ENV["AZURE_OPENAI_SERVICE"], deployment_name=ENV["EMBEDDING_DEPLOYMENT_NAME"]) - embedding_model_name = embedding_deployment.properties.model.name - embedding_model_version = embedding_deployment.properties.model.version + EMBEDDING_MODEL_NAME = embedding_deployment.properties.model.name + EMBEDDING_MODEL_VERSION = embedding_deployment.properties.model.version else: - embedding_model_name = "" - embedding_model_version = "" + EMBEDDING_MODEL_NAME = "" + EMBEDDING_MODEL_VERSION = "" chat_approaches = { Approaches.ReadRetrieveRead: ChatReadRetrieveReadApproach( search_client, ENV["AZURE_OPENAI_ENDPOINT"], - ENV["AZURE_OPENAI_SERVICE_KEY"], ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], ENV["KB_FIELDS_SOURCEFILE"], ENV["KB_FIELDS_CONTENT"], @@ -200,35 +208,39 @@ ENV["AZURE_BLOB_STORAGE_CONTAINER"], blob_client, ENV["QUERY_TERM_LANGUAGE"], - model_name, - model_version, + MODEL_NAME, + MODEL_VERSION, ENV["TARGET_EMBEDDINGS_MODEL"], ENV["ENRICHMENT_APPSERVICE_URL"], ENV["TARGET_TRANSLATION_LANGUAGE"], - ENV["ENRICHMENT_ENDPOINT"], - ENV["ENRICHMENT_KEY"], + ENV["AZURE_AI_ENDPOINT"], + ENV["AZURE_AI_LOCATION"], + token_provider, str_to_bool.get(ENV["USE_SEMANTIC_RERANKER"]) ), Approaches.ChatWebRetrieveRead: ChatWebRetrieveRead( - model_name, + MODEL_NAME, ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], ENV["TARGET_TRANSLATION_LANGUAGE"], ENV["BING_SEARCH_ENDPOINT"], ENV["BING_SEARCH_KEY"], - str_to_bool.get(ENV["ENABLE_BING_SAFE_SEARCH"]) - ), - Approaches.CompareWorkWithWeb: CompareWorkWithWeb( - model_name, + str_to_bool.get(ENV["ENABLE_BING_SAFE_SEARCH"]), + ENV["AZURE_OPENAI_ENDPOINT"], + token_provider + ), + Approaches.CompareWorkWithWeb: CompareWorkWithWeb( + MODEL_NAME, ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], ENV["TARGET_TRANSLATION_LANGUAGE"], ENV["BING_SEARCH_ENDPOINT"], ENV["BING_SEARCH_KEY"], - str_to_bool.get(ENV["ENABLE_BING_SAFE_SEARCH"]) - ), + str_to_bool.get(ENV["ENABLE_BING_SAFE_SEARCH"]), + ENV["AZURE_OPENAI_ENDPOINT"], + token_provider + ), Approaches.CompareWebWithWork: CompareWebWithWork( search_client, ENV["AZURE_OPENAI_ENDPOINT"], - ENV["AZURE_OPENAI_SERVICE_KEY"], ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], ENV["KB_FIELDS_SOURCEFILE"], ENV["KB_FIELDS_CONTENT"], @@ -237,26 +249,28 @@ ENV["AZURE_BLOB_STORAGE_CONTAINER"], blob_client, ENV["QUERY_TERM_LANGUAGE"], - model_name, - model_version, + MODEL_NAME, + MODEL_VERSION, ENV["TARGET_EMBEDDINGS_MODEL"], ENV["ENRICHMENT_APPSERVICE_URL"], ENV["TARGET_TRANSLATION_LANGUAGE"], - ENV["ENRICHMENT_ENDPOINT"], - ENV["ENRICHMENT_KEY"], + ENV["AZURE_AI_ENDPOINT"], + ENV["AZURE_AI_LOCATION"], + token_provider, str_to_bool.get(ENV["USE_SEMANTIC_RERANKER"]) ), Approaches.GPTDirect: GPTDirectApproach( - ENV["AZURE_OPENAI_SERVICE"], - ENV["AZURE_OPENAI_SERVICE_KEY"], + token_provider, ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], ENV["QUERY_TERM_LANGUAGE"], - model_name, - model_version, + MODEL_NAME, + MODEL_VERSION, ENV["AZURE_OPENAI_ENDPOINT"] ) } +IS_READY = True + # Create API app = FastAPI( title="IA Web API", @@ -270,6 +284,25 @@ async def root(): """Redirect to the index.html page""" return RedirectResponse(url="/index.html") +@app.get("/health", response_model=StatusResponse, tags=["health"]) +def health(): + """Returns the health of the API + + Returns: + StatusResponse: The health of the API + """ + + uptime = datetime.now() - start_time + uptime_seconds = uptime.total_seconds() + + output = {"status": None, "uptime_seconds": uptime_seconds, "version": app.version} + + if IS_READY: + output["status"] = "ready" + else: + output["status"] = "loading" + + return output @app.post("/chat") async def chat(request: Request): @@ -290,49 +323,25 @@ async def chat(request: Request): impl = chat_approaches.get(Approaches(int(approach))) if not impl: return {"error": "unknown approach"}, 400 - - if (Approaches(int(approach)) == Approaches.CompareWorkWithWeb or Approaches(int(approach)) == Approaches.CompareWebWithWork): - r = impl.run(json_body.get("history", []), json_body.get("overrides", {}), json_body.get("citation_lookup", {}), json_body.get("thought_chain", {})) + + if (Approaches(int(approach)) == Approaches.CompareWorkWithWeb or + Approaches(int(approach)) == Approaches.CompareWebWithWork): + r = impl.run(json_body.get("history", []), + json_body.get("overrides", {}), + json_body.get("citation_lookup", {}), + json_body.get("thought_chain", {})) else: - r = impl.run(json_body.get("history", []), json_body.get("overrides", {}), {}, json_body.get("thought_chain", {})) - + r = impl.run(json_body.get("history", []), + json_body.get("overrides", {}), + {}, + json_body.get("thought_chain", {})) + return StreamingResponse(r, media_type="application/x-ndjson") except Exception as ex: - log.error(f"Error in chat:: {ex}") + log.error("Error in chat:: %s", ex) raise HTTPException(status_code=500, detail=str(ex)) from ex - - - -@app.get("/getblobclienturl") -async def get_blob_client_url(): - """Get a URL for a file in Blob Storage with SAS token. - - This function generates a Shared Access Signature (SAS) token for accessing a file in Blob Storage. - The generated URL includes the SAS token as a query parameter. - - Returns: - dict: A dictionary containing the URL with the SAS token. - """ - sas_token = generate_account_sas( - ENV["AZURE_BLOB_STORAGE_ACCOUNT"], - ENV["AZURE_BLOB_STORAGE_KEY"], - resource_types=ResourceTypes(object=True, service=True, container=True), - permission=AccountSasPermissions( - read=True, - write=True, - list=True, - delete=False, - add=True, - create=True, - update=True, - process=False, - ), - expiry=datetime.utcnow() + timedelta(hours=1), - ) - return {"url": f"{blob_client.url}?{sas_token}"} - @app.post("/getalluploadstatus") async def get_all_upload_status(request: Request): """ @@ -348,40 +357,41 @@ async def get_all_upload_status(request: Request): timeframe = json_body.get("timeframe") state = json_body.get("state") folder = json_body.get("folder") - tag = json_body.get("tag") + tag = json_body.get("tag") try: - results = statusLog.read_files_status_by_timeframe(timeframe, - State[state], - folder, + results = statusLog.read_files_status_by_timeframe(timeframe, + State[state], + folder, tag, os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"]) # retrieve tags for each file # Initialize an empty list to hold the tags - items = [] - cosmos_client = CosmosClient(url=statusLog._url, credential=statusLog._key) + items = [] + cosmos_client = CosmosClient(url=statusLog._url, + credential=azure_credential, + consistency_level='Session') database = cosmos_client.get_database_client(statusLog._database_name) container = database.get_container_client(statusLog._container_name) query_string = "SELECT DISTINCT VALUE t FROM c JOIN t IN c.tags" items = list(container.query_items( query=query_string, enable_cross_partition_query=True - )) + )) # Extract and split tags unique_tags = set() for item in items: tags = item.split(',') - unique_tags.update(tags) + unique_tags.update(tags) - except Exception as ex: log.exception("Exception in /getalluploadstatus") raise HTTPException(status_code=500, detail=str(ex)) from ex return results @app.post("/getfolders") -async def get_folders(request: Request): +async def get_folders(): """ Get all folders. @@ -459,8 +469,15 @@ async def resubmit_Items(request: Request): blob_container = blob_client.get_container_client(os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"]) # Read the blob content into memory blob_data = blob_container.download_blob(path).readall() - # Overwrite the blob with the modified data - blob_container.upload_blob(name=path, data=blob_data, overwrite=True) + + submitted_blob_client = blob_container.get_blob_client(blob=path) + blob_properties = submitted_blob_client.get_blob_properties() + metadata = blob_properties.metadata + blob_container.upload_blob(name=path, data=blob_data, overwrite=True, metadata=metadata) + + + + # add the container to the path to avoid adding another doc in the status db full_path = os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"] + '/' + path statusLog.upsert_document(document_path=full_path, @@ -490,7 +507,7 @@ async def get_tags(request: Request): try: # Initialize an empty list to hold the tags items = [] - cosmos_client = CosmosClient(url=statusLog._url, credential=statusLog._key) + cosmos_client = CosmosClient(url=statusLog._url, credential=azure_credential, consistency_level='Session') database = cosmos_client.get_database_client(statusLog._database_name) container = database.get_container_client(statusLog._container_name) query_string = "SELECT DISTINCT VALUE t FROM c JOIN t IN c.tags" @@ -562,16 +579,16 @@ async def get_info_data(): """ response = { "AZURE_OPENAI_CHATGPT_DEPLOYMENT": ENV["AZURE_OPENAI_CHATGPT_DEPLOYMENT"], - "AZURE_OPENAI_MODEL_NAME": f"{model_name}", - "AZURE_OPENAI_MODEL_VERSION": f"{model_version}", + "AZURE_OPENAI_MODEL_NAME": f"{MODEL_NAME}", + "AZURE_OPENAI_MODEL_VERSION": f"{MODEL_VERSION}", "AZURE_OPENAI_SERVICE": ENV["AZURE_OPENAI_SERVICE"], "AZURE_SEARCH_SERVICE": ENV["AZURE_SEARCH_SERVICE"], "AZURE_SEARCH_INDEX": ENV["AZURE_SEARCH_INDEX"], "TARGET_LANGUAGE": ENV["QUERY_TERM_LANGUAGE"], "USE_AZURE_OPENAI_EMBEDDINGS": ENV["USE_AZURE_OPENAI_EMBEDDINGS"], "EMBEDDINGS_DEPLOYMENT": ENV["EMBEDDING_DEPLOYMENT_NAME"], - "EMBEDDINGS_MODEL_NAME": f"{embedding_model_name}", - "EMBEDDINGS_MODEL_VERSION": f"{embedding_model_version}", + "EMBEDDINGS_MODEL_NAME": f"{EMBEDDING_MODEL_NAME}", + "EMBEDDINGS_MODEL_VERSION": f"{EMBEDDING_MODEL_VERSION}", } return response @@ -673,26 +690,27 @@ async def getHint(question: Optional[str] = None): @app.post("/posttd") async def posttd(csv: UploadFile = File(...)): try: - global dffinal + global DF_FINAL # Read the file into a pandas DataFrame content = await csv.read() - df = pd.read_csv(StringIO(content.decode('latin-1'))) + df = pd.read_csv(StringIO(content.decode('utf-8-sig'))) - dffinal = df + DF_FINAL = df # Process the DataFrame... save_df(df) except Exception as ex: - raise HTTPException(status_code=500, detail=str(ex)) from ex + raise HTTPException(status_code=500, detail=str(ex)) from ex #return {"filename": csv.filename} @app.get("/process_td_agent_response") async def process_td_agent_response(retries=3, delay=1000, question: Optional[str] = None): + save_df(DF_FINAL) if question is None: raise HTTPException(status_code=400, detail="Question is required") for i in range(retries): try: - results = td_agent_response(question) + results = td_agent_response(question,DF_FINAL) return results except AttributeError as ex: log.exception(f"Exception in /process_tabular_data_agent_response:{str(ex)}") @@ -712,14 +730,14 @@ async def process_td_agent_response(retries=3, delay=1000, question: Optional[st @app.get("/getTdAnalysis") async def getTdAnalysis(retries=3, delay=1, question: Optional[str] = None): - global dffinal + global DF_FINAL if question is None: - raise HTTPException(status_code=400, detail="Question is required") + raise HTTPException(status_code=400, detail="Question is required") for i in range(retries): try: - save_df(dffinal) - results = td_agent_scratch_pad(question, dffinal) + save_df(DF_FINAL) + results = td_agent_scratch_pad(question, DF_FINAL) return results except AttributeError as ex: log.exception(f"Exception in /getTdAnalysis:{str(ex)}") @@ -769,11 +787,11 @@ async def stream_response(question: str): @app.get("/tdstream") async def td_stream_response(question: str): - save_df(dffinal) + save_df(DF_FINAL) try: - stream = td_agent_scratch_pad(question, dffinal) + stream = td_agent_scratch_pad(question, DF_FINAL) return StreamingResponse(stream, media_type="text/event-stream") except Exception as ex: log.exception("Exception in /stream") @@ -806,7 +824,7 @@ async def stream_agent_response(question: str): results = process_agent_response(question) except Exception as e: print(f"Error processing agent response: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e return results @@ -821,17 +839,64 @@ async def get_feature_flags(): - "ENABLE_UNGROUNDED_CHAT": Flag indicating whether ungrounded chat is enabled. - "ENABLE_MATH_ASSISTANT": Flag indicating whether the math assistant is enabled. - "ENABLE_TABULAR_DATA_ASSISTANT": Flag indicating whether the tabular data assistant is enabled. - - "ENABLE_MULTIMEDIA": Flag indicating whether multimedia is enabled. """ response = { "ENABLE_WEB_CHAT": str_to_bool.get(ENV["ENABLE_WEB_CHAT"]), "ENABLE_UNGROUNDED_CHAT": str_to_bool.get(ENV["ENABLE_UNGROUNDED_CHAT"]), "ENABLE_MATH_ASSISTANT": str_to_bool.get(ENV["ENABLE_MATH_ASSISTANT"]), "ENABLE_TABULAR_DATA_ASSISTANT": str_to_bool.get(ENV["ENABLE_TABULAR_DATA_ASSISTANT"]), - "ENABLE_MULTIMEDIA": str_to_bool.get(ENV["ENABLE_MULTIMEDIA"]), } return response +@app.post("/file") +async def upload_file( + file: UploadFile = File(...), + file_path: str = Form(...), + tags: str = Form(None) +): + """ + Upload a file to Azure Blob Storage. + Parameters: + - file: The file to upload. + - file_path: The path to save the file in Blob Storage. + - tags: The tags to associate with the file. + Returns: + - response: A message indicating the result of the upload. + """ + try: + blob_upload_client = blob_upload_container_client.get_blob_client(file_path) + + blob_upload_client.upload_blob( + file.file, + overwrite=True, + content_settings=ContentSettings(content_type=file.content_type), + metadata= {"tags": tags} + ) + + return {"message": f"File '{file.filename}' uploaded successfully"} + + except Exception as ex: + log.exception("Exception in /file") + raise HTTPException(status_code=500, detail=str(ex)) from ex + +@app.post("/get-file") +async def get_file(request: Request): + data = await request.json() + file_path = data['path'] + + # Extract container name and blob name from the file path + container_name, blob_name = file_path.split('/', 1) + + # Download the blob to a local file + + citation_blob_client = blob_upload_container_client.get_blob_client(blob=blob_name) + stream = citation_blob_client.download_blob().chunks() + blob_properties = citation_blob_client.get_blob_properties() + + return StreamingResponse(stream, + media_type=blob_properties.content_settings.content_type, + headers={"Content-Disposition": f"inline; filename={blob_name}"}) + app.mount("/", StaticFiles(directory="static"), name="static") if __name__ == "__main__": diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index a41ac4c08..2720b3163 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -6,20 +6,19 @@ import logging import urllib.parse from datetime import datetime, timedelta -from typing import Any, AsyncGenerator, Coroutine, Sequence +from typing import Any, Sequence import openai -from openai import AzureOpenAI from openai import AsyncAzureOpenAI +from openai import BadRequestError from approaches.approach import Approach from azure.search.documents import SearchClient from azure.search.documents.models import RawVectorQuery from azure.search.documents.models import QueryType from azure.storage.blob import ( - AccountSasPermissions, + BlobSasPermissions, BlobServiceClient, - ResourceTypes, - generate_account_sas, + generate_blob_sas, ) from text import nonewlines from core.modelhelper import get_token_limit @@ -88,7 +87,6 @@ def __init__( self, search_client: SearchClient, oai_endpoint: str, - oai_service_key: str, chatgpt_deployment: str, source_file_field: str, content_field: str, @@ -102,10 +100,10 @@ def __init__( target_embedding_model: str, enrichment_appservice_uri: str, target_translation_language: str, - enrichment_endpoint:str, - enrichment_key:str, + azure_ai_endpoint:str, + azure_ai_location:str, + azure_ai_token_provider:str, use_semantic_reranker: bool - ): self.search_client = search_client self.chatgpt_deployment = chatgpt_deployment @@ -120,20 +118,20 @@ def __init__( #escape target embeddiong model name self.escaped_target_model = re.sub(r'[^a-zA-Z0-9_\-.]', '_', target_embedding_model) self.target_translation_language=target_translation_language - self.enrichment_endpoint=enrichment_endpoint - self.enrichment_key=enrichment_key + self.azure_ai_endpoint=azure_ai_endpoint + self.azure_ai_location=azure_ai_location + self.azure_ai_token_provider=azure_ai_token_provider self.oai_endpoint=oai_endpoint self.embedding_service_url = enrichment_appservice_uri self.use_semantic_reranker=use_semantic_reranker openai.api_base = oai_endpoint openai.api_type = 'azure' - openai.api_key = oai_service_key openai.api_version = "2024-02-01" self.client = AsyncAzureOpenAI( - azure_endpoint = openai.api_base, - api_key=openai.api_key, + azure_endpoint = openai.api_base, + azure_ad_token_provider=azure_ai_token_provider, api_version=openai.api_version) @@ -190,7 +188,23 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] # max_tokens=32, # setting it too low may cause malformed JSON max_tokens=100, n=1) - + # Initialize a list to collect filter reasons + filter_reasons = [] + + # Check for content filtering + if chat_completion.choices[0].finish_reason == 'content_filter': + for category, details in chat_completion.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) + except BadRequestError as e: + log.error(f"Error generating optimized keyword search: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating optimized keyword search: {str(e.body['message'])}"}) + "\n" + return except Exception as e: log.error(f"Error generating optimized keyword search: {str(e)}") yield json.dumps({"error": f"Error generating optimized keyword search: {str(e)}"}) + "\n" @@ -218,10 +232,10 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] response = requests.post(url, json=data,headers=headers,timeout=60) if response.status_code == 200: response_data = response.json() - embedded_query_vector =response_data.get('data') + embedded_query_vector =response_data.get('data') else: # Generate an error message if the embedding generation fails - log.error(f"Error generating embedding:: {response.status_code}") + log.error(f"Error generating embedding:: {response.status_code} - {response.text}") yield json.dumps({"error": "Error generating embedding"}) + "\n" return # Go no further except Exception as e: @@ -290,7 +304,7 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] f"File{idx} " + "| " + nonewlines(doc[self.content_field]) ) data_points.append( - "/".join(urllib.parse.unquote(doc[self.source_file_field]).split("/")[4:] + "/".join(urllib.parse.unquote(doc[self.source_file_field]).split("/")[1:] ) + "| " + nonewlines(doc[self.content_field]) ) # uncomment to debug size of each search result content_field @@ -299,8 +313,8 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] # add the "FileX" moniker and full file name to the citation lookup citation_lookup[f"File{idx}"] = { - "citation": urllib.parse.unquote("https://" + doc[self.source_file_field].split("/")[2] + f"/{self.content_storage_container}/" + doc[self.chunk_file_field]), - "source_path": self.get_source_file_with_sas(doc[self.source_file_field]), + "citation": urllib.parse.unquote("https://" + self.blob_client.url.split("/")[2] + f"/{self.content_storage_container}/" + doc[self.chunk_file_field]), + "source_path": doc[self.source_file_field], "page_number": str(doc[self.page_number_field][0]) or "0", } @@ -357,57 +371,18 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] try: # STEP 3: Generate a contextual and content-specific answer using the search results and chat history. #Added conditional block to use different system messages for different models. - if self.model_name.startswith("gpt-35-turbo"): - messages = self.get_messages_from_history( - system_message, - self.model_name, - history, - history[-1]["user"] + "Sources:\n" + content + "\n\n", # 3.5 has recency Bias that is why this is here - self.RESPONSE_PROMPT_FEW_SHOTS, - max_tokens=self.chatgpt_token_limit - 500 - ) - - #Uncomment to debug token usage. - #print(messages) - #message_string = "" - #for message in messages: - # # enumerate the messages and add the role and content elements of the dictoinary to the message_string - # message_string += f"{message['role']}: {message['content']}\n" - #print("Content Tokens: ", self.num_tokens_from_string("Sources:\n" + content + "\n\n", "cl100k_base")) - #print("System Message Tokens: ", self.num_tokens_from_string(system_message, "cl100k_base")) - #print("Few Shot Tokens: ", self.num_tokens_from_string(self.response_prompt_few_shots[0]['content'], "cl100k_base")) - #print("Message Tokens: ", self.num_tokens_from_string(message_string, "cl100k_base")) - chat_completion= await self.client.chat.completions.create( - model=self.chatgpt_deployment, - messages=messages, - temperature=float(overrides.get("response_temp")) or 0.6, - n=1, - stream=True - ) - - elif self.model_name.startswith("gpt-4"): - messages = self.get_messages_from_history( - system_message, - # "Sources:\n" + content + "\n\n" + system_message, - self.model_name, - history, - # history[-1]["user"], - history[-1]["user"] + "Sources:\n" + content + "\n\n", # GPT 4 starts to degrade with long system messages. so moving sources here - self.RESPONSE_PROMPT_FEW_SHOTS, - max_tokens=self.chatgpt_token_limit - ) - - #Uncomment to debug token usage. - #print(messages) - #message_string = "" - #for message in messages: - # # enumerate the messages and add the role and content elements of the dictoinary to the message_string - # message_string += f"{message['role']}: {message['content']}\n" - #print("Content Tokens: ", self.num_tokens_from_string("Sources:\n" + content + "\n\n", "cl100k_base")) - #print("System Message Tokens: ", self.num_tokens_from_string(system_message, "cl100k_base")) - #print("Few Shot Tokens: ", self.num_tokens_from_string(self.response_prompt_few_shots[0]['content'], "cl100k_base")) - #print("Message Tokens: ", self.num_tokens_from_string(message_string, "cl100k_base")) + messages = self.get_messages_from_history( + system_message, + # "Sources:\n" + content + "\n\n" + system_message, + self.model_name, + history, + # history[-1]["user"], + history[-1]["user"] + "Sources:\n" + content + "\n\n", # GPT 4 starts to degrade with long system messages. so moving sources here + self.RESPONSE_PROMPT_FEW_SHOTS, + max_tokens=self.chatgpt_token_limit + ) + # Generate the chat completion chat_completion= await self.client.chat.completions.create( model=self.chatgpt_deployment, messages=messages, @@ -425,12 +400,27 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] "thought_chain": thought_chain, "work_citation_lookup": citation_lookup, "web_citation_lookup": {}}) + "\n" - + # STEP 4: Format the response async for chunk in chat_completion: # Check if there is at least one element and the first element has the key 'delta' if len(chunk.choices) > 0: + filter_reasons = [] + # Check for content filtering + if chunk.choices[0].finish_reason == 'content_filter': + for category, details in chunk.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) yield json.dumps({"content": chunk.choices[0].delta.content}) + "\n" + except BadRequestError as e: + log.error(f"Error generating chat completion: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating chat completion: {str(e.body['message'])}"}) + "\n" + return except Exception as e: log.error(f"Error generating chat completion: {str(e)}") yield json.dumps({"error": f"Error generating chat completion: {str(e)}"}) + "\n" @@ -440,10 +430,11 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] def detect_language(self, text: str) -> str: """ Function to detect the language of the text""" try: - api_detect_endpoint = f"{self.enrichment_endpoint}language/:analyze-text?api-version=2023-04-01" + api_detect_endpoint = f"{self.azure_ai_endpoint}language/:analyze-text?api-version=2023-04-01" headers = { - 'Ocp-Apim-Subscription-Key': self.enrichment_key, + 'Authorization': f'Bearer {self.azure_ai_token_provider()}', 'Content-type': 'application/json', + 'Ocp-Apim-Subscription-Region': self.azure_ai_location } data = { @@ -464,16 +455,17 @@ def detect_language(self, text: str) -> str: detected_language = response.json()["results"]["documents"][0]["detectedLanguage"]["iso6391Name"] return detected_language else: - raise Exception(f"Error detecting language: {response.status_code}") + raise Exception(f"Error detecting language: {response.status_code} - {response.text}") except Exception as e: raise Exception(f"An error occurred during language detection: {str(e)}") from e def translate_response(self, response: str, target_language: str) -> str: """ Function to translate the response to target language""" - api_translate_endpoint = f"{self.enrichment_endpoint}translator/text/v3.0/translate?api-version=3.0" + api_translate_endpoint = f"{self.azure_ai_endpoint}translator/text/v3.0/translate?api-version=3.0" headers = { - 'Ocp-Apim-Subscription-Key': self.enrichment_key, + 'Authorization': f'Bearer {self.azure_ai_token_provider()}', 'Content-type': 'application/json', + 'Ocp-Apim-Subscription-Region': self.azure_ai_location } params={'to': target_language } data = [{ @@ -490,20 +482,20 @@ def translate_response(self, response: str, target_language: str) -> str: def get_source_file_with_sas(self, source_file: str) -> str: """ Function to return the source file with a SAS token""" try: - sas_token = generate_account_sas( - self.blob_client.account_name, - self.blob_client.credential.account_key, - resource_types=ResourceTypes(object=True, service=True, container=True), - permission=AccountSasPermissions( - read=True, - write=True, - list=True, - delete=False, - add=True, - create=True, - update=True, - process=False, - ), + separator = "/" + file_path_w_name_no_cont = separator.join( + source_file.split(separator)[4:]) + container_name = separator.join( + source_file.split(separator)[3:4]) + # Obtain the user delegation key + user_delegation_key = self.blob_client.get_user_delegation_key(key_start_time=datetime.utcnow(), key_expiry_time=datetime.utcnow() + timedelta(hours=2)) + + sas_token = generate_blob_sas( + account_name=self.blob_client.account_name, + container_name=container_name, + blob_name=file_path_w_name_no_cont, + user_delegation_key=user_delegation_key, + permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) return source_file + "?" + sas_token diff --git a/app/backend/approaches/chatwebretrieveread.py b/app/backend/approaches/chatwebretrieveread.py index 66141c4a3..2485a28ec 100644 --- a/app/backend/approaches/chatwebretrieveread.py +++ b/app/backend/approaches/chatwebretrieveread.py @@ -12,6 +12,7 @@ from azure.core.credentials import AzureKeyCredential import openai from openai import AzureOpenAI +from openai import BadRequestError from openai import AsyncAzureOpenAI from approaches.approach import Approach from core.messagebuilder import MessageBuilder @@ -74,7 +75,15 @@ class ChatWebRetrieveRead(Approach): citations = {} approach_class = "" - def __init__(self, model_name: str, chatgpt_deployment: str, query_term_language: str, bing_search_endpoint: str, bing_search_key: str, bing_safe_search: bool): + def __init__(self, model_name: str, + chatgpt_deployment: str, + query_term_language: str, + bing_search_endpoint: str, + bing_search_key: str, + bing_safe_search: bool, + oai_endpoint: str, + azure_ai_token_provider:str + ): self.name = "ChatBingSearch" self.model_name = model_name self.chatgpt_deployment = chatgpt_deployment @@ -84,14 +93,14 @@ def __init__(self, model_name: str, chatgpt_deployment: str, query_term_language self.bing_search_key = bing_search_key self.bing_safe_search = bing_safe_search - # openai.api_base = oai_endpoint + openai.api_base = oai_endpoint openai.api_type = 'azure' openai.api_version = "2024-02-01" self.client = AsyncAzureOpenAI( azure_endpoint = openai.api_base , - api_key=openai.api_key, + azure_ad_token_provider=azure_ai_token_provider, api_version=openai.api_version) @@ -135,6 +144,10 @@ async def run(self, history: Sequence[dict[str, str]],overrides: dict[str, Any], try: query_resp = await self.make_chat_completion(messages) + except BadRequestError as e: + log.error(f"Error generating optimized keyword search: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating optimized keyword search: {str(e.body['message'])}"}) + "\n" + return except Exception as e: log.error(f"Error generating optimized keyword search: {str(e)}") yield json.dumps({"error": f"Error generating optimized keyword search: {str(e)}"}) + "\n" @@ -187,8 +200,22 @@ async def run(self, history: Sequence[dict[str, str]],overrides: dict[str, Any], async for chunk in resp: # Check if there is at least one element and the first element has the key 'delta' if len(chunk.choices) > 0: + filter_reasons = [] + # Check for content filtering + if chunk.choices[0].finish_reason == 'content_filter': + for category, details in chunk.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) yield json.dumps({"content": chunk.choices[0].delta.content}) + "\n" - + except BadRequestError as e: + log.error(f"Error generating chat completion: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating chat completion: {str(e.body['message'])}"}) + "\n" + return except Exception as e: log.error(f"Error generating chat completion: {str(e)}") yield json.dumps({"error": f"Error generating chat completion: {str(e)}"}) + "\n" @@ -257,6 +284,18 @@ async def make_chat_completion(self, messages): temperature=0.6, n=1 ) + filter_reasons = [] + + # Check for content filtering + if chat_completion.choices[0].finish_reason == 'content_filter': + for category, details in chat_completion.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) return chat_completion.choices[0].message.content def get_messages_builder( diff --git a/app/backend/approaches/comparewebwithwork.py b/app/backend/approaches/comparewebwithwork.py index 77becaeb0..3c47a765d 100644 --- a/app/backend/approaches/comparewebwithwork.py +++ b/app/backend/approaches/comparewebwithwork.py @@ -8,7 +8,7 @@ import urllib.parse from typing import Any, Sequence import openai -from openai import AzureOpenAI +from openai import AzureOpenAI, BadRequestError from openai import AsyncAzureOpenAI from approaches.chatreadretrieveread import ChatReadRetrieveReadApproach from approaches.approach import Approach @@ -46,7 +46,6 @@ def __init__( self, search_client: SearchClient, oai_service_name: str, - oai_service_key: str, chatgpt_deployment: str, source_file_field: str, content_field: str, @@ -60,8 +59,9 @@ def __init__( target_embedding_model: str, enrichment_appservice_url: str, target_translation_language: str, - enrichment_endpoint:str, - enrichment_key:str, + azure_ai_endpoint:str, + azure_ai_location: str, + azure_ai_token_provider: str, use_semantic_reranker: bool ): self.search_client = search_client @@ -76,10 +76,10 @@ def __init__( self.chatgpt_token_limit = get_token_limit(model_name) self.escaped_target_model = re.sub(r'[^a-zA-Z0-9_\-.]', '_', target_embedding_model) self.target_translation_language=target_translation_language - self.enrichment_endpoint=enrichment_endpoint - self.enrichment_key=enrichment_key + self.azure_ai_endpoint=azure_ai_endpoint + self.azure_ai_location = azure_ai_location + self.azure_ai_token_provider=azure_ai_token_provider self.oai_service_name = oai_service_name - self.oai_service_key = oai_service_key self.model_name = model_name self.model_version = model_version self.enrichment_appservice_url = enrichment_appservice_url @@ -91,7 +91,7 @@ def __init__( self.client = AsyncAzureOpenAI( azure_endpoint = openai.api_base, - api_key=openai.api_key, + azure_ad_token_provider=azure_ai_token_provider, api_version=openai.api_version) async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any], web_citation_lookup: dict[str, Any], thought_chain: dict[str, Any]) -> Any: @@ -108,7 +108,6 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] chat_rrr_approach = ChatReadRetrieveReadApproach( self.search_client, self.oai_service_name, - self.oai_service_key, self.chatgpt_deployment, self.source_file_field, self.content_field, @@ -122,8 +121,9 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] self.escaped_target_model, self.enrichment_appservice_url, self.target_translation_language, - self.enrichment_endpoint, - self.enrichment_key, + self.azure_ai_endpoint, + self.azure_ai_location, + self.azure_ai_token_provider, self.use_semantic_reranker ) rrr_response = chat_rrr_approach.run(history, overrides, {}, thought_chain) @@ -181,10 +181,25 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] async for chunk in chat_completion: # Check if there is at least one element and the first element has the key 'delta' if len(chunk.choices) > 0: + filter_reasons = [] + # Check for content filtering + if chunk.choices[0].finish_reason == 'content_filter': + for category, details in chunk.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) yield json.dumps({"content": chunk.choices[0].delta.content}) + "\n" # Step 4: Append web citations from the Bing Search approach for idx, url in enumerate(work_citations.keys(), start=1): yield json.dumps({"content": f"[File{idx}]"}) + "\n" + except BadRequestError as e: + logging.error(f"Error generating chat completion: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating chat completion: {str(e.body['message'])}"}) + "\n" + return except Exception as e: logging.error(f"Error in compare web with work: {e}") yield json.dumps({"error": "An error occurred while generating the completion."}) + "\n" @@ -208,6 +223,18 @@ async def make_chat_completion(self, messages) -> str: temperature=0.6, n=1 ) + filter_reasons = [] + + # Check for content filtering + if chat_completion.choices[0].finish_reason == 'content_filter': + for category, details in chat_completion.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) return chat_completion.choices[0].message.content def get_messages_builder(self, system_prompt: str, model_id: str, user_conv: str, few_shots = [dict[str, str]], max_tokens: int = 4096) -> []: diff --git a/app/backend/approaches/compareworkwithweb.py b/app/backend/approaches/compareworkwithweb.py index a58b76b57..46a69b7fc 100644 --- a/app/backend/approaches/compareworkwithweb.py +++ b/app/backend/approaches/compareworkwithweb.py @@ -7,7 +7,7 @@ from typing import Any, Sequence import urllib.parse import openai -from openai import AzureOpenAI +from openai import AzureOpenAI, BadRequestError from openai import AsyncAzureOpenAI from approaches.chatwebretrieveread import ChatWebRetrieveRead from approaches.approach import Approach @@ -39,7 +39,15 @@ class CompareWorkWithWeb(Approach): web_citations = {} - def __init__(self, model_name: str, chatgpt_deployment: str, query_term_language: str, bing_search_endpoint: str, bing_search_key: str, bing_safe_search: bool): + def __init__(self, model_name: str, + chatgpt_deployment: str, + query_term_language: str, + bing_search_endpoint: str, + bing_search_key: str, + bing_safe_search: bool, + oai_endpoint: str, + azure_ai_token_provider:str + ): """ Initializes the CompareWorkWithWeb approach. @@ -59,14 +67,16 @@ def __init__(self, model_name: str, chatgpt_deployment: str, query_term_language self.bing_search_endpoint = bing_search_endpoint self.bing_search_key = bing_search_key self.bing_safe_search = bing_safe_search + self.oai_endpoint = oai_endpoint + self.azure_ai_token_provider = azure_ai_token_provider # openai.api_base = oai_endpoint openai.api_type = 'azure' openai.api_version = "2024-02-01" self.client = AsyncAzureOpenAI( - azure_endpoint = openai.api_base, - api_key=openai.api_key, + azure_endpoint = openai.api_base, + azure_ad_token_provider=azure_ai_token_provider, api_version=openai.api_version) async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any], work_citation_lookup: dict[str, Any], thought_chain: dict[str, Any]) -> Any: @@ -81,7 +91,14 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] Any: The result of the comparative analysis. """ # Step 1: Call bing Search Approach for a Bing LLM Response and Citations - chat_bing_search = ChatWebRetrieveRead(self.model_name, self.chatgpt_deployment, self.query_term_language, self.bing_search_endpoint, self.bing_search_key, self.bing_safe_search) + chat_bing_search = ChatWebRetrieveRead(self.model_name, + self.chatgpt_deployment, + self.query_term_language, + self.bing_search_endpoint, + self.bing_search_key, + self.bing_safe_search, + self.oai_endpoint, + self.azure_ai_token_provider) bing_search_response = chat_bing_search.run(history, overrides, {}, thought_chain) content = "" @@ -137,10 +154,25 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] async for chunk in chat_completion: # Check if there is at least one element and the first element has the key 'delta' if len(chunk.choices) > 0: + filter_reasons = [] + # Check for content filtering + if chunk.choices[0].finish_reason == 'content_filter': + for category, details in chunk.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) yield json.dumps({"content": chunk.choices[0].delta.content}) + "\n" # Step 4: Append web citations from the Bing Search approach for idx, url in enumerate(self.web_citations.keys(), start=1): yield json.dumps({"content": f"[url{idx}]"}) + "\n" + except BadRequestError as e: + logging.error(f"Error generating chat completion: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating chat completion: {str(e.body['message'])}"}) + "\n" + return except Exception as e: logging.error(f"Error in compare work with web: {e}") yield json.dumps({"error": "An error occurred while generating the completion."}) + "\n" @@ -160,6 +192,18 @@ async def make_chat_completion(self, messages): temperature=0.6, n=1 ) + filter_reasons = [] + + # Check for content filtering + if chat_completion.choices[0].finish_reason == 'content_filter': + for category, details in chat_completion.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) return chat_completion.choices[0].message.content def get_messages_builder(self, system_prompt: str, model_id: str, user_conv: str, few_shots = [dict[str, str]], max_tokens: int = 4096,) -> []: diff --git a/app/backend/approaches/gpt_direct_approach.py b/app/backend/approaches/gpt_direct_approach.py index 1ba7ee298..bef428f53 100644 --- a/app/backend/approaches/gpt_direct_approach.py +++ b/app/backend/approaches/gpt_direct_approach.py @@ -9,7 +9,7 @@ from typing import Any, Sequence import openai -from openai import AzureOpenAI +from openai import AzureOpenAI, BadRequestError from openai import AsyncAzureOpenAI from approaches.approach import Approach @@ -58,27 +58,15 @@ class GPTDirectApproach(Approach): If you cannot generate a search query, return just the number 0. """ - #Few Shot prompting for Keyword Search Query - query_prompt_few_shots = [ - {'role' : USER, 'content' : 'What are the future plans for public transportation development?' }, - {'role' : ASSISTANT, 'content' : 'Future plans for public transportation' }, - {'role' : USER, 'content' : 'how much renewable energy was generated last year?' }, - {'role' : ASSISTANT, 'content' : 'Renewable energy generation last year' } - ] - #Few Shot prompting for Response. This will feed into Chain of thought system message. - response_prompt_few_shots = [ - {'role': USER, 'content': 'What steps are being taken to promote energy conservation?'}, - {'role': USER, 'content': 'Several steps are being taken to promote energy conservation including reducing energy consumption, increasing energy efficiency, and increasing the use of renewable energy sources. Citations[info1.json]'} - ] + response_prompt_few_shots = [] # # Define a class variable for the base URL # EMBEDDING_SERVICE_BASE_URL = 'https://infoasst-cr-{}.azurewebsites.net' def __init__( self, - oai_service_name: str, - oai_service_key: str, + azure_openai_token_provider: str, chatgpt_deployment: str, query_term_language: str, model_name: str, @@ -90,19 +78,16 @@ def __init__( self.chatgpt_token_limit = get_token_limit(model_name) openai.api_base = azure_openai_endpoint - openai.api_type = 'azure' - openai.api_key = oai_service_key + openai.api_type = "azure_ad" + openai.azure_ad_token_provider = azure_openai_token_provider + openai.api_version = "2024-02-01" self.model_name = model_name self.model_version = model_version - - openai.api_type = 'azure' - openai.api_version = "2024-02-01" - self.client = AsyncAzureOpenAI( azure_endpoint = openai.api_base, - api_key=openai.api_key, + azure_ad_token_provider=azure_openai_token_provider, api_version=openai.api_version) # def run(self, history: list[dict], overrides: dict) -> any: @@ -161,7 +146,22 @@ async def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any] async for chunk in chat_completion: # Check if there is at least one element and the first element has the key 'delta' if len(chunk.choices) > 0: + filter_reasons = [] + # Check for content filtering + if chunk.choices[0].finish_reason == 'content_filter': + for category, details in chunk.choices[0].content_filter_results.items(): + if details['filtered']: + filter_reasons.append(f"{category} ({details['severity']})") + + # Raise an error if any filters are triggered + if filter_reasons: + error_message = "The generated content was filtered due to triggering Azure OpenAI's content filtering system. Reason(s): The response contains content flagged as " + ", ".join(filter_reasons) + raise ValueError(error_message) yield json.dumps({"content": chunk.choices[0].delta.content}) + "\n" + except BadRequestError as e: + logging.error(f"Error generating chat completion: {str(e.body['message'])}") + yield json.dumps({"error": f"Error generating chat completion: {str(e.body['message'])}"}) + "\n" + return except Exception as e: logging.error(f"Error in GPTDirectApproach: {e}") yield json.dumps({"error": f"An error occurred while generating the completion. {e}"}) + "\n" diff --git a/app/backend/approaches/mathassistant.py b/app/backend/approaches/mathassistant.py index 44ebe978d..f03e8e444 100644 --- a/app/backend/approaches/mathassistant.py +++ b/app/backend/approaches/mathassistant.py @@ -3,49 +3,36 @@ #Turn warnings off #from st_pages import Page, show_pages, add_page_title -import warnings -warnings.filterwarnings('ignore') import os -# import openai +import warnings from dotenv import load_dotenv +from langchain_openai import AzureChatOpenAI +from langchain.agents import initialize_agent, load_tools, AgentType +from langchain.prompts import ChatPromptTemplate +from azure.identity import ManagedIdentityCredential, AzureAuthorityHosts, DefaultAzureCredential, get_bearer_token_provider -#-------------------------------------------------------------------------- -#variables needed for testing -OPENAI_API_TYPE = "azure" -OPENAI_API_VERSION = "2024-02-01" -OPENAI_API_BASE = " " -OPENAI_API_KEY = " " -OPENAI_DEPLOYMENT_NAME = " " -MODEL_NAME = " " -AZURE_OPENAI_ENDPOINT = ' ' -AZURE_OPENAI_SERVICE_KEY = ' ' - -os.environ["OPENAI_API_TYPE"] = OPENAI_API_TYPE -os.environ["OPENAI_API_VERSION"] = OPENAI_API_VERSION - - +warnings.filterwarnings('ignore') load_dotenv() - -azure_openai_chatgpt_deployment = os.getenv("AZURE_OPENAI_CHATGPT_DEPLOYMENT") - -deployment_name = azure_openai_chatgpt_deployment -OPENAI_DEPLOYMENT_NAME = deployment_name - OPENAI_API_BASE = os.environ.get("AZURE_OPENAI_ENDPOINT") -OPENAI_API_KEY = os.environ.get("AZURE_OPENAI_SERVICE_KEY") -OPENAI_DEPLOYMENT_NAME = azure_openai_chatgpt_deployment +OPENAI_DEPLOYMENT_NAME = os.getenv("AZURE_OPENAI_CHATGPT_DEPLOYMENT") -from langchain_openai import AzureChatOpenAI -from langchain.agents import initialize_agent, load_tools, AgentType -from langchain.prompts import ChatPromptTemplate +if os.environ.get("AZURE_OPENAI_AUTHORITY_HOST") == "AzureUSGovernment": + AUTHORITY = AzureAuthorityHosts.AZURE_GOVERNMENT +else: + AUTHORITY = AzureAuthorityHosts.AZURE_PUBLIC_CLOUD +if os.environ.get("LOCAL_DEBUG") == "true": + azure_credential = DefaultAzureCredential(authority=AUTHORITY) +else: + azure_credential = ManagedIdentityCredential(authority=AUTHORITY) +token_provider = get_bearer_token_provider(azure_credential, f'https://{os.environ.get("AZURE_AI_CREDENTIAL_DOMAIN")}/.default') model = AzureChatOpenAI( - api_key= OPENAI_API_KEY, + azure_ad_token_provider=token_provider, azure_endpoint=OPENAI_API_BASE, - openai_api_version=OPENAI_API_VERSION , - deployment_name=OPENAI_DEPLOYMENT_NAME) + openai_api_version="2024-02-01" , + deployment_name=OPENAI_DEPLOYMENT_NAME) #-------------------------------------------------------------------------------------------------------------------------------------------------- # Addition of custom tools @@ -192,12 +179,7 @@ def process_agent_response( question): #Function to process clues -def generate_response(question): - model = AzureChatOpenAI( - api_key= OPENAI_API_KEY, - azure_endpoint=OPENAI_API_BASE, - openai_api_version=OPENAI_API_VERSION , - deployment_name=OPENAI_DEPLOYMENT_NAME) +def generate_response(question): prompt_template = ChatPromptTemplate.from_template(template=prompt) messages = prompt_template.format_messages( question=question diff --git a/app/backend/approaches/tabulardataassistant.py b/app/backend/approaches/tabulardataassistant.py index d55a5c866..12ff446f2 100644 --- a/app/backend/approaches/tabulardataassistant.py +++ b/app/backend/approaches/tabulardataassistant.py @@ -4,53 +4,38 @@ import base64 import os import glob -import re import warnings -from PIL import Image import io -import pandas as pd +import tempfile +from dotenv import load_dotenv +from PIL import Image from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent from langchain.agents.agent_types import AgentType from langchain_openai import AzureChatOpenAI -from langchain.agents import load_tools -import matplotlib.pyplot as plt -import tempfile -warnings.filterwarnings('ignore') -from dotenv import load_dotenv - - - -#-------------------------------------------------------------------------- -#variables needed for testing -OPENAI_API_TYPE = "azure" -OPENAI_API_VERSION = "2024-02-01" -OPENAI_API_BASE = " " -OPENAI_API_KEY = " " -OPENAI_DEPLOYMENT_NAME = " " -MODEL_NAME = " " -AZURE_OPENAI_ENDPOINT = ' ' -AZURE_OPENAI_SERVICE_KEY = ' ' - -os.environ["OPENAI_API_TYPE"] = OPENAI_API_TYPE -os.environ["OPENAI_API_VERSION"] = OPENAI_API_VERSION - +from azure.identity import ManagedIdentityCredential, AzureAuthorityHosts, DefaultAzureCredential, get_bearer_token_provider +warnings.filterwarnings('ignore') load_dotenv() -#Environment variables when integrated into the app -#_________________________________________________________________________ - - - -azure_openai_chatgpt_deployment = os.getenv("AZURE_OPENAI_CHATGPT_DEPLOYMENT") -deployment_name = azure_openai_chatgpt_deployment -OPENAI_DEPLOYMENT_NAME = deployment_name OPENAI_API_BASE = os.environ.get("AZURE_OPENAI_ENDPOINT") -OPENAI_API_KEY = os.environ.get("AZURE_OPENAI_SERVICE_KEY") +OPENAI_DEPLOYMENT_NAME = os.getenv("AZURE_OPENAI_CHATGPT_DEPLOYMENT") +if os.environ.get("AZURE_OPENAI_AUTHORITY_HOST") == "AzureUSGovernment": + AUTHORITY = AzureAuthorityHosts.AZURE_GOVERNMENT +else: + AUTHORITY = AzureAuthorityHosts.AZURE_PUBLIC_CLOUD -# Page title +if os.environ.get("LOCAL_DEBUG") == "true": + azure_credential = DefaultAzureCredential(authority=AUTHORITY) +else: + azure_credential = ManagedIdentityCredential(authority=AUTHORITY) +token_provider = get_bearer_token_provider(azure_credential, f'https://{os.environ.get("AZURE_AI_CREDENTIAL_DOMAIN")}/.default') +model = AzureChatOpenAI( + azure_ad_token_provider=token_provider, + azure_endpoint=OPENAI_API_BASE, + openai_api_version="2024-02-01" , + deployment_name=OPENAI_DEPLOYMENT_NAME) dffinal = None pdagent = None @@ -69,15 +54,15 @@ def get_image_data(image_path): def save_chart(query): temp_dir = tempfile.gettempdir() - q_s = f""" you are CSV Assistant, you are a dataframe ally. you analyze every row, addressing all queries with unwavering precision. - You DO NOT answer based on subset of dataframe or top 5 or based on head() output. You need to look at all rows and then answer questions. data is case insensitive. - If any charts or graphs or plots were created save them in the {temp_dir} directory + q_s = f'''You are a assistant to help analyze CSV data that is placed in a dataframe and are a dataframe ally. You analyze every row, addressing all queries with unwavering precision. + You DO NOT answer based on subset of the dataframe or top 5 or based on head() output. Do not create an example dataframe. Use the dataframe provided to you. You need to look at all rows and then answer questions based on the entire dataframe and ensure the input to any tool is valid. Data is case insensitive. + Normalize column names by converting them to lowercase and replacing spaces with underscores to handle discrepancies in column naming conventions. + If any charts or graphs or plots were created save them in the {temp_dir} directory. Make sure the output of the result includes the final result and not just the chart or graph. Put the charts in the {temp_dir} directory and not the final output. + Remember, you can handle both singular and plural forms of queries. - Remember, you can handle both singular and plural forms of queries. For example: - - If you ask "How many thinkpads do we have?" or "How many thinkpad do we have?", you will address both forms in the same manner. - - Similarly, for other queries involving counts, averages, or any other operations. - - """ + For example: + - If you ask \'How many thinkpads do we have?\' or \'How many thinkpad do we have?\', you will address both forms in the same manner. + - Similarly, for other queries involving counts, averages, or any other operations.''' query += ' . '+ q_s return query @@ -99,11 +84,6 @@ def save_df(dff): # function to stream agent response def process_agent_scratch_pad(question, df): - chat = AzureChatOpenAI( - api_key= OPENAI_API_KEY, - azure_endpoint=OPENAI_API_BASE, - openai_api_version=OPENAI_API_VERSION , - deployment_name=OPENAI_DEPLOYMENT_NAME) question = save_chart(question) # This agent relies on access to a python repl tool which can execute arbitrary code. @@ -112,7 +92,7 @@ def process_agent_scratch_pad(question, df): # which can lead to data breaches, data loss, or other security incidents. You must opt in # to use this functionality by setting allow_dangerous_code=True. # https://api.python.langchain.com/en/latest/agents/langchain_experimental.agents.agent_toolkits.pandas.base.create_pandas_dataframe_agent.html - pdagent = create_pandas_dataframe_agent(chat, df, verbose=True,agent_type=AgentType.OPENAI_FUNCTIONS,allow_dangerous_code=True , handle_parsing_errors=True ) + pdagent = create_pandas_dataframe_agent(model, df, verbose=True,agent_type=AgentType.OPENAI_FUNCTIONS,allow_dangerous_code=True , agent_executor_kwargs={"handle_parsing_errors": True}) for chunk in pdagent.stream({"input": question}): if "actions" in chunk: for action in chunk["actions"]: @@ -130,17 +110,15 @@ def process_agent_scratch_pad(question, df): raise ValueError() #Function to stream final output -def process_agent_response(question): +def process_agent_response(question, df): question = save_chart(question) - - chat = AzureChatOpenAI( - api_key= OPENAI_API_KEY, - azure_endpoint=OPENAI_API_BASE, - openai_api_version=OPENAI_API_VERSION , - deployment_name=OPENAI_DEPLOYMENT_NAME) - - - pdagent = create_pandas_dataframe_agent(chat, dffinal, verbose=True,handle_parsing_errors=True,agent_type=AgentType.OPENAI_FUNCTIONS, allow_dangerous_code=True) + + pdagent = create_pandas_dataframe_agent(model, + df, + verbose=True, + agent_type=AgentType.OPENAI_FUNCTIONS, + allow_dangerous_code=True, + agent_executor_kwargs={"handle_parsing_errors": True}) for chunk in pdagent.stream({"input": question}): if "output" in chunk: output = f'Final Output: ```{chunk["output"]}```' diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index 1644f8506..a38c9c79d 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -3,17 +3,17 @@ azure-identity==1.16.1 Flask==2.3.2 langchain==0.2.9 azure-mgmt-cognitiveservices==13.5.0 -openai==1.24.0 +openai==1.35.8 # azure-search-documents==11.4.0 azure-search-documents==11.4.0b11 -azure-storage-blob==12.16.0 -azure-cosmos == 4.3.1 +azure-storage-blob==12.20.0 +azure-cosmos == 4.7.0 tiktoken == 0.7.0 fastapi == 0.109.1 fastapi-utils == 0.2.1 uvicorn == 0.23.2 numexpr == 2.10.0 -langchain-experimental==0.0.61 +langchain-experimental==0.0.62 microsoft-bing-websearch==1.0.0 tabulate==0.9.0 matplotlib==3.8.3 @@ -22,8 +22,7 @@ pandas==2.2.1 python-multipart==0.0.9 Pillow==10.3.0 wikipedia==1.4.0 -langchain-openai == 0.1.7 +langchain-openai == 0.1.14 pytest==8.2.1 python-dotenv==1.0.1 -langchain-community==0.2.9 - +langchain-community==0.2.9 diff --git a/app/backend/testsuite.py b/app/backend/testsuite.py index 9a1c56e9c..97f3e31e3 100644 --- a/app/backend/testsuite.py +++ b/app/backend/testsuite.py @@ -2,9 +2,11 @@ import re import pytest from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient +from azure.identity import DefaultAzureCredential import os from fastapi.testclient import TestClient from dotenv import load_dotenv +import io dir = current_working_directory = os.getcwd() # We're running from MAKE file, so we need to change directory to app/backend @@ -13,6 +15,8 @@ load_dotenv(dotenv_path=f'../../scripts/environments/infrastructure.debug.env') +azure_credentials = DefaultAzureCredential() + from app import app client = TestClient(app) @@ -210,10 +214,10 @@ def test_work_compare_web_chat_api(): assert "Satya" in content or "I am not sure." in content -def test_get_blob_client_url(): - response = client.get("/getblobclienturl") +def test_get_blob_client(): + response = client.get("/getblobclient") assert response.status_code == 200 - assert "blob.core.windows.net" in response.json()["url"] + assert "blob.core.windows.net" in response.json()["client"].url def test_get_all_upload_status(): response = client.post("/getalluploadstatus", json={ @@ -306,16 +310,14 @@ def test_get_feature_flags(): "ENABLE_UNGROUNDED_CHAT": os.getenv("ENABLE_UNGROUNDED_CHAT") == "true", "ENABLE_MATH_ASSISTANT": os.getenv("ENABLE_MATH_ASSISTANT") == "true", "ENABLE_TABULAR_DATA_ASSISTANT": os.getenv("ENABLE_TABULAR_DATA_ASSISTANT") == "true", - "ENABLE_MULTIMEDIA": os.getenv("ENABLE_MULTIMEDIA") == "true", } assert response.json() == expected_response def test_upload_blob(): - account_name = os.getenv("AZURE_BLOB_STORAGE_ACCOUNT") - account_key = os.getenv("AZURE_BLOB_STORAGE_KEY") + storage_account_url=os.getenv("BLOB_STORAGE_ACCOUNT_ENDPOINT") container_name = os.getenv("AZURE_BLOB_STORAGE_UPLOAD_CONTAINER") - blob_service_client = BlobServiceClient(account_url=f"https://{account_name}.blob.core.windows.net", credential=account_key) + blob_service_client = BlobServiceClient(account_url=storage_account_url, credential=azure_credentials) # Create a container client container_client = blob_service_client.get_container_client(container_name) @@ -330,6 +332,8 @@ def test_upload_blob(): # Upload the file with open(local_file_path, "rb") as data: blob_client.upload_blob(data, overwrite=True) + + return blob_name def test_log_status(): response = client.post("/logstatus", json={ @@ -348,9 +352,121 @@ def test_resubmit_item(): def test_delete_item(): response = client.post("/deleteItems", json={"path": "/parts_inventory.csv"}) assert response.status_code == 200 + assert response.json() == True + +def test_get_file(): + blob_name = test_upload_blob() + + try: + file_path = blob_name + response = client.post("/get-file", json={"path": file_path}) + + assert response.status_code == 200 + assert response.headers["Content-Disposition"] == f"inline; filename=parts_inventory.csv" + assert "text/csv" in response.headers["Content-Type"] + finally: + test_delete_item() assert response.json() == True +def test_upload_file_one_tag(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} + + +def test_uploadfilenotagsnofolder(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "parts_inventory.csv", "tags": ""} + ) + print(response.json()) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} + +def test_uploadfiletags(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "parts_inventory.csv", "tags": "test,inventory"} + ) + print(response.json()) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} +def test_uploadfilespecificfolder(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "Finance/parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} +def test_uploadfilespecificfoldernested(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "Finance/new/parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} + +def test_upload_file_no_file(): + response = client.post( + "/file", + data={"file_path": "parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 422 # Unprocessable Entity + +def test_upload_file_large_file(): + file_content = b"a" * (10 * 1024 * 1024) # 10 MB file + file = io.BytesIO(file_content) + file.name = "large_parts_inventory.csv" + + response = client.post( + "/file", + files={"file": (file.name, file, "text/csv")}, + data={"file_path": "large_parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'large_parts_inventory.csv' uploaded successfully"} + +def test_upload_file_missing_file_path(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"tags": "test"} + ) + assert response.status_code == 422 # Unprocessable Entity +def test_upload_file_special_characters_in_file_path(): + with open("test_data/parts_inventory.csv", "rb") as file: + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "Finance/@new/parts_inventory.csv", "tags": "test"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} +def test_upload_file_long_tags(): + with open("test_data/parts_inventory.csv", "rb") as file: + long_tags = ",".join(["tag"] * 1000) # Very long tags string + response = client.post( + "/file", + files={"file": ("parts_inventory.csv", file, "text/csv")}, + data={"file_path": "parts_inventory.csv", "tags": long_tags} + ) + assert response.status_code == 200 + assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"} # This test requires some amount of data to be present and processed in IA # It is commented out because processing the data takes time and the test will fail if the data is not processed # Change the question to a valid question that will produce citations if you want to run this test diff --git a/app/enrichment/app.py b/app/enrichment/app.py index 608d5a319..cd26eeb4d 100644 --- a/app/enrichment/app.py +++ b/app/enrichment/app.py @@ -14,12 +14,11 @@ import random from azure.storage.queue import QueueClient, TextBase64EncodePolicy from azure.search.documents import SearchClient -from azure.core.credentials import AzureKeyCredential +from azure.identity import ManagedIdentityCredential, DefaultAzureCredential, get_bearer_token_provider, AzureAuthorityHosts from data_model import (EmbeddingResponse, ModelInfo, ModelListResponse, StatusResponse) from fastapi import FastAPI, HTTPException from fastapi.responses import RedirectResponse -from fastapi_utils.tasks import repeat_every from model_handling import load_models import openai from openai import AzureOpenAI @@ -33,32 +32,30 @@ # === ENV Setup === ENV = { - "AZURE_BLOB_STORAGE_KEY": None, "EMBEDDINGS_QUEUE": None, "LOG_LEVEL": "DEBUG", # Will be overwritten by LOG_LEVEL in Environment "DEQUEUE_MESSAGE_BATCH_SIZE": 1, "AZURE_BLOB_STORAGE_ACCOUNT": None, "AZURE_BLOB_STORAGE_CONTAINER": None, "AZURE_BLOB_STORAGE_ENDPOINT": None, + "AZURE_QUEUE_STORAGE_ENDPOINT": None, "AZURE_BLOB_STORAGE_UPLOAD_CONTAINER": None, "COSMOSDB_URL": None, - "COSMOSDB_KEY": None, "COSMOSDB_LOG_DATABASE_NAME": None, "COSMOSDB_LOG_CONTAINER_NAME": None, "MAX_EMBEDDING_REQUEUE_COUNT": 5, "EMBEDDING_REQUEUE_BACKOFF": 60, "AZURE_OPENAI_SERVICE": None, - "AZURE_OPENAI_SERVICE_KEY": None, "AZURE_OPENAI_ENDPOINT": None, "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME": None, "AZURE_SEARCH_INDEX": None, - "AZURE_SEARCH_SERVICE_KEY": None, - "AZURE_SEARCH_SERVICE": None, - "BLOB_CONNECTION_STRING": None, "TARGET_EMBEDDINGS_MODEL": None, "EMBEDDING_VECTOR_SIZE": None, "AZURE_SEARCH_SERVICE_ENDPOINT": None, - "AZURE_BLOB_STORAGE_ENDPOINT": None + "AZURE_SEARCH_AUDIENCE": None, + "LOCAL_DEBUG": "false", + "AZURE_AI_CREDENTIAL_DOMAIN": None, + "AZURE_OPENAI_AUTHORITY_HOST": None } for key, value in ENV.items(): @@ -68,16 +65,33 @@ elif value is None: raise ValueError(f"Environment variable {key} not set") -search_creds = AzureKeyCredential(ENV["AZURE_SEARCH_SERVICE_KEY"]) - openai.api_base = ENV["AZURE_OPENAI_ENDPOINT"] openai.api_type = "azure" -openai.api_key = ENV["AZURE_OPENAI_SERVICE_KEY"] +if ENV["AZURE_OPENAI_AUTHORITY_HOST"] == "AzureUSGovernment": + AUTHORITY = AzureAuthorityHosts.AZURE_GOVERNMENT +else: + AUTHORITY = AzureAuthorityHosts.AZURE_PUBLIC_CLOUD openai.api_version = "2024-02-01" +# When debugging in VSCode, use the current user identity to authenticate with Azure OpenAI, +# Cognitive Search and Blob Storage (no secrets needed, just use 'az login' locally) +# Use managed identity when deployed on Azure. +# If you encounter a blocking error during a DefaultAzureCredntial resolution, you can exclude +# the problematic credential by using a parameter (ex. exclude_shared_token_cache_credential=True) +if ENV["LOCAL_DEBUG"] == "true": + azure_credential = DefaultAzureCredential(authority=AUTHORITY) +else: + azure_credential = ManagedIdentityCredential(authority=AUTHORITY) +# Comment these two lines out if using keys, set your API key in the OPENAI_API_KEY environment variable instead +openai.api_type = "azure_ad" +token_provider = get_bearer_token_provider(azure_credential, + f'https://{ENV["AZURE_AI_CREDENTIAL_DOMAIN"]}/.default') +openai.azure_ad_token_provider = token_provider +#openai.api_key = ENV["AZURE_OPENAI_SERVICE_KEY"] + client = AzureOpenAI( - azure_endpoint = openai.api_base, - api_key=openai.api_key, + azure_endpoint = openai.api_base, + azure_ad_token_provider=token_provider, api_version=openai.api_version) class AzOAIEmbedding(object): @@ -119,10 +133,10 @@ def encode(self, texts) -> None: utilities_helper = UtilitiesHelper( azure_blob_storage_account=ENV["AZURE_BLOB_STORAGE_ACCOUNT"], azure_blob_storage_endpoint=ENV["AZURE_BLOB_STORAGE_ENDPOINT"], - azure_blob_storage_key=ENV["AZURE_BLOB_STORAGE_KEY"], + credential=azure_credential ) -statusLog = StatusLog(ENV["COSMOSDB_URL"], ENV["COSMOSDB_KEY"], ENV["COSMOSDB_LOG_DATABASE_NAME"], ENV["COSMOSDB_LOG_CONTAINER_NAME"]) +statusLog = StatusLog(ENV["COSMOSDB_URL"], azure_credential, ENV["COSMOSDB_LOG_DATABASE_NAME"], ENV["COSMOSDB_LOG_CONTAINER_NAME"]) # === API Setup === start_time = datetime.now() @@ -257,7 +271,8 @@ def index_sections(chunks): """ search_client = SearchClient(endpoint=ENV["AZURE_SEARCH_SERVICE_ENDPOINT"], index_name=ENV["AZURE_SEARCH_INDEX"], - credential=search_creds) + credential=azure_credential, + audience=ENV["AZURE_SEARCH_AUDIENCE"]) results = search_client.upload_documents(documents=chunks) succeeded = sum([1 for r in results if r.succeeded]) @@ -280,16 +295,13 @@ def get_tags(blob_path): # Remove the container prefix path_parts = blob_path.split('/') blob_path = '/'.join(path_parts[1:]) - - blob_service_client = BlobServiceClient.from_connection_string(ENV["BLOB_CONNECTION_STRING"]) - # container_client = blob_service_client.get_container_client(ENV["AZURE_BLOB_STORAGE_CONTAINER"]) + + blob_service_client = BlobServiceClient(ENV["AZURE_BLOB_STORAGE_ENDPOINT"], + credential=azure_credential) blob_client = blob_service_client.get_blob_client( container=ENV["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"], blob=blob_path) - - # blob_client = container_client.get_blob_client( - # blob_client = container_client.get_blob_client(container_client=container_client, blob=blob_path) blob_properties = blob_client.get_blob_properties() tags = blob_properties.metadata.get("tags") if tags != '' and tags is not None: @@ -309,9 +321,9 @@ def poll_queue() -> None: log.debug("Skipping poll_queue call, models not yet loaded") return - queue_client = QueueClient.from_connection_string( - conn_str=ENV["BLOB_CONNECTION_STRING"], queue_name=ENV["EMBEDDINGS_QUEUE"] - ) + queue_client = QueueClient(account_url=ENV["AZURE_QUEUE_STORAGE_ENDPOINT"], + queue_name=ENV["EMBEDDINGS_QUEUE"], + credential=azure_credential) log.debug("Polling embeddings queue for messages...") response = queue_client.receive_messages(max_messages=int(ENV["DEQUEUE_MESSAGE_BATCH_SIZE"])) @@ -335,21 +347,25 @@ def poll_queue() -> None: try: statusLog.upsert_document(blob_path, f'Embeddings process started with model {target_embeddings_model}', StatusClassification.INFO, State.PROCESSING) + log.debug("Processing file: %s", blob_path) file_name, file_extension, file_directory = utilities_helper.get_filename_and_extension(blob_path) chunk_folder_path = file_directory + file_name + file_extension - blob_service_client = BlobServiceClient.from_connection_string(ENV["BLOB_CONNECTION_STRING"]) + blob_service_client = BlobServiceClient(ENV["AZURE_BLOB_STORAGE_ENDPOINT"], + credential=azure_credential) container_client = blob_service_client.get_container_client(ENV["AZURE_BLOB_STORAGE_CONTAINER"]) index_chunks = [] # get tags to apply to the chunk tag_list = get_tags(blob_path) + log.debug("Successfully pulled tags for %s. %d tags found.", blob_path, len(tag_list)) # Iterate over the chunks in the container chunk_list = container_client.list_blobs(name_starts_with=chunk_folder_path) chunks = list(chunk_list) i = 0 - + log.debug("Processing %d chunks", len(chunks)) for chunk in chunks: + log.debug("Processing chunk %s", chunk.name) statusLog.update_document_state( blob_path, f"Indexing {i+1}/{len(chunks)}", State.INDEXING) # statusLog.update_document_state( blob_path, f"Indexing {i+1}/{len(chunks)}", State.PROCESSING # open the file and extract the content @@ -411,11 +427,13 @@ def poll_queue() -> None: # push batch of content to index, rather than each individual chunk if i % 200 == 0: + log.debug("Indexing %d chunks", i) index_sections(index_chunks) index_chunks = [] # push remainder chunks content to index if len(index_chunks) > 0: + log.debug("Indexing last %d chunks", len(index_chunks)) index_sections(index_chunks) statusLog.upsert_document(blob_path, @@ -423,6 +441,7 @@ def poll_queue() -> None: StatusClassification.INFO, State.COMPLETE) except Exception as error: + log.debug("An error occurred: %s", str(error)) # Dequeue message and update the embeddings queued count to limit the max retries try: requeue_count = message_json['embeddings_queued_count'] @@ -433,10 +452,10 @@ def poll_queue() -> None: if requeue_count <= int(ENV["MAX_EMBEDDING_REQUEUE_COUNT"]): message_json['embeddings_queued_count'] = requeue_count # Requeue with a random backoff within limits - queue_client = QueueClient.from_connection_string( - ENV["BLOB_CONNECTION_STRING"], - ENV["EMBEDDINGS_QUEUE"], - message_encode_policy=TextBase64EncodePolicy()) + queue_client = QueueClient(account_url=ENV["AZURE_QUEUE_STORAGE_ENDPOINT"], + queue_name=ENV["EMBEDDINGS_QUEUE"], + credential=azure_credential, + message_encode_policy=TextBase64EncodePolicy()) message_string = json.dumps(message_json) max_seconds = int(ENV["EMBEDDING_REQUEUE_BACKOFF"]) * (requeue_count**2) backoff = random.randint( diff --git a/app/enrichment/requirements.txt b/app/enrichment/requirements.txt index 1a1058a37..b192ea074 100644 --- a/app/enrichment/requirements.txt +++ b/app/enrichment/requirements.txt @@ -1,13 +1,17 @@ #### Any version change made here should also be made and tested for the webapp backend and function apps in /functions and /app/backend - +--extra-index-url https://download.pytorch.org/whl/cpu +torch +torchvision +torchaudio sentence-transformers == 2.2.2 fastapi == 0.109.1 fastapi-utils == 0.2.1 uvicorn == 0.23.2 azure-storage-queue == 12.6.0 azure-storage-blob==12.16.0 -azure.search.documents==11.4.0b11 -azure-cosmos == 4.3.1 -azure-core == 1.26.4 +azure.search.documents==11.5.1 +azure-cosmos == 4.7.0 +azure-core == 1.30.2 +azure-identity==1.16.1 tenacity == 8.2.3 openai==1.17.0 diff --git a/app/frontend/index.html b/app/frontend/index.html index 218795599..8c94d90e2 100644 --- a/app/frontend/index.html +++ b/app/frontend/index.html @@ -7,7 +7,7 @@ - Information Assistant Accelerator, built with Azure OpenAI + Information Assistant copilot template
diff --git a/app/frontend/package.json b/app/frontend/package.json index 3987602b0..dfd973414 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -9,7 +9,6 @@ "watch": "tsc && vite build --watch" }, "dependencies": { - "@azure/storage-blob": "^12.13.0", "@fluentui/react": "^8.110.7", "@fluentui/react-icons": "^2.0.195", "@react-spring/web": "^9.7.1", @@ -43,6 +42,6 @@ "prettier": "^2.8.3", "typescript": "^4.9.3", "vite": "^5.0.10", - "vite-plugin-node-polyfills": "^0.22.0" + "vite-plugin-node-polyfills": "^0.2.0" } } diff --git a/app/frontend/src/api/api.ts b/app/frontend/src/api/api.ts index 91b072f15..b9261f88a 100644 --- a/app/frontend/src/api/api.ts +++ b/app/frontend/src/api/api.ts @@ -3,7 +3,6 @@ import { ChatResponse, ChatRequest, - BlobClientUrlResponse, AllFilesUploadStatus, GetUploadStatusRequest, GetInfoResponse, @@ -17,6 +16,7 @@ import { ChatResponse, ResubmitItemRequest, GetFeatureFlagsResponse, getMaxCSVFileSizeType, + FetchCitationFileResponse, } from "./models"; export async function chatApi(options: ChatRequest, signal: AbortSignal): Promise { @@ -64,22 +64,6 @@ export function getCitationFilePath(citation: string): string { return `${encodeURIComponent(citation)}`; } -export async function getBlobClientUrl(): Promise { - const response = await fetch("/getblobclienturl", { - method: "GET", - headers: { - "Content-Type": "application/json" - } - }); - - const parsedResponse: BlobClientUrlResponse = await response.json(); - if (response.status > 299 || !response.ok) { - throw Error(parsedResponse.error || "Unknown error"); - } - - return parsedResponse.url; -} - export async function getAllUploadStatus(options: GetUploadStatusRequest): Promise { const response = await fetch("/getalluploadstatus", { method: "POST", @@ -486,4 +470,21 @@ export async function getFeatureFlags(): Promise { } console.log(parsedResponse); return parsedResponse; +} + +export async function fetchCitationFile(filePath: string) : Promise { + const response = await fetch('/get-file', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: filePath }), + }); + + if (response.status > 299 || !response.ok) { + console.log(response); + throw Error('Failed to fetch file' + response.statusText); + } + const fileResponse : FetchCitationFileResponse = {file_blob : await response.blob()}; + return fileResponse; } \ No newline at end of file diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index 34a13191e..667916a29 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -214,6 +214,10 @@ export type GetFeatureFlagsResponse = { ENABLE_UNGROUNDED_CHAT: boolean; ENABLE_MATH_ASSISTANT: boolean; ENABLE_TABULAR_DATA_ASSISTANT: boolean; - ENABLE_MULTIMEDIA: boolean; + error?: string; +} + +export type FetchCitationFileResponse = { + file_blob: Blob; error?: string; } \ No newline at end of file diff --git a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx index a290c6867..99fa65ae2 100644 --- a/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx +++ b/app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx @@ -2,18 +2,18 @@ // Licensed under the MIT license. import { useEffect, useState } from "react"; -import { Pivot, PivotItem, Text } from "@fluentui/react"; +import { IPivotItemProps, IRefObject, ITooltipHost, Pivot, PivotItem, Text, TooltipHost} from "@fluentui/react"; import { Label } from '@fluentui/react/lib/Label'; import { Separator } from '@fluentui/react/lib/Separator'; import DOMPurify from "dompurify"; import ReactMarkdown from 'react-markdown'; -import remarkGfm from 'remark-gfm' import styles from "./AnalysisPanel.module.css"; import { SupportingContent } from "../SupportingContent"; -import { ChatResponse, ActiveCitation, getCitationObj } from "../../api"; +import { ChatResponse, ActiveCitation, getCitationObj, fetchCitationFile, FetchCitationFileResponse } from "../../api"; import { AnalysisPanelTabs } from "./AnalysisPanelTabs"; +import React from "react"; interface Props { className: string; @@ -26,20 +26,69 @@ interface Props { answer: ChatResponse; } -const pivotItemDisabledStyle = { disabled: true, style: { color: "grey" } }; +const pivotItemDisabledStyle: React.CSSProperties = { + color: 'grey' + +}; export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, pageNumber, citationHeight, className, onActiveTabChanged }: Props) => { + + const [innerPivotTab, setInnerPivotTab] = useState('indexedFile'); const [activeCitationObj, setActiveCitationObj] = useState(); const [markdownContent, setMarkdownContent] = useState(''); const [plainTextContent, setPlainTextContent] = useState(''); - + const [sourceFileBlob, setSourceFileBlob] = useState(); + const [sourceFileUrl, setSourceFileUrl] = useState(''); + const [isFetchingSourceFileBlob, setIsFetchingSourceFileBlob] = useState(false); const isDisabledThoughtProcessTab: boolean = !answer.thoughts; const isDisabledSupportingContentTab: boolean = !answer.data_points?.length; const isDisabledCitationTab: boolean = !activeCitation; // the first split on ? separates the file from the sas token, then the second split on . separates the file extension - const sourceFileExt: any = sourceFile?.split("?")[0].split(".").pop(); + const sourceFileExt: any = sourceFile?.split(".").pop(); const sanitizedThoughts = DOMPurify.sanitize(answer.thoughts!); + const tooltipRef2 = React.useRef(null); + const tooltipRef3 = React.useRef(null); + + const onRenderItemLink = (content: string | JSX.Element | JSX.Element[] | undefined, tooltipRef: IRefObject | undefined, shouldRender: boolean) => (properties: IPivotItemProps | undefined, + nullableDefaultRenderer?: (props: IPivotItemProps) => JSX.Element | null) => { + if (!properties || !nullableDefaultRenderer) { + return null; // or handle the undefined case appropriately + } + return shouldRender ? ( + + {nullableDefaultRenderer(properties)} + + ) : ( + nullableDefaultRenderer(properties) + ); + }; + + let sourceFileBlobPromise: Promise | null = null; + async function fetchCitationSourceFile(): Promise { + if (sourceFile) { + const results = await fetchCitationFile(sourceFile); + setSourceFileBlob(results.file_blob); + setSourceFileUrl(URL.createObjectURL(results.file_blob)); + } + } + + function getCitationURL() { + const fetchSourceFileBlob = async () => { + if (sourceFileBlob === undefined) { + if (!isFetchingSourceFileBlob) { + setIsFetchingSourceFileBlob(true); + sourceFileBlobPromise = fetchCitationSourceFile().finally(() => { + setIsFetchingSourceFileBlob(false); + }); + } + await sourceFileBlobPromise; + } + }; + fetchSourceFileBlob(); + return sourceFileUrl; + } + async function fetchActiveCitationObj() { try { const citationObj = await getCitationObj(activeCitation as string); @@ -51,9 +100,6 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, p } } - useEffect(() => { - fetchActiveCitationObj(); - }, [activeCitation]); useEffect(() => { if (!sourceFile) { @@ -61,7 +107,7 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, p } const fetchMarkdownContent = async () => { try { - const response = await fetch(sourceFile!); + const response = await fetch(getCitationURL()); const content = await response.text(); setMarkdownContent(content); } catch (error) { @@ -70,12 +116,12 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, p }; fetchMarkdownContent(); - }, [sourceFile]); + }, [sourceFileBlob, sourceFileExt]); useEffect(() => { const fetchPlainTextContent = async () => { try { - const response = await fetch(sourceFile!); + const response = await fetch(getCitationURL()); const content = await response.text(); setPlainTextContent(content); } catch (error) { @@ -86,7 +132,27 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, p if (["json", "txt", "xml"].includes(sourceFileExt)) { fetchPlainTextContent(); } - }, [sourceFile, sourceFileExt]); + }, [sourceFileBlob, sourceFileExt]); + + useEffect(() => { + if (activeCitation) { + setInnerPivotTab('indexedFile'); + } + fetchActiveCitationObj(); + const fetchSourceFileBlob = async () => { + + if (!isFetchingSourceFileBlob) { + setIsFetchingSourceFileBlob(true); + sourceFileBlobPromise = fetchCitationSourceFile().finally(() => { + setIsFetchingSourceFileBlob(false); + }); + } + await sourceFileBlobPromise; + + }; + fetchSourceFileBlob(); + + }, [activeCitation]); return (
+ + + - + headerButtonProps={{ + disabled: isDisabledCitationTab, + style: isDisabledCitationTab ? pivotItemDisabledStyle : undefined, + }} + onRenderItemLink = {onRenderItemLink("No active citation selected. Please select a citation from the citations list on the left.", tooltipRef3, isDisabledCitationTab)} + > + + { + if (item) { + setInnerPivotTab(item.props.itemKey!); + } else { + // Handle the case where item is undefined + console.warn('Item is undefined'); + } + }}> {activeCitationObj === undefined ? ( Loading... - ) : ( + ) : + (
Metadata {activeCitationObj.file_name} @@ -132,12 +221,14 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, sourceFile, p )} - {["docx", "xlsx", "pptx"].includes(sourceFileExt) ? ( + {getCitationURL() === '' ? ( + Loading... + ) : ["docx", "xlsx", "pptx"].includes(sourceFileExt) ? ( // Treat other Office formats like "xlsx" for the Office Online Viewer -