Merge pull request #764 from microsoft/dayland/1.1.2-secure-mode

Secure mode changes

.devcontainer/Dockerfile

@@ -22,7 +22,11 @@ RUN sudo apt-get update \
     && sudo apt-get -y install --no-install-recommends apt-utils dialog nano bash-completion sudo bsdmainutils cmake \
     #
     # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed
-    && sudo apt-get -y install git iproute2 procps lsb-release figlet build-essential
+    && sudo apt-get -y install git iproute2 procps lsb-release figlet build-essential dnsutils openvpn
+
+RUN sudo mkdir -p /dev/net \
+    && sudo mknod /dev/net/tun c 10 200 \
+    && sudo chmod 600 /dev/net/tun
 
 # Save command line history
 RUN echo "export HISTFILE=/home/$USERNAME/commandhistory/.bash_history" >> "/home/$USERNAME/.bashrc" \
@@ -49,7 +53,7 @@ RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prom
     && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/home/$USERNAME/.bashrc"
 
 # terraform + tflint
-ARG TERRAFORM_VERSION=1.3.7
+ARG TERRAFORM_VERSION=1.8.5
 ARG TFLINT_VERSION=0.44.1
 RUN mkdir -p /tmp/docker-downloads \
     && curl -sSL -o /tmp/docker-downloads/terraform.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \

.devcontainer/devcontainer.json

@@ -11,7 +11,7 @@
 	},
 	"forwardPorts": [ 7071 ],
 	"runArgs": [
-		"--network", "host" // use host networking so that the dev container can access the API when running the container locally
+		"--network", "host", "--cap-add", "NET_ADMIN" // use host networking so that the dev container can access the API when running the container locally
 	],
 
 	"mounts": [

.gitignore

@@ -400,4 +400,8 @@ infra/sp_config/config.json
 #Upgrade & Migrate Support
 scripts/upgrade_repoint.config.json
 azcopy.tar.gz
-azcopy_dir
+azcopy_dir
+
+#Openvpn setting
+openvpn/*
+!openvpn/info.txt

Makefile

@@ -9,7 +9,7 @@ help: ## Show this help
 		| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%s\033[0m|%s\n", $$1, $$2}' \
         | column -t -s '|'
 
-deploy: build infrastructure extract-env deploy-enrichments deploy-search-indexes deploy-webapp deploy-functions ## Deploy infrastructure and application code
+deploy: build build-containers infrastructure extract-env deploy-enrichments deploy-search-indexes deploy-webapp deploy-functions ## Deploy infrastructure and application code
 
 build-deploy-webapp: build extract-env deploy-webapp ##Build and Deploy the Webapp
 build-deploy-enrichments: build extract-env deploy-enrichments ##Build and Deploy the Enrichment Webapp
@@ -18,15 +18,20 @@ build-deploy-functions: build extract-env deploy-functions ##Build and Deploy th
 build: ## Build application code
 	@./scripts/build.sh
 
-build-containers: extract-env
-	@./app/enrichment/docker-build.sh
+build-containers: ## Build the docker containers for the function, webapp, and enrichment
+	@./container_images/enrichment_container_image/docker-build.sh
+	@./functions/docker-build.sh
+	@./container_images/webapp_container_image/docker-build.sh
 
 infrastructure: check-subscription ## Deploy infrastructure
 	@./scripts/inf-create.sh
 
-extract-env: extract-env-debug-webapp extract-env-debug-functions ## Extract infrastructure.env file from Terraform output
+extract-env: check-secure-mode-connectivity extract-env-debug-webapp extract-env-debug-functions ## Extract infrastructure.env file from Terraform output
 	 @./scripts/json-to-env.sh < inf_output.json > ./scripts/environments/infrastructure.env
 
+check-secure-mode-connectivity: ## Check secure mode connectivity
+	@./scripts/check-secure-mode-connectivity.sh
+
 deploy-webapp: extract-env ## Deploys the web app code to Azure App Service
 	@./scripts/deploy-webapp.sh
 

SUPPORT.md

@@ -1,19 +1,19 @@
-# Support
-
-## How to file issues and get help  
-
-This project uses GitHub Issues to track bugs and feature requests. Please search the existing
-issues before filing new issues to avoid duplicates.  For new issues, file your bug or
-feature request as a new Issue.
-
-For help and questions about using this project, please use the [Discussion](https://github.com/microsoft/PubSec-Info-Assistant/discussions) forums on our GitHub Repo page.
-
-For customer support deploying this accelerator, please reach out to your local Microsoft representative or email the [Industry Solutions Accelerator Team](mailto:[email protected]). 
-
-## Providing Feedback
-
-Please refer to the [Contributing](./CONTRIBUTING.md) guidelines for acceptable methods to provide feedback which are not security related.
-
-## Microsoft Support Policy  
-
-Support for this **PROJECT** is limited to the resources listed above.
+# Support
+
+## How to file issues and get help  
+
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates.  For new issues, file your bug or
+feature request as a new Issue.
+
+For help and questions about using this project, please use the [Discussion](https://github.com/microsoft/PubSec-Info-Assistant/discussions) forums on our GitHub Repo page.
+
+For customer support deploying this accelerator, please reach out to your local Microsoft representative or email the [Industry Solutions Accelerator Team](mailto:[email protected]). 
+
+## Providing Feedback
+
+Please refer to the [Contributing](./CONTRIBUTING.md) guidelines for acceptable methods to provide feedback which are not security related.
+
+## Microsoft Support Policy  
+
+Support for this **PROJECT** is limited to the resources listed above.

app/backend/app.py

@@ -11,6 +11,7 @@
 import json
 import urllib.parse
 import pandas as pd
+import pydantic
 from datetime import datetime, time, timedelta
 from fastapi.staticfiles import StaticFiles
 from fastapi import FastAPI, File, HTTPException, Request, UploadFile
@@ -118,6 +119,15 @@
 log.setLevel('DEBUG')
 log.propagate = True
 
+class StatusResponse(pydantic.BaseModel):
+    status: str
+    uptime_seconds: float
+    version: str
+
+start_time = datetime.now()
+
+IS_READY = False
+
 dffinal = None
 # Used by the OpenAI SDK
 openai.api_type = "azure"
@@ -260,6 +270,8 @@
     )
 }
 
+IS_READY = True
+
 # Create API
 app = FastAPI(
     title="IA Web API",
@@ -273,6 +285,25 @@ async def root():
     """Redirect to the index.html page"""
     return RedirectResponse(url="/index.html")
 
+@app.get("/health", response_model=StatusResponse, tags=["health"])
+def health():
+    """Returns the health of the API
+
+    Returns:
+        StatusResponse: The health of the API
+    """
+
+    uptime = datetime.now() - start_time
+    uptime_seconds = uptime.total_seconds()
+
+    output = {"status": None, "uptime_seconds": uptime_seconds, "version": app.version}
+
+    if IS_READY:
+        output["status"] = "ready"
+    else:
+        output["status"] = "loading"
+
+    return output
 
 @app.post("/chat")
 async def chat(request: Request):
@@ -462,8 +493,15 @@ async def resubmit_Items(request: Request):
         blob_container = blob_client.get_container_client(os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"])
         # Read the blob content into memory
         blob_data = blob_container.download_blob(path).readall()
-        # Overwrite the blob with the modified data
-        blob_container.upload_blob(name=path, data=blob_data, overwrite=True)  
+
+        submitted_blob_client = blob_container.get_blob_client(blob=path)
+        blob_properties = submitted_blob_client.get_blob_properties()
+        metadata = blob_properties.metadata
+        blob_container.upload_blob(name=path, data=blob_data, overwrite=True, metadata=metadata)   
+
+
+
+
         # add the container to the path to avoid adding another doc in the status db
         full_path = os.environ["AZURE_BLOB_STORAGE_UPLOAD_CONTAINER"] + '/' + path
         statusLog.upsert_document(document_path=full_path,

app/enrichment/app.py

@@ -19,7 +19,6 @@
                         StatusResponse)
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import RedirectResponse
-from fastapi_utils.tasks import repeat_every
 from model_handling import load_models
 import openai
 from openai import AzureOpenAI

container_images/enrichment_container_image/Dockerfile

@@ -0,0 +1,16 @@
+# Use the official Python image from Microsoft for Azure App Service
+FROM mcr.microsoft.com/devcontainers/python:3.10
+
+# Set the working directory
+WORKDIR /app
+
+COPY app/enrichment /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install -r /app/requirements.txt
+
+# Expose the port the app runs on
+EXPOSE 6000
+
+# Run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "6000"]

container_images/enrichment_container_image/docker-build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -eo pipefail
+
+# Display banner
+figlet "Build Docker Container Images"
+
+# Build the Docker image with the correct context
+echo "Building Docker image: enrichmentapp"
+echo -e "\n"
+sudo docker build -f ./container_images/enrichment_container_image/Dockerfile -t enrichmentapp . --build-arg BUILDKIT_INLINE_CACHE=1
+
+# Generate a unique tag for the image
+tag=$(date -u +"%Y%m%d-%H%M%S")
+echo "Tagging image with: $tag"
+sudo docker tag enrichmentapp enrichmentapp:$tag
+
+# Output the tag to a file to be used in deployment
+echo -n "$tag" > ./container_images/enrichment_container_image/image_tag.txt
+
+echo "Build and tagging complete. Tag: $tag"
+echo -e "\n"

container_images/webapp_container_image/Dockerfile

@@ -0,0 +1,17 @@
+# Use the official Node.js image
+FROM mcr.microsoft.com/devcontainers/python:3.10
+
+# Set the working directory
+WORKDIR /home/site/wwwroot
+
+# Copy the package.json
+COPY app/backend /home/site/wwwroot
+
+# Install the dependencies
+RUN pip install -r /home/site/wwwroot/requirements.txt
+
+# Expose the port the app runs on
+EXPOSE 6000
+
+# Run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "6000"]

container_images/webapp_container_image/docker-build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eo pipefail
+
+# Build the Docker image with the correct context
+echo "Building Docker image: webapp"
+echo -e "\n"
+sudo docker build -f ./container_images/webapp_container_image/Dockerfile -t webapp . --build-arg BUILDKIT_INLINE_CACHE=1
+
+# Generate a unique tag for the image
+tag=$(date -u +"%Y%m%d-%H%M%S")
+echo "Tagging image with: $tag"
+sudo docker tag webapp webapp:$tag
+
+# Output the tag to a file to be used in deployment
+echo -n "$tag" > ./container_images/webapp_container_image/image_tag.txt
+
+echo "Build and tagging complete. Tag: $tag"
+echo -e "\n"

docs/deployment/deployment.md

@@ -53,7 +53,7 @@ WORKSPACE | Yes  | The workspace name (use something simple and unique to you).
 SUBSCRIPTION_ID | Yes | The GUID that represents the Azure Subscription you want the Accelerator to be deployed into. This can be obtained from the *Subscription* blade in the Azure Portal.
 TENANT_ID | Yes | The GUID that represents the Azure Active Directory Tenant for the Subscription you want the accelerator to be deployed into. This can be obtained from the *Tenant Info* blade in the Azure Portal.
 AZURE_ENVIRONMENT | Yes | This will determine the Azure cloud environment the deployment will target. Information Assistant currently supports, AzureCloud and AzureUSGovernment. Info available at [Azure cloud environments](https://docs.microsoft.com/en-us/cli/azure/manage-clouds-azure-cli?toc=/cli/azure/toc.json&bc=/cli/azure/breadcrumb/toc.json). If you are targeting "AzureUSGovernment" please see our [sovereign deployment support documentation](/docs/deployment/enable_sovereign_deployment.md).
-SECURE_MODE | Yes | Defaults to `false`. This feature flag will determine if the Information Assistant deploys it's Azure Infrastructure in a secure mode or not.</br>:warning: Before enabling secure mode please read the extra instructions on [Enabling Secure Deployment](#tdb)
+SECURE_MODE | Yes | Defaults to `false`. This feature flag will determine if the Information Assistant deploys it's Azure Infrastructure in a secure mode or not.</br>:warning: Before enabling secure mode please read the extra instructions on [Enabling Secure Deployment](/docs/deployment/secure_deployment.md)
 ENABLE_WEB_CHAT | Yes | Defaults to `false`. This feature flag will enable the ability to use Web Search results as a data source for generating answers from the LLM. This feature will also deploy a Bing v7 Search instance in Azure to retrieve web results from, however Bing v7 Search is not available in AzureUSGovernment regions, so this feature flag is **NOT** compatible with `AZURE_ENVIRONMENT=AzureUSGovernment`.
 ENABLE_BING_SAFE_SEARCH | No | Defaults to `true`. If you are using the `ENABLE_WEB_CHAT`feature you can set the following values to enable safe search on the Bing v7 Search APIs.
 ENABLE_UNGROUNDED_CHAT | Defaults to `false`. This feature flag will enable the ability to interact directly with an LLM. This experience will be similar to the Azure OpenAI Playground.
@@ -63,6 +63,7 @@ ENABLE_SHAREPOINT_CONNECTOR | Yes | Defaults to `false`. This feature flag enabl
 SHAREPOINT_TO_SYNC | No | This is a JSON Array of Objects for SharePoint Sites and their entry folders. The app will crawl down from the folder specified for each site. Specifying "/Shared Documents" will crawl all the documents in your SharePoint. `[{"url": "https://SharePoint.com/", "folder": "/Shared Documents"}]` This will **overwrite** any prior changes you've made to config.json. Information on setting up SharePoint Ingestion can be found here [SharePoint Connector](/docs/features/sharepoint.md)
 ENABLE_MULTIMEDIA | Yes | Defaults to `false`. This feature flag should not be changed at this time. The multimedia feature is still in development. Enabling this feature will deploy an Azure Video Indexer instance in your resource group only.
 REQUIRE_WEBSITE_SECURITY_MEMBERSHIP | Yes | Use this setting to determine whether a user needs to be granted explicit access to the website via an Azure AD Enterprise Application membership (true) or allow the website to be available to anyone in the Azure tenant (false). Defaults to false. If set to true, A tenant level administrator will be required to grant the implicit grant workflow for the Azure AD App Registration manually.
+SECRET_EXPIRATION_DAYS | Yes | Defaults to `120`. Use this setting to set the secret expiration to the current day plus the number of days specified. Key Vault secrets require an expiration date to be compatible with Microsoft's recommended guardrails for Azure Key Vault policy. We have NOT included automatic secret rotation in this deployment. Go [here](https://learn.microsoft.com/en-us/azure/key-vault/keys/how-to-configure-key-rotation) for more information on enabling cryptographic key auto-rotation.
 SKIP_PLAN_CHECK | No | If this value is set to 1, then the Terraform deployment will not stop to allow you to review the planned changes. The default value is 0 in the scripts, which will allow the deployment to stop and confirm you accept the proposed changes before continuing.
 USE_EXISTING_AOAI | Yes | Defaults to false. Set this value to "true" if you want to use an existing Azure Open AI service instance in your subscription. This can be useful when there are limits to the number of AOAI instances you can have in one subscription. When the value is set to "false" and Terraform will create a new Azure Open AI service instance in your resource group.
 AZURE_OPENAI_RESOURCE_GROUP | No | If you have set **USE_EXISTING_AOAI** to "true" then use this parameter to provide the name of the resource group that hosts the Azure Open AI service instance in your subscription.