update app insights and function app infra

ingest/src/azure/function_app.py

@@ -1,9 +1,10 @@
 import azure.functions as func
 import azure.data.tables as tab
 from azure.core.exceptions import ResourceExistsError, HttpResponseError
+from azure.monitor.opentelemetry import configure_azure_monitor
+from logging import INFO, getLogger
 import ast
 import os
-import logging
 
 #------------------------------------------------------------------------------
 app = func.FunctionApp()
@@ -25,6 +26,11 @@ def prices_create(azqueue: func.QueueMessage):
             azqueue: the incoming Azure Queue Message
             return: None
     """
+    # setup logging
+    configure_azure_monitor(logger_name = "prices_create")
+    log = getLogger("prices_create")
+    log.setLevel(INFO)
+
     # accept the message.
     msg_str = azqueue.get_body().decode('utf-8')
 
@@ -41,7 +47,7 @@ def prices_create(azqueue: func.QueueMessage):
     cl = tab.TableClient.from_connection_string(conn, 'postcode')
     try:
         cl.create_table()
-        logging.info(f'PRICES_CREATE: created table postcode.')
+        log.info(f'PRICES_CREATE: created table postcode.')
     except ResourceExistsError:
         pass
 
@@ -53,7 +59,7 @@ def prices_create(azqueue: func.QueueMessage):
     try:
         cl.upsert_entity(entity)
     except HttpResponseError as uE:
-        logging.error(uE.message)
+        log.error(uE.message)
         raise uE  
 
-    logging.info(f"PRICES_CREATE: Inserted Postcode {postcode}")
+    log.info(f"PRICES_CREATE: Inserted Postcode {postcode}")

ingest/src/requirements.txt

@@ -0,0 +1,47 @@
+asgiref==3.8.1
+azure-core==1.30.0
+azure-core-tracing-opentelemetry==1.0.0b11
+azure-data-tables==12.5.0
+azure-functions==1.18.0
+azure-monitor-opentelemetry==1.4.0
+azure-monitor-opentelemetry-exporter==1.0.0b25
+azure-storage-queue==12.9.0
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+cryptography==42.0.5
+Deprecated==1.2.14
+fixedint==0.1.6
+idna==3.6
+importlib-metadata==7.0.0
+isodate==0.6.1
+msrest==0.7.1
+multidict==6.0.5
+oauthlib==3.2.2
+opentelemetry-api==1.24.0
+opentelemetry-instrumentation==0.45b0
+opentelemetry-instrumentation-asgi==0.45b0
+opentelemetry-instrumentation-dbapi==0.45b0
+opentelemetry-instrumentation-django==0.45b0
+opentelemetry-instrumentation-fastapi==0.45b0
+opentelemetry-instrumentation-flask==0.45b0
+opentelemetry-instrumentation-psycopg2==0.45b0
+opentelemetry-instrumentation-requests==0.45b0
+opentelemetry-instrumentation-urllib==0.45b0
+opentelemetry-instrumentation-urllib3==0.45b0
+opentelemetry-instrumentation-wsgi==0.45b0
+opentelemetry-resource-detector-azure==0.1.4
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
+opentelemetry-util-http==0.45b0
+packaging==24.0
+psutil==5.9.8
+pycparser==2.21
+requests==2.31.0
+requests-oauthlib==2.0.0
+six==1.16.0
+typing_extensions==4.9.0
+urllib3==2.2.1
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.18.1

platform/01-app-insights/appins.deploy.cli

@@ -1,2 +1,2 @@
 paramFile=appins.params.json
-az deployment group create --resource-group Purple001 --template-file bicep/appins.bicep --parameters $paramFile
+az deployment group create --resource-group Purple001 --template-file appins.bicep --parameters $paramFile

platform/02-function-app/funcapp.bicep

@@ -1,7 +1,89 @@
+targetScope = 'resourceGroup'
+@description('the parameters needed to deploy all Azure resources for a function app')
+param azLocation string
+param azTags object
+param projShortName string 
+param projVersion string 
+param storageName string
 
+var hostPlanName = '${projShortName}ServerFarm${projVersion}'
+var insightsName = '${projShortName}Insights${projVersion}'
+var functionName = '${projShortName}Func${projVersion}'
 
-// Get a reference to the seperate data lake storage account (data lake)
-resource dataLake 'Microsoft.Storage/storageAccounts@2019-06-01' existing = {
-  scope: resourceGroup(dataLakeRg)
-  name: dataLakeName
+// Get a reference to the seperate configuration storage account (table storage)
+resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' existing = {
+  scope: resourceGroup()
+  name: storageName
+}
+
+// Get a reference to the previously created app insights in the same RG.
+resource insights 'Microsoft.Insights/components@2020-02-02' existing = {
+  scope: resourceGroup()
+  name: insightsName
+}
+
+// Specify kind: as 'linux' otherwise it defaults to windows.
+@description('Create the App Service Plan for the Server Farm. sku Y1 is the free tier.')
+resource hostingPlan 'Microsoft.Web/serverfarms@2022-03-01' = {
+  name: hostPlanName
+  location: azLocation
+  sku: {
+    name: 'Y1'
+    tier: 'Dynamic'
+    size: 'Y1'
+    family: 'Y'
+  }
+  properties: {
+    reserved: true
+  }
+  tags: azTags
+}
+
+// Deploy as linux running python 3.11
+@description('Create the Function App with references to all other resources')
+resource functionApp 'Microsoft.Web/sites@2021-03-01' = {
+  name: functionName
+  location: azLocation
+  kind: 'functionapp,linux'
+  properties: {
+    reserved: true
+    serverFarmId: hostingPlan.id
+    siteConfig: {
+      linuxFxVersion: 'python|3.11'
+      appSettings: [
+        {
+          name: 'AzureWebJobsStorage'
+          value: 'DefaultEndpointsProtocol=https;AccountName=${storageName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}'
+        }
+        {
+          name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING'
+          value: 'DefaultEndpointsProtocol=https;AccountName=${storageName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}'
+        }
+        {
+          name: 'WEBSITE_CONTENTSHARE'
+          value: toLower(functionName)
+        }
+        {
+          name: 'FUNCTIONS_EXTENSION_VERSION'
+          value: '~4'
+        }
+        {
+          name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
+          value: insights.properties.ConnectionString
+        }
+        {
+          name: 'FUNCTIONS_WORKER_RUNTIME'
+          value: 'python'
+        }
+        {
+          name: 'LandregDataStorage'
+          value: 'DefaultEndpointsProtocol=https;AccountName=${storageName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}'
+        }
+      ]
+      ftpsState: 'FtpsOnly'
+      minTlsVersion: '1.2'
+    }
+    httpsOnly: true
+  }
+  tags: azTags
 }

platform/02-function-app/funcapp.deploy.cli

@@ -0,0 +1,2 @@
+paramFile=funcapp.params.json
+az deployment group create --resource-group Purple001 --template-file funcapp.bicep --parameters $paramFile

platform/02-function-app/funcapp.params.json

@@ -0,0 +1,28 @@
+{
+    "$schema": "http://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "azLocation": {
+            "value": "uksouth"
+        },
+        "projShortName": {
+            "value": "Purple"
+        },
+        "projVersion": {
+            "value": "001"
+        },
+        "storageName": {
+            "value": "purpledata001"
+        },    
+        "azTags": {
+            "value": {
+                "App": "Purple",
+                "Business": "TMRSM",
+                "Licence": "Open",
+                "CreatedBy": "Andy Roberts",
+                "Service": "LandReg Property Prices",
+                "Region": "UK"
+            }
+        }
+    }
+}

platform/monitor.md

@@ -23,6 +23,9 @@ Examples of using this in Python can be found here:  https://github.com/Azure/az
 To create an application insights component, the Bicep template can be found here:  
 https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/components?pivots=deployment-language-bicep  
 
+## Function App 
+See the [Github Function App Templates](https://github.com/Azure/azure-quickstart-templates/tree/master/quickstarts/microsoft.web) for linux and python bicep quickstart templates.  
+
 ## Function App Logging
 Now the default is to use open telemetry logging in code, you can disable the automatic agent based monitoring on old function apps and switch to the new method.  
 
@@ -31,3 +34,22 @@ To ensure the agent is disabled from an existing app, remove the APPINSIGHTS_INS
 
 
 
+
+// Specify kind: as 'linux' otherwise it defaults to windows.
+@description('Create the App Service Plan for the Server Farm. sku Y1 is the free tier.')
+resource hostingPlan 'Microsoft.Web/serverfarms@2022-03-01' = {
+  name: hostPlanName
+  location: azLocation
+  sku: {
+    name: 'Y1'
+    tier: 'Dynamic'
+  }
+  kind: 'linux'               // needed for linux
+  properties: {
+    reserved: true            // needed for linux
+  }
+  tags: azTags
+}
+
+
+