example_notebooks/detect_case_patterns.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detect Case Patterns\n",
    "\n",
    "Demonstrates use of the Intelligence Toolkit library to detect attribute patterns in a dataset of timestamped case records.\n",
    "\n",
    "See [readme](https://github.com/microsoft/intelligence-toolkit/blob/main/app/workflows/detect_attribute_patterns/README.md) for more details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "import os\n",
    "from intelligence_toolkit.detect_case_patterns.api import DetectCasePatterns\n",
    "import intelligence_toolkit.detect_case_patterns.prompts as prompts\n",
    "from intelligence_toolkit.AI.openai_configuration import OpenAIConfiguration\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n"
     ]
    }
   ],
   "source": [
    "# Create the workflow object\n",
    "from intelligence_toolkit.helpers import df_functions\n",
    "\n",
    "\n",
    "dcp = DetectCasePatterns()\n",
    "# Set the AI configuration\n",
    "ai_configuration = OpenAIConfiguration(\n",
    "    {\n",
    "        \"api_type\": \"OpenAI\",\n",
    "        \"api_key\": os.environ[\"OPENAI_API_KEY\"],\n",
    "        \"model\": \"gpt-4o\",\n",
    "    }\n",
    ")\n",
    "dcp.set_ai_configuration(ai_configuration)\n",
    "# Load the prepared case data\n",
    "data_path = \"../example_outputs/detect_case_patterns/customer_complaints/customer_complaints_prepared.csv\"\n",
    "case_data = pd.read_csv(data_path)\n",
    "# Map missing values and binary False to empty strings, since we only care about the presence of attributes\n",
    "case_data = df_functions.supress_boolean_binary(case_data)\n",
    "print(\"Loaded data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated graph model\n"
     ]
    }
   ],
   "source": [
    "# Generate the graph model\n",
    "dcp.generate_graph_model(df=case_data, period_col=\"period\")\n",
    "print(\"Generated graph model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated embedding model\n"
     ]
    }
   ],
   "source": [
    "# Generate the graph model\n",
    "dcp.generate_embedding_model()\n",
    "print(\"Generated embedding model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Detected case patterns\n"
     ]
    }
   ],
   "source": [
    "# Detect the case patterns\n",
    "dcp.detect_patterns(min_pattern_count=10, max_pattern_length=5)\n",
    "print(\"Detected case patterns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      period                                            pattern  length  \\\n",
      "511  2023-H1  age_range:(50-60] & city:Springfield & deliver...       5   \n",
      "510  2023-H1  age_range:(40-50] & city:Mountainview & delive...       5   \n",
      "509  2023-H1  age_range:(30-40] & city:Forestville & price_i...       5   \n",
      "99   2022-H2  age_range:(30-40] & city:Quartz City & deliver...       5   \n",
      "508  2023-H1  age_range:(20-30] & city:Riverside & descripti...       5   \n",
      "\n",
      "     count  mean  z_score  detections  overall_score  \n",
      "511     18   2.0     3.32           1           0.45  \n",
      "510     18   2.0     3.30           1           0.44  \n",
      "509     16   1.0     3.32           1           0.43  \n",
      "99      15   1.0     3.32           1           0.42  \n",
      "508     13   1.0     3.32           1           0.40  \n"
     ]
    }
   ],
   "source": [
    "# Inspect the top patterns of the maximum length\n",
    "pdf = dcp.patterns_df\n",
    "max_length = pdf[\"length\"].max()\n",
    "top_patterns = pdf[pdf[\"length\"] == max_length].head(10)\n",
    "print(top_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created time series\n"
     ]
    }
   ],
   "source": [
    "# Create the time series\n",
    "dcp.create_time_series_df()\n",
    "print(\"Created time series\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "period                                                     2023-H1\n",
      "pattern          age_range:(50-60] & city:Springfield & deliver...\n",
      "length                                                           5\n",
      "count                                                           18\n",
      "mean                                                           2.0\n",
      "z_score                                                       3.32\n",
      "detections                                                       1\n",
      "overall_score                                                 0.45\n",
      "Name: 511, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Set the example pattern to the top pattern of the longest length\n",
    "example_pattern = top_patterns.iloc[0]\n",
    "print(example_pattern)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing attribute counts for pattern: age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True with period: 2023-H1 for period column: period\n",
      "           AttributeValue  Count\n",
      "0       age_range:(50-60]     18\n",
      "1        city:Springfield     18\n",
      "2     delivery_issue:True     18\n",
      "5          product_code:G     18\n",
      "7      service_issue:True     18\n",
      "6      quality_issue:True      7\n",
      "3  description_issue:True      4\n",
      "4        price_issue:True      4\n"
     ]
    }
   ],
   "source": [
    "# Compute related attribute counts for the example pattern\n",
    "att_counts = dcp.compute_attribute_counts(\n",
    "    selected_pattern=example_pattern[\"pattern\"],\n",
    "    selected_pattern_period=example_pattern[\"period\"],\n",
    ")\n",
    "print(att_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ddesouza/Projects/intelligence-toolkit/.venv/lib/python3.11/site-packages/altair/utils/core.py:384: FutureWarning: the convert_dtype parameter is deprecated and will be removed in a future version.  Do ``ser.astype(object).apply()`` instead if you want ``convert_dtype=False``.\n",
      "  col = df[col_name].apply(to_list_if_array, convert_dtype=False)\n",
      "/home/ddesouza/Projects/intelligence-toolkit/.venv/lib/python3.11/site-packages/altair/utils/core.py:384: FutureWarning: the convert_dtype parameter is deprecated and will be removed in a future version.  Do ``ser.astype(object).apply()`` instead if you want ``convert_dtype=False``.\n",
      "  col = df[col_name].apply(to_list_if_array, convert_dtype=False)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-db6896a8bca1470d8b0a9816e276462b\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-db6896a8bca1470d8b0a9816e276462b\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-db6896a8bca1470d8b0a9816e276462b\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"4.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-e9138573dd84820615f2140dd0b9297a\"}, \"mark\": \"line\", \"encoding\": {\"color\": {\"value\": \"blue\"}, \"x\": {\"field\": \"period\", \"type\": \"ordinal\"}, \"y\": {\"field\": \"count\", \"type\": \"quantitative\"}}, \"height\": 220, \"title\": \"Pattern: age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True (2023-H1)\", \"width\": 600, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.17.0.json\", \"datasets\": {\"data-e9138573dd84820615f2140dd0b9297a\": [{\"period\": \"2020-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2020-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2021-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2021-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2022-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2022-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2023-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 18}, {\"period\": \"2023-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2024-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2024-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2025-H1\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}, {\"period\": \"2025-H2\", \"pattern\": \"age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True\", \"count\": 0}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.Chart(...)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the time series chart\n",
    "chart = dcp.create_time_series_chart(\n",
    "    selected_pattern=example_pattern[\"pattern\"],\n",
    "    selected_pattern_period=example_pattern[\"period\"],\n",
    ")\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing attribute counts for pattern: age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True with period: 2023-H1 for period column: period\n",
      "# Pattern Report\n",
      "\n",
      "**Pattern: age_range:(50-60] & city:Springfield & delivery_issue:True & product_code:G & service_issue:True**\n",
      "\n",
      "This pattern describes a group of individuals aged between 50 and 60, residing in Springfield, who have experienced both delivery and service issues with product code G.\n",
      "\n",
      "## Pattern observation\n",
      "\n",
      "The pattern was observed in the first half of 2023, with 18 cases matching the criteria. This is notable because there were no cases matching this pattern in any other time period from 2020 to 2025. The sudden appearance of this pattern in 2023-H1 suggests a specific issue or change during this period that affected this demographic and product.\n",
      "\n",
      "## Pattern context\n",
      "\n",
      "In addition to the attributes defining the pattern, the most common additional issues reported were quality issues (7 cases), description issues (4 cases), and price issues (4 cases). This information suggests that, beyond delivery and service problems, there were also concerns about the quality, description, and pricing of the product. Understanding these additional issues can help in diagnosing the root causes of the pattern.\n",
      "\n",
      "## Possible explanations\n",
      "\n",
      "1. A change in the supply chain or delivery process in Springfield during 2023-H1 could have led to increased delivery issues.\n",
      "2. A new version or batch of product code G might have been released, leading to service and quality issues.\n",
      "3. There could have been a local event or policy change in Springfield affecting logistics or customer service.\n",
      "4. The demographic aged 50-60 might have been targeted by a specific marketing campaign for product G, leading to increased purchases and subsequent issues.\n",
      "5. A competitor's actions or market changes might have influenced customer expectations or satisfaction.\n",
      "6. There might have been staffing changes or shortages in the service department affecting service quality.\n",
      "7. Economic factors, such as inflation or price changes, could have influenced perceptions of price issues.\n",
      "\n",
      "## Suggested actions\n",
      "\n",
      "1. Investigate the delivery process for product G in Springfield to identify any logistical issues.\n",
      "2. Review the service protocols and training for handling product G to improve service quality.\n",
      "3. Conduct a quality assessment of product G to address any potential defects or inconsistencies.\n",
      "4. Analyze customer feedback to understand specific concerns related to product descriptions.\n",
      "5. Evaluate pricing strategies for product G to ensure they align with customer expectations.\n",
      "6. Consider targeted communication with affected customers to address their concerns and improve satisfaction.\n",
      "7. Monitor future sales and service data for product G to detect any recurring patterns or issues.\n"
     ]
    }
   ],
   "source": [
    "# Explain the top-ranked pattern of longest length\n",
    "explanation = dcp.explain_pattern(\n",
    "    selected_pattern=example_pattern[\"pattern\"],\n",
    "    selected_pattern_period=example_pattern[\"period\"],\n",
    "    ai_instructions=prompts.user_prompt,\n",
    ")\n",
    "print(explanation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}