Skip to content

Commit

Permalink
Run Cost Token Tracking notebook with results
Browse files Browse the repository at this point in the history
  • Loading branch information
@marklysze
marklysze committed Nov 19, 2024
1 parent 59ccaa2 commit 25dd5b8
Showing 1 changed file with 90 additions and 52 deletions.
142 changes: 90 additions & 52 deletions notebook/agentchat_cost_token_tracking.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -110,14 +110,14 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.00020600000000000002\n"
"0.0011125\n"
]
}
],
Expand All @@ -139,14 +139,14 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Price: 109\n"
"Price: 0.144\n"
]
}
],
Expand Down Expand Up @@ -177,7 +177,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand All @@ -198,7 +198,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 8,
"metadata": {},
"outputs": [
{
Expand All @@ -207,20 +207,20 @@
"text": [
"----------------------------------------------------------------------------------------------------\n",
"Usage summary excluding cached usage: \n",
"Total cost: 0.00023\n",
"* Model 'gpt-35-turbo': cost: 0.00023, prompt_tokens: 25, completion_tokens: 142, total_tokens: 167\n",
"Total cost: 0.154\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.154, prompt_tokens: 25, completion_tokens: 129, total_tokens: 154\n",
"\n",
"All completions are non-cached: the total cost with cached completions is the same as actual cost.\n",
"----------------------------------------------------------------------------------------------------\n",
"----------------------------------------------------------------------------------------------------\n",
"Usage summary excluding cached usage: \n",
"Total cost: 0.00023\n",
"* Model 'gpt-35-turbo': cost: 0.00023, prompt_tokens: 25, completion_tokens: 142, total_tokens: 167\n",
"Total cost: 0.154\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.154, prompt_tokens: 25, completion_tokens: 129, total_tokens: 154\n",
"----------------------------------------------------------------------------------------------------\n",
"----------------------------------------------------------------------------------------------------\n",
"Usage summary including cached usage: \n",
"Total cost: 0.00023\n",
"* Model 'gpt-35-turbo': cost: 0.00023, prompt_tokens: 25, completion_tokens: 142, total_tokens: 167\n",
"Total cost: 0.154\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.154, prompt_tokens: 25, completion_tokens: 129, total_tokens: 154\n",
"----------------------------------------------------------------------------------------------------\n"
]
}
Expand All @@ -236,15 +236,15 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'total_cost': 0.0002255, 'gpt-35-turbo': {'cost': 0.0002255, 'prompt_tokens': 25, 'completion_tokens': 142, 'total_tokens': 167}}\n",
"{'total_cost': 0.0002255, 'gpt-35-turbo': {'cost': 0.0002255, 'prompt_tokens': 25, 'completion_tokens': 142, 'total_tokens': 167}}\n"
"{'total_cost': 0.154, 'gpt-4o-2024-08-06': {'cost': 0.154, 'prompt_tokens': 25, 'completion_tokens': 129, 'total_tokens': 154}}\n",
"{'total_cost': 0.154, 'gpt-4o-2024-08-06': {'cost': 0.154, 'prompt_tokens': 25, 'completion_tokens': 129, 'total_tokens': 154}}\n"
]
}
],
Expand All @@ -256,7 +256,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 10,
"metadata": {},
"outputs": [
{
Expand All @@ -265,12 +265,12 @@
"text": [
"----------------------------------------------------------------------------------------------------\n",
"Usage summary excluding cached usage: \n",
"Total cost: 0.00023\n",
"* Model 'gpt-35-turbo': cost: 0.00023, prompt_tokens: 25, completion_tokens: 142, total_tokens: 167\n",
"Total cost: 0.154\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.154, prompt_tokens: 25, completion_tokens: 129, total_tokens: 154\n",
"\n",
"Usage summary including cached usage: \n",
"Total cost: 0.00045\n",
"* Model 'gpt-35-turbo': cost: 0.00045, prompt_tokens: 50, completion_tokens: 284, total_tokens: 334\n",
"Total cost: 0.308\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.308, prompt_tokens: 50, completion_tokens: 258, total_tokens: 308\n",
"----------------------------------------------------------------------------------------------------\n"
]
}
Expand All @@ -284,7 +284,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 11,
"metadata": {},
"outputs": [
{
Expand All @@ -303,7 +303,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -314,8 +314,8 @@
"No actual cost incurred (all completions are using cache).\n",
"\n",
"Usage summary including cached usage: \n",
"Total cost: 0.00023\n",
"* Model 'gpt-35-turbo': cost: 0.00023, prompt_tokens: 25, completion_tokens: 142, total_tokens: 167\n",
"Total cost: 0.154\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.154, prompt_tokens: 25, completion_tokens: 129, total_tokens: 154\n",
"----------------------------------------------------------------------------------------------------\n"
]
}
Expand All @@ -340,7 +340,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 13,
"metadata": {},
"outputs": [
{
Expand All @@ -354,30 +354,68 @@
"--------------------------------------------------------------------------------\n",
"\u001b[33massistant\u001b[0m (to ai_user):\n",
"\n",
"To find x, we need to take the cube root of 125. The cube root of a number is the number that, when multiplied by itself three times, gives the original number.\n",
"To solve the equation \\(x^3 = 125\\), you need to find the value of \\(x\\) that makes this equation true. \n",
"\n",
"You can solve for \\(x\\) by taking the cube root of both sides of the equation:\n",
"\n",
"\\[\n",
"x = \\sqrt[3]{125}\n",
"\\]\n",
"\n",
"In this case, the cube root of 125 is 5 since 5 * 5 * 5 = 125. Therefore, x = 5.\n",
"Since \\(125\\) is \\(5^3\\), the cube root of \\(125\\) is \\(5\\). Thus,\n",
"\n",
"\\[\n",
"x = 5\n",
"\\]\n",
"\n",
"Therefore, the solution to the equation \\(x^3 = 125\\) is \\(x = 5\\).\n",
"\n",
"--------------------------------------------------------------------------------\n",
"\u001b[33mai_user\u001b[0m (to assistant):\n",
"\n",
"That's correct! Well done. The value of x is indeed 5, as you correctly found by taking the cube root of 125. Keep up the good work!\n",
"Can you help me solve the equation \\(2x^2 - 8x = 0\\)?\n",
"\n",
"--------------------------------------------------------------------------------\n",
"\u001b[33massistant\u001b[0m (to ai_user):\n",
"\n",
"Thank you! I'm glad I could help. If you have any more questions, feel free to ask!\n",
"Certainly! To solve the equation \\(2x^2 - 8x = 0\\), you can start by factoring the expression on the left-hand side.\n",
"\n",
"First, factor out the greatest common factor, which is \\(2x\\):\n",
"\n",
"\\[\n",
"2x(x - 4) = 0\n",
"\\]\n",
"\n",
"Now, you have a product of two factors equal to zero. According to the zero product property, if the product of two factors is zero, at least one of the factors must be zero. So, you set each factor equal to zero and solve for \\(x\\):\n",
"\n",
"1. \\(2x = 0\\)\n",
"\n",
" Divide both sides by 2 to solve for \\(x\\):\n",
"\n",
" \\[\n",
" x = 0\n",
" \\]\n",
"\n",
"2. \\(x - 4 = 0\\)\n",
"\n",
" Add 4 to both sides to solve for \\(x\\):\n",
"\n",
" \\[\n",
" x = 4\n",
" \\]\n",
"\n",
"So, the solutions to the equation \\(2x^2 - 8x = 0\\) are \\(x = 0\\) and \\(x = 4\\).\n",
"\n",
"--------------------------------------------------------------------------------\n"
]
},
{
"data": {
"text/plain": [
"ChatResult(chat_id=None, chat_history=[{'content': '$x^3=125$. What is x?', 'role': 'assistant'}, {'content': 'To find x, we need to take the cube root of 125. The cube root of a number is the number that, when multiplied by itself three times, gives the original number.\\n\\nIn this case, the cube root of 125 is 5 since 5 * 5 * 5 = 125. Therefore, x = 5.', 'role': 'user'}, {'content': \"That's correct! Well done. The value of x is indeed 5, as you correctly found by taking the cube root of 125. Keep up the good work!\", 'role': 'assistant'}, {'content': \"Thank you! I'm glad I could help. If you have any more questions, feel free to ask!\", 'role': 'user'}], summary=\"Thank you! I'm glad I could help. If you have any more questions, feel free to ask!\", cost={'usage_including_cached_inference': {'total_cost': 0.000333, 'gpt-35-turbo': {'cost': 0.000333, 'prompt_tokens': 282, 'completion_tokens': 128, 'total_tokens': 410}}, 'usage_excluding_cached_inference': {'total_cost': 0.000333, 'gpt-35-turbo': {'cost': 0.000333, 'prompt_tokens': 282, 'completion_tokens': 128, 'total_tokens': 410}}}, human_input=[])"
"ChatResult(chat_id=None, chat_history=[{'content': '$x^3=125$. What is x?', 'role': 'assistant', 'name': 'ai_user'}, {'content': 'To solve the equation \\\\(x^3 = 125\\\\), you need to find the value of \\\\(x\\\\) that makes this equation true. \\n\\nYou can solve for \\\\(x\\\\) by taking the cube root of both sides of the equation:\\n\\n\\\\[\\nx = \\\\sqrt[3]{125}\\n\\\\]\\n\\nSince \\\\(125\\\\) is \\\\(5^3\\\\), the cube root of \\\\(125\\\\) is \\\\(5\\\\). Thus,\\n\\n\\\\[\\nx = 5\\n\\\\]\\n\\nTherefore, the solution to the equation \\\\(x^3 = 125\\\\) is \\\\(x = 5\\\\).', 'role': 'user', 'name': 'assistant'}, {'content': 'Can you help me solve the equation \\\\(2x^2 - 8x = 0\\\\)?', 'role': 'assistant', 'name': 'ai_user'}, {'content': 'Certainly! To solve the equation \\\\(2x^2 - 8x = 0\\\\), you can start by factoring the expression on the left-hand side.\\n\\nFirst, factor out the greatest common factor, which is \\\\(2x\\\\):\\n\\n\\\\[\\n2x(x - 4) = 0\\n\\\\]\\n\\nNow, you have a product of two factors equal to zero. According to the zero product property, if the product of two factors is zero, at least one of the factors must be zero. So, you set each factor equal to zero and solve for \\\\(x\\\\):\\n\\n1. \\\\(2x = 0\\\\)\\n\\n Divide both sides by 2 to solve for \\\\(x\\\\):\\n\\n \\\\[\\n x = 0\\n \\\\]\\n\\n2. \\\\(x - 4 = 0\\\\)\\n\\n Add 4 to both sides to solve for \\\\(x\\\\):\\n\\n \\\\[\\n x = 4\\n \\\\]\\n\\nSo, the solutions to the equation \\\\(2x^2 - 8x = 0\\\\) are \\\\(x = 0\\\\) and \\\\(x = 4\\\\).', 'role': 'user', 'name': 'assistant'}], summary='Certainly! To solve the equation \\\\(2x^2 - 8x = 0\\\\), you can start by factoring the expression on the left-hand side.\\n\\nFirst, factor out the greatest common factor, which is \\\\(2x\\\\):\\n\\n\\\\[\\n2x(x - 4) = 0\\n\\\\]\\n\\nNow, you have a product of two factors equal to zero. According to the zero product property, if the product of two factors is zero, at least one of the factors must be zero. So, you set each factor equal to zero and solve for \\\\(x\\\\):\\n\\n1. \\\\(2x = 0\\\\)\\n\\n Divide both sides by 2 to solve for \\\\(x\\\\):\\n\\n \\\\[\\n x = 0\\n \\\\]\\n\\n2. \\\\(x - 4 = 0\\\\)\\n\\n Add 4 to both sides to solve for \\\\(x\\\\):\\n\\n \\\\[\\n x = 4\\n \\\\]\\n\\nSo, the solutions to the equation \\\\(2x^2 - 8x = 0\\\\) are \\\\(x = 0\\\\) and \\\\(x = 4\\\\).', cost={'usage_including_cached_inference': {'total_cost': 0.7649999999999999, 'gpt-4o-2024-08-06': {'cost': 0.7649999999999999, 'prompt_tokens': 390, 'completion_tokens': 375, 'total_tokens': 765}}, 'usage_excluding_cached_inference': {'total_cost': 0.7649999999999999, 'gpt-4o-2024-08-06': {'cost': 0.7649999999999999, 'prompt_tokens': 390, 'completion_tokens': 375, 'total_tokens': 765}}}, human_input=[])"
]
},
"execution_count": 9,
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -415,7 +453,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand All @@ -425,17 +463,17 @@
"Agent 'ai_user':\n",
"----------------------------------------------------------------------------------------------------\n",
"Usage summary excluding cached usage: \n",
"Total cost: 0.00011\n",
"* Model 'gpt-35-turbo': cost: 0.00011, prompt_tokens: 114, completion_tokens: 35, total_tokens: 149\n",
"Total cost: 0.193\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.193, prompt_tokens: 172, completion_tokens: 21, total_tokens: 193\n",
"\n",
"All completions are non-cached: the total cost with cached completions is the same as actual cost.\n",
"----------------------------------------------------------------------------------------------------\n",
"\n",
"Agent 'assistant':\n",
"----------------------------------------------------------------------------------------------------\n",
"Usage summary excluding cached usage: \n",
"Total cost: 0.00022\n",
"* Model 'gpt-35-turbo': cost: 0.00022, prompt_tokens: 168, completion_tokens: 93, total_tokens: 261\n",
"Total cost: 0.572\n",
"* Model 'gpt-4o-2024-08-06': cost: 0.572, prompt_tokens: 218, completion_tokens: 354, total_tokens: 572\n",
"\n",
"All completions are non-cached: the total cost with cached completions is the same as actual cost.\n",
"----------------------------------------------------------------------------------------------------\n"
Expand All @@ -450,7 +488,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 15,
"metadata": {},
"outputs": [
{
Expand All @@ -474,17 +512,17 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Actual usage summary for assistant (excluding completion from cache): {'total_cost': 0.0002235, 'gpt-35-turbo': {'cost': 0.0002235, 'prompt_tokens': 168, 'completion_tokens': 93, 'total_tokens': 261}}\n",
"Total usage summary for assistant (including completion from cache): {'total_cost': 0.0002235, 'gpt-35-turbo': {'cost': 0.0002235, 'prompt_tokens': 168, 'completion_tokens': 93, 'total_tokens': 261}}\n",
"Actual usage summary for ai_user_proxy: {'total_cost': 0.0001095, 'gpt-35-turbo': {'cost': 0.0001095, 'prompt_tokens': 114, 'completion_tokens': 35, 'total_tokens': 149}}\n",
"Total usage summary for ai_user_proxy: {'total_cost': 0.0001095, 'gpt-35-turbo': {'cost': 0.0001095, 'prompt_tokens': 114, 'completion_tokens': 35, 'total_tokens': 149}}\n",
"Actual usage summary for assistant (excluding completion from cache): {'total_cost': 0.572, 'gpt-4o-2024-08-06': {'cost': 0.572, 'prompt_tokens': 218, 'completion_tokens': 354, 'total_tokens': 572}}\n",
"Total usage summary for assistant (including completion from cache): {'total_cost': 0.572, 'gpt-4o-2024-08-06': {'cost': 0.572, 'prompt_tokens': 218, 'completion_tokens': 354, 'total_tokens': 572}}\n",
"Actual usage summary for ai_user_proxy: {'total_cost': 0.193, 'gpt-4o-2024-08-06': {'cost': 0.193, 'prompt_tokens': 172, 'completion_tokens': 21, 'total_tokens': 193}}\n",
"Total usage summary for ai_user_proxy: {'total_cost': 0.193, 'gpt-4o-2024-08-06': {'cost': 0.193, 'prompt_tokens': 172, 'completion_tokens': 21, 'total_tokens': 193}}\n",
"Actual usage summary for user_proxy: None\n",
"Total usage summary for user_proxy: None\n"
]
Expand All @@ -503,20 +541,20 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'total_cost': 0.000333,\n",
" 'gpt-35-turbo': {'cost': 0.000333,\n",
" 'prompt_tokens': 282,\n",
" 'completion_tokens': 128,\n",
" 'total_tokens': 410}}"
"{'total_cost': 0.7649999999999999,\n",
" 'gpt-4o-2024-08-06': {'cost': 0.7649999999999999,\n",
" 'prompt_tokens': 390,\n",
" 'completion_tokens': 375,\n",
" 'total_tokens': 765}}"
]
},
"execution_count": 13,
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -535,7 +573,7 @@
]
},
"kernelspec": {
"display_name": "msft",
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
Expand All @@ -549,7 +587,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.19"
"version": "3.11.10"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 25dd5b8

Please sign in to comment.