From c07c9bbf7f9f623e4dd42c5e2226a7671fa15323 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Fri, 19 Jan 2024 23:46:01 +0000 Subject: [PATCH 1/9] complements openai miner with cost logging --- neurons/miner.py | 4 +- neurons/miners/openai/miner.py | 84 ++++++++++++++++++++++------------ 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/neurons/miner.py b/neurons/miner.py index a1251f26..14ca7a25 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -127,7 +127,7 @@ async def priority(self, synapse: PromptingSynapse) -> float: ) return prirority - def log_event(self, timing: float, prompt: str, completion: str, system_prompt: str): + def log_event(self, timing: float, prompt: str, completion: str, system_prompt: str, extra_info: dict): step_log = { "epoch_time": timing, # "block": self.last_epoch_block, @@ -140,8 +140,10 @@ def log_event(self, timing: float, prompt: str, completion: str, system_prompt: "incentive": self.metagraph.I[self.uid].item(), "consensus": self.metagraph.C[self.uid].item(), "dividends": self.metagraph.D[self.uid].item(), + **extra_info } + bt.logging.info('Logging event to wandb...', step_log) wandb.log(step_log) diff --git a/neurons/miners/openai/miner.py b/neurons/miners/openai/miner.py index c19044e4..c547938b 100644 --- a/neurons/miners/openai/miner.py +++ b/neurons/miners/openai/miner.py @@ -29,6 +29,8 @@ from langchain_core.output_parsers import StrOutputParser from langchain.chat_models import ChatOpenAI from dotenv import load_dotenv, find_dotenv +from langchain.callbacks import get_openai_callback + class OpenAIMiner(Miner): @@ -66,8 +68,6 @@ def add_args(cls, parser: argparse.ArgumentParser): help="Wandb project to log to.", ) - - def __init__(self, config=None): super().__init__(config=config) @@ -86,6 +86,32 @@ def __init__(self, config=None): ) self.system_prompt = "You are a friendly chatbot who always responds concisely and helpfully. You are honest about things you don't know." + self.accumulated_total_tokens = 0 + self.accumulated_prompt_tokens = 0 + self.accumulated_completion_tokens = 0 + self.accumulated_total_cost = 0 + + def get_cost_logging(self, cb): + bt.logging.info(f"Total Tokens: {cb.total_tokens}") + bt.logging.info(f"Prompt Tokens: {cb.prompt_tokens}") + bt.logging.info(f"Completion Tokens: {cb.completion_tokens}") + bt.logging.info(f"Total Cost (USD): ${cb.total_cost}") + + self.accumulated_total_tokens += cb.total_tokens + self.accumulated_prompt_tokens += cb.prompt_tokens + self.accumulated_completion_tokens += cb.completion_tokens + self.accumulated_total_cost += cb.total_cost + + return { + 'total_tokens': cb.total_tokens, + 'prompt_tokens': cb.prompt_tokens, + 'completion_tokens': cb.completion_tokens, + 'total_cost': cb.total_cost, + 'accumulated_total_tokens': self.accumulated_total_tokens, + 'accumulated_prompt_tokens': self.accumulated_prompt_tokens, + 'accumulated_completion_tokens': self.accumulated_completion_tokens, + 'accumulated_total_cost': self.accumulated_total_cost, + } async def forward( self, synapse: PromptingSynapse @@ -105,35 +131,35 @@ async def forward( """ # TODO(developer): Replace with actual implementation logic. try: - - t0 = time.time() - bt.logging.debug(f"📧 Message received, forwarding synapse: {synapse}") - - prompt = ChatPromptTemplate.from_messages([ - ("system", self.system_prompt), - ("user", "{input}") - ]) - chain = prompt | self.model | StrOutputParser() - - role = synapse.roles[-1] - message = synapse.messages[-1] - - bt.logging.debug(f"💬 Querying openai: {prompt}") - response = chain.invoke( - {"role": role, "input": message} - ) - - synapse.completion = response - synapse_latency = time.time() - t0 - - if self.config.wandb.on: - self.log_event( - timing=synapse_latency, - prompt=message, - completion=response, - system_prompt=self.system_prompt + with get_openai_callback() as cb: + t0 = time.time() + bt.logging.debug(f"📧 Message received, forwarding synapse: {synapse}") + + prompt = ChatPromptTemplate.from_messages([ + ("system", self.system_prompt), + ("user", "{input}") + ]) + chain = prompt | self.model | StrOutputParser() + + role = synapse.roles[-1] + message = synapse.messages[-1] + + bt.logging.debug(f"💬 Querying openai: {prompt}") + response = chain.invoke( + {"role": role, "input": message} ) + synapse.completion = response + synapse_latency = time.time() - t0 + + if self.config.wandb.on: + self.log_event( + timing=synapse_latency, + prompt=message, + completion=response, + system_prompt=self.system_prompt, + extra_info=self.get_cost_logging(cb) + ) bt.logging.debug(f"✅ Served Response: {response}") return synapse From b3af9c4b37d93f59ff18f6f0cf8afe808e58470d Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Fri, 19 Jan 2024 23:46:34 +0000 Subject: [PATCH 2/9] complements date_qa with float weight --- prompting/tasks/date_qa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/tasks/date_qa.py b/prompting/tasks/date_qa.py index 74a27f2d..3896bf09 100644 --- a/prompting/tasks/date_qa.py +++ b/prompting/tasks/date_qa.py @@ -6,7 +6,7 @@ @dataclass class DateQuestionAnsweringTask(Task): reward_definition = [ - dict(name="date", weight=1), + dict(name="date", weight=1.0), ] penalty_definition = [] From d1f505bc8a1bf27a9f04ece768272f961278b8eb Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Sat, 20 Jan 2024 00:02:47 +0000 Subject: [PATCH 3/9] adds cost callback to wiki agent --- neurons/miners/wiki_agent/miner.py | 72 +++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/neurons/miners/wiki_agent/miner.py b/neurons/miners/wiki_agent/miner.py index 3f72fc18..212be457 100644 --- a/neurons/miners/wiki_agent/miner.py +++ b/neurons/miners/wiki_agent/miner.py @@ -33,6 +33,7 @@ from langchain import OpenAI from langchain.agents import Tool, initialize_agent from agent import WikiAgent +from langchain.callbacks import get_openai_callback class WikipediaAgentMiner(Miner): @@ -45,7 +46,7 @@ def add_args(cls, parser: argparse.ArgumentParser): parser.add_argument( "--openai.model_name", type=str, - default="gpt-4", + default="gpt-3.5-turbo-1106", help="OpenAI model to use for completion.", ) @@ -80,9 +81,35 @@ def __init__(self, config=None): self.wandb_run.tags = self.wandb_run.tags + ("wikipedia_agent_miner", ) + (self.config.openai.model_name, ) _ = load_dotenv(find_dotenv()) - - + self.agent = WikiAgent() + self.accumulated_total_tokens = 0 + self.accumulated_prompt_tokens = 0 + self.accumulated_completion_tokens = 0 + self.accumulated_total_cost = 0 + + + def get_cost_logging(self, cb): + bt.logging.info(f"Total Tokens: {cb.total_tokens}") + bt.logging.info(f"Prompt Tokens: {cb.prompt_tokens}") + bt.logging.info(f"Completion Tokens: {cb.completion_tokens}") + bt.logging.info(f"Total Cost (USD): ${cb.total_cost}") + + self.accumulated_total_tokens += cb.total_tokens + self.accumulated_prompt_tokens += cb.prompt_tokens + self.accumulated_completion_tokens += cb.completion_tokens + self.accumulated_total_cost += cb.total_cost + + return { + 'total_tokens': cb.total_tokens, + 'prompt_tokens': cb.prompt_tokens, + 'completion_tokens': cb.completion_tokens, + 'total_cost': cb.total_cost, + 'accumulated_total_tokens': self.accumulated_total_tokens, + 'accumulated_prompt_tokens': self.accumulated_prompt_tokens, + 'accumulated_completion_tokens': self.accumulated_completion_tokens, + 'accumulated_total_cost': self.accumulated_total_cost, + } async def forward( @@ -103,24 +130,27 @@ async def forward( """ # TODO(developer): Replace with actual implementation logic. try: - t0 = time.time() - bt.logging.debug(f"📧 Message received, forwarding synapse: {synapse}") - - message = synapse.messages[-1] - - bt.logging.debug(f"💬 Querying openai and wikipedia: {message}") - - response = self.agent.run(message) - - synapse.completion = response - synapse_latency = time.time() - t0 - - self.log_event( - timing=synapse_latency, - prompt=message, - completion=response, - system_prompt=None - ) + with get_openai_callback() as cb: + t0 = time.time() + bt.logging.debug(f"📧 Message received, forwarding synapse: {synapse}") + + message = synapse.messages[-1] + + bt.logging.debug(f"💬 Querying openai and wikipedia: {message}") + + response = self.agent.run(message) + + synapse.completion = response + synapse_latency = time.time() - t0 + + if self.config.wandb.on: + self.log_event( + timing=synapse_latency, + prompt=message, + completion=response, + system_prompt=self.system_prompt, + extra_info=self.get_cost_logging(cb) + ) bt.logging.debug(f"✅ Served Response: {response}") return synapse From 55f4ebd85a0e94989c892605c116aefcc14431bc Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Sat, 20 Jan 2024 00:02:59 +0000 Subject: [PATCH 4/9] adds wandb to base miner params --- prompting/utils/config.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index b8c0055d..81c4f197 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -191,6 +191,27 @@ def add_miner_args(cls, parser): help="Nucleus sampling parameter, top_p probability mass.", ) + parser.add_argument( + "--wandb.on", + type=bool, + default=False, + help="Enable wandb logging.", + ) + + parser.add_argument( + "--wandb.entity", + type=str, + default="opentensor-dev", + help="Wandb entity to log to.", + ) + + parser.add_argument( + "--wandb.project_name", + type=str, + default="alpha-miners", + help="Wandb project to log to.", + ) + def add_validator_args(cls, parser): """Add validator specific arguments to the parser.""" From cb6a014ea529b9642f71617df25f55a0b1e612b6 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Sat, 20 Jan 2024 00:10:53 +0000 Subject: [PATCH 5/9] drop doubled params from miners --- neurons/miners/openai/miner.py | 23 +---------------------- neurons/miners/wiki_agent/miner.py | 28 +++------------------------- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/neurons/miners/openai/miner.py b/neurons/miners/openai/miner.py index c547938b..dafc3f5c 100644 --- a/neurons/miners/openai/miner.py +++ b/neurons/miners/openai/miner.py @@ -43,31 +43,10 @@ def add_args(cls, parser: argparse.ArgumentParser): parser.add_argument( "--openai.model_name", type=str, - default="gpt-4", + default="gpt-4-1106-preview", help="OpenAI model to use for completion.", ) - parser.add_argument( - "--wandb.on", - type=bool, - default=False, - help="Enable wandb logging.", - ) - - parser.add_argument( - "--wandb.entity", - type=str, - default="<>", - help="Wandb entity to log to.", - ) - - parser.add_argument( - "--wandb.project_name", - type=str, - default="<>", - help="Wandb project to log to.", - ) - def __init__(self, config=None): super().__init__(config=config) diff --git a/neurons/miners/wiki_agent/miner.py b/neurons/miners/wiki_agent/miner.py index 212be457..c7e060c5 100644 --- a/neurons/miners/wiki_agent/miner.py +++ b/neurons/miners/wiki_agent/miner.py @@ -46,31 +46,9 @@ def add_args(cls, parser: argparse.ArgumentParser): parser.add_argument( "--openai.model_name", type=str, - default="gpt-3.5-turbo-1106", + default="gpt-4-1106-preview", help="OpenAI model to use for completion.", - ) - - parser.add_argument( - "--wandb.on", - type=bool, - default=False, - help="Enable wandb logging.", - ) - - parser.add_argument( - "--wandb.entity", - type=str, - default="<>", - help="Wandb entity to log to.", - ) - - parser.add_argument( - "--wandb.project_name", - type=str, - default="<>", - help="Wandb project to log to.", - ) - + ) def __init__(self, config=None): super().__init__(config=config) @@ -148,7 +126,7 @@ async def forward( timing=synapse_latency, prompt=message, completion=response, - system_prompt=self.system_prompt, + system_prompt='', extra_info=self.get_cost_logging(cb) ) From 85df4486d03d5cd8deb0f0d03bac42f01b45cecc Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 22 Jan 2024 14:24:48 +0000 Subject: [PATCH 6/9] fix task distribution for release --- prompting/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 81c4f197..65ea9025 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -242,7 +242,7 @@ def add_validator_args(cls, parser): type=float, nargs="+", help="The probability of sampling each task.", - default=[0.3, 0.3, 0.1, 0.1, 0.2], + default=[0.5, 0.5, 0.0, 0.0, 0.0], ) parser.add_argument( From f4c27891172e2730b2e11f10447f4765f0fce1c5 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 22 Jan 2024 14:36:27 +0000 Subject: [PATCH 7/9] corrects requirements definition on git workflow --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7f86bd1f..6c9a3fd8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,7 +29,7 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest black pip install -e . - pip install -r prompting/requirements.txt + pip install -r requirements.txt - name: Lint with flake8 run: | From 8f5a4b8e33b6960d4ecd2c49756efacbd5a39e49 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 22 Jan 2024 15:41:40 +0000 Subject: [PATCH 8/9] adds tenacity dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 938c46ef..0f5861f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ rouge scipy==1.10.1 sentencepiece wandb==0.15.10 +tenacity \ No newline at end of file From d23461fc05d49444c030a0d6899457642897b48b Mon Sep 17 00:00:00 2001 From: p-ferreira <38992619+p-ferreira@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:30:35 -0500 Subject: [PATCH 9/9] update python-package --- .github/workflows/python-package.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6c9a3fd8..5db44d16 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,8 +28,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 pytest black - pip install -e . - pip install -r requirements.txt + pip install -e . - name: Lint with flake8 run: |