diff --git a/commune/agent/agent.py b/commune/agent/agent.py deleted file mode 100644 index 6a0a5898..00000000 --- a/commune/agent/agent.py +++ /dev/null @@ -1,76 +0,0 @@ - -import commune as c -import os - -class Agent(c.Module): - - prompt = """ - GIVEN THE FOLLOWING QUERY - ---OBJECTIVE--- - {objective} - ---USER--- - {text} - --- CODE --- - {code} - --- NEW CODE --- - """ - def __init__(self, - model='model.openrouter', - objective='YOU ARE A CODER THAT IS FEARLESS AND CAN SOLVE ANY PROBLEM THE QUERY IS AS FOLLOW, RESPOND IN THE FULL CODE PLEASE AND NOTHING ELSE, COMMENT IF YOU WANT TO ADD ANYTHING ELSE.'): - self.model = c.module(model)() - self.objective = objective - - def forward(self, - text , - file=None , - trials=1, - code = None, - stream=False, - objective=None, - ): - - """ - params: - text: str: the text that you want to generate code from - file: str: the file that you want to append the code to - trials: int: the number of trials to run - code: str: the code that you want to improve - stream: bool: stream the output - - """ - if trials > 1: - for trial in range(trials): - c.print(f"Trial {trial}") - code = self.forward(text=text, - file=file, - code=code, - stream=stream - ) - return code - if file != None and code == None: - code = self.read_code(file) - objective = objective or self.objective - text = self.prompt.format(text=text, code=code, file=file, objective=objective) - code = self.model.generate(text, stream=stream) - if file : - self.write_code(file, code, replace=True) - return code - - def write_code(self, file, code, replace=True): - # if this is a generator - if os.path.exists(file): - os.remove(file) - if c.is_generator(code): - for i, code in enumerate(code): - with open(file, 'a') as f: - f.write(code) - else: - with open(file, 'a') as f: - f.write(code) - - def read_code(self, file): - if not os.path.exists(file): - return None - with open(file, 'r') as f: - code = f.read() - return code diff --git a/commune/agent/app.py b/commune/agent/app.py deleted file mode 100644 index e6c5f3e1..00000000 --- a/commune/agent/app.py +++ /dev/null @@ -1,99 +0,0 @@ -import commune as c -import streamlit as st -import os - -class App(c.Module): - - def title(self): - # Change the title of the app to 'Cyberbunk City' - st.markdown('# Cyberbunk City') - - def app(self): - self.title() - self.agent = c.module('agent')() - - # Define the CSS for different buttons with 'cyberbunk' vibes - st.markdown(""" - - """, unsafe_allow_html=True) - - - resolve_path = lambda p: os.path.abspath(os.path.expanduser(p)) - code = None - - - og_code_col, model_code_col = st.columns(2) - - cols = st.columns([2,5]) - folder_path = './' - folder_path = cols[0].text_input('Folder Path', resolve_path(folder_path)) - folder_path = resolve_path(folder_path) - python_files = [f for f in c.glob(folder_path) if f.endswith('.py')] - num_files = len(python_files) - filepath = cols[1].selectbox(f'Select File (n={num_files})', python_files) - with st.expander(f'Code'): - code = c.get_text(filepath) - code = st.text_area('Code', code, height=400) - - input = st.text_area('Input') - - # Use columns to span the page - col1, col2 = st.columns(2) - - send_button = st.button('Transmit', key='send', use_container_width=True) - st.markdown('
', unsafe_allow_html=True) - - - if send_button: - kwargs = {'text': input, 'code': code, 'file': filepath} - tx_id = c.hash(kwargs) - st.write('Transaction ID:', tx_id) - history_path = self.resolve_path(f'history/{self.key.ss58_address}') - - - content = self.get(history_path, {}) - if 'data' not in content: - response = self.agent.forward(**kwargs) - def generator(response): - response = self.agent.forward(input, code=code, stream=1) - - content['data'] = '' - for r in response: - content['data'] += r - yield r - st.write_stream(generator(response)) - self.put(history_path, content) - - response = content['data'] - - - with st.expander('Save Response'): - - response = response.split('```python')[-1].split('```').pop(0) - st.write(response) - - save_filepath = st.text_input('Save File Path', filepath) - save_button = st.button('Save', key='save', use_container_width=True) - - if save_button: - c.put_text(save_filepath, code) - st.write('Saved to', filepath) - - def process_response(self, code): - return code.split('```python')[-1].split('```').pop(0) - - -App.run(__name__) \ No newline at end of file diff --git a/commune/agent/child.py b/commune/agent/child.py deleted file mode 100644 index 25b9e111..00000000 --- a/commune/agent/child.py +++ /dev/null @@ -1,80 +0,0 @@ -import commune as c -import os - - -class Agent(c.Module): - - prompt = """ - GIVEN THE FOLLOWING QUERY - YOU ARE A CODER THAT IS FEARLESS AND CAN SOLVE ANY PROBLEM - THE QUERY IS AS FOLLOWS - - ---START OF QUERY--- - {text} - -- END OF QUERY - - THIS IS YOUR CURRENT CODEBASE THAT YOU CAN IMPROVE PROVIDED BELOW - --- START OF FILE ({file}) --- - {code} - --- END OF FILE ({file})--- - - RESPOND IN THE FULL CODE PLEASE AND NOTHING ELSE, COMMENT IF YOU WANT TO ADD ANYTHING ELSE. - """ - def __init__(self, model='model.openrouter'): - self.model = c.module(model)() - - def forward(self, - text , - file=None , - trials=1, - code = None, - stream=False - ): - - """ - params: - text: str: the text that you want to generate code from - file: str: the file that you want to append the code to - trials: int: the number of trials to run - code: str: the code that you want to improve - stream: bool: stream the output - - """ - if trials > 1: - for trial in range(trials): - c.print(f"Trial {trial}") - code = self.forward(text=text, - file=file, - code=code, - stream=stream - ) - return code - if file != None and code == None: - code = self.read_code(file) - text = self.prompt.format(text=text, code=code, file=file) - code = self.model.generate(text, stream=stream) - if file : - self.write_code(file, code, replace=True) - return code - - def write_code(self, file, code, replace=True): - # if this is agenerator - if os.path.exists(file): - os.remove(file) - if c.is_generator(code): - for i, code in enumerate(code): - with open(file, 'a') as f: - f.write(code) - else: - - with open(file, 'a') as f: - f.write(code) - - def read_code(self, file): - if not os.path.exists(file): - return None - with open(file, 'r') as f: - code = f.read() - return code - - \ No newline at end of file diff --git a/commune/agent/data/agent_data.py b/commune/agent/data/agent_data.py deleted file mode 100644 index 83205e65..00000000 --- a/commune/agent/data/agent_data.py +++ /dev/null @@ -1,24 +0,0 @@ -import commune as c -import json -class Demo(c.Module): - instruciton = """ - - - - """ - def __init__(self, a=1, b=2): - self.set_config(kwargs=locals()) - - def call(self, timeout=30) -> int: - model = c.connect('model.openai') # connect to the model - - input = json.dumps({ - 'instruction': self.instruction, - 'response': None, - }) - - # get the docs - - return model.generate(input, timeout=timeout) - - \ No newline at end of file diff --git a/commune/agent/factory/agent_factory.py b/commune/agent/factory/agent_factory.py deleted file mode 100644 index 5ad8e026..00000000 --- a/commune/agent/factory/agent_factory.py +++ /dev/null @@ -1,11 +0,0 @@ -import commune as c - -class AgentFactory(c.Module): - def __init__(self, a=1, b=2): - self.set_config(kwargs=locals()) - - def call(self, x:int = 1, y:int = 2) -> int: - c.print(self.config) - c.print(self.config, 'This is the config, it is a Munch object') - return x + y - \ No newline at end of file diff --git a/commune/agent/judge.py b/commune/agent/judge.py deleted file mode 100644 index 93557ab0..00000000 --- a/commune/agent/judge.py +++ /dev/null @@ -1,28 +0,0 @@ - - -import commune as c -class Judge(c.Module): - def __init__(self, model='model.openai'): - self.model = c.module(model)() - - def forward(self, text: str = "was the moon landing fake?") -> str: - - prompt = { - "text": text, - 'question': 'Yay or nay? (Y/N)', - } - return self.model.forward(prompt) - - - def test(self , text: str = "was the moon landing fake?"): - return self.forward(text) - - -if __name__ == "__main__": - - Judge.run() - - - - - diff --git a/commune/agent/maker/agent_maker.py b/commune/agent/maker/agent_maker.py deleted file mode 100644 index a34a7ed9..00000000 --- a/commune/agent/maker/agent_maker.py +++ /dev/null @@ -1,27 +0,0 @@ -import commune as c -import json -class Demo(c.Module): - instruction = "Fill in the template for a gpt agent." - - example = " Make a gpt that can do math." - - template = { - "name": "math", - "description": "A demo agent.", - "prompt": "Make a gpt that can do math.", - - } - def __init__(self, a=1, b=2): - self.set_config(kwargs=locals()) - - def call(self, description) -> int: - x = json.dumps({ - 'instructions': self.instruction, - 'description': description, - 'template': self.template, - 'output_template': "FILL IN THE TEMPLATE", - }) - - - return c.call("model.openai/generate", x) - \ No newline at end of file diff --git a/commune/agent/sumarizer/agent_sumarizer.py b/commune/agent/sumarizer/agent_sumarizer.py deleted file mode 100644 index 0775fe19..00000000 --- a/commune/agent/sumarizer/agent_sumarizer.py +++ /dev/null @@ -1,193 +0,0 @@ -import commune as c -import json - - - - -class Agent(c.Module): - description = """ - Summarize the following content into a more concice representation. - Preferably I want it in a succint knowledge graph - (HEAD, RELATIONSHIP, TAIL) - """ - - tools = [] - - def __init__(self, - name='agent', - description : str = None, - model : str = 'model.openai', - network : str = 'local', - tools:list = tools - ): - self.name = name - self.description = description if description != None else self.description - self.set_model(model, network=network) - self.set_tools(tools) - - - def set_model(self, model:str = 'model.openai ', network:str = 'local'): - self.model_namespace = c.namespace(search=model, netowrk=network) - assert len(self.model_namespace) > 0, f"no models found in {model}, please check the model path" - self.model_addresses = list(self.model_namespace.values()) - self.model_names = list(self.model_namespace.keys()) - self.network = network - self.model = c.connect(c.choice(self.model_addresses)) - return {"success": True, "message": f"set model to {self.model}"} - - - def rm_tools(self, tools:list = None): - if tools == None: - self.tools = {} - else: - for t in tools: - self.rm_tool(t) - return self.tools - - - def resolve_tools(self, tools): - - if isinstance(tools, str): - tools = [tools] - if isinstance(tools, list): - tools = self.get_tools(tools) - if tools == None: - tools = self.tools - - return tools - - - - - - def call(self, - text:str, - model=None, - history=None, - tools=tools, - n = 1, - description:str = None) -> str: - - - - if model != None: - self.model = c.connect(model) - tools = self.resolve_tools(tools) - history = history or [] - description = self.description if description == None else description - - for i in range(n): - prompt = { - 'step': i, - 'max_steps': n, - 'description': description, - 'input': text, - 'history': history, - 'tools': tools, - 'purpose': """ ANSWER THE FOLLOWING""", - 'confidence': 0, - 'call_tool': {'tool': None, 'kwargs': None}, - 'answer': None - } - output = self.model.generate(json.dumps(prompt), max_tokens=512) - c.print(output) - output = json.loads(output) - prompt.update(output) - if 'call_tool' in output: - tool = output['call_tool']['tool'] - kwargs = output['call_tool']['kwargs'] - if kwargs == None: - kwargs = {} - if tool != None: - module = '.'.join(tool.split('.')[:-1]) - fn = tool.split('.')[-1] - module = c.module(module) - fn_type = module.classify_fn(fn) - if fn_type == "self": - module = module() - try: - response = getattr(module, fn)(**kwargs) - except Exception as e: - response = c.detailed_error(e) - - - output['call_tool']['response'] = response - history.append(output['call_tool']) - return output - # prompt tooling - generate = call - - @classmethod - def find_tools(cls, prompt:str): - raise NotImplementedError - - @classmethod - def prompt2agent(cls, prompt:str) -> 'Agent': - cls.find_tools(prompt, topk=5) - - - - - - - - def set_tools(self, tools:list): - self.tools = {} - self.add_tools(tools) - return self.tools - - def add_tools(self, tools:list): - for t in tools: - self.add_tool(t) - return self.tools - - def get_tool(self, tool:str, fn_seperator:str = '.'): - module = fn_seperator.join(tool.split(fn_seperator)[:1]) - fn = tool.split(fn_seperator)[1] - module = c.module(module) - tool_info = module.fn_schema(fn, docs=True) - return tool_info - - - def get_tools(self, tools:list, fn_seperator:str = '.'): - return {t: self.get_tool(t, fn_seperator=fn_seperator) for t in tools} - - def add_tool(self, tool:str): - schema = self.schema(tool) - self.tools[tool] = schema - return self.tools - - def rm_tool(self, tool:str): - del self.tools[tool] - return self.tools - - - - def test_model(self, prompt:str, model=None, history=None, **kwargs): - if model != None: - self.model = c.connect(model) - - prompt = { - 'description': self.description, - 'prompt': prompt, - 'history': history, - 'response': None, - 'instruction': 'complete response' - } - - output = self.model.generate(json.dumps(prompt)) - - prompt.update(json.loads(self.model.generate(json.dumps(prompt)))) - return prompt - - def test(self, prompt:str='hey', model=None, history=None, **kwargs): - response = self.call(prompt, model=model, history=history, **kwargs) - - assert 'response' in response, f"response not in {response}" - assert isinstance(response['response'], str), f"response is not a string: {response['response']}" - return { - 'prompt': prompt, - 'response': response['response'], - 'success': True, - } - diff --git a/commune/api/api.py b/commune/api/api.py deleted file mode 100644 index e0ca646d..00000000 --- a/commune/api/api.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import commune as c -import streamlit as st - -class ApiManager(c.Module): - def __init__(self, path='api_vault', password=None, api_keys=[]): - self.path = self.resolve_path(path) - self.password = password - - @property - def api_keys(self): - return self.get(self.path, {}) - - def add_api_key(self, name , api_key): - api_keys = self.api_keys - api_keys[name] = list(set(api_keys.get(name, []) + [api_key])) - num_keys = len(api_keys[name]) - assert isinstance(api_keys, dict), api_keys - self.save_api_keys(api_keys) - return {'msg': f'api_key {name} added', 'num_keys': num_keys} - - def remove_api_key(self, name, api_key): - api_keys = self.api_keys - assert api_key in api_keys[name], f"api_key {name} does not exist" - api_keys[name].remove(api_key) - self.save_api_keys(api_keys) - return {'msg': f'api_key {name} removed', 'num_keys': len(api_keys[name])} - - def pop_api_key(self, name, index=-1, api_key=None): - api_keys = self.api_keys - api_keys = api_keys.get(name, []) - if len(api_key) == 0: - raise ValueError(f"api_key {name} does not exist") - api_key.pop(index) - - self.save_api_keys(api_keys) - - def get_api_keys(self, name): - return self.api_keys.get(name, []) - - def get_api_key(self, name): - return c.choice(self.get_api_keys(name)) - - def save_api_keys(self, api_keys): - self.put(self.path, api_keys) - - @property - def api_names(self): - return list(self.api_keys.keys()) - - - -ApiManager.run(__name__) \ No newline at end of file diff --git a/commune/api/app.py b/commune/api/app.py deleted file mode 100644 index 84c69ed5..00000000 --- a/commune/api/app.py +++ /dev/null @@ -1,50 +0,0 @@ - -import streamlit as st -import commune as c -from commune.api.api import ApiManager -class App(c.Module): - def app(self): - - st.title("API Key Manager") - - # Create an instance of ApiManager - api_manager = ApiManager() - api_keys = api_manager.api_keys - with st.expander("View API"): - refresh = st.button("Refresh") - st.write(api_keys) - - - # Sidebar menu - menu = ["Add API", "Remove API"] - api_names = list(api_keys.keys()) - choice = st.selectbox("Select an option", menu) - - if choice == "Add API": - new_api_check = st.checkbox("New API") - if new_api_check: - name = st.text_input("Name") - else: - name = st.selectbox("Select API Name", api_names) - api_key = st.text_input("API Key") - if st.button("Add"): - result = api_manager.add_api_key(name, api_key) - st.success(result['msg']) - st.info(f"Number of keys for {name}: {result['num_keys']}") - elif choice == "Remove API": - st.subheader("Remove API") - name = st.selectbox("Select API Name", api_names) - selected_rm_keys = st.multiselect("Select API Keys to remove", api_keys.get(name, [])) - if st.button("Pop"): - try: - for key in selected_rm_keys: - st.success(api_manager.remove_api_key(name, key)) - except ValueError as e: - st.error(str(e)) - - - - - -if __name__ == '__main__': - App.run() \ No newline at end of file diff --git a/commune/base/base.py b/commune/base/base.py deleted file mode 100644 index 1b0418ee..00000000 --- a/commune/base/base.py +++ /dev/null @@ -1,11 +0,0 @@ -import commune as c - -class Demo(c.Module): - def __init__(self, a=1, b=2): - self.set_config(kwargs=locals()) - - def call(self, x:int = 1, y:int = 2) -> int: - c.print(self.config) - c.print(self.config, 'This is the config, it is a Munch object') - return x + y - \ No newline at end of file diff --git a/commune/base/file_module.py b/commune/base/file_module.py deleted file mode 100644 index f97980bb..00000000 --- a/commune/base/file_module.py +++ /dev/null @@ -1,6 +0,0 @@ -import commune as c - -class MyModule(c.Module): - - def __init__(self, **kwargs): - self.init_module(kwargs) \ No newline at end of file diff --git a/commune/cli.py b/commune/cli.py index dd3663a3..4c3e7771 100644 --- a/commune/cli.py +++ b/commune/cli.py @@ -35,23 +35,14 @@ def __init__(self, self.forward(args) - - def forward(self, argv=None): + t0 = time.time() argv = argv or self.argv() self.input_msg = 'c ' + ' '.join(argv) output = None - """ - the cli works as follows - c {module}/{fn} arg1 arg2 arg3 ... argn - if you are calling a function ont he module function (the root module), it is not necessary to specify the module - c {fn} arg1 arg2 arg3 ... argn - """ - # any of the --flags are init kwargs - - init_kwargs = {} + init_kwargs = {} if any([arg.startswith('--') for arg in argv]): for arg in c.copy(argv): if arg.startswith('--'): @@ -62,17 +53,22 @@ def forward(self, argv=None): new_argvs = [key , new_argvs[0]] return self.forward(new_argvs) argv.remove(arg) + if '=' not in arg: + value = True value = arg.split('=')[1] init_kwargs[key] = self.determine_type(value) - + + # any of the --flags are init kwargs if argv[0].endswith('.py'): argv[0] = argv[0][:-3] + if ':' in argv[0]: # {module}:{fn} arg1 arg2 arg3 ... argn argv[0] = argv[0].replace(':', '/') - + if '/' in argv[0]: + # prioritize the module over the function module = '.'.join(argv[0].split('/')[:-1]) fn = argv[0].split('/')[-1] argv = [module , fn , *argv[1:]] @@ -86,24 +82,13 @@ def forward(self, argv=None): else: module = argv.pop(0) fn = argv.pop(0) - + if isinstance(module, str): - try: - module = c.get_module(module) - except Exception as e: - c.print(f'Error: {e}', color='red') - return None - - + module = c.get_module(module) # module = self.base_module.from_object(module) module_name = module.module_name() fn_path = f'{module_name}/{fn}' - try: - fn_obj = getattr(module, fn) - except Exception as e: - - fn_obj =getattr(module(), fn) - + fn_obj = getattr(module, fn) fn_type = c.classify_fn(fn_obj) if fn_type == 'self' or len(init_kwargs) > 0: fn_obj = getattr(module(**init_kwargs), fn) @@ -122,16 +107,16 @@ def forward(self, argv=None): buffer = '⚡️'*4 c.print(buffer+input_msg+buffer, color='yellow') output = output() - latency = time.time() - t0 - is_error = c.is_error(output) + if is_error: buffer = '❌' - msg = f'Error(latency= {latency:.3f})' + msg = f'Error(latency={latency:.3f})' else: buffer = '✅' msg = f'Result(latency={latency:.3f})' + print(buffer + msg + buffer) num_spacers = max(0, len(self.input_msg) - len(msg) ) @@ -139,7 +124,6 @@ def forward(self, argv=None): right_spacers = num_spacers - left_spacers msg = self.seperator*left_spacers + msg + self.seperator*right_spacers buffer = self.buffer_size * buffer - result_header = f'{buffer}{msg}{buffer}' is_generator = c.is_generator(output) if is_generator: @@ -149,10 +133,9 @@ def forward(self, argv=None): c.print(item) else: c.print(item, end='') - else: - - c.print( output) + c.print(output) + return output # c.print( f'Result ✅ (latency={self.latency:.2f}) seconds ✅') diff --git a/commune/module/_misc.py b/commune/module/_misc.py index 8dbdae1f..d94c0d43 100644 --- a/commune/module/_misc.py +++ b/commune/module/_misc.py @@ -941,15 +941,6 @@ def root_key_address(cls) -> str: def is_root_key(cls, address:str)-> str: return address == cls.root_key().ss58_address - - @staticmethod - def repo2module( repo, module = None): - if module == None: - module = os.path.basename(repo).replace('.git','').replace(' ','_').replace('-','_').lower() - - cls.new_module(module=module, repo=repo) - return {'module':module, 'repo':repo, 'status':'success'} - # time within the context @classmethod def context_timer(cls, *args, **kwargs): diff --git a/commune/module/module.py b/commune/module/module.py index 780fc5a6..d499173c 100755 --- a/commune/module/module.py +++ b/commune/module/module.py @@ -6,7 +6,6 @@ import nest_asyncio nest_asyncio.apply() -# YOU CAN DEFINE YOUR CORE MODULES BY JUST adding a class to the core folder # for instance if you have a class called 'os_fam' the file would be ./commune/module/_os_fam.py def get_core_modules(prefix = 'commune.module', core_prefix = '_'): """ @@ -29,7 +28,7 @@ def get_core_modules(prefix = 'commune.module', core_prefix = '_'): CORE_MODULES = get_core_modules() class c(*CORE_MODULES): - core_modules = ['module', 'key', 'client', 'server', 'serializer', ] + core_modules = ['module', 'key', 'client', 'server', 'serializer'] libname = lib_name = lib = 'commune' # the name of the library cost = 1 description = """This is a module""" @@ -403,7 +402,6 @@ def version(cls, lib:str=libname): def forward(self, a=1, b=2): return a+b - ### DICT LAND ### def to_dict(self)-> Dict: @@ -639,7 +637,17 @@ def local_config(cls, filename_options = ['module', 'commune', 'config', 'cfg'], cls._local_config = local_config return cls._local_config - + @classmethod + def local_module(cls, filename_options = ['module', 'agent', 'block'], cache=True): + for filename in filename_options: + path = os.path.dirname(f'./{filename}.py') + for filename in filename_options: + if os.path.exists(path): + classes = cls.find_classes(path) + if len(classes) > 0: + return classes[-1] + return None + # local update @classmethod def update(cls, @@ -668,7 +676,6 @@ def resolve_keypath(cls, key = None): if key == None: key = cls.module_name() return key - def sign(self, data:dict = None, key: str = None, **kwargs) -> bool: key = self.resolve_key(key) @@ -680,6 +687,12 @@ def logs(self, name:str = None, verbose: bool = False): def hardware(self, *args, **kwargs): return c.obj('commune.utils.os.hardware')(*args, **kwargs) + + def set_params(self,*args, **kwargs): + return self.set_config(*args, **kwargs) + + def init_module(self,*args, **kwargs): + return self.set_config(*args, **kwargs) c.enable_routes() Module = c # Module is alias of c diff --git a/commune/modules/peer.py b/commune/modules/peer.py deleted file mode 100644 index 79c36f5d..00000000 --- a/commune/modules/peer.py +++ /dev/null @@ -1,1000 +0,0 @@ -import commune as c -import streamlit as st -from typing import * -import json - -class Peer(c.Module): - @classmethod - def add_host(cls, - cmd:str = None , # in the format of - host:str = '0.0.0.0', - port:int = 22, - user:str = 'root', - pwd:str = None, - name : str = None - ): - - hosts = cls.hosts() - host = { - 'host': host, - 'port': port, - 'user': user, - 'pwd': pwd - } - if name == None: - cnt = 0 - name = f'{user}{cnt}' - - while name in hosts: - name = f'{user}{cnt}' - cnt += 1 - - hosts[name] = host - cls.save_hosts(hosts) - - return {'status': 'success', '': f'Host added', } - - @classmethod - def save_hosts(cls, hosts=None, filetype=filetype, path = None): - if path == None: - path = cls.host_data_path - if hosts == None: - hosts = cls.hosts() - if filetype == 'json': - cls.put_json(path, hosts) - elif filetype == 'yaml': - cls.put_yaml(path, hosts) - - return { - 'status': 'success', - 'msg': f'Hosts saved', - 'hosts': hosts, - 'path': cls.host_data_path, - 'filetype': filetype - } - @classmethod - def load_hosts(cls, path = None, filetype=filetype): - if path == None: - path = cls.host_data_path - if filetype == 'json': - return cls.get_json(path, {}) - elif filetype == 'yaml': - return cls.get_yaml(path, {}) - - @classmethod - def switch_hosts(cls, path): - hosts = c.get_json(path) - cls.save_hosts(hosts) - return {'status': 'success', 'msg': f'Host data path switched to {path}'} - - @classmethod - def rm_host(cls, name): - hosts = cls.hosts() - if name in hosts: - del hosts[name] - cls.save_hosts( hosts) - c.print(cls.hosts()) - return {'status': 'success', 'msg': f'Host {name} removed'} - else: - return {'status': 'error', 'msg': f'Host {name} not found'} - - @classmethod - def hosts(cls, search=None, filetype=filetype, enable_search_terms: bool = True): - hosts = cls.load_hosts(filetype=filetype) - if len(hosts) == 0: - assert False, f'No hosts found, please add your hosts to {cls.host_data_path}' - if search != None: - hosts = {k:v for k,v in hosts.items() if search in k} - - if enable_search_terms: - return cls.filter_hosts(hosts=hosts) - return hosts - - host_map = hosts - - @classmethod - def host2ip(cls, search=None): - hosts = cls.hosts(search=search) - return {k:v['host'] for k,v in hosts.items()} - - @classmethod - def ip2host(cls, search=None): - host2ip = cls.host2ip(search=search) - return {v:k for k,v in host2ip.items()} - @classmethod - def names(cls, search=None): - return list(cls.hosts(search=search).keys()) - - def host2name(self, host): - hosts = self.hosts() - for name, h in hosts.items(): - if h == host: - return name - raise Exception(f'Host {host} not found') - - @classmethod - def n(cls, search=None): - return len(cls.hosts(search=search)) - - def num_servers(self): - return len(self.servers()) - - - @classmethod - def n_servers(self): - return len(self.servers()) - - @classmethod - def host(self, name): - hosts = self.hosts() - - if name not in hosts: - raise Exception(f'Host {name} not found') - - return hosts[name] - @classmethod - def has(cls, name): - return name in cls.hosts() - @classmethod - def host_exists(self, name): - return name in self.hosts() - @classmethod - def install(self): - c.cmd('pip3 install paramiko') - def test(self): - # Test Remote - c.print(self.ssh_cmd('ls')) - @classmethod - def cmd(cls, *commands, search=None, hosts:Union[list, dict, str] = None, cwd=None, host:str=None, timeout=5 , verbose:bool = True, num_trials=1, **kwargs): - output = {} - if hosts == None: - hosts = cls.hosts() - if host != None: - hosts = {host:hosts[host]} - if search != None: - hosts = {k:v for k,v in hosts.items() if search in k} - if isinstance(hosts, list): - hosts = {h:hosts[h] for h in hosts} - elif isinstance(hosts, str): - hosts = {hosts:cls.hosts(hosts)} - - assert isinstance(hosts, dict), f'Hosts must be a dict, got {type(hosts)}' - - results = {} - for host in hosts: - result_future = c.submit(cls.ssh_cmd, args=commands, kwargs=dict(host=host, cwd=cwd, verbose=verbose,**kwargs), return_future=True) - results[host] = result_future - - result_values = c.wait(list(results.values()), timeout=timeout) - results = dict(zip(results.keys(), result_values)) - results = {k:v for k,v in results.items()} - - if all([v == None for v in results.values()]): - raise Exception(f'all results are None') - - for k,v in results.items(): - if isinstance(v, str): - results[k] = v.strip('\n') - - return results - - @classmethod - def add_admin(cls, timeout=10): - root_key_address = c.root_key().ss58_address - return cls.cmd(f'c add_admin {root_key_address}', timeout=timeout) - - @classmethod - def is_admin(cls, timeout=10): - root_key_address = c.root_key().ss58_address - results = cls.cmd(f'c is_admin {root_key_address}', timeout=timeout) - for host, r in results.items(): - results[host] = bool(r) - return results - - def add_server(self, address): - return c.add_server(address, network='remote') - - def host2rootkey(self, **kwargs): - host2rootkey = self.cmd(f'c root_key_address', **kwargs) - return {k: v if isinstance(v, str) else None for k,v in host2rootkey.items()} - - @classmethod - def servers(self,search: str ='module', network='remote'): - return c.servers(search, network=network) - - @classmethod - def serve(cls, *args, update=False, min_memory:int=10, timeout=10, **kwargs): - modules = cls.available_peers(update=update, min_memory=min_memory) - c.print(f'Available modules: {modules}') - module = c.choice(modules) - c.print(f'Serving on {module}') - namespace = c.namespace(network='remote') - address = namespace[module] - module = c.connect(address) - return module.serve(*args, **kwargs, timeout=timeout) - - @classmethod - def fleet(cls, *args, update=False, min_memory:int=20, n=10, tag=None, **kwargs): - responses = [] - for i in range(n): - try: - response = cls.serve(*args, - update=update, - min_memory=min_memory, - tag= '' if tag == None else tag + str(i), - **kwargs - ) - except Exception as e: - response = c.detailed_error(e) - c.print(response) - responses += [response] - return responses - - @classmethod - def logs(cls, module, n=3 , **kwargs): - namespace = cls.namespace(search=module) - c.print(namespace) - for name, address in list(namespace.items())[:n]: - if address == None: - raise Exception(f'Address for {name} not found') - logs = c.call(address, 'logs', name, mode='local') - c.print(f'[bold yellow]{name}[/bold yellow]') - c.print('\n'.join(logs.split('\n')[-10:])) - - - @classmethod - def namespace(cls, search=None, network='remote', update=False): - namespace = {} - if not update: - namespace = c.get_namespace(network=network) - return namespace - - peer2namespace = cls.peer2namespace() - for peer, peer_namespace in peer2namespace.items(): - - for name, address in peer_namespace.items(): - if search != None and search not in name: - continue - if name in namespace: - continue - namespace[name + '_'+ {peer}] = address - c.put_namespace(namespace=namespace, network=network) - return namespace - - @classmethod - def get_address(self, name): - return c.get_address(name) - - - @classmethod - def addresses(self, search=None, network='remote'): - return c.addresses(search=search, network=network) - - @classmethod - def peer2address(self, network='remote'): - return c.namespace(search='module', network=network) - - - - def keys(self): - return [info.get('key', None)for info in self.infos()] - - @classmethod - def infos(self, search='module', network='remote', update=False): - return c.infos(search=search, network=network, update=update) - - - @classmethod - def push(cls,**kwargs): - return [c.push(), cls.pull()] - - - @classmethod - def pull(cls, stash=True, hosts=None): - return c.rcmd(f'c pull stash={stash}', hosts=hosts) - - @classmethod - def hardware(cls, timeout=20, update= True, cache_path:str = 'hardware.json', trials=2): - - - if not update: - peer2hardware = c.get_json(cache_path, {}) - if len(peer2hardware) > 0: - return peer2hardware - peer2hardware = {p:None for p in peers} - - peers = cls.peers() - for i in range(trials): - call_modules = [p for p in peers if peer2hardware[p] == None] - response = cls.call('hardware', timeout=timeout, modules=call_modules) - for peer, hardware in response.items(): - if isinstance(hardware, dict) and 'memory' in hardware: - c.print(f'{peer} {hardware}') - peer2hardware[peer] = hardware - - - c.put_json(cache_path, peer2hardware) - return peer2hardware - - - # peers - - def peers(self, network='remote'): - return self.servers(search='module', network=network) - - def peer2hash(self, search='module', network='remote'): - return self.call('chash', search=search, network=network) - - def peer2memory(self, min_memory:int=0, **kwargs): - peer2hardware = self.hardware(**kwargs) - peer2memory = {} - for server, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - if isinstance(memory_info, dict) and 'available' in memory_info: - if memory_info['available'] >= min_memory: - peer2memory[server] = memory_info['available'] - return peer2memory - - - - def ps(self, update=False, **kwargs): - peer2ps = {} - for peer, ps in self.call('ps', **kwargs).items(): - peer2ps[peer] = ps - return peer2ps - - @classmethod - def available_peers(cls, min_memory:int=10, update: bool=False, address=False): - peer2hardware = cls.hardware(update=update) - peer2memory = {} - - for peer, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - - if isinstance(memory_info, dict) and 'available' in memory_info: - if memory_info['available'] >= min_memory: - peer2memory[peer] = memory_info['available'] - - if address : - namespace = c.namespace(network='remote') - peers = [namespace.get(k) for k in peer2memory] - else : - peers = list(peer2memory.keys()) - return peers - - - @classmethod - def most_available_peer(cls, min_memory:int=8, update: bool=False): - peer2hardware = cls.hardware(update=update) - peer2memory = {} - - for peer, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - if isinstance(memory_info, dict) and 'available' in memory_info: - peer2memory[peer] = memory_info['available'] - - # get the top n peers with the most memory - peer2memory = {k:v for k,v in sorted(peer2memory.items(), key=lambda x: x[1], reverse=True)} - most_available_peer = list(peer2memory.keys())[0] - most_available_peer_memory = peer2memory[most_available_peer] - assert most_available_peer_memory >= min_memory, f'Not enough memory, {most_available_peer_memory} < {min_memory}' - c.print(f'Most available peer: {most_available_peer} with {most_available_peer_memory} GB memory') - return most_available_peer - - @classmethod - def push(self): - c.push() - c.rcmd('c pull', verbose=True) - c.rcmd('c serve', verbose=True) - c.add_peers() - - @classmethod - def check_peers(cls, timeout=10): - futures = [] - for m,a in c.namespace(network='remote').items(): - futures += [c.submit(c.call, args=(a,'info'),return_future=True)] - results = c.wait(futures, timeout=timeout) - return results - - @classmethod - def setup(cls,**kwargs): - repo_url = c.repo_url() - c.print(cls.cmd(f'git clone {repo_url}', **kwargs)) - c.print(cls.cmd(f'apt ', **kwargs)) - c.print(cls.cmd(f'cd commune && pip install -e .', **kwargs)) - c.print(cls.cmd(f'c add_admin {c.root_key().ss58_address} ', **kwargs)) - c.print(cls.cmd(f'c serve', **kwargs)) - - def enter(self, host='root10'): - host2ssh = self.host2ssh() - c.print(host2ssh) - ssh = host2ssh[host] - c.cmd(ssh) - - - def loop(self, timeout=40, interval=30, max_age=360, remote=True, batch_size=10): - if remote: - return self.remote_fn('loop',kwargs = locals()) - while True: - self.sync() - c.sleep(10) - - def save_ssh_config(self, path="~/.ssh/config"): - ssh_config = [] - - for host_name, host in self.hosts().items(): - ssh_config.append(f'Host {host_name}') - ssh_config.append(f' HostName {host["host"]}') - ssh_config.append(f' Port {host["port"]}') - ssh_config.append(f' User {host["user"]}') - - ssh_config = '\n'.join(ssh_config) - - return c.put_text(path, ssh_config) - - - def text2hosts(self, text, model='model.openai'): - prompt = { - 'instruciton': 'given the text place into the following format', - 'text': text, - 'format': list(self.hosts().values())[0], - 'output': None - } - model = c.module(model) - return model.generate(c.python2str(prompt)) - - - def peer_dashboard(self): - import streamlit as st - import pandas as pd - - with st.sidebar: - cols = st.columns(2) - search = cols[0].text_input('Search', 'module') - peer2info = self.peer2info() - - st.write(list(peer2info.values())[0]) - - peer_info_df = [] - for peer, info in peer2info.items(): - memory_fields = ['available', 'total', 'used'] - row = {'peer': peer} - for field in memory_fields: - row['memory_'+field] = info.get('hardware', {}).get('memory', {}).get(field, None) - - # disk fields - disk_fields = ['total', 'used', 'free'] - for field in disk_fields: - row['disk_'+field] = info.get('hardware', {}).get('disk', {}).get(field, None) - peer_info_df += [row] - row['num_modules'] = len(info.get('namespace', {})) - - peer_info_df = pd.DataFrame(peer_info_df) - namespace = c.namespace(search=search, network='remote') - ip2host = self.ip2host() - - with st.expander('Peers', expanded=False): - for peer, info in peer2info.items(): - cols = st.columns([1,4]) - peer = ip2host.get(peer, peer) - cols[0].write('#### '+peer) - - timestamp = info.get('timestamp', None) - lag = c.time() - timestamp if timestamp != None else None - if lag != None: - lag = round(lag, 2) - st.write(f'{lag} seconds ago') - cols[1].write(info.get('hardware', {})) - cols[1].write(info.get('namespace', {})) - - if len(namespace) == 0: - st.error(f'No peers found with search: {search}') - return - n = cols[1].slider('Number of servers', 1, len(namespace), len(namespace)) - module_names = list(namespace.keys())[:n] - module_names = st.multiselect('Modules', module_names, module_names) - namespace = {k:v for k,v in namespace.items() if k in module_names} - module_addresses = list(namespace.values()) - module_names = list(namespace.keys()) - - if len(module_names) == 0: - st.error('No modules found') - return - - cols = st.columns(3) - module_name = cols[0].selectbox('Module', module_names, index=0) - module_address = namespace[module_name] - c.print(f'Connecting to {module_name} {module_address}') - module = c.connect(module_address) - cache = cols[2].checkbox('Cache', True) - - cache_path = f'module_info_cache/{module_address}' - t1 = c.time() - if cache: - module_info = self.get_json(cache_path, {}) - else: - module_info = {} - - if len(module_info) == 0: - st.write('Getting module info') - - module_info = module.info() - self.put_json(cache_path, module_info) - fns = list(module_info['schema'].keys()) - fn_name = st.selectbox('Function', fns, index=0) - fn = getattr(module, fn_name) - with st.expander(fn_name, expanded=False): - kwargs = self.function2streamlit(fn=fn_name, fn_schema=module_info['schema'][fn_name]) - timeout = cols[1].number_input('Timeout', 1, 100, 10, key='timeout_fn') - run = st.button(f'Run {fn_name}') - - if run: - future2module = {} - for module_address in module_addresses: - kwargs['fn'] = fn_name - future = c.submit(c.call, args=[module_address], kwargs=kwargs, return_future=True) - future2module[future] = module_address - - futures = list(future2module.keys()) - modules = list(future2module.values()) - for result in c.wait(futures, timeout=timeout, generator=True, return_dict=True): - if not ('idx' in result and 'result' in result): - continue - - module_name = modules[result['idx']] - result = result['result'] - - with st.expander(f'{module_name}', expanded=False): - - st.markdown(f'### {module_name}') - if c.is_error(result): - result - pass - else: - st.write(result) - - t2 = c.time() - st.write(f'Info took {t2-t1} seconds') - - - - def sidebar(self, **kwargs): - with st.sidebar: - self.filter_hosts_dashboard() - self.manage_hosts_dashboard() - - - # SEARCH TERM LAND - - search_terms_path = 'search_terms' - @classmethod - def set_search_terms(cls, search_terms): - path = cls.search_terms_path - cls.put(path, search_terms) - return {'status': 'success', 'msg': f'Search terms set', 'search_terms': search_terms} - - @classmethod - def clear_terms(cls): - path = cls.search_terms_path - return cls.put(path, {'include': '', 'avoid': ''}) - - - @classmethod - def avoid(cls, *terms): - terms = ','.join(terms) - search_terms = cls.get_search_terms() - search_terms['avoid'] = terms - cls.set_search_terms(search_terms) - return {'status': 'success', 'msg': f'Added {terms} to avoid terms', 'search_terms': search_terms} - - @classmethod - def include(cls, *terms): - terms = ','.join(terms) - search_terms = cls.get_search_terms() - search_terms['include'] = terms - cls.set_search_terms(search_terms) - return {'status': 'success', 'msg': f'Added {terms} to include terms', 'search_terms': search_terms} - - @classmethod - def get_search_terms(cls): - path = cls.search_terms_path - return cls.get(path, {'include': '', 'avoid': ''}) - search_terms = get_search_terms - - @classmethod - def filter_hosts(cls, include=None, avoid=None, hosts=None): - - host_map = hosts or cls.hosts() - search_terms = cls.search_terms() - if avoid != None: - search_terms['avoid'] = avoid - if include != None: - search_terms['include'] = include - - for k, v in search_terms.items(): - # - if v == None: - v = '' - - if len(v) > 0: - if ',' in v: - v = v.split(',') - else: - v = [v] - - v = [a.strip() for a in v] - else: - v = [] - search_terms[k] = v - - def filter_host(host_name): - for avoid_term in search_terms["avoid"]: - if avoid_term in host_name: - return False - for include_term in search_terms["include"]: - if not include_term in host_name: - return False - return True - - return {k:v for k,v in host_map.items() if filter_host(k)} - - - - def filter_hosts_dashboard(self, host_names: list = None): - - host_map = self.hosts() - host_names = list(host_map.keys()) - - # get the search terms - search_terms = self.get_search_terms() - for k, v in search_terms.items(): - search_terms[k] = st.text_input(k, v) - self.set_search_terms(search_terms) - host_map = self.filter_hosts(**search_terms) - host_names = list(host_map.keys()) - n = len(host_names) - - self.filter_hosts() - - - with st.expander(f'Hosts (n={n})', expanded=False): - host_names = st.multiselect('Host', host_names, host_names) - host_map = {k:host_map[k] for k in host_names} - self.host2ssh = self.host2ssh(host_map=host_map) - - - @classmethod - def host2ssh(cls, host_map=None): - host_map = host_map or cls.hosts() - host2ssh = {} - for k, v in host_map.items(): - host2ssh[k] = f'sshpass -p {v["pwd"]} ssh {v["user"]}@{v["host"]} -p {v["port"]}' - return host2ssh - - - def manage_hosts_dashboard(self): - - with st.expander('Add Host', expanded=False): - st.markdown('## Hosts') - cols = st.columns(2) - host = cols[0].text_input('Host', '0.0.0.0') - port = cols[1].number_input('Port', 22, 30000000000, 22) - user = st.text_input('User', 'root') - pwd = st.text_input('Password', type='password') - add_host = st.button('Add Host') - - if add_host: - self.add_host(host=host, port=port, user=user, pwd=pwd) - - with st.expander('Remove Host', expanded=False): - host_names = list(self.hosts().keys()) - rm_host_name = st.selectbox('Host Name', host_names) - rm_host = st.button('Remove Host') - if rm_host: - self.rm_host(rm_host_name) - - - - def ssh_dashboard(self): - import streamlit as st - host_map = self.hosts() - - host_names = list(host_map.keys()) - - - # progress bar - # add splace to cols[2] vertically - - - with st.expander('params', False): - cols = st.columns([4,4,2,2]) - cwd = cols[0].text_input('cwd', '/') - timeout = cols[1].number_input('Timeout', 1, 100, 10) - if cwd == '/': - cwd = None - for i in range(2): - cols[2].write('\n') - filter_bool = cols[2].checkbox('Filter', False) - - st.write('Print Formatting') - expanded = True - cols = st.columns([4,1]) - num_columns = cols[1].number_input('Num Columns', 1, 10, 2) - fn_code = cols[0].text_input('Function', 'x') - - cols = st.columns([5,1]) - cmd = cols[0].text_input('Command', 'ls') - [cols[1].write('') for i in range(2)] - sudo = cols[1].checkbox('Sudo') - - if 'x' not in fn_code: - fn_code = f'x' - fn_code = f'lambda x: {fn_code}' - fn_code = eval(fn_code) - - run_button = st.button('Run') - host2future = {} - - if run_button: - for host in host_names: - future = c.submit(self.ssh_cmd, args=[cmd], kwargs=dict(host=host, verbose=False, sudo=sudo, search=host_names, cwd=cwd), return_future=True, timeout=timeout) - host2future[host] = future - - futures = list(host2future.values()) - num_jobs = len(futures ) - hosts = list(host2future.keys()) - host2error = {} - cols = st.columns(num_columns) - failed_hosts = [] - col_idx = 0 - - try: - for result in c.wait(futures, timeout=timeout, generator=True, return_dict=True): - - host = hosts[result['idx']] - if host == None: - continue - - host2future.pop(host) - result = result['result'] - is_error = c.is_error(result) - emoji = c.emoji('cross') if is_error else c.emoji('check_mark') - msg = result['error'] if is_error else result.strip() - - # get the colkumne - col_idx = (col_idx) % len(cols) - col = cols[col_idx] - col_idx += 1 - - - # if the column is full, add a new column - with col: - with st.expander(f'{host} -> {emoji}', expanded=expanded): - msg = fn_code(msg) - if is_error: - st.write('ERROR') - st.error(msg) - failed_hosts += [host] - else: - st.code(msg) - - - except Exception as e: - pending_hosts = list(host2future.keys()) - st.error(c.detailed_error(e)) - st.error(f"Hosts {pending_hosts} timed out") - failed_hosts += pending_hosts - - failed_hosts2ssh = {h:self.host2ssh[h] for h in failed_hosts} - with st.expander('Failed Hosts', expanded=False): - for host, ssh in failed_hosts2ssh.items(): - st.write(host) - st.code(ssh) - - - @classmethod - def dashboard(cls, module: str = None, **kwargs): - if module: - cls = c.module(module) - c.new_event_loop() - import streamlit as st - - c.load_style() - st.title('Remote Dashboard') - self = cls() - self.sidebar() - self.ssh_dashboard() - - @classmethod - def peer2key(cls, search=None, network:str='remote', update=False): - infos = c.infos(search=search, network=network, update=update) - return {v['name']:v['key'] for v in infos if 'name' in v and 'address' in v} - - @classmethod - def peer_addresses(cls, network:str='remote'): - infos = c.infos(network=network) - return {info['key'] for info in infos if 'key' in info} - - - def check_peers(self, timeout=10): - futures = [] - for m,a in c.namespace(network='remote').items(): - futures += [c.submit(c.call, args=(a,'info'),return_future=True)] - results = c.wait(futures, timeout=timeout) - return results - - - def sync(self, timeout=40, max_age=360): - futures = [] - namespace = c.namespace('module', network='remote') - paths = [] - for name, address in namespace.items(): - path = 'peers/' + name - existing_peer_info = self.get(path, {}) - peer_update_ts = existing_peer_info.get('timestamp', 0) - future = c.submit(c.call, - args = [address, 'info'], - kwargs = dict(schema=True, namespace=True, hardware=True), - timeout=timeout, return_future=True - ) - paths += [path] - futures += [future] - - results = c.wait(futures, timeout=timeout, generator=False) - for i, result in enumerate(results): - path = paths[i] - if c.is_error(result): - c.print(f'Error {result}') - continue - else: - c.print(f'Success {path}') - self.put(path, result) - self.put(path, result) - return {'status': 'success', 'msg': f'Peers synced'} - - def peerpath2name(self, path:str): - return path.split('/')[-1].replace('.json', '') - - - def peer2info(self): - peer_infos = {} - for path in self.ls('peers'): - peer_name = self.peerpath2name(path) - info = self.get(path, {}) - peer_infos[peer_name] = info - peer_infos[peer_name] = info - return peer_infos - - - def peer2lag(self, max_age=1000): - peer2timestamp = self.peer2timestamp() - time = c.time() - ip2host = self.ip2host() - return {ip2host.get(k,k):time - v for k,v in peer2timestamp.items() if time - v < max_age} - - def peer2timestamp(self): - peer2info = self.peer2info() - return {k:v.get('timestamp', 0) for k,v in peer2info.items()} - - def peer2hardware(self): - info_paths = self.ls('peers') - peer_infos = [] - for path in info_paths: - info = self.get(path, {}) - # c.print(info) - peer_infos += [info.get('hardware', {})] - return peer_infos - - @classmethod - def peer2namespace(cls): - info_paths = cls.ls('peers') - peer2namespace = {} - for path in info_paths: - info = cls.get(path, {}) - peer2namespace[path] = info.get('namespace', {}) - return peer2namespace - - - - @classmethod - def add_peers(cls, add_admins:bool=False, timeout=20, update=True, network='remote'): - """ - Adds servers to the network by running `c add_peers` on each server. - - """ - if update: - c.rm_namespace(network=network) - if add_admins: - cls.add_admin(timeout=timeout) - - namespace = c.namespace(network=network) - address2name = {v:k for k,v in namespace.items()} - host2_server_addresses_responses = cls.cmd('c addy', verbose=True, timeout=timeout) - for i, (host,server_address) in enumerate(host2_server_addresses_responses.items()): - if isinstance(server_address, str): - server_address = server_address.split('\n')[-1] - - if server_address in address2name: - server_name = address2name[server_address] - c.print(f'{server_name} already in namespace') - continue - else: - ip = ':'.join(server_address.split(':')[:-1]) - - server_name = 'module' + '_' + host - namespace[server_name] = server_address - - c.put_namespace(network=network, namespace=namespace) - - return {'status': 'success', 'msg': f'Servers added', 'namespace': namespace} - - - - - def peer_info(self, peer): - host2ip = self.host2ip() - peer = host2ip.get(peer, peer) - return self.get(f'peers/{peer}', {}) - - - - @classmethod - def call(cls, fn:str='info' , *args, - search:str='module', - modules=None, - network:str='remote', - avoid_hosts: str = 'root', - n:int=None, - return_future: bool = False, - timeout=4, **kwargs): - futures = {} - kwargs['network'] = network - - namespace = c.namespace(search=search, network=network) - - if modules != None: - assert isinstance(modules, list), f'modules must be a list, got {type(modules)}' - namespace = {k:v for k,v in namespace.items() if k in modules} - if n == None: - n = len(namespace) - - for name, address in c.shuffle(list(namespace.items()))[:n]: - c.print(f'Calling {name} {address}') - futures[name] = c.async_call(address, fn, *args) - - if return_future: - if len(futures) == 1: - return list(futures.values())[0] - return futures - else: - - - - num_futures = len(futures) - results = {} - import tqdm - - - progress_bar = tqdm.tqdm(total=num_futures) - error_progress = tqdm.tqdm(total=num_futures) - - results = c.gather(list(futures.values()), timeout=timeout) - - for i, result in enumerate(results): - if c.is_error(result): - # c.print(f'Error {result}') - error_progress.update(1) - continue - - else: - # c.print(f'Success {result}') - results[i] = result - progress_bar.update(1) - # if len(results) == 1: - # return list(results.values())[0] - - return results - - - -Peer.run(__name__) diff --git a/commune/modules/remote/peer.py b/commune/modules/remote/peer.py deleted file mode 100644 index ff8027f7..00000000 --- a/commune/modules/remote/peer.py +++ /dev/null @@ -1,388 +0,0 @@ - - - -import commune as c - - -class Peer(c.Module): - - - def namespace(self, search=None, network='remote', update=False): - namespace = {} - if not update: - namespace = c.get_namespace(network=network) - return namespace - - peer2namespace = self.peer2namespace() - for peer, peer_namespace in peer2namespace.items(): - - for name, address in peer_namespace.items(): - if search != None and search not in name: - continue - if name in namespace: - continue - namespace[name + '_'+ {peer}] = address - c.put_namespace(namespace=namespace, network=network) - return namespace - - - - def peers(self, network='remote'): - return self.servers(search='module', network=network) - - def peer2hash(self, search='module', network='remote'): - return self.call('chash', search=search, network=network) - - - # peers - - def peer2memory(self, min_memory:int=0, **kwargs): - peer2hardware = self.hardware(**kwargs) - peer2memory = {} - for server, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - if isinstance(memory_info, dict) and 'available' in memory_info: - if memory_info['available'] >= min_memory: - peer2memory[server] = memory_info['available'] - return peer2memory - - @classmethod - def available_peers(cls, min_memory:int=10, update: bool=False, address=False): - peer2hardware = cls.hardware(update=update) - peer2memory = {} - - for peer, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - - if isinstance(memory_info, dict) and 'available' in memory_info: - if memory_info['available'] >= min_memory: - peer2memory[peer] = memory_info['available'] - - if address : - namespace = c.namespace(network='remote') - peers = [namespace.get(k) for k in peer2memory] - else : - peers = list(peer2memory.keys()) - return peers - - - @classmethod - def most_available_peer(cls, min_memory:int=8, update: bool=False): - peer2hardware = cls.hardware(update=update) - peer2memory = {} - - for peer, hardware in peer2hardware.items(): - memory_info = hardware['memory'] - if isinstance(memory_info, dict) and 'available' in memory_info: - peer2memory[peer] = memory_info['available'] - - # get the top n peers with the most memory - peer2memory = {k:v for k,v in sorted(peer2memory.items(), key=lambda x: x[1], reverse=True)} - most_available_peer = list(peer2memory.keys())[0] - most_available_peer_memory = peer2memory[most_available_peer] - assert most_available_peer_memory >= min_memory, f'Not enough memory, {most_available_peer_memory} < {min_memory}' - c.print(f'Most available peer: {most_available_peer} with {most_available_peer_memory} GB memory') - return most_available_peer - - - - @classmethod - def check_peers(cls, timeout=10): - futures = [] - for m,a in c.namespace(network='remote').items(): - futures += [c.submit(c.call, args=(a,'info'),return_future=True)] - results = c.wait(futures, timeout=timeout) - return results - - - @classmethod - def peer2namespace(cls): - info_paths = cls.ls('peers') - peer2namespace = {} - for path in info_paths: - info = cls.get(path, {}) - peer2namespace[path] = info.get('namespace', {}) - return peer2namespace - - - @classmethod - def add_peers(cls, add_admins:bool=False, timeout=20, update=False, network='remote'): - """ - Adds servers to the network by running `c add_peers` on each server. - - """ - if update: - c.rm_namespace(network=network) - if add_admins: - cls.add_admin(timeout=timeout) - - namespace = c.namespace(network=network) - address2name = {v:k for k,v in namespace.items()} - host2_server_addresses_responses = cls.cmd('c addy', verbose=True, timeout=timeout) - for i, (host,server_address) in enumerate(host2_server_addresses_responses.items()): - if isinstance(server_address, str): - server_address = server_address.split('\n')[-1] - - if server_address in address2name: - server_name = address2name[server_address] - c.print(f'{server_name} already in namespace') - continue - else: - ip = ':'.join(server_address.split(':')[:-1]) - - server_name = 'module' + '_' + host - namespace[server_name] = server_address - - c.put_namespace(network=network, namespace=namespace) - - return {'status': 'success', 'msg': f'Servers added', 'namespace': namespace} - - - - def peer_info(self, peer): - host2ip = self.host2ip() - peer = host2ip.get(peer, peer) - return self.get(f'peers/{peer}', {}) - - - @classmethod - def serve(cls, *args, update=False, min_memory:int=10, timeout=10, **kwargs): - modules = cls.available_peers(update=update, min_memory=min_memory) - c.print(f'Available modules: {modules}') - module = c.choice(modules) - c.print(f'Serving on {module}') - namespace = c.namespace(network='remote') - address = namespace[module] - module = c.connect(address) - return module.serve(*args, **kwargs, timeout=timeout) - - @classmethod - def servers(self,search: str ='module', network='remote'): - return c.servers(search, network=network) - - - @classmethod - def fleet(cls, *args, update=False, min_memory:int=20, n=10, tag=None, **kwargs): - responses = [] - for i in range(n): - try: - response = cls.serve(*args, - update=update, - min_memory=min_memory, - tag= '' if tag == None else tag + str(i), - **kwargs - ) - except Exception as e: - response = c.detailed_error(e) - c.print(response) - responses += [response] - return responses - - def num_servers(self): - return len(self.servers()) - - - @classmethod - def n_servers(self): - return len(self.servers()) - - - - @classmethod - def peer2key(cls, search=None, network:str='remote', update=False): - infos = c.infos(search=search, network=network, update=update) - return {v['name']:v['key'] for v in infos if 'name' in v and 'address' in v} - - @classmethod - def peer_addresses(cls, network:str='remote'): - infos = c.infos(network=network) - return {info['key'] for info in infos if 'key' in info} - - def check_peers(self, timeout=10): - futures = [] - for m,a in c.namespace(network='remote').items(): - futures += [c.submit(c.call, args=(a,'info'),return_future=True)] - results = c.wait(futures, timeout=timeout) - return results - - def peerpath2name(self, path:str): - return path.split('/')[-1].replace('.json', '') - - def peer2info(self): - peer_infos = {} - for path in self.ls('peers'): - peer_name = self.peerpath2name(path) - info = self.get(path, {}) - peer_infos[peer_name] = info - peer_infos[peer_name] = info - return peer_infos - - def peer2lag(self, max_age=1000): - peer2timestamp = self.peer2timestamp() - time = c.time() - ip2host = self.ip2host() - return {ip2host.get(k,k):time - v for k,v in peer2timestamp.items() if time - v < max_age} - - def peer2timestamp(self): - peer2info = self.peer2info() - return {k:v.get('timestamp', 0) for k,v in peer2info.items()} - - def peer2hardware(self): - info_paths = self.ls('peers') - peer_infos = [] - for path in info_paths: - c.print(path) - info = self.get(path, {}) - # c.print(info) - peer_infos += [info.get('hardware', {})] - return peer_infos - - - @classmethod - def hardware(cls, timeout=20, update= True, cache_path:str = 'hardware.json', trials=2): - - - if not update: - peer2hardware = c.get_json(cache_path, {}) - if len(peer2hardware) > 0: - return peer2hardware - peer2hardware = {p:None for p in peers} - peers = cls.peers() - for i in range(trials): - call_modules = [p for p in peers if peer2hardware[p] == None] - response = cls.call('hardware', timeout=timeout, modules=call_modules) - for peer, hardware in response.items(): - if isinstance(hardware, dict) and 'memory' in hardware: - c.print(f'{peer} {hardware}') - peer2hardware[peer] = hardware - - - c.put_json(cache_path, peer2hardware) - return peer2hardware - - - @classmethod - def addresses(self, search=None, network='remote'): - return c.addresses(search=search, network=network) - - def keys(self): - return [info.get('key', None)for info in self.infos()] - - @classmethod - def infos(self, search='module', network='remote', update=False): - return c.infos(search=search, network=network, update=update) - - - @classmethod - def peer2address(self, network='remote'): - return c.namespace(search='module', network=network) - - - -# def peer_dashboard(self): - -# cols = st.columns(2) -# search = cols[0].text_input('Search', 'module') -# peer2info = self.remote.peer2info() - -# peer_info_df = [] -# for peer, info in peer2info.items(): -# memory_fields = ['available', 'total', 'used'] -# row = {'peer': peer} -# for field in memory_fields: -# row['memory_'+field] = info.get('hardware', {}).get('memory', {}).get(field, None) -# # disk fields -# disk_fields = ['total', 'used', 'free'] -# for field in disk_fields: -# row['disk_'+field] = info.get('hardware', {}).get('disk', {}).get(field, None) -# peer_info_df += [row] -# row['num_modules'] = len(info.get('namespace', {})) - -# peer_info_df = pd.DataFrame(peer_info_df) -# namespace = c.namespace(search=search, network='remote') -# ip2host = self.remote.ip2host() - -# with st.expander('Peers', expanded=False): -# for peer, info in peer2info.items(): -# cols = st.columns([1,4]) -# peer = ip2host.get(peer, peer) -# cols[0].write('#### '+peer) - -# timestamp = info.get('timestamp', None) -# lag = c.time() - timestamp if timestamp != None else None -# if lag != None: -# lag = round(lag, 2) -# st.write(f'{lag} seconds ago') -# cols[1].write(info.get('hardware', {})) -# cols[1].write(info.get('namespace', {})) - -# if len(namespace) == 0: -# st.error(f'No peers found with search: {search}') -# return -# n = cols[1].slider('Number of servers', 1, len(namespace), len(namespace)) -# module_names = list(namespace.keys())[:n] -# module_names = st.multiselect('Modules', module_names, module_names) -# namespace = {k:v for k,v in namespace.items() if k in module_names} -# module_addresses = list(namespace.values()) -# module_names = list(namespace.keys()) - -# if len(module_names) == 0: -# st.error('No modules found') -# return - -# cols = st.columns(3) -# module_name = cols[0].selectbox('Module', module_names, index=0) -# module_address = namespace[module_name] -# c.print(f'Connecting to {module_name} {module_address}') -# module = c.connect(module_address) -# cache = cols[2].checkbox('Cache', True) - -# cache_path = f'module_info_cache/{module_address}' -# t1 = c.time() -# if cache: -# module_info = self.get_json(cache_path, {}) -# else: -# module_info = {} - -# if len(module_info) == 0: -# st.write('Getting module info') - -# module_info = module.info() -# self.put_json(cache_path, module_info) -# fns = list(module_info['schema'].keys()) -# fn_name = st.selectbox('Function', fns, index=0) -# fn = getattr(module, fn_name) -# with st.expander(fn_name, expanded=False): -# kwargs = self.function2streamlit(fn=fn_name, fn_schema=module_info['schema'][fn_name]) -# timeout = cols[1].number_input('Timeout', 1, 100, 10, key='timeout_fn') -# run = st.button(f'Run {fn_name}') - -# if run: -# future2module = {} -# for module_address in module_addresses: -# kwargs['fn'] = fn_name -# future = c.submit(c.call, args=[module_address], kwargs=kwargs, return_future=True) -# future2module[future] = module_address - -# futures = list(future2module.keys()) -# modules = list(future2module.values()) -# for result in c.wait(futures, timeout=timeout, generator=True, return_dict=True): -# if not ('idx' in result and 'result' in result): -# continue - -# module_name = modules[result['idx']] -# result = result['result'] - -# with st.expander(f'{module_name}', expanded=False): - -# st.markdown(f'### {module_name}') -# if c.is_error(result): -# result -# pass -# else: -# st.write(result) - -# t2 = c.time() -# st.write(f'Info took {t2-t1} seconds') - - diff --git a/commune/sandbox.py b/commune/modules/sandbox.py similarity index 100% rename from commune/sandbox.py rename to commune/modules/sandbox.py diff --git a/commune/test/test.py b/commune/modules/test.py similarity index 100% rename from commune/test/test.py rename to commune/modules/test.py diff --git a/commune/serializer/serializer.md b/commune/serializer/README.md similarity index 100% rename from commune/serializer/serializer.md rename to commune/serializer/README.md diff --git a/commune/server/history.py b/commune/server/history.py deleted file mode 100644 index ba8b4051..00000000 --- a/commune/server/history.py +++ /dev/null @@ -1,52 +0,0 @@ - -import commune as c -from typing import * -import pandas as pd - -class History(c.Module): - def __init__(self, history_path='history', **kwargs): - self.set_history_path(history_path) - # HISTORY - def add_history(self, item:dict): - path = self.history_path + '/' + item['address'] + '/'+ str(item['timestamp']) - self.put(path, item) - - def set_history_path(self, history_path): - assert history_path is not None, f"History path is not set" - self.history_path = history_path - return {'history_path': self.history_path} - - def rm_history(self, server): - dirpath = f'{self.history_path}/{server}' - return c.rm(dirpath) - - @classmethod - def history_paths(cls, server=None, history_path='history', n=100, key=None): - if server == None: - dirpath = f'{history_path}' - paths = cls.glob(dirpath) - else: - - dirpath = f'{history_path}/{server}' - paths = cls.ls(dirpath) - paths = sorted(paths, reverse=True)[:n] - return paths - - def history(self, - key=None, - history_path='history', - features=[ 'module', 'fn', 'seconds_ago', 'latency', 'address'], - to_list=False, - **kwargs - ): - key = c.get_key(key) - history_path = self.history_paths(key=key, history_path=history_path) - df = c.df([self.get(path) for path in history_path]) - now = c.timestamp() - df['seconds_ago'] = df['timestamp'].apply(lambda x: now - x) - df = df[features] - if to_list: - return df.to_dict('records') - - return df - \ No newline at end of file diff --git a/commune/server/manager.py b/commune/server/manager.py deleted file mode 100644 index 752c3838..00000000 --- a/commune/server/manager.py +++ /dev/null @@ -1,363 +0,0 @@ - -import commune as c -from typing import * -class ServerManager: - - @classmethod - def kill(cls, - module, - mode:str = 'pm2', - verbose:bool = False, - update : bool = True, - prefix_match = False, - network = 'local', # local, dev, test, main - **kwargs): - - kill_fn = getattr(cls, f'{mode}_kill') - delete_modules = [] - - try: - killed_module =kill_fn(module, verbose=verbose,prefix_match=prefix_match, **kwargs) - except Exception as e: - return {'error':str(e)} - if isinstance(killed_module, list): - delete_modules.extend(killed_module) - elif isinstance(killed_module, str): - delete_modules.append(killed_module) - else: - delete_modules.append(killed_module) - # update modules - c.deregister_server(module, network=network) - - assert c.server_exists(module, network=network) == False, f'module {module} still exists' - - servers = c.servers() - for m in delete_modules: - if m in servers: - c.deregister_server(m, network=network) - - return {'server_killed': delete_modules, 'update': update} - - - - @classmethod - def kill_prefix(cls, prefix:str, **kwargs): - servers = c.servers(network='local') - killed_servers = [] - for s in servers: - if s.startswith(prefix): - c.kill(s, **kwargs) - killed_servers.append(s) - return {'success':True, 'message':f'Killed servers with prefix {prefix}'} - - - - @classmethod - def kill_many(cls, servers, search:str = None, network='local', timeout=10, **kwargs): - servers = c.servers(network=network) - servers = [s for s in servers if search in s] - futures = [] - for s in servers: - c.print(f'Killing {s}', color='red') - future = c.submit(c.kill, kwargs={'module':s, **kwargs}, imeout=timeout) - futures.append(future) - results = [] - for r in c.as_completed(futures, timeout=timeout): - results += [r.result()] - c.print(f'Killed {len(results)} servers', color='red') - return results - - - @classmethod - def fleet(cls, module, n=5, timeout=10): - futures = [] - if '::' not in module: - module = f'{module}::' - - - for i in range(n): - module_name = f'{module}{i}' - future = c.submit(cls.serve, kwargs=dict(module=module_name), timeout=timeout) - futures.append(future) - results = [] - for future in c.as_completed(futures, timeout=timeout): - result = future.result() - results.append(result) - - return results - - - @classmethod - def serve_many(cls, modules:list, **kwargs): - - if isinstance(modules[0], list): - modules = modules[0] - - futures = [] - for module in modules: - future = c.submit(c.serve, kwargs={'module': module, **kwargs}) - futures.append(future) - - results = [] - for future in c.as_completed(futures): - result = future.result() - results.append(result) - return results - serve_batch = serve_many - - - @classmethod - def wait_for_server(cls, - name: str , - network: str = 'local', - timeout:int = 600, - sleep_interval: int = 1, - verbose:bool = False) -> bool : - - time_waiting = 0 - while time_waiting < timeout: - namespace = c.namespace(network=network) - if name in namespace: - c.print(f'{name} is ready', color='green') - return True - time_waiting += sleep_interval - c.print(f'Waiting for {name} for {time_waiting} seconds', color='red') - c.sleep(sleep_interval) - raise TimeoutError(f'Waited for {timeout} seconds for {name} to start') - - - - @staticmethod - def kill_all_servers( *args, **kwargs): - ''' - Kill all of the servers - ''' - for module in c.servers(*args, **kwargs): - c.kill(module) - - # c.update(network='local') - - @classmethod - def kill_all(cls, network='local', timeout=20, verbose=True): - futures = [] - servers = c.servers(network=network) - n = len(servers) - progress = c.tqdm(n) - for s in servers: - c.print(f'Killing {s}', color='red') - futures += [c.submit(c.kill, kwargs={'module':s, 'update': False}, return_future=True)] - results_list = [] - for f in c.as_completed(futures, timeout=timeout): - result = f.result() - print(result) - progress.update(1) - results_list += [result] - namespace = c.namespace(network=network, update=True) - new_n = len(servers) - c.print(f'Killed {n - new_n} servers, with {n} remaining {servers}', color='red') - return {'success':True, 'old_n':n, 'new_n':new_n, 'servers':servers, 'namespace':namespace} - - - - @classmethod - def serve(cls, - module:Any = None, - kwargs:dict = None, # kwargs for the module - params = None, # kwargs for the module - tag:str=None, - server_network = 'local', # network to run the server - port :int = None, # name of the server if None, it will be the module name - server_name:str=None, # name of the server if None, it will be the module name - name = None, # name of the server if None, it will be the module name - refresh:bool = True, # refreshes the server's key - remote:bool = True, # runs the server remotely (pm2, ray) - tag_seperator:str='::', - max_workers:int = None, - free: bool = False, - mnemonic = None, # mnemonic for the server - key = None, - **extra_kwargs - ): - module = module or c.module_name() - if module.endswith('.py'): - module = module[:-3] - if tag_seperator in str(module): - module, tag = module.split(tag_seperator) - kwargs = {**(params or kwargs or {}), **extra_kwargs} - name = name or server_name or module - if tag_seperator in name: - module, tag = name.split(tag_seperator) - else: - if tag != None: - name = f'{name}{tag_seperator}{tag}' - - if port == None: - # now if we have the server_name, we can repeat the server - address = c.get_address(name, network=server_network) - try: - port = int(address.split(':')[-1]) - if c.port_used(port): - c.kill_port(port) - except Exception as e: - port = c.free_port() - # RESOLVE THE PORT FROM THE ADDRESS IF IT ALREADY EXISTS - - # # NOTE REMOVE is FROM THE KWARGS REMOTE - if remote: - remote_kwargs = c.locals2kwargs(locals()) # GET THE LOCAL KWARGS FOR SENDING TO THE REMOTE - remote_kwargs['remote'] = False # SET THIS TO FALSE TO AVOID RECURSION - for _ in ['extra_kwargs', 'address']: - remote_kwargs.pop(_, None) # WE INTRODUCED THE ADDRES - response = cls.remote_fn('serve', name=name, kwargs=remote_kwargs) - if response['success'] == False: - return response - return {'success':True, - 'name': name, - 'address':c.ip() + ':' + str(remote_kwargs['port']), - 'kwargs':kwargs, - 'module':module - } - - module_class = c.module(module) - kwargs.update(extra_kwargs) - module = module_class(**kwargs) - cls(module=module, - name=name, - port=port, - network=server_network, - max_workers=max_workers, - mnemonic = mnemonic, - free=free, - key=key) - - return {'success':True, - 'address': f'{c.default_ip}:{port}' , - 'name':name, - 'kwargs': kwargs, - 'module':module} - - - - @classmethod - def launch(cls, - module:str = None, - fn: str = 'serve', - name:Optional[str]=None, - tag : str = None, - args : list = None, - kwargs: dict = None, - device:str=None, - interpreter:str='python3', - autorestart: bool = True, - verbose: bool = False , - force:bool = True, - meta_fn: str = 'module_fn', - tag_seperator:str = '::', - cwd = None, - refresh:bool=True ): - import commune as c - - if hasattr(module, 'module_name'): - module = module.module_name() - - # avoid these references fucking shit up - args = args if args else [] - kwargs = kwargs if kwargs else {} - - # convert args and kwargs to json strings - kwargs = { - 'module': module , - 'fn': fn, - 'args': args, - 'kwargs': kwargs - } - - kwargs_str = json.dumps(kwargs).replace('"', "'") - - name = name or module - if refresh: - cls.pm2_kill(name) - module = c.module() - # build command to run pm2 - filepath = c.filepath() - cwd = cwd or module.dirpath() - command = f"pm2 start {filepath} --name {name} --interpreter {interpreter}" - - if not autorestart: - command += ' --no-autorestart' - if force: - command += ' -f ' - command = command + f' -- --fn {meta_fn} --kwargs "{kwargs_str}"' - env = {} - if device != None: - if isinstance(device, int): - env['CUDA_VISIBLE_DEVICES']=str(device) - if isinstance(device, list): - env['CUDA_VISIBLE_DEVICES']=','.join(list(map(str, device))) - if refresh: - cls.pm2_kill(name) - - cwd = cwd or module.dirpath() - - stdout = c.cmd(command, env=env, verbose=verbose, cwd=cwd) - return {'success':True, 'message':f'Launched {module}', 'command': command, 'stdout':stdout} - - - - - - @classmethod - def remote_fn(cls, - fn: str='train', - module: str = None, - args : list = None, - kwargs : dict = None, - name : str =None, - tag: str = None, - refresh : bool =True, - mode = 'pm2', - tag_seperator : str = '::', - cwd = None, - **extra_launch_kwargs - ): - import commune as c - - kwargs = c.locals2kwargs(kwargs) - if 'remote' in kwargs: - kwargs['remote'] = False - if len(fn.split('.'))>1: - module = '.'.join(fn.split('.')[:-1]) - fn = fn.split('.')[-1] - - kwargs = kwargs if kwargs else {} - args = args if args else [] - if 'remote' in kwargs: - kwargs['remote'] = False - - cwd = cwd or cls.dirpath() - kwargs = kwargs or {} - args = args or [] - module = cls.resolve_object(module) - # resolve the name - if name == None: - # if the module has a module_path function, use that as the name - if hasattr(module, 'module_path'): - name = module.module_name() - else: - name = module.__name__.lower() - - c.print(f'[bold cyan]Launching --> <<[/bold cyan][bold yellow]class:{module.__name__}[/bold yellow] [bold white]name[/bold white]:{name} [bold white]fn[/bold white]:{fn} [bold white]mode[/bold white]:{mode}>>', color='green') - - launch_kwargs = dict( - module=module, - fn = fn, - name=name, - tag=tag, - args = args, - kwargs = kwargs, - refresh=refresh, - **extra_launch_kwargs - ) - assert fn != None, 'fn must be specified for pm2 launch' - - return cls.launch(**launch_kwargs) diff --git a/commune/server/namespace.py b/commune/server/namespace.py index fed21481..61e973e4 100644 --- a/commune/server/namespace.py +++ b/commune/server/namespace.py @@ -62,7 +62,6 @@ def update_namespace(cls, network, netuid=None, timeout=5, search=None, verbose= namespace = c.module(network)().namespace(search=search, max_age=1, netuid=netuid) return namespace elif 'local' == network: - print(network, 'FAM') namespace = {} addresses = ['0.0.0.0'+':'+str(p) for p in c.used_ports()] future2address = {} @@ -71,7 +70,6 @@ def update_namespace(cls, network, netuid=None, timeout=5, search=None, verbose= future2address[f] = address futures = list(future2address.keys()) try: - progress = c.tqdm(len(futures)) for f in c.as_completed(futures, timeout=timeout): address = future2address[f] try: @@ -79,18 +77,15 @@ def update_namespace(cls, network, netuid=None, timeout=5, search=None, verbose= namespace[name] = address except Exception as e: c.print(f'Error {e} with {name} and {address}', color='red', verbose=True) - progress.update(1) - except Exception as e: - c.print(f'Timeout error {e}', color='red', verbose=True) - + c.print(f'Error: {e}', color='red', verbose=True) namespace = {k:v for k,v in namespace.items() if 'Error' not in k} ip = c.ip(update=1) namespace = {k: v.replace(ip, '0.0.0.0') for k,v in namespace.items() } else: return {} - return namespace + get_namespace = _namespace = namespace @classmethod @@ -100,15 +95,12 @@ def register_server(cls, name:str, address:str, network=network) -> None: cls.put_namespace(network, namespace) return {'success': True, 'msg': f'Block {name} registered to {network}.'} - @classmethod def deregister_server(cls, name:str, network=network) -> Dict: - namespace = cls.namespace(network=network) address2name = {v: k for k, v in namespace.items()} if name in address2name: name = address2name[name] - if name in namespace: del namespace[name] cls.put_namespace(network, namespace) @@ -128,7 +120,6 @@ def get_address(cls, name:str, network:str=network, external:bool = True) -> dic address = address.replace(c.default_ip, c.ip()) return address - @classmethod def put_namespace(cls, network:str, namespace:dict) -> None: assert isinstance(namespace, dict), 'Namespace must be a dict.' @@ -136,7 +127,6 @@ def put_namespace(cls, network:str, namespace:dict) -> None: add_namespace = put_namespace - @classmethod def rm_namespace(cls,network:str) -> None: if cls.namespace_exists(network): @@ -144,21 +134,7 @@ def rm_namespace(cls,network:str) -> None: return {'success': True, 'msg': f'Namespace {network} removed.'} else: return {'success': False, 'msg': f'Namespace {network} not found.'} - @classmethod - def name2address(cls, name:str, network:str=network ): - namespace = cls.namespace(network=network) - address = namespace.get(name, None) - ip = c.ip() - address = address.replace(c.default_ip, ip) - assert ip in address, f'ip {ip} not in address {address}' - return address - - @classmethod - def address2name(cls, name:str, network:str=network ): - namespace = cls.namespace(network=network) - address2name = {v: k for k, v in namespace.items()} - return address2name - + @classmethod def networks(cls) -> dict: return [p.split('/')[-1].split('.')[0] for p in cls.ls()] @@ -167,7 +143,7 @@ def networks(cls) -> dict: def namespace_exists(cls, network:str) -> bool: path = cls.resolve_network_path( network) return os.path.exists(path) - + @classmethod def modules(cls, network:List=network) -> List[str]: return list(cls.namespace(network=network).keys()) @@ -180,7 +156,6 @@ def addresses(cls, network:str=network, **kwargs) -> List[str]: def module_exists(cls, module:str, network:str=network) -> bool: namespace = cls.namespace(network=network) return bool(module in namespace) - @classmethod def check_servers(self, *args, **kwargs): @@ -192,23 +167,9 @@ def check_servers(self, *args, **kwargs): c.print(c.pm2_restart(server)) return {'success': True, 'msg': 'Servers checked.'} - - - - @classmethod - def merge_namespace(cls, from_network:str, to_network:str, module = None): - from_namespace = cls.namespace(network=from_network) - if module == None: - module = c.module(from_network) - - to_namespace = cls.namespace(network=to_network) - to_namespace.update(from_namespace) - cls.put_namespace(to_network, to_namespace) - return {'success': True, 'msg': f'Namespace {from_network} merged into {to_network}.'} @classmethod def add_server(cls, address:str, name=None, network:str = 'local',timeout:int=4, **kwargs): - """ Add a server to the namespace. """ @@ -234,22 +195,6 @@ def add_server(cls, address:str, name=None, network:str = 'local',timeout:int=4, def remote_servers(cls, network:str = 'remote', **kwargs): return cls.namespace(network=network) - @classmethod - def add_servers(cls, *servers, network:str='local', **kwargs): - if len(servers) == 1 and isinstance(servers[0], list): - servers = servers[0] - responses = [] - for server in servers: - try: - response = cls.add_server(server, network=network) - responses.append(response) - except Exception as e: - e = c.detailed_error(e) - c.print(f'Could not add {e} to {network} modules. {e}', color='red') - responses.append({'success': False, 'msg': f'Could not add {server} to {network} modules. {e}'}) - - return responses - @classmethod def infos(cls, search=None, network=network, servers=None, features=['key', 'address', 'name'], update:str=True, batch_size = 10, timeout=20, hardware=True, namespace=True, schema=True) -> List[str]: path = f'infos/{network}' @@ -284,10 +229,6 @@ def infos(cls, search=None, network=network, servers=None, features=['key', 'add infos = [{k:v for k,v in s.items() if k in features} for s in infos] return infos - @classmethod - def server2info(cls, *args, **kwargs): - return {m['name']:m for m in cls.infos(*args, **kwargs)} - @classmethod def rm_server(cls, name, network:str = 'local', **kwargs): namespace = cls.namespace(network=network) @@ -307,7 +248,6 @@ def rm_server(cls, name, network:str = 'local', **kwargs): else: return {'success': False, 'msg': f'{name} does not exist'} - @classmethod def servers(cls, search=None, network:str = 'local', **kwargs): namespace = cls.namespace(search=search, network=network, **kwargs) @@ -360,7 +300,6 @@ def server_exists(cls, name:str, network:str = None, prefix_match:bool=False, * server_exists = bool(name in servers) return server_exists - @classmethod def clean(cls, network='local'): @@ -377,21 +316,6 @@ def clean(cls, network='local'): namespace = {v:k for k,v in address2name.items()} return namespace - @classmethod - def port2module(cls, *args, **kwargs): - namespace = c.namespace(*args, **kwargs) - port2module = {} - for name, address in namespace.items(): - port = int(address.split(':')[1]) - port2module[port] = name - return port2module - port2name = port2module - - @classmethod - def module2port(cls, *args, **kwargs): - port2module = cls.port2module(*args, **kwargs) - return {v:k for k,v in port2module.items()} - name2port = m2p = module2port Namespace.run(__name__) diff --git a/commune/server/server.py b/commune/server/server.py index 27c43a3e..1ec880b0 100644 --- a/commune/server/server.py +++ b/commune/server/server.py @@ -5,11 +5,9 @@ from fastapi.middleware.cors import CORSMiddleware import uvicorn from .middleware import ServerMiddleware -from .manager import ServerManager from sse_starlette.sse import EventSourceResponse -class Server(ServerManager, c.Module): - +class Server(c.Module): def __init__( self, module: Union[c.Module, object] = None, @@ -156,4 +154,365 @@ def __del__(self): c.deregister_server(self.name) + + + @classmethod + def kill(cls, + module, + mode:str = 'pm2', + verbose:bool = False, + update : bool = True, + prefix_match = False, + network = 'local', # local, dev, test, main + **kwargs): + + kill_fn = getattr(cls, f'{mode}_kill') + delete_modules = [] + + try: + killed_module =kill_fn(module, verbose=verbose,prefix_match=prefix_match, **kwargs) + except Exception as e: + return {'error':str(e)} + if isinstance(killed_module, list): + delete_modules.extend(killed_module) + elif isinstance(killed_module, str): + delete_modules.append(killed_module) + else: + delete_modules.append(killed_module) + # update modules + c.deregister_server(module, network=network) + + assert c.server_exists(module, network=network) == False, f'module {module} still exists' + + servers = c.servers() + for m in delete_modules: + if m in servers: + c.deregister_server(m, network=network) + + return {'server_killed': delete_modules, 'update': update} + + + + @classmethod + def kill_prefix(cls, prefix:str, **kwargs): + servers = c.servers(network='local') + killed_servers = [] + for s in servers: + if s.startswith(prefix): + c.kill(s, **kwargs) + killed_servers.append(s) + return {'success':True, 'message':f'Killed servers with prefix {prefix}'} + + + + @classmethod + def kill_many(cls, servers, search:str = None, network='local', timeout=10, **kwargs): + servers = c.servers(network=network) + servers = [s for s in servers if search in s] + futures = [] + for s in servers: + c.print(f'Killing {s}', color='red') + future = c.submit(c.kill, kwargs={'module':s, **kwargs}, imeout=timeout) + futures.append(future) + results = [] + for r in c.as_completed(futures, timeout=timeout): + results += [r.result()] + c.print(f'Killed {len(results)} servers', color='red') + return results + + + @classmethod + def fleet(cls, module, n=5, timeout=10): + futures = [] + if '::' not in module: + module = f'{module}::' + + + for i in range(n): + module_name = f'{module}{i}' + future = c.submit(cls.serve, kwargs=dict(module=module_name), timeout=timeout) + futures.append(future) + results = [] + for future in c.as_completed(futures, timeout=timeout): + result = future.result() + results.append(result) + + return results + + + @classmethod + def serve_many(cls, modules:list, **kwargs): + + if isinstance(modules[0], list): + modules = modules[0] + + futures = [] + for module in modules: + future = c.submit(c.serve, kwargs={'module': module, **kwargs}) + futures.append(future) + + results = [] + for future in c.as_completed(futures): + result = future.result() + results.append(result) + return results + serve_batch = serve_many + + + @classmethod + def wait_for_server(cls, + name: str , + network: str = 'local', + timeout:int = 600, + sleep_interval: int = 1, + verbose:bool = False) -> bool : + + time_waiting = 0 + while time_waiting < timeout: + namespace = c.namespace(network=network) + if name in namespace: + c.print(f'{name} is ready', color='green') + return True + time_waiting += sleep_interval + c.print(f'Waiting for {name} for {time_waiting} seconds', color='red') + c.sleep(sleep_interval) + raise TimeoutError(f'Waited for {timeout} seconds for {name} to start') + + + + @staticmethod + def kill_all_servers( *args, **kwargs): + ''' + Kill all of the servers + ''' + for module in c.servers(*args, **kwargs): + c.kill(module) + + # c.update(network='local') + + @classmethod + def kill_all(cls, network='local', timeout=20, verbose=True): + futures = [] + servers = c.servers(network=network) + n = len(servers) + progress = c.tqdm(n) + for s in servers: + c.print(f'Killing {s}', color='red') + futures += [c.submit(c.kill, kwargs={'module':s, 'update': False}, return_future=True)] + results_list = [] + for f in c.as_completed(futures, timeout=timeout): + result = f.result() + print(result) + progress.update(1) + results_list += [result] + namespace = c.namespace(network=network, update=True) + new_n = len(servers) + c.print(f'Killed {n - new_n} servers, with {n} remaining {servers}', color='red') + return {'success':True, 'old_n':n, 'new_n':new_n, 'servers':servers, 'namespace':namespace} + + + + @classmethod + def serve(cls, + module:Any = None, + kwargs:dict = None, # kwargs for the module + params = None, # kwargs for the module + tag:str=None, + server_network = 'local', # network to run the server + port :int = None, # name of the server if None, it will be the module name + server_name:str=None, # name of the server if None, it will be the module name + name = None, # name of the server if None, it will be the module name + refresh:bool = True, # refreshes the server's key + remote:bool = True, # runs the server remotely (pm2, ray) + tag_seperator:str='::', + max_workers:int = None, + free: bool = False, + mnemonic = None, # mnemonic for the server + key = None, + **extra_kwargs + ): + module = module or c.module_name() + if module.endswith('.py'): + module = module[:-3] + if tag_seperator in str(module): + module, tag = module.split(tag_seperator) + kwargs = {**(params or kwargs or {}), **extra_kwargs} + name = name or server_name or module + if tag_seperator in name: + module, tag = name.split(tag_seperator) + else: + if tag != None: + name = f'{name}{tag_seperator}{tag}' + + if port == None: + # now if we have the server_name, we can repeat the server + address = c.get_address(name, network=server_network) + try: + port = int(address.split(':')[-1]) + if c.port_used(port): + c.kill_port(port) + except Exception as e: + port = c.free_port() + # RESOLVE THE PORT FROM THE ADDRESS IF IT ALREADY EXISTS + + # # NOTE REMOVE is FROM THE KWARGS REMOTE + if remote: + remote_kwargs = c.locals2kwargs(locals()) # GET THE LOCAL KWARGS FOR SENDING TO THE REMOTE + remote_kwargs['remote'] = False # SET THIS TO FALSE TO AVOID RECURSION + for _ in ['extra_kwargs', 'address']: + remote_kwargs.pop(_, None) # WE INTRODUCED THE ADDRES + response = cls.remote_fn('serve', name=name, kwargs=remote_kwargs) + if response['success'] == False: + return response + return {'success':True, + 'name': name, + 'address':c.ip() + ':' + str(remote_kwargs['port']), + 'kwargs':kwargs, + 'module':module + } + + module_class = c.module(module) + kwargs.update(extra_kwargs) + module = module_class(**kwargs) + cls(module=module, + name=name, + port=port, + network=server_network, + max_workers=max_workers, + mnemonic = mnemonic, + free=free, + key=key) + + return {'success':True, + 'address': f'{c.default_ip}:{port}' , + 'name':name, + 'kwargs': kwargs, + 'module':module} + + + + @classmethod + def launch(cls, + module:str = None, + fn: str = 'serve', + name:Optional[str]=None, + tag : str = None, + args : list = None, + kwargs: dict = None, + device:str=None, + interpreter:str='python3', + autorestart: bool = True, + verbose: bool = False , + force:bool = True, + meta_fn: str = 'module_fn', + tag_seperator:str = '::', + cwd = None, + refresh:bool=True ): + import commune as c + + if hasattr(module, 'module_name'): + module = module.module_name() + + # avoid these references fucking shit up + args = args if args else [] + kwargs = kwargs if kwargs else {} + + # convert args and kwargs to json strings + kwargs = { + 'module': module , + 'fn': fn, + 'args': args, + 'kwargs': kwargs + } + + kwargs_str = json.dumps(kwargs).replace('"', "'") + + name = name or module + if refresh: + cls.pm2_kill(name) + module = c.module() + # build command to run pm2 + filepath = c.filepath() + cwd = cwd or module.dirpath() + command = f"pm2 start {filepath} --name {name} --interpreter {interpreter}" + + if not autorestart: + command += ' --no-autorestart' + if force: + command += ' -f ' + command = command + f' -- --fn {meta_fn} --kwargs "{kwargs_str}"' + env = {} + if device != None: + if isinstance(device, int): + env['CUDA_VISIBLE_DEVICES']=str(device) + if isinstance(device, list): + env['CUDA_VISIBLE_DEVICES']=','.join(list(map(str, device))) + if refresh: + cls.pm2_kill(name) + + cwd = cwd or module.dirpath() + + stdout = c.cmd(command, env=env, verbose=verbose, cwd=cwd) + return {'success':True, 'message':f'Launched {module}', 'command': command, 'stdout':stdout} + + + + @classmethod + def remote_fn(cls, + fn: str='train', + module: str = None, + args : list = None, + kwargs : dict = None, + name : str =None, + tag: str = None, + refresh : bool =True, + mode = 'pm2', + tag_seperator : str = '::', + cwd = None, + **extra_launch_kwargs + ): + import commune as c + + kwargs = c.locals2kwargs(kwargs) + if 'remote' in kwargs: + kwargs['remote'] = False + if len(fn.split('.'))>1: + module = '.'.join(fn.split('.')[:-1]) + fn = fn.split('.')[-1] + + kwargs = kwargs if kwargs else {} + args = args if args else [] + if 'remote' in kwargs: + kwargs['remote'] = False + + cwd = cwd or cls.dirpath() + kwargs = kwargs or {} + args = args or [] + module = cls.resolve_object(module) + # resolve the name + if name == None: + # if the module has a module_path function, use that as the name + if hasattr(module, 'module_path'): + name = module.module_name() + else: + name = module.__name__.lower() + + c.print(f'[bold cyan]Launching --> <<[/bold cyan][bold yellow]class:{module.__name__}[/bold yellow] [bold white]name[/bold white]:{name} [bold white]fn[/bold white]:{fn} [bold white]mode[/bold white]:{mode}>>', color='green') + + launch_kwargs = dict( + module=module, + fn = fn, + name=name, + tag=tag, + args = args, + kwargs = kwargs, + refresh=refresh, + **extra_launch_kwargs + ) + assert fn != None, 'fn must be specified for pm2 launch' + + return cls.launch(**launch_kwargs) + + + Server.run(__name__) \ No newline at end of file diff --git a/commune/subspace/client.py b/commune/subspace/client.py index ae1420aa..cef14fc6 100644 --- a/commune/subspace/client.py +++ b/commune/subspace/client.py @@ -12,8 +12,8 @@ from substrateinterface.storage import StorageKey # type: ignore from commune.subspace.utils import transform_stake_dmap -from commune.errors import ChainTransactionError, NetworkQueryError -from commune.types import NetworkParams, Ss58Address, SubnetParams +from commune.utils.errors import ChainTransactionError, NetworkQueryError +from commune.utils.types import NetworkParams, Ss58Address, SubnetParams # TODO: InsufficientBalanceError, MismatchedLengthError etc diff --git a/commune/subspace/global.py b/commune/subspace/global.py deleted file mode 100644 index e69de29b..00000000 diff --git a/commune/subspace/subnet.py b/commune/subspace/subnet.py deleted file mode 100644 index da5f034d..00000000 --- a/commune/subspace/subnet.py +++ /dev/null @@ -1,1056 +0,0 @@ -from typing import * -import commune as c -import requests -class SubspaceSubnet: - - subnet_param_features = [ - "ImmunityPeriod", - "MinAllowedWeights", - "MaxAllowedWeights", - "Tempo", - "MaxAllowedUids", - "Founder", - "FounderShare", - "IncentiveRatio", - "TrustRatio", - "SubnetNames", - "MaxWeightAge", - "BondsMovingAverage", - "MaximumSetWeightCallsPerEpoch", - "MinValidatorStake", - "MaxAllowedValidators", - "ModuleBurnConfig", - "SubnetMetadata", - 'SubnetGovernanceConfig' - ] - - - - def stake_to(self, block=None, max_age=1000, update=False, fmt='nano', **kwargs): - stake_to = self.query_map('StakeTo', block=block, max_age=max_age, update=update, **kwargs) - format_value = lambda v: {v_k: self.format_amount(v_v, fmt=fmt) for v_k, v_v in v.items()} - stake_to = {k: format_value(v) for k,v in stake_to.items()} - return stake_to - - - - def netuid2founder(self, fmt='j', **kwargs): - netuid2founder = self.query_map('Founder', **kwargs) - return netuid2founder - - - def stake_from(self, - block=None, - update=False, - max_age=10000, - fmt='nano', - **kwargs) -> List[Dict[str, Union[str, int]]]: - - stake_from = self.query_map('StakeFrom', block=block, update=update, max_age=max_age ) - format_value = lambda v: {v_k: self.format_amount(v_v, fmt=fmt) for v_k, v_v in v.items()} - stake_from = {k: format_value(v) for k,v in stake_from.items()} - return stake_from - - """ Returns network Tempo hyper parameter """ - def stakes(self, fmt:str='j', max_age = 100, update=False, stake_from=None, **kwargs) -> int: - stake_from = stake_from or self.stake_from( update=update, max_age=max_age, fmt=fmt) - stakes = {k: sum(v.values()) for k,v in stake_from.items()} - return stakes - - def leaderboard(self, netuid = 0, block=None, update=False, columns = ['emission', 'name', 'incentive', 'dividends'], **kwargs): - modules = self.get_modules(netuid=netuid, block=block, update=update, **kwargs) - return c.df(modules)[columns] - - - def min_stake(self, netuid: int = 0, fmt:str='j', **kwargs) -> int: - min_stake = self.query('MinStake', netuid=netuid, **kwargs) - return self.format_amount(min_stake, fmt=fmt) - - def regblock(self, netuid: int = 0, block: Optional[int] = None, update=False ) -> Optional[float]: - regblock = self.query_map('RegistrationBlock',block=block, update=update ) - if isinstance(netuid, int): - regblock = regblock[netuid] - return regblock - - def emissions(self, netuid = None, block=None, update=False, fmt = 'nanos', **kwargs): - netuid = self.resolve_netuid(netuid) - emissions = self.query_vector('Emission', netuid=netuid, block=block, update=update, **kwargs) - if netuid == 'all': - for netuid, netuid_emissions in emissions.items(): - emissions[netuid] = [self.format_amount(e, fmt=fmt) for e in netuid_emissions] - else: - emissions = [self.format_amount(e, fmt=fmt) for e in emissions] - - return emissions - - emission = emissions - - - def total_emission( self, netuid: int = 0, block: Optional[int] = None, fmt:str = 'j', **kwargs ) -> Optional[float]: - total_emission = sum(self.emission(netuid=netuid, block=block, **kwargs)) - return self.format_amount(total_emission, fmt=fmt) - - def addresses(self, netuid: int = 0, update=False, **kwargs) -> List[str]: - netuid = self.resolve_netuid(netuid) - addresses = self.query_map('Address',netuid=netuid, update=update, **kwargs) - - if isinstance(netuid, int): - addresses = list(addresses.values()) - else: - for k,v in addresses.items(): - addresses[k] = list(v.values()) - return addresses - - def namespace(self, search=None, netuid: int = 0, update:bool = False, timeout=30, local=False, max_age=1000, **kwargs) -> Dict[str, str]: - namespace = {} - results = { - 'names': None, - 'addresses': None - } - netuid = self.resolve_netuid(netuid) - while any([v == None for v in results.values()]): - future2key = {} - for k,v in results.items(): - if v == None: - f = c.submit(getattr(self, k), kwargs=dict(netuid=netuid, update=update, max_age=max_age, **kwargs)) - future2key[f] = k - for future in c.as_completed(list(future2key.keys()), timeout=timeout): - key = future2key.pop(future) - r = future.result() - if not c.is_error(r) and r != None: - results[key] = r - - if netuid == 'all': - netuid2subnet = self.netuid2subnet() - namespace = {} - for netuid, netuid_addresses in results['addresses'].items(): - for uid,address in enumerate(netuid_addresses): - name = results['names'][netuid][uid] - subnet = netuid2subnet[netuid] - namespace[f'{subnet}/{name}'] = address - - else: - namespace = {k:v for k,v in zip(results['names'], results['addresses'])} - - if search != None: - namespace = {k:v for k,v in namespace.items() if search in str(k)} - - if local: - ip = c.ip() - namespace = {k:v for k,v in namespace.items() if ip in str(v)} - - return namespace - - def emissions(self, netuid = None, network = "main", block=None, update=False, **kwargs): - netuid = self.resolve_netuid(netuid) - return self.query_vector('Emission', network=network, netuid=netuid, block=block, update=update, **kwargs) - - def key2name(self, key: str = None, netuid: int = 0) -> str: - modules = self.keys(netuid=netuid) - key2name = { m['key']: m['name']for m in modules} - if key != None: - return key2name[key] - - def uid2name(self, netuid: int = 0, update=False, **kwargs) -> List[str]: - netuid = self.resolve_netuid(netuid) - names = self.query_map('Name', netuid=netuid, update=update,**kwargs) - return names - - def is_registered(self, key: str, netuid: int = 0, update=False, **kwargs) -> bool: - key_address = self.resolve_key_ss58(key) - try: - uid = self.get_uid(key_address, netuid=netuid, update=update, **kwargs) - if isinstance(uid, int): - return True - except Exception as e: - return False - - def keys(self, - netuid = None, - update=False, - max_age=1000, - **kwargs) -> List[str]: - netuid = self.resolve_netuid(netuid) - keys = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) - if netuid == 'all': - for netuid, netuid_keys in keys.items(): - keys[netuid] = list(netuid_keys.values()) - else: - keys = list(keys.values()) - return keys - - def delegation_fee(self, netuid = None, block=None, update=False, fmt='j'): - netuid = self.resolve_netuid(netuid) - delegation_fee = self.query_map('DelegationFee', netuid=netuid, block=block ,update=update) - return delegation_fee - - - def feature2name(self, feature='MinStake'): - translations = { - 'subnet_names': 'name' - } - name = '' - for i, ch in enumerate(feature): - if ch.isupper(): - if i == 0: - name += ch.lower() - else: - name += f'_{ch.lower()}' - else: - name += ch - name = translations.get(name, name) - return name - - - def subnet_params(self, - netuid=0, - update = False, - max_age = 1000, - timeout=40, - fmt:str='j', - features = None, - value_features = [], - **kwargs - ) -> list: - if netuid == 'all': - return self.all_subnet_params(update=update, - max_age=max_age, - timeout=timeout, - fmt=fmt, - features=features, - value_features=value_features, - **kwargs) - - - default_params = { - 'maximum_set_weight_calls_per_epoch': 30, - 'max_allowed_validators': 50 - } - - features = features or self.subnet_param_features - netuid = self.resolve_netuid(netuid) - path = f'query/{self.network}/SubnetParams.{netuid}' - subnet_params = self.get(path, max_age=max_age, update=update) - if subnet_params == None: - - names = [self.feature2name(f) for f in features] - future2name = {} - for name, feature in dict(zip(names, features)).items(): - query_kwargs = dict(name=feature, netuid=netuid,block=None, max_age=max_age, update=update) - if name in ['SubnetGovernanceConfig']: - fn = self.query_map - else: - fn = self.query - - f = c.submit(fn, kwargs=query_kwargs, timeout=timeout) - future2name[f] = name - subnet_params = {} - for f in c.as_completed(future2name, timeout=timeout): - result = f.result() - subnet_params[future2name.pop(f)] = result - for k in subnet_params.keys(): - v = subnet_params[k] - if v == None: - v = default_params.get(k, v) - if k in value_features: - v = self.format_amount(v, fmt=fmt) - subnet_params[k] = v - - self.put(path, subnet_params) - - subnet_params.update(subnet_params.pop('subnet_governance_config')) - translation = { - 'subnet_names': 'name', - 'bonds_moving_average': 'bonds_ma' - } - for k,v in translation.items(): - if k in subnet_params: - subnet_params[v] = subnet_params.pop(k) - return subnet_params - - - def all_subnet_params(self, - update = False, - max_age = 1000, - features = None, - **kwargs - ) -> list: - - features = features or self.subnet_param_features - netuid = self.resolve_netuid(netuid) - path = f'query/{self.network}/SubnetParams.all' - all_subnet_params = self.get(path, max_age=max_age, update=update) - if all_subnet_params == None: - all_subnet_params = {} - for netuid in self.netuids(update=update): - all_subnet_params[netuid] = self.subnet_params(netuid=netuid, update=update, max_age=max_age, **kwargs) - return all_subnet_params - - def pending_deregistrations(self, netuid = None, update=False, **kwargs): - netuid = self.resolve_netuid(netuid) - pending_deregistrations = self.query_map('PendingDeregisterUids',update=update,**kwargs)[netuid] - return pending_deregistrations - - def num_pending_deregistrations(self, netuid = 0, **kwargs): - pending_deregistrations = self.pending_deregistrations(netuid=netuid, **kwargs) - return len(pending_deregistrations) - - def subnet_names(self , search=None, update=False, block=None, max_age=60, **kwargs) -> Dict[str, str]: - records = self.query_map('SubnetNames', update=update, block=block, max_age=max_age, **kwargs) - subnet_names = sorted(list(map(lambda x: str(x), records.values()))) - if search != None: - subnet_names = [s for s in subnet_names if search in s] - return subnet_names - - def subnets(self, **kwargs) -> Dict[int, str]: - return self.subnet_names(**kwargs) - - def num_subnets(self, **kwargs) -> int: - return len(self.subnets(**kwargs)) - - def subnet2stake(self, fmt='j'): - netuid2subnet = self.netuid2subnet() - netuid2stake = self.netuid2stake(fmt=fmt) - subnet2stake = {} - for netuid, subnet in netuid2subnet.items(): - subnet2stake[subnet] = netuid2stake[netuid] - return subnet2stake - - def netuid2stake(self, fmt='j', **kwargs): - netuid2stake = self.query_map('TotalStake', **kwargs) - for netuid, stake in netuid2stake.items(): - netuid2stake[netuid] = self.format_amount(stake, fmt=fmt) - return netuid2stake - - def netuid2n(self, fmt='j', **kwargs): - netuid2n = self.query_map('N', **kwargs) - return netuid2n - - def trust(self, - netuid = 0, - block=None, - update:bool = False, - **kwargs): - return self.query_vector('Trust', netuid=netuid, block=block, update=update, **kwargs) - - def incentives(self, - netuid = 0, - block=None, - update:bool = False, - **kwargs): - return self.query_vector('Incentive', netuid=netuid, block=block, update=update, **kwargs) - incentive = incentives - - def last_update(self, netuid = 0, update=False, **kwargs): - return self.query_vector('LastUpdate', netuid=netuid, update=update, **kwargs) - - def dividends(self, netuid = 0, update=False, **kwargs): - return self.query_vector('Dividends', netuid=netuid, update=update, **kwargs) - - dividend = dividends - - def names(self, - netuid: int = 0, - update=False, - **kwargs) -> List[str]: - netuid = self.resolve_netuid(netuid) - names = self.query_map('Name', update=update, netuid=netuid,**kwargs) - if netuid == 'all': - for netuid, netuid_names in names.items(): - names[netuid] = list(netuid_names.values()) - else: - names = list(names.values()) - return names - - def uids(self, - netuid = 0, - update=False, - max_age=1000, - **kwargs) -> List[str]: - netuid = self.resolve_netuid(netuid) - keys = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) - if netuid == 'all': - for netuid, netuid_keys in keys.items(): - keys[netuid] = list(netuid_keys.keys ()) - else: - keys = list(keys.keys()) - return keys - - def subnet2n(self, fmt='j', **kwargs): - netuid2n = self.netuid2n(fmt=fmt, **kwargs) - netuid2subnet = self.netuid2subnet() - subnet2n = {} - for netuid, subnet in netuid2subnet.items(): - subnet2n[subnet] = netuid2n[netuid] - return subnet2n - - def subnet2stakes(self, block=None, update=False, fmt='j', **kwargs): - subnet2stakes = {} - for netuid in self.netuids( update=update): - subnet2stakes[netuid] = self.stakes(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) - return subnet2stakes - - def subnet_state(self, netuid='all', block=None, update=False, fmt='j', **kwargs): - - subnet_state = { - 'params': self.subnet_params(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs), - 'modules': self.modules(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs), - } - return subnet_state - - def subnet2emission(self, fmt='j', **kwargs): - subnet2params = self.subnet_params(netuid='all') - netuid2emission = self.netuid2emission(fmt=fmt, **kwargs) - netuid2subnet = self.netuid2subnet() - subnet2emission = {} - for netuid, subnet in netuid2subnet.items(): - subnet2emission[subnet] = netuid2emission[netuid] - # sort by emission - subnet2emission = dict(sorted(subnet2emission.items(), key=lambda x: x[1], reverse=True)) - - return subnet2emission - - - def uid2key(self, uid=None, - netuid = 0, - update=False, - - max_age= 1000, - **kwargs): - netuid = self.resolve_netuid(netuid) - uid2key = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) - # sort by uid - if uid != None: - return uid2key[uid] - return uid2key - - - def key2uid(self, key = None, netuid: int = 0, update=False, netuids=None , **kwargs): - uid2key = self.uid2key( netuid=netuid, update=update, **kwargs) - reverse_map = lambda x: {v: k for k,v in x.items()} - if netuid == 'all': - key2uid = {netuid: reverse_map(_key2uid) for netuid, _key2uid in uid2key.items() if netuids == None or netuid in netuids } - else: - key2uid = reverse_map(uid2key) - if key != None: - key_ss58 = self.resolve_key_ss58(key) - return key2uid[key_ss58] - return key2uid - - """ Returns network SubnetN hyper parameter """ - def n(self, netuid: int = 0,block: Optional[int] = None, max_age=100, update=False, **kwargs ) -> int: - if netuid == 'all': - return sum(self.query_map('N', block=block , update=update, max_age=max_age, **kwargs).values()) - else: - return self.query( 'N', params=[netuid], block=block , update=update, **kwargs) - - def subnet_exists(self, subnet:str) -> bool: - subnets = self.subnets() - return bool(subnet in subnets) - - def subnet_emission(self, netuid:str = 0, block=None, update=False, **kwargs): - emissions = self.emission(block=block, update=update, netuid=netuid, **kwargs) - if isinstance(emissions[0], list): - emissions = [sum(e) for e in emissions] - return sum(emissions) - - def get_modules(self, - keys : list = None, - netuid=None, - timeout=30, - min_emission=0, - max_age = 1000, - update=False, - **kwargs): - netuid = self.resolve_netuid(netuid) - modules = None - path = None - if keys == None : - path = f'subnet/{self.network}/{netuid}/modules' - modules = self.get(path, None, max_age=max_age, update=update) - keys = self.keys(netuid=netuid) - - n = len(keys) - if modules == None: - modules = [] - print(f'Getting modules {n}') - futures = [c.submit(self.get_module, kwargs=dict(module=k, netuid=netuid, **kwargs)) for k in keys] - progress = c.tqdm(n) - modules = [] - - - should_pass = lambda x: isinstance(x, dict) \ - and 'name' in x \ - and len(x['name']) > 0 \ - and x['emission'] >= min_emission - - for future in c.as_completed(futures, timeout=timeout): - module = future.result() - if should_pass(module): - modules += [module] - progress.update(1) - if path != None: - self.put(path, modules) - - - return modules - - module_param_features = [ - 'key', - 'name', - 'address', - 'emission', - 'incentive', - 'dividends', - 'last_update', - 'stake_from', - 'delegation_fee' - ] - - def get_module(self, - module=None, - netuid=None, - trials = 4, - fmt='j', - mode = 'http', - block = None, - max_age = None, - lite = False, - update = False, - **kwargs ) -> 'ModuleInfo': - U16_MAX = 2**16 - 1 - - netuid = self.resolve_netuid(netuid) - - if module == None: - module = self.keys(netuid=netuid, update=update, max_age=max_age)[0] - c.print(f'No module specified, using {module}') - - module = c.key2address().get(module, module) - url = self.resolve_url( mode=mode) - module_key = module - is_valid_key = c.valid_ss58_address(module) - if not is_valid_key: - module_key = self.name2key(name=module, netuid=netuid, **kwargs) - netuid = self.resolve_netuid(netuid) - json={'id':1, 'jsonrpc':'2.0', 'method': 'subspace_getModuleInfo', 'params': [module_key, netuid]} - module = None - for i in range(trials): - try: - module = requests.post(url, json=json).json() - break - except Exception as e: - c.print(e) - continue - assert module != None, f"Failed to get module {module_key} after {trials} trials" - if not 'result' in module: - return module - module = {**module['result']['stats'], **module['result']['params']} - # convert list of u8 into a string Vector to a string - module['name'] = self.vec82str(module['name']) - module['address'] = self.vec82str(module['address']) - module['dividends'] = module['dividends'] / (U16_MAX) - module['incentive'] = module['incentive'] / (U16_MAX) - module['stake_from'] = {k:self.format_amount(v, fmt=fmt) for k,v in module['stake_from']} - module['stake'] = sum([v for k,v in module['stake_from'].items() ]) - module['emission'] = self.format_amount(module['emission'], fmt=fmt) - module['key'] = module.pop('controller', None) - module['metadata'] = module.pop('metadata', {}) - - module['vote_staleness'] = (block or self.block) - module['last_update'] - if lite : - features = self.module_param_features + ['stake', 'vote_staleness'] - module = {f: module[f] for f in features} - assert module['key'] == module_key, f"Key mismatch {module['key']} != {module_key}" - return module - - - def root_valis(self, search=None, netuid = 0, update=False, **kwargs): - root_valis = [] - for module in self.get_modules(netuid=netuid, update=update, **kwargs): - if search != None: - if search not in module['name']: - continue - module.pop('stake_from') - root_valis += [module ] - - return c.df(root_valis)[['name', 'key', 'stake']] - - - def root_keys(self, netuid = 0, update=False, **kwargs): - return self.keys(netuid=netuid, update=update, **kwargs) - - - - - def registration_block(self, netuid: int = 0, update=False, **kwargs): - registration_blocks = self.query_map('RegistrationBlock', netuid=netuid, update=update, **kwargs) - return registration_blocks - - regblocks = registration_blocks = registration_block - - - - - - - - def key2name(self, key=None, netuid: int = None, update=False) -> Dict[str, str]: - - key2name = {v:k for k,v in self.name2key(netuid=netuid, update=update).items()} - if key != None: - return key2name[key] - return key2name - - - - - def name2key(self, name:str=None, - max_age=1000, - timeout=30, - netuid: int = 0, - update=False, - trials=3, - **kwargs ) -> Dict[str, str]: - # netuid = self.resolve_netuid(netuid) - netuid = self.resolve_netuid(netuid) - - names = c.submit(self.names, kwargs={'feature': 'names', 'netuid':netuid, 'update':update, 'max_age':max_age, 'network': self.network}) - keys = c.submit(self.keys, kwargs={'feature': 'keys', 'netuid':netuid, 'update':update, 'max_age':max_age, 'network': self.network}) - names, keys = c.wait([names, keys], timeout=timeout) - name2key = dict(zip(names, keys)) - if name != None: - if name in name2key: - return name2key[name] - else: - trials -= 1 - if trials == 0: - return None - else: - return self.name2key(name=name, - timeout=timeout, netuid=netuid, update=True, - trials=trials, **kwargs) - - return name2key - - - - - def name2uid(self, name = None, netuid: int = 0, search=None) -> int: - netuid = self.resolve_netuid(netuid) - uid2name = self.uid2name(netuid=netuid) - - if netuid == 'all': - netuid2name2uid = {} - for netuid, netuid_uid2name in uid2name.items(): - name2uid = self.search_dict(netuid_uid2name) - if name != None: - name2uid = name2uid[name] - netuid2name2uid[netuid] = name2uid - return netuid2name2uid - - else: - name2uid = {v:k for k,v in uid2name.items()} - if search != None: - name2uid = self.search_dict(name2uid, search=search) - if name != None: - return name2uid[name] - - return name2uid - - def netuids(self, update=False, block=None) -> Dict[int, str]: - return list(self.netuid2subnet( update=update, block=block).keys()) - - def netuid2subnet(self, netuid=None, update=False, block=None, **kwargs ) -> Dict[str, str]: - netuid2subnet = self.query_map('SubnetNames', update=update, block=block, **kwargs) - netuid2subnet = dict(sorted(netuid2subnet.items(), key=lambda x: x[0])) - if netuid != None: - return netuid2subnet[netuid] - return netuid2subnet - netuid2name = netuid2subnet - - def subnet2netuid(self, subnet=None, update=False, **kwargs ) -> Dict[str, str]: - subnet2netuid = {v:k for k,v in self.netuid2subnet( update=update, **kwargs).items()} - # sort by subnet - if subnet != None: - return subnet2netuid[subnet] if subnet in subnet2netuid else len(subnet2netuid) - return subnet2netuid - name2netuid = subnet2netuid - - - - def get_uid( self, key: str, netuid: int = 0, block: Optional[int] = None, update=False, **kwargs) -> int: - return self.query( 'Uids', block=block, params=[ netuid, key ] , update=update, **kwargs) - - - def weights(self, netuid = 0, update=False, **kwargs) -> list: - weights = self.query_map('Weights',netuid=netuid, update=update, **kwargs) - - tuples2list = lambda x: [list(v) for v in x] - if netuid == 'all': - for netuid, netuid_weights in weights.items(): - weights[netuid] = {k: tuples2list(v) for k,v in netuid_weights.items()} - else: - weights = {k: tuples2list(v) for k,v in weights.items()} - - return weights - - def resolve_uid(self, uid=None, netuid=None, **kwargs) -> int: - netuid = self.resolve_netuid(netuid) - if isinstance(uid, int): - return uid - elif isinstance(uid, str): - if c.key_exists(uid): - # for key - uid = self.resolve_key_ss58(uid) - uid = self.key2uid(netuid=netuid,**kwargs)[uid] - else: - # resolve name - uid = self.name2uid(name=uid, netuid=netuid, **kwargs) - - return uid - - - - def get_weights(self, key=None, netuid = 0, update=False, **kwargs) -> list: - uid = self.resolve_uid(key, netuid=netuid) - weights = self.query('Weights', params=[netuid, uid], update=update, **kwargs) - return weights - - - - - - - def total_emissions(self, netuid = 9, block=None, update=False, fmt = 'j', **kwargs): - - emissions = self.query_vector('Emission', netuid=netuid, block=block, update=update, **kwargs) - if netuid == 'all': - for netuid, netuid_emissions in emissions.items(): - emissions[netuid] = [self.format_amount(e, fmt=fmt) for e in netuid_emissions] - else: - emissions = [self.format_amount(e, fmt=fmt) for e in emissions] - - return sum(emissions) - - - - # def state(self, block=None, netuid='all', update=False, max_age=10000, fmt='j', **kwargs): - # subnet_params = self.subnet_params(block=block, netuid=netuid, max_age=max_age, **kwargs) - # subnet2emissions = self.emissions(netuid=netuid, max_age=max_age, block=block, **kwargs) - # subnet2staketo = self.stake_to(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) - # subnet2incentives = self.incentives(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) - # subnet2trust = self.trust(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) - # subnet2keys = self.keys(netuid=netuid, block=block, update=update, **kwargs) - - # subnet2state = {} - # for netuid, params in subnet_params.items(): - # subnet_state = { - # 'params': params, - # 'incentives': subnet2incentives[netuid], - # 'emissions': subnet2emissions[netuid], - # '' - # 'stake_to': subnet2staketo[netuid], - # 'keys': subnet2keys[netuid], - - # } - # subnet_state[netuid] = subnet_state - - # return subnet2state - - - def netuid2emission(self, fmt='j', period='day', names=None, **kwargs): - netuid2emission = {} - netuid2tempo = None - emissions = self.query_vector('Emission', netuid='all', **kwargs) - for netuid, netuid_emissions in emissions.items(): - - if period == 'day': - if netuid2tempo == None: - netuid2tempo = self.query_map('Tempo', netuid='all', **kwargs) - tempo = netuid2tempo.get(netuid, 100) - multiplier = self.blocks_per_day() / tempo - else: - multiplier = 1 - netuid2emission[netuid] = self.format_amount(sum(netuid_emissions), fmt=fmt) * multiplier - - netuid2emission = {k: v for k,v in netuid2emission.items()} - if names: - netuid2emission = {self.netuid2name(netuid=k): v for k,v in netuid2emission.items()} - return netuid2emission - - def subnet2emission(self, fmt='j', period='day', **kwargs): - return self.netuid2emission(fmt=fmt, period=period, names=1, **kwargs) - - - def global_emissions(self, **kwargs): - return sum(list(self.subnet2emissions( **kwargs).values())) - - - - - def subnet2params( self, block: Optional[int] = None ) -> Optional[float]: - netuids = self.netuids() - subnet2params = {} - netuid2subnet = self.netuid2subnet() - for netuid in netuids: - subnet = netuid2subnet[netuid] - subnet2params[subnet] = self.subnet_params(netuid=netuid, block=block) - return subnet2params - - - - - ################# - #### UPDATE SUBNET #### - ################# - def update_subnet( - self, - params: dict= None, - netuid: int = 0, - key: str = None, - nonce = None, - update= True, - **extra_params, - ) -> bool: - - params = {**(params or {}), **extra_params} - - netuid = self.resolve_netuid(netuid) - subnet_params = self.subnet_params( netuid=netuid , update=update, fmt='nanos') - # infer the key if you have it - for k in ['min_immunity_stake']: - if k in params: - params[k] = params[k] * 1e9 - if key == None: - key2address = c.address2key() - if subnet_params['founder'] not in key2address: - return {'success': False, 'message': f"Subnet {netuid} not found in local namespace, please deploy it "} - key = c.get_key(key2address.get(subnet_params['founder'])) - c.print(f'Using key: {key}') - - # remove the params that are the same as the module info - params = {**subnet_params, **params} - for k in ['name']: - params[k] = params[k].encode('utf-8') - - params['netuid'] = netuid - return self.compose_call(fn='update_subnet', params=params, key=key, nonce=nonce) - - - ################# - #### Serving #### - ################# - def propose_subnet_update( - self, - netuid: int = None, - key: str = None, - nonce = None, - **params, - ) -> bool: - - netuid = self.resolve_netuid(netuid) - c.print(f'Adding proposal to subnet {netuid}') - subnet_params = self.subnet_params( netuid=netuid , update=True) - # remove the params that are the same as the module info - params = {**subnet_params, **params} - for k in ['name', 'vote_mode']: - params[k] = params[k].encode('utf-8') - params['netuid'] = netuid - - response = self.compose_call(fn='add_subnet_proposal', - params=params, - key=key, - nonce=nonce) - - - return response - - - - def get_stake( self, key_ss58: str, block: Optional[int] = None, netuid:int = None , fmt='j', update=True ) -> Optional['Balance']: - - key_ss58 = self.resolve_key_ss58( key_ss58) - netuid = self.resolve_netuid( netuid ) - stake = self.query( 'Stake',params=[netuid, key_ss58], block=block , update=update) - return self.format_amount(stake, fmt=fmt) - - - - def min_register_stake(self, netuid: int = 0, fmt='j', **kwargs) -> float: - netuid = self.resolve_netuid(netuid) - min_burn = self.min_burn( fmt=fmt) - min_stake = self.min_stake(netuid=netuid, fmt=fmt) - return min_stake + min_burn - - - - - def resolve_netuid(self, netuid: int = None) -> int: - ''' - Resolves a netuid to a subnet name. - ''' - if netuid == 'all': - return netuid - if netuid == None : - # If the netuid is not specified, use the default. - return self.netuid - if isinstance(netuid, str): - subnet2netuid = self.subnet2netuid() - if netuid not in subnet2netuid: # if still not found, try lower case - subnet2netuid =self.subnet2netuid(update=True) - assert netuid in subnet2netuid, f"Subnet {netuid} not found in {subnet2netuid}" - return subnet2netuid[netuid] - - elif isinstance(netuid, int): - if netuid == 0: - return netuid - # If the netuid is an integer, ensure it is valid. - - assert isinstance(netuid, int), "netuid must be an integer" - return netuid - - - def blocks_until_vote(self, netuid=None, **kwargs): - netuid = self.resolve_netuid(netuid) - tempo = self.subnet_params(netuid=netuid, **kwargs)['tempo'] - block = self.block - return tempo - ((block + netuid) % tempo) - - def emission_per_epoch(self, netuid=None): - return self.subnet(netuid=netuid)['emission']*self.epoch_time(netuid=netuid) - - - - - - def get_stake_to( self, - key: str = None, - module_key=None, - block: Optional[int] = None, - fmt='j' , update=False, - max_age = 60, - timeout = 10, - **kwargs) -> Optional['Balance']: - - key_address = self.resolve_key_ss58( key ) - stake_to = self.query_map( 'StakeTo', params=[ key_address], block=block, update=update, max_age=max_age) - stake_to = {k: self.format_amount(v, fmt=fmt) for k, v in stake_to.items()} - return stake_to - - def get_stake_from( self, key: str, block: Optional[int] = None, fmt='j', update=True ) -> Optional['Balance']: - key = self.resolve_key_ss58( key ) - stake_from = self.query_map( 'StakeFrom', params=[key], block=block, update=update ) - stake_from = {k: self.format_amount(v, fmt=fmt) for k, v in stake_from.items()} - return stake_from - - - def epoch_time(self, netuid=None, update=False, **kwargs): - netuid = self.resolve_netuid(netuid) - return self.subnet_params(netuid=netuid, update=update, **kwargs)['tempo']*self.block_time - - - def seconds_per_day(self ): - return 24*60*60 - - def epochs_per_day(self, netuid=None): - netuid = self.resolve_netuid(netuid) - return self.seconds_per_day()/self.epoch_time(netuid=netuid) - - def seconds_per_epoch(self, netuid=0): - netuid =self.resolve_netuid(netuid) - return self.block_time * self.subnet_params(netuid=netuid)['tempo'] - - - def format_module(self, module: 'ModuleInfo', fmt:str='j') -> 'ModuleInfo': - U16_MAX = 2**16 - 1 - for k in ['emission']: - module[k] = self.format_amount(module[k], fmt=fmt) - for k in ['incentive', 'dividends']: - module[k] = module[k] / (U16_MAX) - - module['stake_from'] = {k: self.format_amount(v, fmt=fmt) for k, v in module['stake_from']} - return module - - - - def netuid2module(self, update=False, fmt:str='j', **kwargs) -> 'ModuleInfo': - netuids = self.netuids(update=update) - future2netuid = {} - for netuid in netuids: - f = c.submit(self.get_module, dict(netuid=netuid, update=update, fmt=fmt, **kwargs)) - future2netuid[f] = netuid - netuid2module = {} - progress = c.tqdm(len(netuids)) - - for future in c.as_completed(future2netuid): - netuid = future2netuid.pop(future) - module = future.result() - if not c.is_error(module): - netuid2module[netuid] = module - progress.update(1) - - return netuid2module - - - - - - def netuid2uid(self, key=None, update=False, **kwargs) -> Dict[str, str]: - key = self.resolve_key_ss58(key) - netuids = self.netuids(update=update) - netuid2uid = {} - progress = c.tqdm(len(netuids)) - - future2netuid = {} - for netuid in netuids: - f = c.submit(self.get_uid, kwargs=dict(key=key, netuid=netuid, **kwargs)) - future2netuid[f] = netuid - - for future in c.as_completed(future2netuid): - netuid = future2netuid.pop(future) - uid = future.result() - if uid != None: - netuid2uid[netuid] = uid - progress.update(1) - # sort by netuid key - netuid2uid = dict(sorted(netuid2uid.items(), key=lambda x: x[0])) - - return netuid2uid - - - def subnet_state(self, netuid=0, update=False, **kwargs): - - modules = self.get_modules(netuid=netuid, update=update, **kwargs) - - return { - 'params': self.subnet_params(netuid=netuid, - update=update, - fmt='nanos'), - 'modules': modules - } - - def register_subnet(self, key: 'Keypair', name: str, metadata: str | None = None) -> 'c': - """ - Registers a new subnet in the network. - - Args: - key (Keypair): The keypair used for registering the subnet. - name (str): The name of the subnet to be registered. - metadata (str | None, optional): Additional metadata for the subnet. Defaults to None. - - Returns: - ExtrinsicReceipt: A receipt of the subnet registration transaction. - - Raises: - ChainTransactionError: If the transaction fails. - """ - - params = { - "name": name, - "metadata": metadata, - } - - response = self.compose_call("register_subnet", params=params, key=key) - - return response diff --git a/commune/subspace/subspace.py b/commune/subspace/subspace.py index cc6c2535..18449ce1 100644 --- a/commune/subspace/subspace.py +++ b/commune/subspace/subspace.py @@ -5,11 +5,9 @@ import os import commune as c import requests -from .subnet import SubspaceSubnet -from .wallet import SubspaceWallet from substrateinterface import SubstrateInterface -class Subspace( SubspaceSubnet, SubspaceWallet, c.Module): +class Subspace(c.Module): """ Handles interactions with the subspace chain. """ @@ -38,9 +36,6 @@ def __init__(self, sync_loop = False, **kwargs, ): - - - self.config = self.set_config(locals()) self.url_path = self.dirpath() + '/urls.yaml' # merge the config with the subspace config @@ -913,5 +908,2218 @@ def sync_loop(self,max_age=None): c.print('Synced all subnets, sleeping') c.sleep(self.config.max_age) + + + + ################## + #### Transfer #### + ################## + def transfer( + self, + dest: str, + amount: float , + key: str = None, + nonce= None, + **kwargs + + ) -> bool: + # this is a bit of a hack to allow for the amount to be a string for c send 500 0x1234 instead of c send 0x1234 500 + if type(dest) in [int, float]: + assert isinstance(amount, str), f"Amount must be a string" + new_amount = int(dest) + dest = amount + amount = new_amount + key = self.resolve_key(key) + dest = self.resolve_key_ss58(dest) + amount = self.to_nanos(amount) # convert to nano (10^9 nanos = 1 token) + response = self.compose_call( + module='Balances', + fn='transfer_keep_alive', + params={ + 'dest': dest, + 'value': amount + }, + key=key, + nonce = nonce, + **kwargs + ) + + return response + + + + + + def add_profit_shares( + self, + keys: List[str], # the keys to add profit shares to + shares: List[float] = None , # the shares to add to the keys + key: str = None, + netuid : int = 0, + ) -> bool: + + key = self.resolve_key(key) + assert len(keys) > 0, f"Must provide at least one key" + key2address = c.key2address() + keys = [key2address.get(k, k) for k in keys] + assert all([c.valid_ss58_address(k) for k in keys]), f"All keys must be valid ss58 addresses" + shares = shares or [1 for _ in keys] + + assert len(keys) == len(shares), f"Length of keys {len(keys)} must be equal to length of shares {len(shares)}" + + response = self.compose_call( + module='SubspaceModule', + fn='add_profit_shares', + params={ + 'keys': keys, + 'shares': shares, + 'netuid': netuid + }, + key=key + ) + + return response + + + def stake_many( self, + modules:List[str] = None, + amounts:Union[List[str], float, int] = None, + key: str = None, + netuid:int = 0, + min_balance = 100_000_000_000, + n:str = 100) -> Optional['Balance']: + + netuid = self.resolve_netuid( netuid ) + key = self.resolve_key( key ) + + if modules == None: + my_modules = self.my_modules(netuid=netuid, update=False) + modules = [m['key'] for m in my_modules if 'vali' in m['name']] + + modules = modules[:n] # only stake to the first n modules + + assert len(modules) > 0, f"No modules found with name {modules}" + module_keys = modules + + if amounts == None: + balance = self.get_balance(key=key, fmt='nanos') - min_balance + amounts = [(balance // len(modules))] * len(modules) + assert sum(amounts) < balance, f'The total amount is {sum(amounts)} > {balance}' + else: + if isinstance(amounts, (float, int)): + amounts = [amounts] * len(modules) + for i, amount in enumerate(amounts): + amounts[i] = self.to_nanos(amount) + + assert len(modules) == len(amounts), f"Length of modules and amounts must be the same. Got {len(modules)} and {len(amounts)}." + + params = { + "module_keys": module_keys, + "amounts": amounts + } + + response = self.compose_call('add_stake_multiple', params=params, key=key) + + return response + + def transfer_multiple( self, + destinations:List[str], + amounts:Union[List[str], float, int], + key: str = None, + n:str = 10) -> Optional['Balance']: + key2address = c.key2address() + key = self.resolve_key( key ) + balance = self.get_balance(key=key, fmt='j') + for i, destination in enumerate(destinations): + if not c.valid_ss58_address(destination): + if destination in key2address: + destinations[i] = key2address[destination] + else: + raise Exception(f"Invalid destination address {destination}") + if type(amounts) in [float, int]: + amounts = [amounts] * len(destinations) + assert len(set(destinations)) == len(destinations), f"Duplicate destinations found" + assert len(destinations) == len(amounts), f"Length of modules and amounts must be the same. Got {len(destinations)} and {len(amounts)}." + assert all([c.valid_ss58_address(d) for d in destinations]), f"Invalid destination address {destinations}" + total_amount = sum(amounts) + assert total_amount < balance, f'The total amount is {total_amount} > {balance}' + + # convert the amounts to their interger amount (1e9) + amounts = [a*(10**9) for a in amounts] + + params = { + "destinations": destinations, + "amounts": amounts + } + + return self.compose_call('transfer_multiple', params=params, key=key) + + transfer_many = transfer_multiple + + def my_modules(self, + modules : list = None, + netuid=0, + timeout=30, + **kwargs): + if modules == None: + modules = self.my_keys(netuid=netuid) + futures = [c.submit(self.get_module, kwargs=dict(module=module, netuid=netuid, **kwargs)) for module in modules] + for future in c.as_completed(futures, timeout=timeout): + module = future.result() + print(module) + if not c.is_error(module): + modules += [module] + return modules + + def unstake_many( self, + + modules:Union[List[str], str] = None, + amounts:Union[List[str], float, int] = None, + key: str = None, + netuid=0, + update=True, + + ) -> Optional['Balance']: + + key = self.resolve_key( key ) + name2key = {} + + module_keys = [] + for i, module in enumerate(modules): + if c.valid_ss58_address(module): + module_keys += [module] + else: + if name2key == {}: + name2key = self.name2key(netuid=netuid, update=update) + assert module in name2key, f"Invalid module {module} not found in SubNetwork {netuid}" + module_keys += [name2key[module]] + + # RESOLVE AMOUNTS + if amounts == None: + stake_to = self.get_stake_to(key=key, names=False, update=update, fmt='j') # name to amounts + amounts = [stake_to[m] for m in module_keys] + + if isinstance(amounts, (float, int)): + amounts = [amounts] * len(module_keys) + + for i, amount in enumerate(amounts): + amounts[i] = self.to_nanos(amount) + + assert len(module_keys) == len(amounts), f"Length of modules and amounts must be the same. Got {len(module_keys)} and {len(amounts)}." + + params = { + "netuid": netuid, + "module_keys": module_keys, + "amounts": amounts + } + response = self.compose_call('remove_stake_multiple', params=params, key=key) + return response + + + def update_module( + self, + module: str, # the module you want to change + address: str = None, # the address of the new module + name: str = None, # the name of the new module + delegation_fee: float = None, # the delegation fee of the new module + metadata = None, # the metadata of the new module + fee : float = None, # the fee of the new module + netuid: int = 0, # the netuid of the new module + nonce = None, # the nonce of the new module + tip: int = 0, # the tip of the new module + params = None + ) -> bool: + key = self.resolve_key(module) + netuid = self.resolve_netuid(netuid) + assert self.is_registered(key.ss58_address, netuid=netuid), f"Module {module} is not registered in SubNetwork {netuid}" + if params == None: + params = { + 'name': name , # defaults to module.tage + 'address': address , # defaults to module.tage + 'delegation_fee': fee or delegation_fee, # defaults to module.delegate_fee + 'metadata': c.python2str(metadata or {}), # defaults to module.metadata + } + + should_update_module = False + module_info = self.get_module(key.ss58_address, netuid=netuid) + + for k,v in params.items(): + if params[k] == None: + params[k] = module_info[k] + if k in module_info and params[k] != module_info[k]: + should_update_module = True + + if not should_update_module: + return {'success': False, 'message': f"Module {module} is already up to date"} + + c.print('Updating with', params, color='cyan') + params['netuid'] = netuid + + reponse = self.compose_call('update_module', params=params, key=key, nonce=nonce, tip=tip) + + # IF SUCCESSFUL, MOVE THE KEYS, AS THIS IS A NON-REVERSIBLE OPERATION + + + return reponse + + update = update_server = update_module + + def stake_transfer( + self, + module_key: str , + new_module_key: str , + amount: Union[int, float] = None, + key: str = None, + ) -> bool: + # STILL UNDER DEVELOPMENT, DO NOT USE + key = c.get_key(key) + netuid = 0 + module_key = self.resolve_module_key(module_key, netuid=netuid) + new_module_key = self.resolve_module_key(new_module_key, netuid=netuid) + + assert module_key != new_module_key, f"Module key {module_key} is the same as new_module_key {new_module_key}" + + if amount == None: + amount = self.get_stake_to( key=key , fmt='j', max_age=0).get(module_key, 0) + + # Get current stake + params={ + 'amount': int(amount * 10**9), + 'module_key': module_key, + 'new_module_key': new_module_key + + } + + return self.compose_call('transfer_stake',params=params, key=key) + + def unstake( + self, + module : str = None, # defaults to most staked module + amount: float =None, # defaults to all of the amount + key : 'c.Key' = None, # defaults to first key + netuid=0, + **kwargs + ) -> dict: + """ + description: + Unstakes the specified amount from the module. + If no amount is specified, it unstakes all of the amount. + If no module is specified, it unstakes from the most staked module. + params: + amount: float = None, # defaults to all + module : str = None, # defaults to most staked module + key : 'c.Key' = None, # defaults to first key + netuid : Union[str, int] = 0, # defaults to module.netuid + network: str= main, # defaults to main + return: + response: dict + """ + + key = c.get_key(key) + # get most stake from the module + + if isinstance(module, int): + module = amount + amount = module + + assert module != None or amount != None, f"Must provide a module or an amount" + key2address = c.key2address() + if module in key2address: + module_key = key2address[module] + else: + name2key = self.name2key(netuid=netuid) + if module in name2key: + module_key = name2key[module] + else: + module_key = module + assert self.is_registered(module_key, netuid=netuid), f"Module {module} is not registered in SubNetwork {netuid}" + + if amount == None: + stake_to = self.get_stake_to(names = False, fmt='nano', key=module_key) + amount = stake_to[module_key] - 100000 + else: + amount = int(self.to_nanos(amount)) + # convert to nanos + params={ + 'amount': amount , + 'module_key': module_key + } + response = self.compose_call(fn='remove_stake',params=params, key=key, **kwargs) + + return response + + + + def my_servers(self, search=None, **kwargs): + servers = [m['name'] for m in self.my_modules(**kwargs)] + if search != None: + servers = [s for s in servers if search in s] + return servers + + def my_modules_names(self, *args, **kwargs): + my_modules = self.my_modules(*args, **kwargs) + return [m['name'] for m in my_modules] + + def my_module_keys(self, *args, **kwargs): + modules = self.my_modules(*args, **kwargs) + return [m['key'] for m in modules] + + def my_key2uid(self, *args, netuid=0, update=False, **kwargs): + key2uid = self.key2uid(*args, netuid=netuid, **kwargs) + + key2address = c.key2address(update=update ) + key_addresses = list(key2address.values()) + if netuid == 'all': + for netuid, netuid_keys in key2uid.items(): + key2uid[netuid] = {k: v for k,v in netuid_keys.items() if k in key_addresses} + + my_key2uid = { k: v for k,v in key2uid.items() if k in key_addresses} + return my_key2uid + + def my_keys(self, + netuid=0, + search=None, + max_age=None, + names = False, + update=False, **kwargs): + netuid = self.resolve_netuid(netuid) + keys = self.keys(netuid=netuid, max_age=max_age, update=update, **kwargs) + key2address = c.key2address(search=search, max_age=max_age, update=update) + if search != None: + key2address = {k: v for k,v in key2address.items() if search in k} + address2key = {v:k for k,v in key2address.items()} + addresses = list(key2address.values()) + convert_fn = lambda x : address2key.get(x, x) if names else x + if netuid == 'all': + my_keys = {} + for netuid, netuid_keys in keys.items(): + + my_netuid_keys = [convert_fn(k) for k in netuid_keys if k in addresses] + + if len(my_netuid_keys) > 0: + my_keys[netuid] = my_netuid_keys + else: + my_keys = [convert_fn(k) for k in keys if k in addresses] + return my_keys + + + def my_value( self, *args, **kwargs ): + return sum(list(self.key2value( *args, **kwargs).values())) + + + def my_total_stake(self, netuid='all', fmt='j', update=False): + my_stake_to = self.my_stake_to(netuid=netuid, fmt=fmt, update=update) + return sum([sum(list(v.values())) for k,v in my_stake_to.items()]) + + def my_staked_module_keys(self, netuid = 0, **kwargs): + my_stake_to = self.my_stake_to(netuid=netuid, **kwargs) + module_keys = {} if netuid == 'all' else [] + for subnet_netuid, stake_to_key in my_stake_to.items(): + if netuid == 'all': + for _netuid, stake_to_subnet in stake_to_key.items(): + module_keys[_netuid] = list(stake_to_subnet.keys()) + module_keys.get(_netuid, []) + else: + module_keys += list(stake_to_key.keys()) + return module_keys + + def my_stake_to(self, fmt='j', **kwargs): + stake_to = self.stake_to(fmt=fmt, **kwargs) + key2address = c.key2address() + my_stake_to = {} + for key, address in key2address.items(): + my_stake_to[address] = {k:v for k,v in stake_to.get(address, {}).items()} + stake_to_keys = list(my_stake_to.keys()) + for key in stake_to_keys: + if len(my_stake_to[key]) == 0: + del my_stake_to[key] + + return my_stake_to + + + + + def my_stake_from(self, netuid = 0, block=None, update=False, fmt='j', max_age=1000 , **kwargs): + stake_from_tuples = self.stake_from(netuid=netuid, + block=block, + update=update, + tuples = True, + fmt=fmt, max_age=max_age, **kwargs) + + address2key = c.address2key() + stake_from_total = {} + if netuid == 'all': + for netuid, stake_from_tuples_subnet in stake_from_tuples.items(): + for module_key,staker_tuples in stake_from_tuples_subnet.items(): + for staker_key, stake in staker_tuples: + if module_key in address2key: + stake_from_total[staker_key] = stake_from_total.get(staker_key, 0) + stake + + else: + for module_key,staker_tuples in stake_from_tuples.items(): + for staker_key, stake in staker_tuples: + if module_key in address2key: + stake_from_total[staker_key] = stake_from_total.get(staker_key, 0) + stake + + + for staker_address in address2key.keys(): + if staker_address in stake_from_total: + stake_from_total[staker_address] = self.format_amount(stake_from_total[staker_address], fmt=fmt) + return stake_from_total + + + + def my_total_stake_to( self, + key: str = None, + block: Optional[int] = None, + timeout=20, + names = False, + fmt='j' , + update=False, + max_age = 1000, + **kwargs) -> Optional['Balance']: + return sum(list(self.get_stake_to(key=key, block=block, timeout=timeout, names=names, fmt=fmt, + update=update, + max_age=max_age, **kwargs).values())) + + + def my_modules(self, search=None, netuid=0, generator=False, **kwargs): + keys = self.my_keys(netuid=netuid, search=search) + if netuid == 'all': + modules = {} + all_keys = keys + for netuid, keys in enumerate(all_keys): + try: + modules[netuid]= self.get_modules(keys=keys, netuid=netuid, **kwargs) + except Exception as e: + c.print(e) + modules = {k: v for k,v in modules.items() if len(v) > 0 } + return modules + modules = self.get_modules(keys=keys, netuid=netuid, **kwargs) + return modules + + def stats(self, + search = None, + netuid=0, + df:bool=True, + update:bool = False , + features : list = ['name', 'emission','incentive', 'dividends', 'stake', 'vote_staleness', 'serving', 'address'], + sort_features = ['emission', 'stake'], + fmt : str = 'j', + modules = None, + servers = None, + **kwargs + ): + + if isinstance(netuid, str): + netuid = self.subnet2netuid(netuid) + + if search == 'all': + netuid = search + search = None + + + if netuid == 'all': + all_modules = self.my_modules(netuid=netuid, update=update, fmt=fmt, search=search) + servers = c.servers() + stats = {} + netuid2subnet = self.netuid2subnet(update=update) + for netuid, modules in all_modules.items(): + subnet_name = netuid2subnet[netuid] + stats[netuid] = self.stats(modules=modules, netuid=netuid, servers=servers) + + color = c.random_color() + c.print(f'\n {subnet_name.upper()} :: (netuid:{netuid})\n', color=color) + c.print(stats[netuid], color=color) + + + modules = modules or self.my_modules(netuid=netuid, update=update, fmt=fmt, search=search) + + stats = [] + + local_key_addresses = list(c.key2address().values()) + servers = servers or c.servers() + for i, m in enumerate(modules): + if m['key'] not in local_key_addresses : + continue + # sum the stake_from + # we want to round these values to make them look nice + for k in ['emission', 'dividends', 'incentive']: + m[k] = c.round(m[k], sig=4) + + m['serving'] = bool(m['name'] in servers) + m['stake'] = int(m['stake']) + stats.append(m) + df_stats = c.df(stats) + if len(stats) > 0: + df_stats = df_stats[features] + if 'emission' in features: + epochs_per_day = self.epochs_per_day(netuid=netuid) + df_stats['emission'] = df_stats['emission'] * epochs_per_day + sort_features = [c for c in sort_features if c in df_stats.columns] + df_stats.sort_values(by=sort_features, ascending=False, inplace=True) + if search is not None: + df_stats = df_stats[df_stats['name'].str.contains(search, case=True)] + + if not df: + return df_stats.to_dict('records') + else: + return df_stats + + + + + + def update_modules(self, search=None, + timeout=60, + netuid=0, + **kwargs) -> List[str]: + + netuid = self.resolve_netuid(netuid) + my_modules = self.my_modules(search=search, netuid=netuid, **kwargs) + + self.keys() + futures = [] + namespace = c.namespace() + for m in my_modules: + + name = m['name'] + if name in namespace: + address = namespace[name] + else: + address = c.serve(name)['address'] + + if m['address'] == address and m['name'] == name: + c.print(f"Module {m['name']} already up to date") + + f = c.submit(c.update_module, kwargs={'module': name, + 'name': name, + 'netuid': netuid, + 'address': address, + **kwargs}, timeout=timeout) + futures+= [f] + + + results = [] + + for future in c.as_completed(futures, timeout=timeout): + results += [future.result()] + c.print(future.result()) + return results + + + + + + def stake( + self, + module: Optional[str] = None, # defaults to key if not provided + amount: Union['Balance', float] = None, + key: str = None, # defaults to first key + existential_deposit: float = 0, + netuid=0, + **kwargs + ) -> bool: + """ + description: + Unstakes the specified amount from the module. + If no amount is specified, it unstakes all of the amount. + If no module is specified, it unstakes from the most staked module. + params: + amount: float = None, # defaults to all + module : str = None, # defaults to most staked module + key : 'c.Key' = None, # defaults to first key + netuid : Union[str, int] = 0, # defaults to module.netuid + network: str= main, # defaults to main + return: + response: dict + + """ + key = c.get_key(key) + if c.valid_ss58_address(module): + module_key = module + else: + module_key = self.name2key(netuid=netuid).get(module) + + # Flag to indicate if we are using the wallet's own hotkey. + + if amount == None: + amount = self.get_balance( key.ss58_address , fmt='nano') - existential_deposit*10**9 + else: + amount = int(self.to_nanos(amount - existential_deposit)) + assert amount > 0, f"Amount must be greater than 0 and greater than existential deposit {existential_deposit}" + + # Get current stake + params={ + 'amount': amount, + 'module_key': module_key + } + + return self.compose_call('add_stake',params=params, key=key) + + + + + def key_info(self, key:str = None, netuid='all', detail=0, timeout=10, update=False, **kwargs): + key_info = { + 'balance': self.get_balance(key=key, **kwargs), + 'stake_to': self.get_stake_to(key=key, **kwargs), + } + if detail: + pass + else: + for netuid, stake_to in key_info['stake_to'].items(): + key_info['stake_to'][netuid] = sum(stake_to.values()) + + + return key_info + + + + + def subnet2modules(self, **kwargs): + subnet2modules = {} + + for netuid in self.netuids(): + c.print(f'Getting modules for SubNetwork {netuid}') + subnet2modules[netuid] = self.my_modules(netuid=netuid, **kwargs) + + return subnet2modules + + def staking_rewards( self, + key: str = None, + module_key=None, + block: Optional[int] = None, + timeout=20, + period = 100, + names = False, + fmt='j' , update=False, + max_age = 1000, + **kwargs) -> Optional['Balance']: + + block = int(block or self.block) + block_yesterday = int(block - period) + day_before_stake = self.my_total_stake_to(key=key, module_key=module_key, block=block_yesterday, timeout=timeout, names=names, fmt=fmt, update=update, max_age=max_age, **kwargs) + day_after_stake = self.my_total_stake_to(key=key, module_key=module_key, block=block, timeout=timeout, names=names, fmt=fmt, update=update, max_age=max_age, **kwargs) + return (day_after_stake - day_before_stake) + + def registered_servers(self, netuid = 0, **kwargs): + netuid = self.resolve_netuid(netuid) + servers = c.servers() + keys = self.keys(netuid=netuid) + registered_keys = [] + key2address = c.key2address() + for s in servers: + key_address = key2address[s] + if key_address in keys: + registered_keys += [s] + return registered_keys + + + def key2balance(self, search=None, + batch_size = 64, + timeout = 10, + max_age = 1000, + fmt = 'j', + update=False, + names = False, + min_value=0.0001, + **kwargs): + + input_hash = c.hash(c.locals2kwargs(locals())) + path = f'key2balance/{input_hash}' + key2balance = self.get(path, max_age=max_age, update=update) + + if key2balance == None: + key2balance = self.get_balances(search=search, + batch_size=batch_size, + timeout=timeout, + fmt = 'nanos', + update=1, + min_value=min_value, + **kwargs) + self.put(path, key2balance) + + for k,v in key2balance.items(): + key2balance[k] = self.format_amount(v, fmt=fmt) + key2balance = sorted(key2balance.items(), key=lambda x: x[1], reverse=True) + key2balance = {k: v for k,v in key2balance if v > min_value} + if names: + address2key = c.address2key() + key2balance = {address2key[k]: v for k,v in key2balance.items()} + return key2balance + + def empty_keys(self, block=None, update=False, max_age=1000, fmt='j'): + key2address = c.key2address() + key2value = self.key2value( block=block, update=update, max_age=max_age, fmt=fmt) + empty_keys = [] + for key,key_address in key2address.items(): + key_value = key2value.get(key_address, 0) + if key_value == 0: + empty_keys.append(key) + return empty_keys + + def profit_shares(self, key=None, **kwargs) -> List[Dict[str, Union[str, int]]]: + key = self.resolve_module_key(key) + return self.query_map('ProfitShares', **kwargs) + + def key2stake(self, + block=None, + update=False, + names = True, + max_age = 1000,fmt='j'): + stake_to = self.stake_to( + block=block, + max_age=max_age, + update=update, + + fmt=fmt) + address2key = c.address2key() + stake_to_total = {} + + for staker_address in address2key.keys(): + if staker_address in stake_to: + stake_to_total[staker_address] = sum(stake_to.get(staker_address, {}).values()) + # sort the dictionary by value + stake_to_total = dict(sorted(stake_to_total.items(), key=lambda x: x[1], reverse=True)) + if names: + stake_to_total = {address2key.get(k, k): v for k,v in stake_to_total.items()} + return stake_to_total + + def key2value(self, block=None, update=False, max_age=1000, fmt='j', min_value=0, **kwargs): + key2balance = self.key2balance(block=block, update=update, max_age=max_age, fmt=fmt) + key2stake = self.key2stake( block=block, update=update, max_age=max_age, fmt=fmt) + key2value = {} + keys = set(list(key2balance.keys()) + list(key2stake.keys())) + for key in keys: + key2value[key] = key2balance.get(key, 0) + key2stake.get(key, 0) + key2value = {k:v for k,v in key2value.items()} + key2value = dict(sorted(key2value.items(), key=lambda x: x[1], reverse=True)) + return key2value + + def resolve_module_key(self, x, netuid=0, max_age=60): + if not c.valid_ss58_address(x): + name2key = self.name2key(netuid=netuid, max_age=max_age) + x = name2key.get(x) + assert c.valid_ss58_address(x), f"Module key {x} is not a valid ss58 address" + return x + + def global_params(self, + update = False, + max_age = 1000, + timeout=30, + fmt:str='j', + features = None, + value_features = [], + path = f'global_params', + **kwargs + ) -> list: + + features = features or self.global_param_features + subnet_params = self.get(path, None, max_age=max_age, update=update) + names = [self.feature2name(f) for f in features] + future2name = {} + name2feature = dict(zip(names, features)) + for name, feature in name2feature.items(): + c.print(f'Getting {name} for {feature}') + query_kwargs = dict(name=feature, params=[], block=None, max_age=max_age, update=update) + f = c.submit(self.query, kwargs=query_kwargs, timeout=timeout) + future2name[f] = name + + subnet_params = {} + + for f in c.as_completed(future2name): + result = f.result() + subnet_params[future2name.pop(f)] = result + for k in value_features: + subnet_params[k] = self.format_amount(subnet_params[k], fmt=fmt) + return subnet_params + + def get_balance(self, + key: str = None , + block: int = None, + fmt='j', + max_age=0, + update=False) -> Optional['Balance']: + r""" Returns the token balance for the passed ss58_address address + Args: + address (Substrate address format, default = 42): + ss58 chain address. + Return: + balance (bittensor.utils.balance.Balance): + account balance + """ + key_ss58 = self.resolve_key_ss58( key ) + + result = self.query( + module='System', + name='Account', + params=[key_ss58], + block = block, + update=update, + max_age=max_age + ) + + return self.format_amount(result['data']['free'] , fmt=fmt) + + balance = get_balance + + def accounts(self, key = None, update=True, block=None, max_age=100000, **kwargs): + key = self.resolve_key_ss58(key) + accounts = self.query_map( + module='System', + name='Account', + update=update, + block = block, + max_age=max_age, + **kwargs + ) + return accounts + + def balances(self,fmt:str = 'n', block: int = None, n = None, update=False , **kwargs) -> Dict[str, 'Balance']: + accounts = self.accounts( update=update, block=block) + balances = {k:v['data']['free'] for k,v in accounts.items()} + balances = {k: self.format_amount(v, fmt=fmt) for k,v in balances.items()} + return balances + + def blocks_per_day(self): + return 24*60*60/self.block_time + + def min_burn(self, block=None, update=False, fmt='j'): + min_burn = self.query('MinBurn', block=block, update=update) + return self.format_amount(min_burn, fmt=fmt) + + + def get_balances(self, + keys=None, + search=None, + batch_size = 128, + fmt = 'j', + n = 100, + max_trials = 3, + names = False, + **kwargs): + + key2balance = {} + key2address = c.key2address(search=search) + + if keys == None: + keys = list(key2address.keys()) + + if len(keys) > n: + c.print(f'Getting balances for {len(keys)} keys > {n} keys, using batch_size {batch_size}') + balances = self.balances(**kwargs) + key2balance = {} + for k,a in key2address.items(): + if a in balances: + key2balance[k] = balances[a] + else: + future2key = {} + for key in keys: + f = c.submit(self.get_balance, kwargs=dict(key=key, fmt=fmt, **kwargs)) + future2key[f] = key + + for f in c.as_completed(future2key): + key = future2key.pop(f) + key2balance[key] = f.result() + + for k,v in key2balance.items(): + key2balance[k] = self.format_amount(v, fmt=fmt) + + + if names: + address2key = c.address2key() + key2balance = {address2key[k]: v for k,v in key2balance.items()} + + return key2balance + + def total_balance(self, **kwargs): + balances = self.balances(**kwargs) + return sum(balances.values()) + + def num_holders(self, **kwargs): + balances = self.balances(**kwargs) + return len(balances) + + def proposals(self, block=None, nonzero:bool=False, update:bool = False, **kwargs): + proposals = [v for v in self.query_map('Proposals', block=block, update=update, **kwargs)] + return proposals + + def registrations_per_block(self,**kwargs) -> int: + return self.query('RegistrationsPerBlock', params=[], **kwargs) + regsperblock = registrations_per_block + + def max_registrations_per_block(self, **kwargs) -> int: + return self.query('MaxRegistrationsPerBlock', params=[], **kwargs) + + ################# + #### Serving #### + ################# + def update_global( + self, + key: str = None, + network : str = 'main', + sudo: bool = True, + **params, + ) -> bool: + + key = self.resolve_key(key) + network = self.resolve_network(network) + global_params = self.global_params(fmt='nanos') + global_params.update(params) + params = global_params + for k,v in params.items(): + if isinstance(v, str): + params[k] = v.encode('utf-8') + # this is a sudo call + return self.compose_call(fn='update_global', + params=params, + key=key, + sudo=sudo) + + + ################# + #### Serving #### + ################# + def vote_proposal( + self, + proposal_id: int = None, + key: str = None, + network = 'main', + nonce = None, + netuid = 0, + **params, + + ) -> bool: + + self.resolve_network(network) + # remove the params that are the same as the module info + params = { + 'proposal_id': proposal_id, + 'netuid': netuid, + } + + response = self.compose_call(fn='add_subnet_proposal', + params=params, + key=key, + nonce=nonce) + + return response + + + + + + + def register( + self, + name: str , # defaults to module.tage + address : str = None, + stake : float = None, + netuid = 0, + network_name : str = None, + key : str = None, + module_key : str = None, + wait_for_inclusion: bool = True, + wait_for_finalization: bool = True, + max_address_characters = 32, + metadata = None, + network = None, + nonce=None, + **kwargs + ) -> bool: + module_key = c.get_key(module_key or name).ss58_address + netuid2subnet = self.netuid2subnet(update=False) + subnet2netuid = {v:k for k,v in netuid2subnet.items()} + + if network_name == None and netuid != 0: + network_name = netuid2subnet[netuid] + else: + assert isinstance(network_name, str), f"Subnet must be a string" + if not network_name in subnet2netuid: + subnet2netuid = self.subnet2netuid(update=True) + if network_name not in subnet2netuid: + subnet2netuid[network_name] = len(subnet2netuid) + response = input(f"Do you want to create a new subnet ({network_name}) (yes or y or dope): ") + if response.lower() not in ["yes", 'y', 'dope']: + return {'success': False, 'msg': 'Subnet not found and not created'} + + # require prompt to create new subnet + stake = (stake or 0) * 1e9 + + if c.server_exists(name): + address = c.namespace().get(name) + else: + address = address or 'NA' + + params = { + 'network_name': network_name.encode('utf-8'), + 'address': address[-max_address_characters:].replace('0.0.0.0', c.ip()).encode('utf-8'), + 'name': name.encode('utf-8'), + 'stake': stake, + 'module_key': module_key, + 'metadata': json.dumps(metadata or {}).encode('utf-8'), + } + + # create extrinsic call + response = self.compose_call('register', params=params, key=key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, nonce=nonce) + return response + + def resolve_uids(self, uids: Union['torch.LongTensor', list], netuid: int = 0, update=False) -> None: + name2uid = None + key2uid = None + for i, uid in enumerate(uids): + if isinstance(uid, str): + if name2uid == None: + name2uid = self.name2uid(netuid=netuid, update=update) + if uid in name2uid: + uids[i] = name2uid[uid] + else: + if key2uid == None: + key2uid = self.key2uid(netuid=netuid, update=update) + if uid in key2uid: + uids[i] = key2uid[uid] + return uids + + def set_weights( + self, + uids: Union['torch.LongTensor', list] , + weights: Union['torch.FloatTensor', list] , + netuid: int = 0, + key: 'c.key' = None, + update=False, + modules = None, + vector_length = 2**16 - 1, + nonce=None, + **kwargs + ) -> bool: + import torch + netuid = self.resolve_netuid(netuid) + key = self.resolve_key(key) + + # ENSURE THE UIDS ARE VALID UIDS, CONVERTING THE NAMES AND KEYS ASSOCIATED WITH THEM + uids = self.resolve_uids(modules or uids, netuid=netuid, update=update) + weights = weights or [1] * len(uids) + + # CHECK WEIGHTS LENGHTS + assert len(uids) == len(weights), f"Length of uids {len(uids)} must be equal to length of weights {len(weights)}" + + # NORMALIZE WEIGHTS + weights = torch.tensor(weights) + weights = (weights / weights.sum()) * vector_length # normalize the weights between 0 and 1 + weights = torch.clamp(weights, 0, vector_length) # min_value and max_value are between 0 and 1 + + params = {'uids': list(map(int, uids)), + 'weights': list(map(int, weights.tolist())), + 'netuid': netuid} + + return self.compose_call('set_weights',params = params , key=key, nonce=nonce, **kwargs) + + vote = set_weights + + def unstake_all( self, + key: str = None, + existential_deposit = 1, + min_stake = 0.5, + ) -> Optional['Balance']: + + key = self.resolve_key( key ) + key_stake_to = self.get_stake_to(key=key, names=False, update=True, fmt='nanos') # name to amount + + min_stake = min_stake * 1e9 + key_stake_to = {k:v for k,v in key_stake_to.items() if v > min_stake } + + params = { + "module_keys": list(key_stake_to.keys()), + "amounts": list(key_stake_to.values()) + } + + + + + + response = {} + + if len(key_stake_to) > 0: + c.print(f'Unstaking all of {len(key_stake_to)} modules') + response['stake'] = self.compose_call('remove_stake_multiple', params=params, key=key) + total_stake = (sum(key_stake_to.values())) / 1e9 + else: + c.print(f'No modules found to unstake') + total_stake = self.get_balance(key) + total_stake = total_stake - existential_deposit + + return response + + def registered_keys(self, netuid='all'): + key2address = c.key2address() + address2key = {v:k for k,v in key2address.items()} + + if netuid == 'all': + registered_keys = {} + netuid2keys = self.keys(netuid=netuid) + for netuid, keys in netuid2keys.items(): + registered_keys[netuid] = [] + for k in keys: + if k in address2key: + registered_keys[netuid].append(k) + else: + registered_keys = [k for k in self.keys(netuid=netuid) if k in address2key] + + return registered_keys + + + subnet_param_features = [ + "ImmunityPeriod", + "MinAllowedWeights", + "MaxAllowedWeights", + "Tempo", + "MaxAllowedUids", + "Founder", + "FounderShare", + "IncentiveRatio", + "TrustRatio", + "SubnetNames", + "MaxWeightAge", + "BondsMovingAverage", + "MaximumSetWeightCallsPerEpoch", + "MinValidatorStake", + "MaxAllowedValidators", + "ModuleBurnConfig", + "SubnetMetadata", + 'SubnetGovernanceConfig' + ] + + + + def stake_to(self, block=None, max_age=1000, update=False, fmt='nano', **kwargs): + stake_to = self.query_map('StakeTo', block=block, max_age=max_age, update=update, **kwargs) + format_value = lambda v: {v_k: self.format_amount(v_v, fmt=fmt) for v_k, v_v in v.items()} + stake_to = {k: format_value(v) for k,v in stake_to.items()} + return stake_to + + + + def netuid2founder(self, fmt='j', **kwargs): + netuid2founder = self.query_map('Founder', **kwargs) + return netuid2founder + + + def stake_from(self, + block=None, + update=False, + max_age=10000, + fmt='nano', + **kwargs) -> List[Dict[str, Union[str, int]]]: + + stake_from = self.query_map('StakeFrom', block=block, update=update, max_age=max_age ) + format_value = lambda v: {v_k: self.format_amount(v_v, fmt=fmt) for v_k, v_v in v.items()} + stake_from = {k: format_value(v) for k,v in stake_from.items()} + return stake_from + + """ Returns network Tempo hyper parameter """ + def stakes(self, fmt:str='j', max_age = 100, update=False, stake_from=None, **kwargs) -> int: + stake_from = stake_from or self.stake_from( update=update, max_age=max_age, fmt=fmt) + stakes = {k: sum(v.values()) for k,v in stake_from.items()} + return stakes + + def leaderboard(self, netuid = 0, block=None, update=False, columns = ['emission', 'name', 'incentive', 'dividends'], **kwargs): + modules = self.get_modules(netuid=netuid, block=block, update=update, **kwargs) + return c.df(modules)[columns] + + + def min_stake(self, netuid: int = 0, fmt:str='j', **kwargs) -> int: + min_stake = self.query('MinStake', netuid=netuid, **kwargs) + return self.format_amount(min_stake, fmt=fmt) + + def regblock(self, netuid: int = 0, block: Optional[int] = None, update=False ) -> Optional[float]: + regblock = self.query_map('RegistrationBlock',block=block, update=update ) + if isinstance(netuid, int): + regblock = regblock[netuid] + return regblock + + def emissions(self, netuid = None, block=None, update=False, fmt = 'nanos', **kwargs): + netuid = self.resolve_netuid(netuid) + emissions = self.query_vector('Emission', netuid=netuid, block=block, update=update, **kwargs) + if netuid == 'all': + for netuid, netuid_emissions in emissions.items(): + emissions[netuid] = [self.format_amount(e, fmt=fmt) for e in netuid_emissions] + else: + emissions = [self.format_amount(e, fmt=fmt) for e in emissions] + + return emissions + + emission = emissions + + + def total_emission( self, netuid: int = 0, block: Optional[int] = None, fmt:str = 'j', **kwargs ) -> Optional[float]: + total_emission = sum(self.emission(netuid=netuid, block=block, **kwargs)) + return self.format_amount(total_emission, fmt=fmt) + + def addresses(self, netuid: int = 0, update=False, **kwargs) -> List[str]: + netuid = self.resolve_netuid(netuid) + addresses = self.query_map('Address',netuid=netuid, update=update, **kwargs) + + if isinstance(netuid, int): + addresses = list(addresses.values()) + else: + for k,v in addresses.items(): + addresses[k] = list(v.values()) + return addresses + + def namespace(self, search=None, netuid: int = 0, update:bool = False, timeout=30, local=False, max_age=1000, **kwargs) -> Dict[str, str]: + namespace = {} + results = { + 'names': None, + 'addresses': None + } + netuid = self.resolve_netuid(netuid) + while any([v == None for v in results.values()]): + future2key = {} + for k,v in results.items(): + if v == None: + f = c.submit(getattr(self, k), kwargs=dict(netuid=netuid, update=update, max_age=max_age, **kwargs)) + future2key[f] = k + for future in c.as_completed(list(future2key.keys()), timeout=timeout): + key = future2key.pop(future) + r = future.result() + if not c.is_error(r) and r != None: + results[key] = r + + if netuid == 'all': + netuid2subnet = self.netuid2subnet() + namespace = {} + for netuid, netuid_addresses in results['addresses'].items(): + for uid,address in enumerate(netuid_addresses): + name = results['names'][netuid][uid] + subnet = netuid2subnet[netuid] + namespace[f'{subnet}/{name}'] = address + + else: + namespace = {k:v for k,v in zip(results['names'], results['addresses'])} + + if search != None: + namespace = {k:v for k,v in namespace.items() if search in str(k)} + + if local: + ip = c.ip() + namespace = {k:v for k,v in namespace.items() if ip in str(v)} + + return namespace + + def emissions(self, netuid = None, network = "main", block=None, update=False, **kwargs): + netuid = self.resolve_netuid(netuid) + return self.query_vector('Emission', network=network, netuid=netuid, block=block, update=update, **kwargs) + + def key2name(self, key: str = None, netuid: int = 0) -> str: + modules = self.keys(netuid=netuid) + key2name = { m['key']: m['name']for m in modules} + if key != None: + return key2name[key] + + def uid2name(self, netuid: int = 0, update=False, **kwargs) -> List[str]: + netuid = self.resolve_netuid(netuid) + names = self.query_map('Name', netuid=netuid, update=update,**kwargs) + return names + + def is_registered(self, key: str, netuid: int = 0, update=False, **kwargs) -> bool: + key_address = self.resolve_key_ss58(key) + try: + uid = self.get_uid(key_address, netuid=netuid, update=update, **kwargs) + if isinstance(uid, int): + return True + except Exception as e: + return False + + def keys(self, + netuid = None, + update=False, + max_age=1000, + **kwargs) -> List[str]: + netuid = self.resolve_netuid(netuid) + keys = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) + if netuid == 'all': + for netuid, netuid_keys in keys.items(): + keys[netuid] = list(netuid_keys.values()) + else: + keys = list(keys.values()) + return keys + + def delegation_fee(self, netuid = None, block=None, update=False, fmt='j'): + netuid = self.resolve_netuid(netuid) + delegation_fee = self.query_map('DelegationFee', netuid=netuid, block=block ,update=update) + return delegation_fee + + + def feature2name(self, feature='MinStake'): + translations = { + 'subnet_names': 'name' + } + name = '' + for i, ch in enumerate(feature): + if ch.isupper(): + if i == 0: + name += ch.lower() + else: + name += f'_{ch.lower()}' + else: + name += ch + name = translations.get(name, name) + return name + + + def subnet_params(self, + netuid=0, + update = False, + max_age = 1000, + timeout=40, + fmt:str='j', + features = None, + value_features = [], + **kwargs + ) -> list: + if netuid == 'all': + return self.all_subnet_params(update=update, + max_age=max_age, + timeout=timeout, + fmt=fmt, + features=features, + value_features=value_features, + **kwargs) + + + default_params = { + 'maximum_set_weight_calls_per_epoch': 30, + 'max_allowed_validators': 50 + } + + features = features or self.subnet_param_features + netuid = self.resolve_netuid(netuid) + path = f'query/{self.network}/SubnetParams.{netuid}' + subnet_params = self.get(path, max_age=max_age, update=update) + if subnet_params == None: + + names = [self.feature2name(f) for f in features] + future2name = {} + for name, feature in dict(zip(names, features)).items(): + query_kwargs = dict(name=feature, netuid=netuid,block=None, max_age=max_age, update=update) + if name in ['SubnetGovernanceConfig']: + fn = self.query_map + else: + fn = self.query + + f = c.submit(fn, kwargs=query_kwargs, timeout=timeout) + future2name[f] = name + subnet_params = {} + for f in c.as_completed(future2name, timeout=timeout): + result = f.result() + subnet_params[future2name.pop(f)] = result + for k in subnet_params.keys(): + v = subnet_params[k] + if v == None: + v = default_params.get(k, v) + if k in value_features: + v = self.format_amount(v, fmt=fmt) + subnet_params[k] = v + + self.put(path, subnet_params) + + subnet_params.update(subnet_params.pop('subnet_governance_config')) + translation = { + 'subnet_names': 'name', + 'bonds_moving_average': 'bonds_ma' + } + for k,v in translation.items(): + if k in subnet_params: + subnet_params[v] = subnet_params.pop(k) + return subnet_params + + + def all_subnet_params(self, + update = False, + max_age = 1000, + features = None, + **kwargs + ) -> list: + + features = features or self.subnet_param_features + netuid = self.resolve_netuid(netuid) + path = f'query/{self.network}/SubnetParams.all' + all_subnet_params = self.get(path, max_age=max_age, update=update) + if all_subnet_params == None: + all_subnet_params = {} + for netuid in self.netuids(update=update): + all_subnet_params[netuid] = self.subnet_params(netuid=netuid, update=update, max_age=max_age, **kwargs) + return all_subnet_params + + def pending_deregistrations(self, netuid = None, update=False, **kwargs): + netuid = self.resolve_netuid(netuid) + pending_deregistrations = self.query_map('PendingDeregisterUids',update=update,**kwargs)[netuid] + return pending_deregistrations + + def num_pending_deregistrations(self, netuid = 0, **kwargs): + pending_deregistrations = self.pending_deregistrations(netuid=netuid, **kwargs) + return len(pending_deregistrations) + + def subnet_names(self , search=None, update=False, block=None, max_age=60, **kwargs) -> Dict[str, str]: + records = self.query_map('SubnetNames', update=update, block=block, max_age=max_age, **kwargs) + subnet_names = sorted(list(map(lambda x: str(x), records.values()))) + if search != None: + subnet_names = [s for s in subnet_names if search in s] + return subnet_names + + def subnets(self, **kwargs) -> Dict[int, str]: + return self.subnet_names(**kwargs) + + def num_subnets(self, **kwargs) -> int: + return len(self.subnets(**kwargs)) + + def subnet2stake(self, fmt='j'): + netuid2subnet = self.netuid2subnet() + netuid2stake = self.netuid2stake(fmt=fmt) + subnet2stake = {} + for netuid, subnet in netuid2subnet.items(): + subnet2stake[subnet] = netuid2stake[netuid] + return subnet2stake + + def netuid2stake(self, fmt='j', **kwargs): + netuid2stake = self.query_map('TotalStake', **kwargs) + for netuid, stake in netuid2stake.items(): + netuid2stake[netuid] = self.format_amount(stake, fmt=fmt) + return netuid2stake + + def netuid2n(self, fmt='j', **kwargs): + netuid2n = self.query_map('N', **kwargs) + return netuid2n + + def trust(self, + netuid = 0, + block=None, + update:bool = False, + **kwargs): + return self.query_vector('Trust', netuid=netuid, block=block, update=update, **kwargs) + + def incentives(self, + netuid = 0, + block=None, + update:bool = False, + **kwargs): + return self.query_vector('Incentive', netuid=netuid, block=block, update=update, **kwargs) + incentive = incentives + + def last_update(self, netuid = 0, update=False, **kwargs): + return self.query_vector('LastUpdate', netuid=netuid, update=update, **kwargs) + + def dividends(self, netuid = 0, update=False, **kwargs): + return self.query_vector('Dividends', netuid=netuid, update=update, **kwargs) + + dividend = dividends + + def names(self, + netuid: int = 0, + update=False, + **kwargs) -> List[str]: + netuid = self.resolve_netuid(netuid) + names = self.query_map('Name', update=update, netuid=netuid,**kwargs) + if netuid == 'all': + for netuid, netuid_names in names.items(): + names[netuid] = list(netuid_names.values()) + else: + names = list(names.values()) + return names + + def uids(self, + netuid = 0, + update=False, + max_age=1000, + **kwargs) -> List[str]: + netuid = self.resolve_netuid(netuid) + keys = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) + if netuid == 'all': + for netuid, netuid_keys in keys.items(): + keys[netuid] = list(netuid_keys.keys ()) + else: + keys = list(keys.keys()) + return keys + + def subnet2n(self, fmt='j', **kwargs): + netuid2n = self.netuid2n(fmt=fmt, **kwargs) + netuid2subnet = self.netuid2subnet() + subnet2n = {} + for netuid, subnet in netuid2subnet.items(): + subnet2n[subnet] = netuid2n[netuid] + return subnet2n + + def subnet2stakes(self, block=None, update=False, fmt='j', **kwargs): + subnet2stakes = {} + for netuid in self.netuids( update=update): + subnet2stakes[netuid] = self.stakes(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) + return subnet2stakes + + def subnet_state(self, netuid='all', block=None, update=False, fmt='j', **kwargs): + + subnet_state = { + 'params': self.subnet_params(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs), + 'modules': self.modules(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs), + } + return subnet_state + + def subnet2emission(self, fmt='j', **kwargs): + subnet2params = self.subnet_params(netuid='all') + netuid2emission = self.netuid2emission(fmt=fmt, **kwargs) + netuid2subnet = self.netuid2subnet() + subnet2emission = {} + for netuid, subnet in netuid2subnet.items(): + subnet2emission[subnet] = netuid2emission[netuid] + # sort by emission + subnet2emission = dict(sorted(subnet2emission.items(), key=lambda x: x[1], reverse=True)) + + return subnet2emission + + + def uid2key(self, uid=None, + netuid = 0, + update=False, + + max_age= 1000, + **kwargs): + netuid = self.resolve_netuid(netuid) + uid2key = self.query_map('Keys', netuid=netuid, update=update, max_age=max_age, **kwargs) + # sort by uid + if uid != None: + return uid2key[uid] + return uid2key + + + def key2uid(self, key = None, netuid: int = 0, update=False, netuids=None , **kwargs): + uid2key = self.uid2key( netuid=netuid, update=update, **kwargs) + reverse_map = lambda x: {v: k for k,v in x.items()} + if netuid == 'all': + key2uid = {netuid: reverse_map(_key2uid) for netuid, _key2uid in uid2key.items() if netuids == None or netuid in netuids } + else: + key2uid = reverse_map(uid2key) + if key != None: + key_ss58 = self.resolve_key_ss58(key) + return key2uid[key_ss58] + return key2uid + + """ Returns network SubnetN hyper parameter """ + def n(self, netuid: int = 0,block: Optional[int] = None, max_age=100, update=False, **kwargs ) -> int: + if netuid == 'all': + return sum(self.query_map('N', block=block , update=update, max_age=max_age, **kwargs).values()) + else: + return self.query( 'N', params=[netuid], block=block , update=update, **kwargs) + + def subnet_exists(self, subnet:str) -> bool: + subnets = self.subnets() + return bool(subnet in subnets) + + def subnet_emission(self, netuid:str = 0, block=None, update=False, **kwargs): + emissions = self.emission(block=block, update=update, netuid=netuid, **kwargs) + if isinstance(emissions[0], list): + emissions = [sum(e) for e in emissions] + return sum(emissions) + + def get_modules(self, + keys : list = None, + netuid=None, + timeout=30, + min_emission=0, + max_age = 1000, + update=False, + **kwargs): + netuid = self.resolve_netuid(netuid) + modules = None + path = None + if keys == None : + path = f'subnet/{self.network}/{netuid}/modules' + modules = self.get(path, None, max_age=max_age, update=update) + keys = self.keys(netuid=netuid) + + n = len(keys) + if modules == None: + modules = [] + print(f'Getting modules {n}') + futures = [c.submit(self.get_module, kwargs=dict(module=k, netuid=netuid, **kwargs)) for k in keys] + progress = c.tqdm(n) + modules = [] + + + should_pass = lambda x: isinstance(x, dict) \ + and 'name' in x \ + and len(x['name']) > 0 \ + and x['emission'] >= min_emission + + for future in c.as_completed(futures, timeout=timeout): + module = future.result() + if should_pass(module): + modules += [module] + progress.update(1) + if path != None: + self.put(path, modules) + + + return modules + + module_param_features = [ + 'key', + 'name', + 'address', + 'emission', + 'incentive', + 'dividends', + 'last_update', + 'stake_from', + 'delegation_fee' + ] + + def get_module(self, + module=None, + netuid=None, + trials = 4, + fmt='j', + mode = 'http', + block = None, + max_age = None, + lite = False, + update = False, + **kwargs ) -> 'ModuleInfo': + U16_MAX = 2**16 - 1 + + netuid = self.resolve_netuid(netuid) + + if module == None: + module = self.keys(netuid=netuid, update=update, max_age=max_age)[0] + c.print(f'No module specified, using {module}') + + module = c.key2address().get(module, module) + url = self.resolve_url( mode=mode) + module_key = module + is_valid_key = c.valid_ss58_address(module) + if not is_valid_key: + module_key = self.name2key(name=module, netuid=netuid, **kwargs) + netuid = self.resolve_netuid(netuid) + json={'id':1, 'jsonrpc':'2.0', 'method': 'subspace_getModuleInfo', 'params': [module_key, netuid]} + module = None + for i in range(trials): + try: + module = requests.post(url, json=json).json() + break + except Exception as e: + c.print(e) + continue + assert module != None, f"Failed to get module {module_key} after {trials} trials" + if not 'result' in module: + return module + module = {**module['result']['stats'], **module['result']['params']} + # convert list of u8 into a string Vector to a string + module['name'] = self.vec82str(module['name']) + module['address'] = self.vec82str(module['address']) + module['dividends'] = module['dividends'] / (U16_MAX) + module['incentive'] = module['incentive'] / (U16_MAX) + module['stake_from'] = {k:self.format_amount(v, fmt=fmt) for k,v in module['stake_from']} + module['stake'] = sum([v for k,v in module['stake_from'].items() ]) + module['emission'] = self.format_amount(module['emission'], fmt=fmt) + module['key'] = module.pop('controller', None) + module['metadata'] = module.pop('metadata', {}) + + module['vote_staleness'] = (block or self.block) - module['last_update'] + if lite : + features = self.module_param_features + ['stake', 'vote_staleness'] + module = {f: module[f] for f in features} + assert module['key'] == module_key, f"Key mismatch {module['key']} != {module_key}" + return module + + + def root_valis(self, search=None, netuid = 0, update=False, **kwargs): + root_valis = [] + for module in self.get_modules(netuid=netuid, update=update, **kwargs): + if search != None: + if search not in module['name']: + continue + module.pop('stake_from') + root_valis += [module ] + + return c.df(root_valis)[['name', 'key', 'stake']] + + + def root_keys(self, netuid = 0, update=False, **kwargs): + return self.keys(netuid=netuid, update=update, **kwargs) + + + + + def registration_block(self, netuid: int = 0, update=False, **kwargs): + registration_blocks = self.query_map('RegistrationBlock', netuid=netuid, update=update, **kwargs) + return registration_blocks + + regblocks = registration_blocks = registration_block + + + + + + + + def key2name(self, key=None, netuid: int = None, update=False) -> Dict[str, str]: + + key2name = {v:k for k,v in self.name2key(netuid=netuid, update=update).items()} + if key != None: + return key2name[key] + return key2name + + + + + def name2key(self, name:str=None, + max_age=1000, + timeout=30, + netuid: int = 0, + update=False, + trials=3, + **kwargs ) -> Dict[str, str]: + # netuid = self.resolve_netuid(netuid) + netuid = self.resolve_netuid(netuid) + + names = c.submit(self.names, kwargs={'feature': 'names', 'netuid':netuid, 'update':update, 'max_age':max_age, 'network': self.network}) + keys = c.submit(self.keys, kwargs={'feature': 'keys', 'netuid':netuid, 'update':update, 'max_age':max_age, 'network': self.network}) + names, keys = c.wait([names, keys], timeout=timeout) + name2key = dict(zip(names, keys)) + if name != None: + if name in name2key: + return name2key[name] + else: + trials -= 1 + if trials == 0: + return None + else: + return self.name2key(name=name, + timeout=timeout, netuid=netuid, update=True, + trials=trials, **kwargs) + + return name2key + + + + + def name2uid(self, name = None, netuid: int = 0, search=None) -> int: + netuid = self.resolve_netuid(netuid) + uid2name = self.uid2name(netuid=netuid) + + if netuid == 'all': + netuid2name2uid = {} + for netuid, netuid_uid2name in uid2name.items(): + name2uid = self.search_dict(netuid_uid2name) + if name != None: + name2uid = name2uid[name] + netuid2name2uid[netuid] = name2uid + return netuid2name2uid + + else: + name2uid = {v:k for k,v in uid2name.items()} + if search != None: + name2uid = self.search_dict(name2uid, search=search) + if name != None: + return name2uid[name] + + return name2uid + + def netuids(self, update=False, block=None) -> Dict[int, str]: + return list(self.netuid2subnet( update=update, block=block).keys()) + + def netuid2subnet(self, netuid=None, update=False, block=None, **kwargs ) -> Dict[str, str]: + netuid2subnet = self.query_map('SubnetNames', update=update, block=block, **kwargs) + netuid2subnet = dict(sorted(netuid2subnet.items(), key=lambda x: x[0])) + if netuid != None: + return netuid2subnet[netuid] + return netuid2subnet + netuid2name = netuid2subnet + + def subnet2netuid(self, subnet=None, update=False, **kwargs ) -> Dict[str, str]: + subnet2netuid = {v:k for k,v in self.netuid2subnet( update=update, **kwargs).items()} + # sort by subnet + if subnet != None: + return subnet2netuid[subnet] if subnet in subnet2netuid else len(subnet2netuid) + return subnet2netuid + name2netuid = subnet2netuid + + + + def get_uid( self, key: str, netuid: int = 0, block: Optional[int] = None, update=False, **kwargs) -> int: + return self.query( 'Uids', block=block, params=[ netuid, key ] , update=update, **kwargs) + + + def weights(self, netuid = 0, update=False, **kwargs) -> list: + weights = self.query_map('Weights',netuid=netuid, update=update, **kwargs) + + tuples2list = lambda x: [list(v) for v in x] + if netuid == 'all': + for netuid, netuid_weights in weights.items(): + weights[netuid] = {k: tuples2list(v) for k,v in netuid_weights.items()} + else: + weights = {k: tuples2list(v) for k,v in weights.items()} + + return weights + + def resolve_uid(self, uid=None, netuid=None, **kwargs) -> int: + netuid = self.resolve_netuid(netuid) + if isinstance(uid, int): + return uid + elif isinstance(uid, str): + if c.key_exists(uid): + # for key + uid = self.resolve_key_ss58(uid) + uid = self.key2uid(netuid=netuid,**kwargs)[uid] + else: + # resolve name + uid = self.name2uid(name=uid, netuid=netuid, **kwargs) + + return uid + + + + def get_weights(self, key=None, netuid = 0, update=False, **kwargs) -> list: + uid = self.resolve_uid(key, netuid=netuid) + weights = self.query('Weights', params=[netuid, uid], update=update, **kwargs) + return weights + + + + + + + def total_emissions(self, netuid = 9, block=None, update=False, fmt = 'j', **kwargs): + + emissions = self.query_vector('Emission', netuid=netuid, block=block, update=update, **kwargs) + if netuid == 'all': + for netuid, netuid_emissions in emissions.items(): + emissions[netuid] = [self.format_amount(e, fmt=fmt) for e in netuid_emissions] + else: + emissions = [self.format_amount(e, fmt=fmt) for e in emissions] + + return sum(emissions) + + + + # def state(self, block=None, netuid='all', update=False, max_age=10000, fmt='j', **kwargs): + # subnet_params = self.subnet_params(block=block, netuid=netuid, max_age=max_age, **kwargs) + # subnet2emissions = self.emissions(netuid=netuid, max_age=max_age, block=block, **kwargs) + # subnet2staketo = self.stake_to(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) + # subnet2incentives = self.incentives(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) + # subnet2trust = self.trust(netuid=netuid, block=block, update=update, fmt=fmt, **kwargs) + # subnet2keys = self.keys(netuid=netuid, block=block, update=update, **kwargs) + + # subnet2state = {} + # for netuid, params in subnet_params.items(): + # subnet_state = { + # 'params': params, + # 'incentives': subnet2incentives[netuid], + # 'emissions': subnet2emissions[netuid], + # '' + # 'stake_to': subnet2staketo[netuid], + # 'keys': subnet2keys[netuid], + + # } + # subnet_state[netuid] = subnet_state + + # return subnet2state + + + def netuid2emission(self, fmt='j', period='day', names=None, **kwargs): + netuid2emission = {} + netuid2tempo = None + emissions = self.query_vector('Emission', netuid='all', **kwargs) + for netuid, netuid_emissions in emissions.items(): + + if period == 'day': + if netuid2tempo == None: + netuid2tempo = self.query_map('Tempo', netuid='all', **kwargs) + tempo = netuid2tempo.get(netuid, 100) + multiplier = self.blocks_per_day() / tempo + else: + multiplier = 1 + netuid2emission[netuid] = self.format_amount(sum(netuid_emissions), fmt=fmt) * multiplier + + netuid2emission = {k: v for k,v in netuid2emission.items()} + if names: + netuid2emission = {self.netuid2name(netuid=k): v for k,v in netuid2emission.items()} + return netuid2emission + + def subnet2emission(self, fmt='j', period='day', **kwargs): + return self.netuid2emission(fmt=fmt, period=period, names=1, **kwargs) + + + def global_emissions(self, **kwargs): + return sum(list(self.subnet2emissions( **kwargs).values())) + + + + + def subnet2params( self, block: Optional[int] = None ) -> Optional[float]: + netuids = self.netuids() + subnet2params = {} + netuid2subnet = self.netuid2subnet() + for netuid in netuids: + subnet = netuid2subnet[netuid] + subnet2params[subnet] = self.subnet_params(netuid=netuid, block=block) + return subnet2params + + + + + ################# + #### UPDATE SUBNET #### + ################# + def update_subnet( + self, + params: dict= None, + netuid: int = 0, + key: str = None, + nonce = None, + update= True, + **extra_params, + ) -> bool: + + params = {**(params or {}), **extra_params} + + netuid = self.resolve_netuid(netuid) + subnet_params = self.subnet_params( netuid=netuid , update=update, fmt='nanos') + # infer the key if you have it + for k in ['min_immunity_stake']: + if k in params: + params[k] = params[k] * 1e9 + if key == None: + key2address = c.address2key() + if subnet_params['founder'] not in key2address: + return {'success': False, 'message': f"Subnet {netuid} not found in local namespace, please deploy it "} + key = c.get_key(key2address.get(subnet_params['founder'])) + c.print(f'Using key: {key}') + + # remove the params that are the same as the module info + params = {**subnet_params, **params} + for k in ['name']: + params[k] = params[k].encode('utf-8') + + params['netuid'] = netuid + return self.compose_call(fn='update_subnet', params=params, key=key, nonce=nonce) + + + ################# + #### Serving #### + ################# + def propose_subnet_update( + self, + netuid: int = None, + key: str = None, + nonce = None, + **params, + ) -> bool: + + netuid = self.resolve_netuid(netuid) + c.print(f'Adding proposal to subnet {netuid}') + subnet_params = self.subnet_params( netuid=netuid , update=True) + # remove the params that are the same as the module info + params = {**subnet_params, **params} + for k in ['name', 'vote_mode']: + params[k] = params[k].encode('utf-8') + params['netuid'] = netuid + + response = self.compose_call(fn='add_subnet_proposal', + params=params, + key=key, + nonce=nonce) + + + return response + + + + def get_stake( self, key_ss58: str, block: Optional[int] = None, netuid:int = None , fmt='j', update=True ) -> Optional['Balance']: + + key_ss58 = self.resolve_key_ss58( key_ss58) + netuid = self.resolve_netuid( netuid ) + stake = self.query( 'Stake',params=[netuid, key_ss58], block=block , update=update) + return self.format_amount(stake, fmt=fmt) + + + + def min_register_stake(self, netuid: int = 0, fmt='j', **kwargs) -> float: + netuid = self.resolve_netuid(netuid) + min_burn = self.min_burn( fmt=fmt) + min_stake = self.min_stake(netuid=netuid, fmt=fmt) + return min_stake + min_burn + + + + + def resolve_netuid(self, netuid: int = None) -> int: + ''' + Resolves a netuid to a subnet name. + ''' + if netuid == 'all': + return netuid + if netuid == None : + # If the netuid is not specified, use the default. + return self.netuid + if isinstance(netuid, str): + subnet2netuid = self.subnet2netuid() + if netuid not in subnet2netuid: # if still not found, try lower case + subnet2netuid =self.subnet2netuid(update=True) + assert netuid in subnet2netuid, f"Subnet {netuid} not found in {subnet2netuid}" + return subnet2netuid[netuid] + + elif isinstance(netuid, int): + if netuid == 0: + return netuid + # If the netuid is an integer, ensure it is valid. + + assert isinstance(netuid, int), "netuid must be an integer" + return netuid + + + def blocks_until_vote(self, netuid=None, **kwargs): + netuid = self.resolve_netuid(netuid) + tempo = self.subnet_params(netuid=netuid, **kwargs)['tempo'] + block = self.block + return tempo - ((block + netuid) % tempo) + + def emission_per_epoch(self, netuid=None): + return self.subnet(netuid=netuid)['emission']*self.epoch_time(netuid=netuid) + + + + + + def get_stake_to( self, + key: str = None, + module_key=None, + block: Optional[int] = None, + fmt='j' , update=False, + max_age = 60, + timeout = 10, + **kwargs) -> Optional['Balance']: + + key_address = self.resolve_key_ss58( key ) + stake_to = self.query_map( 'StakeTo', params=[ key_address], block=block, update=update, max_age=max_age) + stake_to = {k: self.format_amount(v, fmt=fmt) for k, v in stake_to.items()} + return stake_to + + def get_stake_from( self, key: str, block: Optional[int] = None, fmt='j', update=True ) -> Optional['Balance']: + key = self.resolve_key_ss58( key ) + stake_from = self.query_map( 'StakeFrom', params=[key], block=block, update=update ) + stake_from = {k: self.format_amount(v, fmt=fmt) for k, v in stake_from.items()} + return stake_from + + + def epoch_time(self, netuid=None, update=False, **kwargs): + netuid = self.resolve_netuid(netuid) + return self.subnet_params(netuid=netuid, update=update, **kwargs)['tempo']*self.block_time + + + def seconds_per_day(self ): + return 24*60*60 + + def epochs_per_day(self, netuid=None): + netuid = self.resolve_netuid(netuid) + return self.seconds_per_day()/self.epoch_time(netuid=netuid) + + def seconds_per_epoch(self, netuid=0): + netuid =self.resolve_netuid(netuid) + return self.block_time * self.subnet_params(netuid=netuid)['tempo'] + + + def format_module(self, module: 'ModuleInfo', fmt:str='j') -> 'ModuleInfo': + U16_MAX = 2**16 - 1 + for k in ['emission']: + module[k] = self.format_amount(module[k], fmt=fmt) + for k in ['incentive', 'dividends']: + module[k] = module[k] / (U16_MAX) + + module['stake_from'] = {k: self.format_amount(v, fmt=fmt) for k, v in module['stake_from']} + return module + + + + def netuid2module(self, update=False, fmt:str='j', **kwargs) -> 'ModuleInfo': + netuids = self.netuids(update=update) + future2netuid = {} + for netuid in netuids: + f = c.submit(self.get_module, dict(netuid=netuid, update=update, fmt=fmt, **kwargs)) + future2netuid[f] = netuid + netuid2module = {} + progress = c.tqdm(len(netuids)) + + for future in c.as_completed(future2netuid): + netuid = future2netuid.pop(future) + module = future.result() + if not c.is_error(module): + netuid2module[netuid] = module + progress.update(1) + + return netuid2module + + + + + + def netuid2uid(self, key=None, update=False, **kwargs) -> Dict[str, str]: + key = self.resolve_key_ss58(key) + netuids = self.netuids(update=update) + netuid2uid = {} + progress = c.tqdm(len(netuids)) + + future2netuid = {} + for netuid in netuids: + f = c.submit(self.get_uid, kwargs=dict(key=key, netuid=netuid, **kwargs)) + future2netuid[f] = netuid + + for future in c.as_completed(future2netuid): + netuid = future2netuid.pop(future) + uid = future.result() + if uid != None: + netuid2uid[netuid] = uid + progress.update(1) + # sort by netuid key + netuid2uid = dict(sorted(netuid2uid.items(), key=lambda x: x[0])) + + return netuid2uid + + + def subnet_state(self, netuid=0, update=False, **kwargs): + + modules = self.get_modules(netuid=netuid, update=update, **kwargs) + + return { + 'params': self.subnet_params(netuid=netuid, + update=update, + fmt='nanos'), + 'modules': modules + } + + def register_subnet(self, key: 'Keypair', name: str, metadata: str | None = None) -> 'c': + """ + Registers a new subnet in the network. + + Args: + key (Keypair): The keypair used for registering the subnet. + name (str): The name of the subnet to be registered. + metadata (str | None, optional): Additional metadata for the subnet. Defaults to None. + + Returns: + ExtrinsicReceipt: A receipt of the subnet registration transaction. + + Raises: + ChainTransactionError: If the transaction fails. + """ + + params = { + "name": name, + "metadata": metadata, + } + + response = self.compose_call("register_subnet", params=params, key=key) + + return response + + Subspace.run(__name__) diff --git a/commune/subspace/wallet.py b/commune/subspace/wallet.py index 6bef7022..7ce13a0c 100644 --- a/commune/subspace/wallet.py +++ b/commune/subspace/wallet.py @@ -5,1243 +5,4 @@ class SubspaceWallet: - ################## - #### Transfer #### - ################## - def transfer( - self, - dest: str, - amount: float , - key: str = None, - nonce= None, - **kwargs - - ) -> bool: - # this is a bit of a hack to allow for the amount to be a string for c send 500 0x1234 instead of c send 0x1234 500 - if type(dest) in [int, float]: - assert isinstance(amount, str), f"Amount must be a string" - new_amount = int(dest) - dest = amount - amount = new_amount - key = self.resolve_key(key) - dest = self.resolve_key_ss58(dest) - amount = self.to_nanos(amount) # convert to nano (10^9 nanos = 1 token) - response = self.compose_call( - module='Balances', - fn='transfer_keep_alive', - params={ - 'dest': dest, - 'value': amount - }, - key=key, - nonce = nonce, - **kwargs - ) - - return response - - - - - - def add_profit_shares( - self, - keys: List[str], # the keys to add profit shares to - shares: List[float] = None , # the shares to add to the keys - key: str = None, - netuid : int = 0, - ) -> bool: - - key = self.resolve_key(key) - assert len(keys) > 0, f"Must provide at least one key" - key2address = c.key2address() - keys = [key2address.get(k, k) for k in keys] - assert all([c.valid_ss58_address(k) for k in keys]), f"All keys must be valid ss58 addresses" - shares = shares or [1 for _ in keys] - - assert len(keys) == len(shares), f"Length of keys {len(keys)} must be equal to length of shares {len(shares)}" - - response = self.compose_call( - module='SubspaceModule', - fn='add_profit_shares', - params={ - 'keys': keys, - 'shares': shares, - 'netuid': netuid - }, - key=key - ) - - return response - - - def stake_many( self, - modules:List[str] = None, - amounts:Union[List[str], float, int] = None, - key: str = None, - netuid:int = 0, - min_balance = 100_000_000_000, - n:str = 100) -> Optional['Balance']: - - netuid = self.resolve_netuid( netuid ) - key = self.resolve_key( key ) - - if modules == None: - my_modules = self.my_modules(netuid=netuid, update=False) - modules = [m['key'] for m in my_modules if 'vali' in m['name']] - - modules = modules[:n] # only stake to the first n modules - - assert len(modules) > 0, f"No modules found with name {modules}" - module_keys = modules - - if amounts == None: - balance = self.get_balance(key=key, fmt='nanos') - min_balance - amounts = [(balance // len(modules))] * len(modules) - assert sum(amounts) < balance, f'The total amount is {sum(amounts)} > {balance}' - else: - if isinstance(amounts, (float, int)): - amounts = [amounts] * len(modules) - for i, amount in enumerate(amounts): - amounts[i] = self.to_nanos(amount) - - assert len(modules) == len(amounts), f"Length of modules and amounts must be the same. Got {len(modules)} and {len(amounts)}." - - params = { - "module_keys": module_keys, - "amounts": amounts - } - - response = self.compose_call('add_stake_multiple', params=params, key=key) - - return response - - def transfer_multiple( self, - destinations:List[str], - amounts:Union[List[str], float, int], - key: str = None, - n:str = 10) -> Optional['Balance']: - key2address = c.key2address() - key = self.resolve_key( key ) - balance = self.get_balance(key=key, fmt='j') - for i, destination in enumerate(destinations): - if not c.valid_ss58_address(destination): - if destination in key2address: - destinations[i] = key2address[destination] - else: - raise Exception(f"Invalid destination address {destination}") - if type(amounts) in [float, int]: - amounts = [amounts] * len(destinations) - assert len(set(destinations)) == len(destinations), f"Duplicate destinations found" - assert len(destinations) == len(amounts), f"Length of modules and amounts must be the same. Got {len(destinations)} and {len(amounts)}." - assert all([c.valid_ss58_address(d) for d in destinations]), f"Invalid destination address {destinations}" - total_amount = sum(amounts) - assert total_amount < balance, f'The total amount is {total_amount} > {balance}' - - # convert the amounts to their interger amount (1e9) - amounts = [a*(10**9) for a in amounts] - - params = { - "destinations": destinations, - "amounts": amounts - } - - return self.compose_call('transfer_multiple', params=params, key=key) - - transfer_many = transfer_multiple - - def my_modules(self, - modules : list = None, - netuid=0, - timeout=30, - **kwargs): - if modules == None: - modules = self.my_keys(netuid=netuid) - futures = [c.submit(self.get_module, kwargs=dict(module=module, netuid=netuid, **kwargs)) for module in modules] - for future in c.as_completed(futures, timeout=timeout): - module = future.result() - print(module) - if not c.is_error(module): - modules += [module] - return modules - - def unstake_many( self, - - modules:Union[List[str], str] = None, - amounts:Union[List[str], float, int] = None, - key: str = None, - netuid=0, - update=True, - - ) -> Optional['Balance']: - - key = self.resolve_key( key ) - name2key = {} - - module_keys = [] - for i, module in enumerate(modules): - if c.valid_ss58_address(module): - module_keys += [module] - else: - if name2key == {}: - name2key = self.name2key(netuid=netuid, update=update) - assert module in name2key, f"Invalid module {module} not found in SubNetwork {netuid}" - module_keys += [name2key[module]] - - # RESOLVE AMOUNTS - if amounts == None: - stake_to = self.get_stake_to(key=key, names=False, update=update, fmt='j') # name to amounts - amounts = [stake_to[m] for m in module_keys] - - if isinstance(amounts, (float, int)): - amounts = [amounts] * len(module_keys) - - for i, amount in enumerate(amounts): - amounts[i] = self.to_nanos(amount) - - assert len(module_keys) == len(amounts), f"Length of modules and amounts must be the same. Got {len(module_keys)} and {len(amounts)}." - - params = { - "netuid": netuid, - "module_keys": module_keys, - "amounts": amounts - } - response = self.compose_call('remove_stake_multiple', params=params, key=key) - return response - - - def update_module( - self, - module: str, # the module you want to change - address: str = None, # the address of the new module - name: str = None, # the name of the new module - delegation_fee: float = None, # the delegation fee of the new module - metadata = None, # the metadata of the new module - fee : float = None, # the fee of the new module - netuid: int = 0, # the netuid of the new module - nonce = None, # the nonce of the new module - tip: int = 0, # the tip of the new module - params = None - ) -> bool: - key = self.resolve_key(module) - netuid = self.resolve_netuid(netuid) - assert self.is_registered(key.ss58_address, netuid=netuid), f"Module {module} is not registered in SubNetwork {netuid}" - if params == None: - params = { - 'name': name , # defaults to module.tage - 'address': address , # defaults to module.tage - 'delegation_fee': fee or delegation_fee, # defaults to module.delegate_fee - 'metadata': c.python2str(metadata or {}), # defaults to module.metadata - } - - should_update_module = False - module_info = self.get_module(key.ss58_address, netuid=netuid) - - for k,v in params.items(): - if params[k] == None: - params[k] = module_info[k] - if k in module_info and params[k] != module_info[k]: - should_update_module = True - - if not should_update_module: - return {'success': False, 'message': f"Module {module} is already up to date"} - - c.print('Updating with', params, color='cyan') - params['netuid'] = netuid - - reponse = self.compose_call('update_module', params=params, key=key, nonce=nonce, tip=tip) - - # IF SUCCESSFUL, MOVE THE KEYS, AS THIS IS A NON-REVERSIBLE OPERATION - - - return reponse - - update = update_server = update_module - - def stake_transfer( - self, - module_key: str , - new_module_key: str , - amount: Union[int, float] = None, - key: str = None, - ) -> bool: - # STILL UNDER DEVELOPMENT, DO NOT USE - key = c.get_key(key) - netuid = 0 - module_key = self.resolve_module_key(module_key, netuid=netuid) - new_module_key = self.resolve_module_key(new_module_key, netuid=netuid) - - assert module_key != new_module_key, f"Module key {module_key} is the same as new_module_key {new_module_key}" - - if amount == None: - amount = self.get_stake_to( key=key , fmt='j', max_age=0).get(module_key, 0) - - # Get current stake - params={ - 'amount': int(amount * 10**9), - 'module_key': module_key, - 'new_module_key': new_module_key - - } - - return self.compose_call('transfer_stake',params=params, key=key) - - def unstake( - self, - module : str = None, # defaults to most staked module - amount: float =None, # defaults to all of the amount - key : 'c.Key' = None, # defaults to first key - netuid=0, - **kwargs - ) -> dict: - """ - description: - Unstakes the specified amount from the module. - If no amount is specified, it unstakes all of the amount. - If no module is specified, it unstakes from the most staked module. - params: - amount: float = None, # defaults to all - module : str = None, # defaults to most staked module - key : 'c.Key' = None, # defaults to first key - netuid : Union[str, int] = 0, # defaults to module.netuid - network: str= main, # defaults to main - return: - response: dict - """ - - key = c.get_key(key) - # get most stake from the module - - if isinstance(module, int): - module = amount - amount = module - - assert module != None or amount != None, f"Must provide a module or an amount" - key2address = c.key2address() - if module in key2address: - module_key = key2address[module] - else: - name2key = self.name2key(netuid=netuid) - if module in name2key: - module_key = name2key[module] - else: - module_key = module - assert self.is_registered(module_key, netuid=netuid), f"Module {module} is not registered in SubNetwork {netuid}" - - if amount == None: - stake_to = self.get_stake_to(names = False, fmt='nano', key=module_key) - amount = stake_to[module_key] - 100000 - else: - amount = int(self.to_nanos(amount)) - # convert to nanos - params={ - 'amount': amount , - 'module_key': module_key - } - response = self.compose_call(fn='remove_stake',params=params, key=key, **kwargs) - - return response - - - - def my_servers(self, search=None, **kwargs): - servers = [m['name'] for m in self.my_modules(**kwargs)] - if search != None: - servers = [s for s in servers if search in s] - return servers - - def my_modules_names(self, *args, **kwargs): - my_modules = self.my_modules(*args, **kwargs) - return [m['name'] for m in my_modules] - - def my_module_keys(self, *args, **kwargs): - modules = self.my_modules(*args, **kwargs) - return [m['key'] for m in modules] - - def my_key2uid(self, *args, netuid=0, update=False, **kwargs): - key2uid = self.key2uid(*args, netuid=netuid, **kwargs) - - key2address = c.key2address(update=update ) - key_addresses = list(key2address.values()) - if netuid == 'all': - for netuid, netuid_keys in key2uid.items(): - key2uid[netuid] = {k: v for k,v in netuid_keys.items() if k in key_addresses} - - my_key2uid = { k: v for k,v in key2uid.items() if k in key_addresses} - return my_key2uid - - def my_keys(self, - netuid=0, - search=None, - max_age=None, - names = False, - update=False, **kwargs): - netuid = self.resolve_netuid(netuid) - keys = self.keys(netuid=netuid, max_age=max_age, update=update, **kwargs) - key2address = c.key2address(search=search, max_age=max_age, update=update) - if search != None: - key2address = {k: v for k,v in key2address.items() if search in k} - address2key = {v:k for k,v in key2address.items()} - addresses = list(key2address.values()) - convert_fn = lambda x : address2key.get(x, x) if names else x - if netuid == 'all': - my_keys = {} - for netuid, netuid_keys in keys.items(): - - my_netuid_keys = [convert_fn(k) for k in netuid_keys if k in addresses] - - if len(my_netuid_keys) > 0: - my_keys[netuid] = my_netuid_keys - else: - my_keys = [convert_fn(k) for k in keys if k in addresses] - return my_keys - - def register_servers(self, - search=None, - infos=None, - netuid = 0, - timeout=60, - max_age=None, - key=None, update=False, - parallel = True, - **kwargs): - ''' - key2address : dict - A dictionary of module names to their keys - timeout : int - The timeout for each registration - netuid : int - The netuid of the modules - - ''' - keys = c.submit(self.keys, dict(netuid=netuid, update=update, max_age=max_age)) - names = c.submit(self.names, dict(netuid=netuid, update=update, max_age=max_age)) - keys, names = c.wait([keys, names], timeout=timeout) - - if infos==None: - infos = c.infos(search=search, **kwargs) - should_register_fn = lambda x: x['key'] not in keys and x['name'] not in names - infos = [i for i in infos if should_register_fn(i)] - c.print(f'Found {infos} modules to register') - if parallel: - launcher2balance = c.key2balance() - min_stake = self.min_register_stake(netuid=netuid) - launcher2balance = {k: v for k,v in launcher2balance.items() if v > min_stake} - launcher_keys = list(launcher2balance.keys()) - futures = [] - for i, info in enumerate(infos): - if info['key'] in keys: - continue - - launcher_key = launcher_keys[i % len(launcher_keys)] - c.print(f"Registering {info['name']} with module_key {info['key']} using launcher {launcher_key}") - f = c.submit(c.register, kwargs=dict(name=info['name'], - address= info['address'], - netuid = netuid, - module_key=info['key'], - key=launcher_key), timeout=timeout) - futures+= [f] - - if len(futures) == len(launcher_keys): - for future in c.as_completed(futures, timeout=timeout): - r = future.result() - c.print(r, color='green') - futures.remove(future) - break - - for future in c.as_completed(futures, timeout=timeout): - r = future.result() - c.print(r, color='green') - futures.remove(future) - - return infos - - else: - - for info in infos: - r = c.register(name=info['name'], - address= info['address'], - module_key=info['key'], - key=key) - c.print(r, color='green') - - return {'success': True, 'message': 'All modules registered'} - - - def unregistered_servers(self, search=None, netuid = 0, key=None, max_age=None, update=False, transfer_multiple=True,**kwargs): - netuid = self.resolve_netuid(netuid) - servers = c.servers(search=search) - key2address = c.key2address(update=update) - keys = self.keys(netuid=netuid, max_age=max_age, update=update) - unregister_servers = [] - for s in servers: - if key2address[s] not in keys: - unregister_servers += [s] - return unregister_servers - - - - - def my_value( self, *args, **kwargs ): - return sum(list(self.key2value( *args, **kwargs).values())) - - - def my_total_stake(self, netuid='all', fmt='j', update=False): - my_stake_to = self.my_stake_to(netuid=netuid, fmt=fmt, update=update) - return sum([sum(list(v.values())) for k,v in my_stake_to.items()]) - - def my_staked_module_keys(self, netuid = 0, **kwargs): - my_stake_to = self.my_stake_to(netuid=netuid, **kwargs) - module_keys = {} if netuid == 'all' else [] - for subnet_netuid, stake_to_key in my_stake_to.items(): - if netuid == 'all': - for _netuid, stake_to_subnet in stake_to_key.items(): - module_keys[_netuid] = list(stake_to_subnet.keys()) + module_keys.get(_netuid, []) - else: - module_keys += list(stake_to_key.keys()) - return module_keys - - def my_stake_to(self, fmt='j', **kwargs): - stake_to = self.stake_to(fmt=fmt, **kwargs) - key2address = c.key2address() - my_stake_to = {} - for key, address in key2address.items(): - my_stake_to[address] = {k:v for k,v in stake_to.get(address, {}).items()} - stake_to_keys = list(my_stake_to.keys()) - for key in stake_to_keys: - if len(my_stake_to[key]) == 0: - del my_stake_to[key] - - return my_stake_to - - - - - def my_stake_from(self, netuid = 0, block=None, update=False, fmt='j', max_age=1000 , **kwargs): - stake_from_tuples = self.stake_from(netuid=netuid, - block=block, - update=update, - tuples = True, - fmt=fmt, max_age=max_age, **kwargs) - - address2key = c.address2key() - stake_from_total = {} - if netuid == 'all': - for netuid, stake_from_tuples_subnet in stake_from_tuples.items(): - for module_key,staker_tuples in stake_from_tuples_subnet.items(): - for staker_key, stake in staker_tuples: - if module_key in address2key: - stake_from_total[staker_key] = stake_from_total.get(staker_key, 0) + stake - - else: - for module_key,staker_tuples in stake_from_tuples.items(): - for staker_key, stake in staker_tuples: - if module_key in address2key: - stake_from_total[staker_key] = stake_from_total.get(staker_key, 0) + stake - - - for staker_address in address2key.keys(): - if staker_address in stake_from_total: - stake_from_total[staker_address] = self.format_amount(stake_from_total[staker_address], fmt=fmt) - return stake_from_total - - - - def my_total_stake_to( self, - key: str = None, - block: Optional[int] = None, - timeout=20, - names = False, - fmt='j' , - update=False, - max_age = 1000, - **kwargs) -> Optional['Balance']: - return sum(list(self.get_stake_to(key=key, block=block, timeout=timeout, names=names, fmt=fmt, - update=update, - max_age=max_age, **kwargs).values())) - - - def my_modules(self, search=None, netuid=0, generator=False, **kwargs): - keys = self.my_keys(netuid=netuid, search=search) - if netuid == 'all': - modules = {} - all_keys = keys - for netuid, keys in enumerate(all_keys): - try: - modules[netuid]= self.get_modules(keys=keys, netuid=netuid, **kwargs) - except Exception as e: - c.print(e) - modules = {k: v for k,v in modules.items() if len(v) > 0 } - return modules - modules = self.get_modules(keys=keys, netuid=netuid, **kwargs) - return modules - - def stats(self, - search = None, - netuid=0, - df:bool=True, - update:bool = False , - features : list = ['name', 'emission','incentive', 'dividends', 'stake', 'vote_staleness', 'serving', 'address'], - sort_features = ['emission', 'stake'], - fmt : str = 'j', - modules = None, - servers = None, - **kwargs - ): - - if isinstance(netuid, str): - netuid = self.subnet2netuid(netuid) - - if search == 'all': - netuid = search - search = None - - - if netuid == 'all': - all_modules = self.my_modules(netuid=netuid, update=update, fmt=fmt, search=search) - servers = c.servers() - stats = {} - netuid2subnet = self.netuid2subnet(update=update) - for netuid, modules in all_modules.items(): - subnet_name = netuid2subnet[netuid] - stats[netuid] = self.stats(modules=modules, netuid=netuid, servers=servers) - - color = c.random_color() - c.print(f'\n {subnet_name.upper()} :: (netuid:{netuid})\n', color=color) - c.print(stats[netuid], color=color) - - - modules = modules or self.my_modules(netuid=netuid, update=update, fmt=fmt, search=search) - - stats = [] - - local_key_addresses = list(c.key2address().values()) - servers = servers or c.servers() - for i, m in enumerate(modules): - if m['key'] not in local_key_addresses : - continue - # sum the stake_from - # we want to round these values to make them look nice - for k in ['emission', 'dividends', 'incentive']: - m[k] = c.round(m[k], sig=4) - - m['serving'] = bool(m['name'] in servers) - m['stake'] = int(m['stake']) - stats.append(m) - df_stats = c.df(stats) - if len(stats) > 0: - df_stats = df_stats[features] - if 'emission' in features: - epochs_per_day = self.epochs_per_day(netuid=netuid) - df_stats['emission'] = df_stats['emission'] * epochs_per_day - sort_features = [c for c in sort_features if c in df_stats.columns] - df_stats.sort_values(by=sort_features, ascending=False, inplace=True) - if search is not None: - df_stats = df_stats[df_stats['name'].str.contains(search, case=True)] - - if not df: - return df_stats.to_dict('records') - else: - return df_stats - - - - - - def update_modules(self, search=None, - timeout=60, - netuid=0, - **kwargs) -> List[str]: - - netuid = self.resolve_netuid(netuid) - my_modules = self.my_modules(search=search, netuid=netuid, **kwargs) - - self.keys() - futures = [] - namespace = c.namespace() - for m in my_modules: - - name = m['name'] - if name in namespace: - address = namespace[name] - else: - address = c.serve(name)['address'] - - if m['address'] == address and m['name'] == name: - c.print(f"Module {m['name']} already up to date") - - f = c.submit(c.update_module, kwargs={'module': name, - 'name': name, - 'netuid': netuid, - 'address': address, - **kwargs}, timeout=timeout) - futures+= [f] - - - results = [] - - for future in c.as_completed(futures, timeout=timeout): - results += [future.result()] - c.print(future.result()) - return results - - - - - - def stake( - self, - module: Optional[str] = None, # defaults to key if not provided - amount: Union['Balance', float] = None, - key: str = None, # defaults to first key - existential_deposit: float = 0, - netuid=0, - **kwargs - ) -> bool: - """ - description: - Unstakes the specified amount from the module. - If no amount is specified, it unstakes all of the amount. - If no module is specified, it unstakes from the most staked module. - params: - amount: float = None, # defaults to all - module : str = None, # defaults to most staked module - key : 'c.Key' = None, # defaults to first key - netuid : Union[str, int] = 0, # defaults to module.netuid - network: str= main, # defaults to main - return: - response: dict - - """ - key = c.get_key(key) - if c.valid_ss58_address(module): - module_key = module - else: - module_key = self.name2key(netuid=netuid).get(module) - - # Flag to indicate if we are using the wallet's own hotkey. - - if amount == None: - amount = self.get_balance( key.ss58_address , fmt='nano') - existential_deposit*10**9 - else: - amount = int(self.to_nanos(amount - existential_deposit)) - assert amount > 0, f"Amount must be greater than 0 and greater than existential deposit {existential_deposit}" - - # Get current stake - params={ - 'amount': amount, - 'module_key': module_key - } - - return self.compose_call('add_stake',params=params, key=key) - - - - - def key_info(self, key:str = None, netuid='all', detail=0, timeout=10, update=False, **kwargs): - key_info = { - 'balance': self.get_balance(key=key, **kwargs), - 'stake_to': self.get_stake_to(key=key, **kwargs), - } - if detail: - pass - else: - for netuid, stake_to in key_info['stake_to'].items(): - key_info['stake_to'][netuid] = sum(stake_to.values()) - - - return key_info - - - - - def subnet2modules(self, **kwargs): - subnet2modules = {} - - for netuid in self.netuids(): - c.print(f'Getting modules for SubNetwork {netuid}') - subnet2modules[netuid] = self.my_modules(netuid=netuid, **kwargs) - - return subnet2modules - - def staking_rewards( self, - key: str = None, - module_key=None, - block: Optional[int] = None, - timeout=20, - period = 100, - names = False, - fmt='j' , update=False, - max_age = 1000, - **kwargs) -> Optional['Balance']: - - block = int(block or self.block) - block_yesterday = int(block - period) - day_before_stake = self.my_total_stake_to(key=key, module_key=module_key, block=block_yesterday, timeout=timeout, names=names, fmt=fmt, update=update, max_age=max_age, **kwargs) - day_after_stake = self.my_total_stake_to(key=key, module_key=module_key, block=block, timeout=timeout, names=names, fmt=fmt, update=update, max_age=max_age, **kwargs) - return (day_after_stake - day_before_stake) - - def registered_servers(self, netuid = 0, **kwargs): - netuid = self.resolve_netuid(netuid) - servers = c.servers() - keys = self.keys(netuid=netuid) - registered_keys = [] - key2address = c.key2address() - for s in servers: - key_address = key2address[s] - if key_address in keys: - registered_keys += [s] - return registered_keys - - - def key2balance(self, search=None, - batch_size = 64, - timeout = 10, - max_age = 1000, - fmt = 'j', - update=False, - names = False, - min_value=0.0001, - **kwargs): - - input_hash = c.hash(c.locals2kwargs(locals())) - path = f'key2balance/{input_hash}' - key2balance = self.get(path, max_age=max_age, update=update) - - if key2balance == None: - key2balance = self.get_balances(search=search, - batch_size=batch_size, - timeout=timeout, - fmt = 'nanos', - update=1, - min_value=min_value, - **kwargs) - self.put(path, key2balance) - - for k,v in key2balance.items(): - key2balance[k] = self.format_amount(v, fmt=fmt) - key2balance = sorted(key2balance.items(), key=lambda x: x[1], reverse=True) - key2balance = {k: v for k,v in key2balance if v > min_value} - if names: - address2key = c.address2key() - key2balance = {address2key[k]: v for k,v in key2balance.items()} - return key2balance - - def empty_keys(self, block=None, update=False, max_age=1000, fmt='j'): - key2address = c.key2address() - key2value = self.key2value( block=block, update=update, max_age=max_age, fmt=fmt) - empty_keys = [] - for key,key_address in key2address.items(): - key_value = key2value.get(key_address, 0) - if key_value == 0: - empty_keys.append(key) - return empty_keys - - def profit_shares(self, key=None, **kwargs) -> List[Dict[str, Union[str, int]]]: - key = self.resolve_module_key(key) - return self.query_map('ProfitShares', **kwargs) - - def key2stake(self, - block=None, - update=False, - names = True, - max_age = 1000,fmt='j'): - stake_to = self.stake_to( - block=block, - max_age=max_age, - update=update, - - fmt=fmt) - address2key = c.address2key() - stake_to_total = {} - - for staker_address in address2key.keys(): - if staker_address in stake_to: - stake_to_total[staker_address] = sum(stake_to.get(staker_address, {}).values()) - # sort the dictionary by value - stake_to_total = dict(sorted(stake_to_total.items(), key=lambda x: x[1], reverse=True)) - if names: - stake_to_total = {address2key.get(k, k): v for k,v in stake_to_total.items()} - return stake_to_total - - def key2value(self, block=None, update=False, max_age=1000, fmt='j', min_value=0, **kwargs): - key2balance = self.key2balance(block=block, update=update, max_age=max_age, fmt=fmt) - key2stake = self.key2stake( block=block, update=update, max_age=max_age, fmt=fmt) - key2value = {} - keys = set(list(key2balance.keys()) + list(key2stake.keys())) - for key in keys: - key2value[key] = key2balance.get(key, 0) + key2stake.get(key, 0) - key2value = {k:v for k,v in key2value.items()} - key2value = dict(sorted(key2value.items(), key=lambda x: x[1], reverse=True)) - return key2value - - def resolve_module_key(self, x, netuid=0, max_age=60): - if not c.valid_ss58_address(x): - name2key = self.name2key(netuid=netuid, max_age=max_age) - x = name2key.get(x) - assert c.valid_ss58_address(x), f"Module key {x} is not a valid ss58 address" - return x - - def global_params(self, - update = False, - max_age = 1000, - timeout=30, - fmt:str='j', - features = None, - value_features = [], - path = f'global_params', - **kwargs - ) -> list: - - features = features or self.config.global_features - subnet_params = self.get(path, None, max_age=max_age, update=update) - names = [self.feature2name(f) for f in features] - future2name = {} - name2feature = dict(zip(names, features)) - for name, feature in name2feature.items(): - c.print(f'Getting {name} for {feature}') - query_kwargs = dict(name=feature, params=[], block=None, max_age=max_age, update=update) - f = c.submit(self.query, kwargs=query_kwargs, timeout=timeout) - future2name[f] = name - - subnet_params = {} - - for f in c.as_completed(future2name): - result = f.result() - subnet_params[future2name.pop(f)] = result - for k in value_features: - subnet_params[k] = self.format_amount(subnet_params[k], fmt=fmt) - return subnet_params - - def get_balance(self, - key: str = None , - block: int = None, - fmt='j', - max_age=0, - update=False) -> Optional['Balance']: - r""" Returns the token balance for the passed ss58_address address - Args: - address (Substrate address format, default = 42): - ss58 chain address. - Return: - balance (bittensor.utils.balance.Balance): - account balance - """ - key_ss58 = self.resolve_key_ss58( key ) - - result = self.query( - module='System', - name='Account', - params=[key_ss58], - block = block, - update=update, - max_age=max_age - ) - - return self.format_amount(result['data']['free'] , fmt=fmt) - - balance = get_balance - - def accounts(self, key = None, update=True, block=None, max_age=100000, **kwargs): - key = self.resolve_key_ss58(key) - accounts = self.query_map( - module='System', - name='Account', - update=update, - block = block, - max_age=max_age, - **kwargs - ) - return accounts - - def balances(self,fmt:str = 'n', block: int = None, n = None, update=False , **kwargs) -> Dict[str, 'Balance']: - accounts = self.accounts( update=update, block=block) - balances = {k:v['data']['free'] for k,v in accounts.items()} - balances = {k: self.format_amount(v, fmt=fmt) for k,v in balances.items()} - return balances - - def blocks_per_day(self): - return 24*60*60/self.block_time - - def min_burn(self, block=None, update=False, fmt='j'): - min_burn = self.query('MinBurn', block=block, update=update) - return self.format_amount(min_burn, fmt=fmt) - - - def get_balances(self, - keys=None, - search=None, - batch_size = 128, - fmt = 'j', - n = 100, - max_trials = 3, - names = False, - **kwargs): - - key2balance = {} - key2address = c.key2address(search=search) - - if keys == None: - keys = list(key2address.keys()) - - if len(keys) > n: - c.print(f'Getting balances for {len(keys)} keys > {n} keys, using batch_size {batch_size}') - balances = self.balances(**kwargs) - key2balance = {} - for k,a in key2address.items(): - if a in balances: - key2balance[k] = balances[a] - else: - future2key = {} - for key in keys: - f = c.submit(self.get_balance, kwargs=dict(key=key, fmt=fmt, **kwargs)) - future2key[f] = key - - for f in c.as_completed(future2key): - key = future2key.pop(f) - key2balance[key] = f.result() - - for k,v in key2balance.items(): - key2balance[k] = self.format_amount(v, fmt=fmt) - - - if names: - address2key = c.address2key() - key2balance = {address2key[k]: v for k,v in key2balance.items()} - - return key2balance - - def total_balance(self, **kwargs): - balances = self.balances(**kwargs) - return sum(balances.values()) - - def num_holders(self, **kwargs): - balances = self.balances(**kwargs) - return len(balances) - - def proposals(self, block=None, nonzero:bool=False, update:bool = False, **kwargs): - proposals = [v for v in self.query_map('Proposals', block=block, update=update, **kwargs)] - return proposals - - def registrations_per_block(self,**kwargs) -> int: - return self.query('RegistrationsPerBlock', params=[], **kwargs) - regsperblock = registrations_per_block - - def max_registrations_per_block(self, **kwargs) -> int: - return self.query('MaxRegistrationsPerBlock', params=[], **kwargs) - - ################# - #### Serving #### - ################# - def update_global( - self, - key: str = None, - network : str = 'main', - sudo: bool = True, - **params, - ) -> bool: - - key = self.resolve_key(key) - network = self.resolve_network(network) - global_params = self.global_params(fmt='nanos') - global_params.update(params) - params = global_params - for k,v in params.items(): - if isinstance(v, str): - params[k] = v.encode('utf-8') - # this is a sudo call - return self.compose_call(fn='update_global', - params=params, - key=key, - sudo=sudo) - - - ################# - #### Serving #### - ################# - def vote_proposal( - self, - proposal_id: int = None, - key: str = None, - network = 'main', - nonce = None, - netuid = 0, - **params, - - ) -> bool: - - self.resolve_network(network) - # remove the params that are the same as the module info - params = { - 'proposal_id': proposal_id, - 'netuid': netuid, - } - - response = self.compose_call(fn='add_subnet_proposal', - params=params, - key=key, - nonce=nonce) - - return response - - - - - - - def register( - self, - name: str , # defaults to module.tage - address : str = None, - stake : float = None, - netuid = 0, - network_name : str = None, - key : str = None, - module_key : str = None, - wait_for_inclusion: bool = True, - wait_for_finalization: bool = True, - max_address_characters = 32, - metadata = None, - network = None, - nonce=None, - **kwargs - ) -> bool: - module_key = c.get_key(module_key or name).ss58_address - netuid2subnet = self.netuid2subnet(update=False) - subnet2netuid = {v:k for k,v in netuid2subnet.items()} - - if network_name == None and netuid != 0: - network_name = netuid2subnet[netuid] - else: - assert isinstance(network_name, str), f"Subnet must be a string" - if not network_name in subnet2netuid: - subnet2netuid = self.subnet2netuid(update=True) - if network_name not in subnet2netuid: - subnet2netuid[network_name] = len(subnet2netuid) - response = input(f"Do you want to create a new subnet ({network_name}) (yes or y or dope): ") - if response.lower() not in ["yes", 'y', 'dope']: - return {'success': False, 'msg': 'Subnet not found and not created'} - - # require prompt to create new subnet - stake = (stake or 0) * 1e9 - - if c.server_exists(name): - address = c.namespace().get(name) - else: - address = address or 'NA' - - params = { - 'network_name': network_name.encode('utf-8'), - 'address': address[-max_address_characters:].replace('0.0.0.0', c.ip()).encode('utf-8'), - 'name': name.encode('utf-8'), - 'stake': stake, - 'module_key': module_key, - 'metadata': json.dumps(metadata or {}).encode('utf-8'), - } - - # create extrinsic call - response = self.compose_call('register', params=params, key=key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, nonce=nonce) - return response - - def resolve_uids(self, uids: Union['torch.LongTensor', list], netuid: int = 0, update=False) -> None: - name2uid = None - key2uid = None - for i, uid in enumerate(uids): - if isinstance(uid, str): - if name2uid == None: - name2uid = self.name2uid(netuid=netuid, update=update) - if uid in name2uid: - uids[i] = name2uid[uid] - else: - if key2uid == None: - key2uid = self.key2uid(netuid=netuid, update=update) - if uid in key2uid: - uids[i] = key2uid[uid] - return uids - - def set_weights( - self, - uids: Union['torch.LongTensor', list] , - weights: Union['torch.FloatTensor', list] , - netuid: int = 0, - key: 'c.key' = None, - update=False, - modules = None, - vector_length = 2**16 - 1, - nonce=None, - **kwargs - ) -> bool: - import torch - netuid = self.resolve_netuid(netuid) - key = self.resolve_key(key) - - # ENSURE THE UIDS ARE VALID UIDS, CONVERTING THE NAMES AND KEYS ASSOCIATED WITH THEM - uids = self.resolve_uids(modules or uids, netuid=netuid, update=update) - weights = weights or [1] * len(uids) - - # CHECK WEIGHTS LENGHTS - assert len(uids) == len(weights), f"Length of uids {len(uids)} must be equal to length of weights {len(weights)}" - - # NORMALIZE WEIGHTS - weights = torch.tensor(weights) - weights = (weights / weights.sum()) * vector_length # normalize the weights between 0 and 1 - weights = torch.clamp(weights, 0, vector_length) # min_value and max_value are between 0 and 1 - - params = {'uids': list(map(int, uids)), - 'weights': list(map(int, weights.tolist())), - 'netuid': netuid} - - return self.compose_call('set_weights',params = params , key=key, nonce=nonce, **kwargs) - - vote = set_weights - - def unstake_all( self, - key: str = None, - existential_deposit = 1, - min_stake = 0.5, - ) -> Optional['Balance']: - - key = self.resolve_key( key ) - key_stake_to = self.get_stake_to(key=key, names=False, update=True, fmt='nanos') # name to amount - - min_stake = min_stake * 1e9 - key_stake_to = {k:v for k,v in key_stake_to.items() if v > min_stake } - - params = { - "module_keys": list(key_stake_to.keys()), - "amounts": list(key_stake_to.values()) - } - - - - - - response = {} - - if len(key_stake_to) > 0: - c.print(f'Unstaking all of {len(key_stake_to)} modules') - response['stake'] = self.compose_call('remove_stake_multiple', params=params, key=key) - total_stake = (sum(key_stake_to.values())) / 1e9 - else: - c.print(f'No modules found to unstake') - total_stake = self.get_balance(key) - total_stake = total_stake - existential_deposit - - return response - - def registered_keys(self, netuid='all'): - key2address = c.key2address() - address2key = {v:k for k,v in key2address.items()} - - if netuid == 'all': - registered_keys = {} - netuid2keys = self.keys(netuid=netuid) - for netuid, keys in netuid2keys.items(): - registered_keys[netuid] = [] - for k in keys: - if k in address2key: - registered_keys[netuid].append(k) - else: - registered_keys = [k for k in self.keys(netuid=netuid) if k in address2key] - - return registered_keys + \ No newline at end of file diff --git a/commune/user/user.py b/commune/user/user.py index d8fb5294..352c7a7e 100644 --- a/commune/user/user.py +++ b/commune/user/user.py @@ -164,6 +164,6 @@ def app(self): response = getattr(self, f'rm_user')(add_user_address) st.write(response) - -User.run(__name__) +if __name__ == '__main__': + User.run() diff --git a/commune/errors.py b/commune/utils/errors.py similarity index 100% rename from commune/errors.py rename to commune/utils/errors.py diff --git a/commune/utils/misc.py b/commune/utils/misc.py index 57fb0c5e..3b16cf55 100644 --- a/commune/utils/misc.py +++ b/commune/utils/misc.py @@ -966,31 +966,17 @@ def path2text(cls, path:str, relative=False): def root_key(cls): return cls.get_key() - def root_key_address(cls) -> str: return cls.root_key().ss58_address - - def is_root_key(cls, address:str)-> str: return address == cls.root_key().ss58_address - -@staticmethod -def repo2module( repo, module = None): - if module == None: - module = os.path.basename(repo).replace('.git','').replace(' ','_').replace('-','_').lower() - - cls.new_module(module=module, repo=repo) - return {'module':module, 'repo':repo, 'status':'success'} - # time within the context def context_timer(cls, *args, **kwargs): return cls.timer(*args, **kwargs) - - def folder_structure(cls, path:str='./', search='py', max_depth:int=5, depth:int=0)-> dict: import glob files = cls.glob(path + '/**') diff --git a/commune/types.py b/commune/utils/types.py similarity index 100% rename from commune/types.py rename to commune/utils/types.py diff --git a/commune/subspace/README.md b/docs/8_avoiding_recursion.md similarity index 100% rename from commune/subspace/README.md rename to docs/8_avoiding_recursion.md