From 03fe258c2385f2c11a5a7e5396eabd5c2a129b11 Mon Sep 17 00:00:00 2001 From: latentvector Date: Tue, 19 Nov 2024 14:15:57 -0500 Subject: [PATCH] v1 --- commune/__init__.py | 1 + commune/cli.py | 13 +- commune/key.py | 300 +++++++++------------------ commune/module.py | 64 +++--- commune/modules/agent/agent.py | 4 +- commune/modules/chat/chat.py | 1 - commune/modules/sandbox.py | 2 +- commune/modules/select/select.py | 38 ++++ commune/modules/selector/selector.py | 0 commune/network/network.py | 15 +- commune/network/subspace/subspace.py | 184 +++++++++++----- commune/server.py | 9 +- commune/utils/git.py | 33 --- commune/utils/misc.py | 122 +++++++---- commune/utils/os.py | 5 - commune/utils/utils.py | 21 -- commune/vali.py | 1 + 17 files changed, 408 insertions(+), 405 deletions(-) create mode 100644 commune/modules/select/select.py delete mode 100644 commune/modules/selector/selector.py delete mode 100644 commune/utils/git.py delete mode 100644 commune/utils/utils.py diff --git a/commune/__init__.py b/commune/__init__.py index db1c79a5..a69a6efb 100755 --- a/commune/__init__.py +++ b/commune/__init__.py @@ -10,4 +10,5 @@ M = IDONTGIVEAFUCKWHATYOUCALLTHIS = c = Block = Lego = M = Agent = Module # alias c.Module as c.Block, c.Lego, c.M c.add_to_globals(globals()) key = c.get_key # override key function with file key in commune/key.py TODO: remove this line with a better solution +utils = c.utils # set utils as a global diff --git a/commune/cli.py b/commune/cli.py index 8b9228d5..90120eb1 100644 --- a/commune/cli.py +++ b/commune/cli.py @@ -48,12 +48,16 @@ def determine_type(x): except ValueError: pass return x -def forward(sep = '--', +def forward(argv = None, + sep = '--', fn_splitters = [':', '/', '//', '::'], base = 'module', - helper_fns = ['code', 'schema', 'fn_schema', 'help', 'fn_info', 'fn_hash']): + helper_fns = ['code', 'schema', 'fn_schema', 'help', 'fn_info', 'fn_hash'], + default_fn = 'vs'): t0 = time.time() - argv = sys.argv[1:] + argv = argv or sys.argv[1:] + if len(argv) == 0: + argv = [default_fn] output = None init_kwargs = {} if any([arg.startswith(sep) for arg in argv]): @@ -67,7 +71,6 @@ def forward(sep = '--', value = arg.split('=')[-1] if '=' in arg else True argv.remove(arg) init_kwargs[key] = determine_type(value) - # any of the --flags are init kwargs fn = argv.pop(0).replace('-', '_') module = c.module(base) @@ -75,7 +78,7 @@ def forward(sep = '--', if len(fs) == 1: module, fn = fn.split(fs[0]) module = c.shortcuts.get(module, module) - modules = c.modules() + modules = c.get_modules() module_options = [] for m in modules: if module == m: diff --git a/commune/key.py b/commune/key.py index b079e3ae..b6589d92 100644 --- a/commune/key.py +++ b/commune/key.py @@ -22,7 +22,6 @@ from commune.network.substrate.key import extract_derive_path from commune.network.substrate.utils.ecdsa_helpers import mnemonic_to_ecdsa_private_key, ecdsa_verify, ecdsa_sign from commune.network.substrate.utils.encrypted_json import decode_pair_from_encrypted_json, encode_pair - from bip39 import bip39_to_mini_secret, bip39_generate, bip39_validate import sr25519 import ed25519_zebra @@ -33,6 +32,8 @@ class KeyType: SR25519 = 1 ECDSA = 2 +KeyType.crypto_types = [k for k in KeyType.__dict__.keys() if not k.startswith('_')] +KeyType.crypto_type_map = {k.lower():v for k,v in KeyType.__dict__.items() if k in KeyType.crypto_types } class MnemonicLanguageCode: ENGLISH = 'en' CHINESE_SIMPLIFIED = 'zh-hans' @@ -44,12 +45,38 @@ class MnemonicLanguageCode: SPANISH = 'es' class Key(c.Module): + crypto_types = KeyType.crypto_types + crypto_type_map = KeyType.crypto_type_map ss58_format = 42 crypto_type = 'sr25519' - def __init__(self, + + def __init__(self,private_key: Union[bytes, str] = None, + ss58_format: int = ss58_format, + crypto_type: int = crypto_type, + derive_path: str = None, + path:str = None, + **kwargs): + self.set_private_key(private_key=private_key, + ss58_format=ss58_format, + crypto_type=crypto_type, + derive_path=derive_path, + path=path, **kwargs) + + def set_crypto_type(self, crypto_type): + crypto_type = self.resolve_crypto_type(crypto_type) + if crypto_type != self.crypto_type: + return self.set_private_key(private_key=self.private_key, + crypto_type=crypto_type, + ss58_format=self.ss58_format, + derive_path=self.derive_path, + path=self.path) + else: + raise ValueError(f'crypto_type {crypto_type} is already set') + + def set_private_key(self, private_key: Union[bytes, str] = None, ss58_format: int = ss58_format, - crypto_type: int = KeyType.SR25519, + crypto_type: int = crypto_type, derive_path: str = None, path:str = None, **kwargs @@ -98,7 +125,7 @@ def __init__(self, self.hash_type = hash_type self.public_key = public_key - self.key_address = self.ss58_address = key_address + self.address = self.key_address = self.ss58_address = key_address self.private_key = private_key self.crypto_type = crypto_type self.derive_path = derive_path @@ -106,15 +133,11 @@ def __init__(self, self.ss58_format = ss58_format self.key_address = self.ss58_address - @property - def address(self): - return self.ss58_address - @classmethod def add_key(cls, path:str, mnemonic:str = None, password:str=None, refresh:bool=False, private_key=None, **kwargs): if cls.key_exists(path) and not refresh : - c.print(f'key already exists at {path}', color='red') + c.print(f'key already exists at {path}') return json.loads(cls.get(path)) key = cls.new_key(mnemonic=mnemonic, private_key=private_key, **kwargs) key.path = path @@ -130,8 +153,7 @@ def rename_key(self, new_path): return self.mv_key(self.path, new_path) def ticket(self, data=None, **kwargs): - data = {'data':data, 'time': c.time()} - return self.sign(data , return_json=True, **kwargs) + return self.sign({'data':data, 'time': c.time()} , to_json=True, **kwargs) @classmethod def mv_key(cls, path, new_path): @@ -139,12 +161,22 @@ def mv_key(cls, path, new_path): cls.put(new_path, cls.get_key(path).to_json()) cls.rm_key(path) assert cls.key_exists(new_path), f'key does not exist at {new_path}' + assert not cls.key_exists(path), f'key still exists at {path}' new_key = cls.get_key(new_path) return {'success': True, 'from': path , 'to': new_path, 'key': new_key} rename_key = mv_key - + @classmethod + def copy_key(cls, path, new_path): + assert cls.key_exists(path), f'key does not exist at {path}' + cls.put(new_path, cls.get_key(path).to_json()) + assert cls.key_exists(new_path), f'key does not exist at {new_path}' + assert cls.get_key(path) == cls.get_key(new_path), f'key does not match' + new_key = cls.get_key(new_path) + return {'success': True, 'from': path , 'to': new_path, 'key': new_key} + + @classmethod def add_keys(cls, name, n=100, verbose:bool = False, **kwargs): response = [] @@ -174,44 +206,46 @@ def load_key(cls, path=None): def save_keys(cls, path='saved_keys.json', **kwargs): path = cls.resolve_path(path) c.print(f'saving mems to {path}') - mems = cls.mems() - c.put_json(path, mems) - return {'saved_mems':list(mems.keys()), 'path':path} + key2mnemonic = cls.key2mnemonic() + c.put_json(path, key2mnemonic) + return {'success': True, 'msg': 'saved keys', 'path':path, 'n': len(key2mnemonic)} - savemems = savekeys = save_keys - @classmethod def load_keys(cls, path='saved_keys.json', refresh=False, **kwargs): - - """""" - mems = c.get_json(path) - for k,mnemonic in mems.items(): + key2mnemonic = c.get_json(path) + for k,mnemonic in key2mnemonic.items(): try: cls.add_key(k, mnemonic=mnemonic, refresh=refresh, **kwargs) except Exception as e: # c.print(f'failed to load mem {k} due to {e}', color='red') pass - return {'loaded_mems':list(mems.keys()), 'path':path} + return {'loaded_mems':list(key2mnemonic.keys()), 'path':path} loadkeys = loadmems = load_keys @classmethod - def mems(cls, search=None): + def key2mnemonic(cls, search=None) -> dict[str, str]: + """ + keyname (str) --> mnemonic (str) + + """ mems = {} for key in cls.keys(search): try: - mems[key] = cls.getmem(key) + mems[key] = cls.get_mnemonic(key) except Exception as e: - c.print(f'failed to get mem for {key} due to {e}', color='red') + c.print(f'failed to get mem for {key} due to {e}') if search: mems = {k:v for k,v in mems.items() if search in k or search in v} return mems - - mnemonics = mems @classmethod - def get_key(cls, path:str,password:str=None, crypto_type=None, json:bool=False,create_if_not_exists:bool = True, **kwargs): - if hasattr(path, 'ss58_address'): - key = path + def get_key(cls, path:str,password:str=None, create_if_not_exists:bool = True, crypto_type=crypto_type, **kwargs): + for k in ['crypto_type', 'key_type', 'type']: + if k in kwargs: + crypto_type = kwargs.pop(k) + break + if hasattr(path, 'key_address'): + key = path return key path = path or 'module' # if ss58_address is provided, get key from address @@ -225,20 +259,17 @@ def get_key(cls, path:str,password:str=None, crypto_type=None, json:bool=False,c print(path) raise ValueError(f'key does not exist at --> {path}') key_json = cls.get(path) - # if key is encrypted, decrypt it if cls.is_encrypted(key_json): key_json = c.decrypt(data=key_json, password=password) if key_json == None: - c.print({'status': 'error', 'message': f'key is encrypted, please {path} provide password'}, color='red') + c.print({'status': 'error', 'message': f'key is encrypted, please {path} provide password'}) return None - key_json = c.jload(key_json) if isinstance(key_json, str) else key_json key = cls.from_json(key_json, crypto_type=crypto_type) key.path = path return key - @classmethod def get_keys(cls, search=None, clean_failed_keys=False): keys = {} @@ -251,8 +282,7 @@ def get_keys(cls, search=None, clean_failed_keys=False): if keys[key] == None: if clean_failed_keys: cls.rm_key(key) - keys.pop(key) - + keys.pop(key) return keys @classmethod @@ -336,41 +366,39 @@ def rm_key(cls, key=None): def crypto_type_name(self): return self.crypto_type2name(self.crypto_type).lower() - - crypto_types = ['ED25519', 'SR25519', 'ECDSA'] + @property + def key_type(self): + return self.crypto_type2name(self.crypto_type).lower() - @classmethod - def crypto_type_map(cls): - crypto_type_map = {k.lower():v for k,v in KeyType.__dict__.items() if k in cls.crypto_types } - return crypto_type_map @classmethod def crypto_name2type(cls, name:str): - crypto_type_map = cls.crypto_type_map() + crypto_type_map = cls.crypto_type_map name = name.lower() for k,v in crypto_type_map.items(): if k.startswith(name.lower()): return v - return crypto_type_map[name.lower()] + name = name.lower() + if not name in crypto_type_map: + raise ValueError(f'crypto_type {name} not supported {crypto_type_map}') + return crypto_type_map[name] @classmethod def crypto_type2name(cls, crypto_type:str): - crypto_type_map ={v:k for k,v in cls.crypto_type_map().items()} + crypto_type_map ={v:k for k,v in cls.crypto_type_map.items()} return crypto_type_map[crypto_type] @classmethod def resolve_crypto_type(cls, crypto_type): - - if isinstance(crypto_type, int) or c.is_int(crypto_type): + if isinstance(crypto_type, int) or (isinstance(crypto_type, str) and c.is_int(crypto_type)): crypto_type = int(crypto_type) - crypto_type_map = cls.crypto_type_map() + crypto_type_map = cls.crypto_type_map reverse_crypto_type_map = {v:k for k,v in crypto_type_map.items()} assert crypto_type in reverse_crypto_type_map, f'crypto_type {crypto_type} not supported {crypto_type_map}' crypto_type = reverse_crypto_type_map[crypto_type] if isinstance(crypto_type, str): crypto_type = crypto_type.lower() crypto_type = cls.crypto_name2type(crypto_type) - return int(crypto_type) @classmethod @@ -385,7 +413,7 @@ def new_key(cls, yo rody, this is a class method you can gen keys whenever fam ''' if verbose: - c.print(f'generating {crypto_type} keypair, {suri}', color='green') + c.print(f'generating {crypto_type} keypair, {suri}') crypto_type = cls.resolve_crypto_type(crypto_type) @@ -423,7 +451,6 @@ def from_json(cls, obj: Union[str, dict], password: str = None, crypto_type=None obj = json.loads(obj) if obj == None: return None - for k,v in obj.items(): if cls.is_encrypted(obj[k]) and password != None: obj[k] = cls.decrypt(data=obj[k], password=password) @@ -443,7 +470,9 @@ def generate_mnemonic(cls, words: int = 12, language_code: str = MnemonicLanguag 'zh-hant', 'fr', 'it', 'ja', 'ko', 'es'. Defaults to `MnemonicLanguageCode.ENGLISH` """ - return bip39_generate(words, language_code) + mnemonic = bip39_generate(words, language_code) + assert cls.validate_mnemonic(mnemonic, language_code), 'mnemonic is invalid' + return mnemonic @classmethod def validate_mnemonic(cls, mnemonic: str, language_code: str = MnemonicLanguageCode.ENGLISH) -> bool: @@ -462,12 +491,8 @@ def validate_mnemonic(cls, mnemonic: str, language_code: str = MnemonicLanguageC return bip39_validate(mnemonic, language_code) - # def resolve_crypto_type() @classmethod - def create_from_mnemonic(cls, mnemonic: str = None, ss58_format=ss58_format, - crypto_type=KeyType.SR25519, - language_code: str = MnemonicLanguageCode.ENGLISH, - return_kwargs:bool = False) -> 'Key': + def create_from_mnemonic(cls, mnemonic: str = None, ss58_format=ss58_format, crypto_type=KeyType.SR25519, language_code: str = MnemonicLanguageCode.ENGLISH) -> 'Key': """ Create a Key for given memonic @@ -496,12 +521,7 @@ def create_from_mnemonic(cls, mnemonic: str = None, ss58_format=ss58_format, seed_hex=binascii.hexlify(bytearray(bip39_to_mini_secret(mnemonic, "", language_code))).decode("ascii"), ss58_format=ss58_format, crypto_type=crypto_type, - return_kwargs=return_kwargs ) - - if return_kwargs: - kwargs = keypair - return kwargs keypair.mnemonic = mnemonic @@ -510,14 +530,7 @@ def create_from_mnemonic(cls, mnemonic: str = None, ss58_format=ss58_format, from_mnemonic = from_mem = create_from_mnemonic @classmethod - def create_from_seed( - cls, - seed_hex: Union[bytes, str], - ss58_format: Optional[int] = ss58_format, - crypto_type=KeyType.SR25519, - return_kwargs:bool = False - - ) -> 'Key': + def create_from_seed(cls, seed_hex: Union[bytes, str], ss58_format: Optional[int] = ss58_format, crypto_type=KeyType.SR25519) -> 'Key': """ Create a Key for given seed @@ -551,10 +564,7 @@ def create_from_seed( crypto_type=crypto_type, ) - if return_kwargs: - return kwargs - else: - return cls(**kwargs) + return cls(**kwargs) @classmethod def create_from_password(cls, password:str, **kwargs): return cls.create_from_uri(password, **kwargs) @@ -648,8 +658,12 @@ def create_from_uri( from_mnem = from_mnemonic = create_from_mnemonic @classmethod def create_from_private_key( - cls, private_key: Union[bytes, str], public_key: Union[bytes, str] = None, ss58_address: str = None, - ss58_format: int = ss58_format, crypto_type: int = KeyType.SR25519 + cls, + private_key: Union[bytes, str], + public_key: Union[bytes, str] = None, + ss58_address: str = None, + ss58_format: int = ss58_format, + crypto_type: int = KeyType.SR25519 ) -> 'Key': """ Creates Key for specified public/private keys @@ -674,7 +688,6 @@ def create_from_private_key( ) from_private_key = create_from_private_key - @classmethod def create_from_encrypted_json(cls, json_data: Union[str, dict], passphrase: str, ss58_format: int = None) -> 'Key': @@ -750,27 +763,22 @@ def export_to_encrypted_json(self, passphrase: str, name: str = None) -> dict: return json_data seperator = "::signature=" - + + def sign(self, data: Union[ScaleBytes, bytes, str], - return_json:bool=False, - to_dict = False, to_json = False, - seperator : str = seperator ) -> bytes: """ Creates a signature for given data - Parameters ---------- data: data to sign in `Scalebytes`, bytes or hex string format - Returns ------- signature in bytes """ - return_json = return_json or to_dict or to_json if not isinstance(data, str): data = c.python2str(data) if type(data) is ScaleBytes: @@ -782,7 +790,7 @@ def sign(self, if not self.private_key: raise ConfigurationError('No private key set to create signatures') - + if self.crypto_type == KeyType.SR25519: signature = sr25519.sign((self.public_key, self.private_key), data) @@ -795,13 +803,11 @@ def sign(self, else: raise ConfigurationError("Crypto type not supported") - if return_json: - return { - 'data': data.decode(), - 'crypto_type': self.crypto_type, # optional - 'signature': signature.hex(), - 'address': self.ss58_address, - } + if to_json: + return {'data':data.decode(), + 'crypto_type':self.crypto_type, + 'signature':signature.hex(), + 'address': self.ss58_address,} return signature def is_ticket(self, data): @@ -947,8 +953,6 @@ def encrypt_message( raise ConfigurationError('No private key set to encrypt') if self.crypto_type != KeyType.ED25519: raise ConfigurationError('Only ed25519 keypair type supported') - - curve25519_public_key = nacl.bindings.crypto_sign_ed25519_pk_to_curve25519(recipient_public_key) recipient = nacl.public.PublicKey(curve25519_public_key) private_key = nacl.bindings.crypto_sign_ed25519_sk_to_curve25519(self.private_key + self.public_key) @@ -999,7 +1003,6 @@ def is_key_encrypted(cls, path, data=None): @classmethod def decrypt_key(cls, path = 'test.enc', password=None): - data = cls.get(path) assert cls.is_encrypted(data), f'{path} not encrypted' enc_text = c.decrypt(data['data'], password=password) @@ -1007,13 +1010,11 @@ def decrypt_key(cls, path = 'test.enc', password=None): return {'encrypted':enc_text, 'path':path , 'password':password} @classmethod - def getmem(cls, key): + def get_mnemonic(cls, key): return cls.get_key(key).mnemonic - - mem = getmem def __str__(self): - return f'' + return f'' def save(self, path=None): if path == None: @@ -1024,31 +1025,6 @@ def save(self, path=None): def __repr__(self): return self.__str__() - def state_dict(self): - return self.__dict__ - - to_dict = state_dict - @classmethod - def dashboard(cls): - import streamlit as st - self = cls.new_key() - keys = self.keys() - selected_keys = st.multiselect('Keys', keys) - buttons = {} - for key_name in selected_keys: - key = cls.get_key(key_name) - with st.expander('Key Info'): - st.write(key.to_dict()) - buttons[key_name] = {} - buttons[key_name]['sign'] = st.button('Sign', key_name) - st.write(self.keys()) - - @classmethod - def key2mem(cls, search=None): - keys = cls.keys(search, object=True) - key2mem = {k.path: k.mnemonic for k in keys} - return key2mem - @classmethod def from_private_key(cls, private_key:str): return cls(private_key=private_key) @@ -1062,55 +1038,7 @@ def valid_ss58_address(cls, address: str, ss58_format=ss58_format ) -> bool: return ss58.is_valid_ss58_address( address , valid_ss58_format =ss58_format ) except Exception as e: return False - - @classmethod - def is_valid_ed25519_pubkey(cls, public_key: Union[str, bytes], ss58_format=ss58_format) -> bool: - """ - Checks if the given public_key is a valid ed25519 key. - """ - try: - if isinstance( public_key, str ): - if len(public_key) != 64 and len(public_key) != 66: - raise ValueError( "a public_key should be 64 or 66 characters" ) - elif isinstance( public_key, bytes ): - if len(public_key) != 32: - raise ValueError( "a public_key should be 32 bytes" ) - else: - raise ValueError( "public_key must be a string or bytes" ) - - keypair = Key(public_key=public_key, ss58_format=ss58_format) - - ss58_addr = keypair.ss58_address - return ss58_addr is not None - - except (ValueError, IndexError): - return False - - @classmethod - def is_valid_address_or_public_key(cls, address: Union[str, bytes] ) -> bool: - """ - Checks if the given address is a valid destination address. - - Args: - address(Union[str, bytes]): The address to check. - - Returns: - True if the address is a valid destination address, False otherwise. - """ - if isinstance( address, str ): - # Check if ed25519 - if address.startswith('0x'): - return cls.is_valid_ed25519_pubkey( address ) - else: - # Assume ss58 address - return cls.valid_ss58_address( address ) - elif isinstance( address, bytes ): - # Check if ed25519 - return cls.is_valid_ed25519_pubkey( address ) - else: - # Invalid address type - return False - + @classmethod def is_encrypted(cls, data): if isinstance(data, str): @@ -1130,7 +1058,6 @@ def is_encrypted(cls, data): def ss58_encode(*args, **kwargs): return ss58_encode(*args, **kwargs) - @staticmethod def ss58_decode(*args, **kwargs): return ss58_decode(*args, **kwargs) @@ -1192,31 +1119,6 @@ def valid_h160_address(cls, address): return True - # @staticmethod - # def convert_h160_to_ss58( eth_address): - # from eth_utils import to_bytes - # # Ensure the eth_address starts with '0x' - # if not eth_address.startswith('0x'): - # eth_address = '0x' + eth_address - - # # Convert the prefix to bytes - # prefix = b'evm:' - - # # Convert the Ethereum address to bytes - # address_bytes = to_bytes(hexstr=eth_address) - - # # Combine prefix and address - # combined = prefix + address_bytes - - # # Hash the combined data using Blake2 256-bit - # blake2_hash = hashlib.blake2b(combined, digest_size=32).digest() - - # # Convert the public key to SS58 format - # ss58_address = ss58_encode(blake2_hash, ss58_format=42) # Using 42 as the network ID, adjust as needed - - # return ss58_address - - # if __name__ == "__main__": # Key.run() diff --git a/commune/module.py b/commune/module.py index d03f9fd7..e6529125 100755 --- a/commune/module.py +++ b/commune/module.py @@ -50,7 +50,11 @@ class c: 'network.local': 'network', } @classmethod - def module(cls, path:str = 'module', cache=True, verbose = False, tree = None, trials=1, **_kwargs ) -> str: + def module(cls, + path:str = 'module', + cache=True, + trials=1, + **_kwargs ) -> str: if path == None: path = 'module' if path.endswith('.py'): @@ -60,23 +64,23 @@ def module(cls, path:str = 'module', cache=True, verbose = False, tree = None, t0 = time.time() og_path = path if path in c.module_cache and cache: - module = c.module_cache[path] - else: - if path in ['module', 'c']: - module = c - else: + return c.module_cache[path] - tree = tree or c.tree() - path = c.shortcuts.get(path, path) - path = tree.get(path, path) - try: - module = c.import_object(path) - except Exception as e: - if trials == 0: - raise ValueError(f'Error in module {og_path} {e}') - return c.module(path, cache=cache, verbose=verbose, tree=tree, trials=trials-1) - if cache: - c.module_cache[path] = module + if path in ['module', 'c']: + return c + + tree = c.tree() + path = c.shortcuts.get(path, path) + path = tree.get(path, path) + try: + module = c.import_object(path) + except Exception as e: + if trials == 0: + raise ValueError(f'Error in module {og_path} {e}') + return c.module(path, + cache=cache, + tree=tree, + trials=trials-1) latency = c.round(time.time() - t0, 3) if not hasattr(module, 'module_name'): module.module_name = module.name = lambda *args, **kwargs : c.module_name(module) @@ -91,7 +95,8 @@ def module(cls, path:str = 'module', cache=True, verbose = False, tree = None, module.key = c.get_key(module.module_name(), create_if_not_exists=True) module.fn2code = lambda *args, **kwargs : c.fn2code(module) module.help = lambda *args, **kwargs : c.help(*args, module=module, **kwargs) - c.print(f'Module({og_path}->{path})({latency}s)', verbose=verbose) + if cache: + c.module_cache[path] = module return module get_module = module @@ -109,7 +114,7 @@ def filepath(cls, obj=None) -> str: module_path = inspect.getfile(cls) return module_path - path = file_path = filepath + file_path = filepath @classmethod def dirpath(cls, obj=None) -> str: return os.path.dirname(cls.filepath(obj)) @@ -474,6 +479,7 @@ def utils(cls, search=None): utils = [u for u in utils if search in u] return sorted(utils) + @classmethod def num_utils(cls, search=None): return len(cls.utils(search)) @@ -491,7 +497,7 @@ def util2code(cls, search=None): @classmethod def util2path(cls, search=None): - utils_paths = cls.utils_paths(search=search) + utils_paths = cls.utils(search=search) util2path = {} for f in utils_paths: util2path[f.split('.')[-1]] = f @@ -524,11 +530,9 @@ def get_yaml( path:str=None, default={}, **kwargs) -> Dict: return data @classmethod - def get_routes(cls, cache=True): + def get_routes(cls): if not hasattr(cls, 'routes'): - if cls.route_cache is not None and cache: - return cls.route_cache routes_path = os.path.dirname(__file__)+ '/routes.json' routes = cls.get_yaml(routes_path) else: @@ -546,8 +550,6 @@ def add_utils(): return routes add_utils() - cls.route_cache = routes - return routes @classmethod @@ -1225,8 +1227,12 @@ def lines_of_code(cls, code:str=None): @classmethod def is_fn(cls, fn:str): return '/' in str(fn) or hasattr(cls, str(fn)) or (c.object_exists(fn) and callable(c.obj(fn))) + + @classmethod def code(cls, module = None, search=None, *args, **kwargs): + if module in c.core_modules(): + return c.get_text(c.filepath(module), *args, **kwargs) if cls.is_fn(module): return cls.fn_code(module) module = cls.resolve_object(module) @@ -1924,11 +1930,11 @@ def tree(cls, search=None, max_age=60,update=False, **kwargs): @classmethod def overlapping_modules(cls, search:str=None, **kwargs): local_modules = cls.local_modules(search=search) - lib_modules = cls.lib_modules(search=search) - return [m for m in local_modules if m in lib_modules] + core_modules = cls.core_modules(search=search) + return [m for m in local_modules if m in core_modules] @classmethod - def lib_modules(cls, search=None, depth=10000, **kwargs): + def core_modules(cls, search=None, depth=10000, **kwargs): object_paths = cls.find_classes(cls.libpath, depth=depth ) object_paths = [cls.objectpath2name(p) for p in object_paths] if search != None: @@ -2216,7 +2222,7 @@ def module2fns(cls, path=None): try: module2fns[m] = c.module(m).fns() except Exception as e: - print(f'Error in module2fns {m}', e) + pass return module2fns @classmethod diff --git a/commune/modules/agent/agent.py b/commune/modules/agent/agent.py index 10fa2836..59ee98fa 100644 --- a/commune/modules/agent/agent.py +++ b/commune/modules/agent/agent.py @@ -123,8 +123,6 @@ def batch_context(self, path='./', batch_size=20000): current_size += s files_batch[f] = c.get_text(path + f ) return batch_list - - def tools(self): - return c.fns("fn") + diff --git a/commune/modules/chat/chat.py b/commune/modules/chat/chat.py index 302b7062..2f7dc9bc 100644 --- a/commune/modules/chat/chat.py +++ b/commune/modules/chat/chat.py @@ -60,7 +60,6 @@ def generate(self, ): context = context or path # text = self.process_text(text, context=context) - print(text) output = self.model.generate(text, stream=stream, model=model, max_tokens=max_tokens,temperature=temperature ) for token in output: yield token diff --git a/commune/modules/sandbox.py b/commune/modules/sandbox.py index faf867a7..510d0844 100644 --- a/commune/modules/sandbox.py +++ b/commune/modules/sandbox.py @@ -1,3 +1,3 @@ import commune as c -c.print(c.modules()) \ No newline at end of file +c.print(c.get_key('subspace')) \ No newline at end of file diff --git a/commune/modules/select/select.py b/commune/modules/select/select.py new file mode 100644 index 00000000..b38a6bd5 --- /dev/null +++ b/commune/modules/select/select.py @@ -0,0 +1,38 @@ +import commune as c +import json +class Select: + + def forward(self, + query="update a subnet", + module='subspace'): + options = list(c.module(module).fns()) + prompt = f""" + ---- + QUERY + ---- + {query} + ---- + INSTRUCTION + ---- + SELECT FROM THE OPTIONS + {options} + ---- + OUTPUT FORMAT (JSON STRING BETWEEN THE ANCHORS) + ---- + RETURN THE INDICES OF THE SELECTED OPTIONS + DICT(data:LIST[LIST(name:str, score:int[0,100])]) + ---- + OUTPUT + ---- + """ + output = '' + for ch in c.ask(prompt, model='sonnet'): + print(ch, end='') + if ch == '<': + break + output += ch + output = output.split('')[1].split('')[0] + output = json.loads(output)["data"] + output_schema = {k:c.fn_schema(module + '/'+k) for k,v in output} + return output_schema + diff --git a/commune/modules/selector/selector.py b/commune/modules/selector/selector.py deleted file mode 100644 index e69de29b..00000000 diff --git a/commune/network/network.py b/commune/network/network.py index bd33a98b..c1cfdc70 100644 --- a/commune/network/network.py +++ b/commune/network/network.py @@ -11,13 +11,13 @@ class Network(c.Module): # the default endpoints = ['namespace'] def __init__(self, network:str='local', tempo=tempo, path=None, **kwargs): - self.modules_path = self.resolve_path(path or f'modules_{self.network}') - self.set_network(network=network, tempo=tempo) + self.set_network(network=network, tempo=tempo, path=path) - def set_network(self, network:str, tempo:int=60): + def set_network(self, network:str, tempo:int=60, path=None, **kwargs): self.network = network self.tempo = tempo - return {'network': self.network, 'tempo': self.tempo} + self.modules_path = self.resolve_path(path or f'{self.network}/modules') + return {'network': self.network, 'tempo': self.tempo, 'modules_path': self.modules_path} def params(self,*args, **kwargs): return { 'network': self.network, 'tempo' : self.tempo,'n': self.n} @@ -71,13 +71,6 @@ def deregister_server(self, name:str, features=['name', 'key', 'address']) -> Di def resolve_network(self, network:str) -> str: return network or self.network - def resolve_network_path(self, network:str) -> str: - return self.resolve_path(self.resolve_network(network)) - - def namespace_exists(self, network:str) -> bool: - path = self.resolve_network_path( network) - return os.path.exists(path) - def names(self, *args, **kwargs) -> List[str]: return list(self.namespace(*args, **kwargs).keys()) diff --git a/commune/network/subspace/subspace.py b/commune/network/subspace/subspace.py index 10aab9de..6755a895 100644 --- a/commune/network/subspace/subspace.py +++ b/commune/network/subspace/subspace.py @@ -53,7 +53,7 @@ def __init__( network=network, url: str = None, mode = 'wss', - num_connections: int = 1, + num_connections: int = 5, wait_for_finalization: bool = False, test = False, ws_options = {}, @@ -110,21 +110,25 @@ def set_network(self, network = network or self.network if timeout != None: ws_options["timeout"] = timeout - self.ws_options = ws_options self.url = url or (mode + '://' + self.url_map.get(network)[0]) self.num_connections = num_connections self.wait_for_finalization = wait_for_finalization self.network = network - self.connections_queue = queue.Queue(self.num_connections) + self.set_connections(num_connections) + self.connection_latency = c.time() - t0 + c.print(f'Network(name={self.network} url={self.url} connections={self.num_connections} latency={c.round(self.connection_latency, 2)})', color='blue') + + def set_connections(self, num_connections: int): + self.connections_queue = queue.Queue(num_connections) + self.num_connections = num_connections try: for _ in range(self.num_connections): self.connections_queue.put(SubstrateInterface(self.url, ws_options=self.ws_options)) except Exception as e: c.print('ERROR IN CONNECTIONS QUEUE:', e) - self.connection_latency = c.time() - t0 - c.print(f'Network(name={self.network} url={self.url} connections={self.num_connections} latency={c.round(self.connection_latency, 2)})', color='blue') - + + return {'connections': self.num_connections, 'url': self.url} def get_url(self, mode='wss', **kwargs): prefix = mode + '://' url = c.choice(self.url_map[self.network]) @@ -622,6 +626,7 @@ def concat_hash_len(key_hasher: str) -> int: def query_batch( self, functions: dict[str, list[tuple[str, list[Any]]]], block_hash: str | None = None, + verbose=False, ) -> dict[str, str]: """ Executes batch queries on a substrate and returns results in a dictionary format. @@ -641,7 +646,7 @@ def query_batch( {'function_name': 'query_result', ...} """ - c.print(f'QueryBatch({functions})') + c.print(f'QueryBatch({functions})', verbose=verbose) result: dict[str, str] = {} if not functions: raise Exception("No result") @@ -674,6 +679,7 @@ def query_batch_map( path = None, max_age=None, update=False, + verbose = False, ) -> dict[str, dict[Any, Any]]: """ Queries multiple storage functions using a map batch approach and returns the combined result. @@ -689,7 +695,7 @@ def query_batch_map( >>> query_batch_map(substrate_instance, {'module_name': [('function_name', ['param1', 'param2'])]}) # Returns the combined result of the map batch query """ - c.print(f'QueryBatchMap({functions})') + c.print(f'QueryBatchMap({functions})', verbose=verbose) if path != None: return self.get(path, max_age=max_age, update=update) multi_result: dict[str, dict[Any, Any]] = {} @@ -1126,6 +1132,15 @@ def transfer_multiple( def get_stake(self, key=None): return sum(list(self.get_stake_from(key).values())) + def my_tokens(self, min_value=0): + my_stake = self.my_stake() + my_balance = self.my_balance() + my_tokens = {k:my_stake.get(k,0) + my_balance.get(k,0) for k in set(my_stake) | set(my_balance)} + return dict(sorted({k:v for k,v in my_tokens.items() if v > min_value}.items(), key=lambda x: x[1], reverse=True)) + + def my_total(self): + return sum(self.my_tokens().values()) + def my_stakefrom(self): stakefrom = self.stakefrom() key2address = c.key2address() @@ -1143,7 +1158,11 @@ def my_staketo(self, update=False, max_age=60): if address in staketo: my_stakefrom[key] = staketo[address] return my_stakefrom - + + def my_stake(self, update=False, max_age=60): + my_stake = {k:sum(v.values()) for k,v in self.my_staketo(update=update, max_age=max_age).items()} + my_stake = dict(sorted(my_stake.items(), key=lambda x: x[1], reverse=True)) + return {k:v for k,v in my_stake.items() if v > 0} def stake( self, @@ -1346,9 +1365,9 @@ def register_subnet(self, name: str, metadata: str | None = None, key: Keypair= def set_weights( self, - key: Keypair, modules: list[int], # uids, keys or names weights: list[int], # any value, relative is takens + key: Keypair, subnet = 0, ) -> ExtrinsicReceipt: """ @@ -1375,11 +1394,7 @@ def set_weights( assert len(modules) == len(weights) key2uid = self.key2uid(subnet) uids = [key2uid.get(m, m) for m in modules] - params = { - "uids": uids, - "weights": weights, - "netuid": subnet, - } + params = {"uids": uids,"weights": weights,"netuid": subnet} response = self.compose_call("set_weights", params=params, key=key) return response @@ -2036,8 +2051,43 @@ def get_balances( key_addresses = [key2address.get(a, a) for a in key_addresses] with self.get_conn(init=True) as substrate: balances = substrate.query_multi( [substrate.create_storage_key(pallet='System', storage_function='Account', params=[ka]) for ka in key_addresses if not ka.startswith('0x')]) - return len(balances) + return balances + def my_balance(self, batch_size=128, timeout=60, max_age=6000, update=False): + path = f'{self.network}/balances' + + balances = self.get(path, None, update=update, max_age=max_age) + if balances == None: + + key2address = c.key2address() + addresses = list(key2address.values()) + batched_addresses = [addresses[i:i + batch_size] for i in range(0, len(addresses), batch_size)] + hash2batch = {} + future2hash = {} + n = len(addresses) + print(f'Getting baances for {n } addresses in {len(batched_addresses)} batches') + for batch in batched_addresses: + batch_hash : str = c.hash(batch) + hash2batch[batch_hash] = batch + f = c.submit(self.get_balances, dict(key_addresses=batch), timeout=timeout) + future2hash[f] = batch_hash + balances = {} + progress_bar = c.tqdm(total=n, desc='Getting Balances') + for f in c.as_completed(future2hash.keys(), timeout=timeout): + batch_hash = future2hash[f] + balances_batch = f.result() + addresses_batch = hash2batch[batch_hash] + for address, balance in zip(addresses_batch, balances_batch): + balances[address] = balance[1].value['data']['free'] + progress_bar.update(1) + # c.print(f'Balance for {address}: {balance} ({counter}/{len(addresses)})') + # self.put(f'balances/{batch_hash}', balances) + address2key = c.address2key() + balances = {address2key.get(k, k):v for k,v in balances.items()} + self.put(path, balances) + balances = {k: v for k, v in balances.items() if v > 0} + balances = dict(sorted(balances.items(), key=lambda x: x[1], reverse=True)) + return self.format_amount(balances, fmt='j') def names( self, subnet: int = 0, extract_value: bool = False, max_age=60, update=False ) -> dict[int, str]: @@ -2289,7 +2339,7 @@ def params(self, subnet = None, block_hash: str | None = None, max_age=tempo, u path = f'{self.network}/params_map' results = self.get(path,None, max_age=max_age, update=update) if results == None: - print("Updating Subnet Params") + print(f"PARAMS(subnet=all update=True)") params = [] bulk_query = self.query_batch_map( { @@ -2310,6 +2360,7 @@ def params(self, subnet = None, block_hash: str | None = None, max_age=tempo, u ("MaxAllowedValidators", params), ("ModuleBurnConfig", params), ("SubnetMetadata", params), + ("TrustRatio", params), ], "GovernanceModule": [ ("SubnetGovernanceConfig", params), @@ -2341,6 +2392,7 @@ def params(self, subnet = None, block_hash: str | None = None, max_age=tempo, u "max_allowed_validators": bulk_query.get("MaxAllowedValidators", {}), "module_burn_config": bulk_query.get("ModuleBurnConfig", {}), "metadata": bulk_query.get("SubnetMetadata", {}), + "trust_ratio": bulk_query.get("TrustRatio", {}), } results: dict[int, SubnetParamsWithEmission] = {} @@ -2360,6 +2412,7 @@ def params(self, subnet = None, block_hash: str | None = None, max_age=tempo, u results = {int(k):v for k,v in results.items()} if subnet != None: subnet = self.resolve_subnet(subnet) + print(f"PARAMS(subnet={subnet})") results = results[subnet] return results @@ -2428,37 +2481,33 @@ def global_params(self, max_age=60, update=False) -> NetworkParams: self.put(path, result) return result - - def clean_feature_name(self, x): - new_x = '' - for i, ch in enumerate(x): - if ch == ch.upper(): - ch = ch.lower() - if i > 0: - ch = '_' + ch - new_x += ch - return new_x def founders(self): return self.query_map("Founder", module="SubspaceModule") - def my_subnets(self, key=None): - founders = self.founders() + def my_subnets(self, update=False): + c.print("===========MY SUBNETS=========") + subnet2params = self.params(update=update) address2key = c.address2key() - netuid2subnet = self.netuid2subnet() results = [] - for netuid,v in founders.items(): - if v in address2key: - row = {'subnet': netuid2subnet[netuid], + for netuid,params in subnet2params.items(): + if params['founder'] in address2key: + row = {'name': params['name'], 'netuid': netuid, - 'founder': address2key[v] + 'founder': address2key[params['founder']], + 'tempo': params['tempo'], + 'incentive_ratio': params['incentive_ratio'], + 'immunity_period': params['immunity_period'], + 'emission': params['emission'], } results += [row] # group by founder - return c.df(results).sort_values('subnet') + return c.df(results).sort_values('name') + def mynets(self, update=False): + return self.my_subnets(update=update) def my_modules(self, subnet="all", max_age=60, @@ -2514,7 +2563,7 @@ def all_modules(self, }, **kwargs): - path = f'{self.network}/all_modules' + path = f'{self.network}/modules/all' modules = self.get(path, None, max_age=max_age, update=update) if modules == None: results = self.query_batch_map({module:[(f, []) for f in features]},self.block_hash()) @@ -2535,7 +2584,7 @@ def all_modules(self, module[f] = module[f].get(uid, default_module.get(f, None)) elif isinstance(module[f], list): module[f] = module[f][uid] - module = {self.clean_feature_name(k):v for k,v in module.items()} + module = {self.storage2name(k):v for k,v in module.items()} modules[_netuid].append(module) self.put(path, modules) modules = {int(k):v for k,v in modules.items()} @@ -2543,10 +2592,45 @@ def all_modules(self, def validators(self, subnet=0): - return self.modules(subnet=subnet ) + return self.modules(subnet=subnet ) + + + + name2storage_exceptions = {'key': 'Keys'} + storage2name_exceptions = {v:k for k,v in name2storage_exceptions.items()} + def storage2name(self, name): + new_name = '' + if name in self.storage2name_exceptions: + return self.storage2name_exceptions[name] + for i, ch in enumerate(name): + if ch == ch.upper(): + ch = ch.lower() + if i > 0: + ch = '_' + ch + new_name += ch + return new_name + + def name2storage(self, name): + new_name = '' + next_char_upper = False + if name in self.name2storage_exceptions: + return self.name2storage_exceptions[name] + for i, ch in enumerate(name): + if next_char_upper: + ch = ch.upper() + next_char_upper = False + if ch == '_': + next_char_upper = True + ch = '' + if i == 0: + ch = ch.upper() + new_name += ch + return new_name + + def modules(self, - subnet=None, + subnet=0, max_age = tempo, update=False, timeout=30, @@ -2556,27 +2640,21 @@ def modules(self, lite = True, vector_fetures = ['Incentive', 'Dividends', 'Emission'], num_connections = 4, - default_module = { - 'Weights': [], - 'Incentive': 0, - 'Emissions': 0, - 'Dividends': 0, - 'DelegationFee': 30, - 'LastUpdate': 0, + default_module = {'Weights': [], 'Incentive': 0, 'Emissions': 0, 'Dividends': 0, 'DelegationFee': 30, 'LastUpdate': 0, }, **kwargs): - if subnet == None: + if subnet == 'all': return self.all_modules(max_age=max_age, update=update, module=module, features=features, default_module=default_module, **kwargs) subnet = self.resolve_subnet(subnet) if not lite: features += extra_features - - if subnet in [0] and lite == False: - features += ['StakeFrom', 'DelegationFee'] path = f'{self.network}/modules/{subnet}' modules = self.get(path, None, max_age=max_age, update=update) - if modules == None: + update = bool(modules == None) + c.print(f'MODULES(subnet={subnet} update={update})') + + if update: self.set_network(num_connections=num_connections) future2feature = {} params = [subnet] if subnet != None else [] @@ -2608,7 +2686,7 @@ def modules(self, module[f] = results[f].get(uid, default_module.get(f, None)) elif isinstance(results[f], list): module[f] = results[f][uid] - module = {self.clean_feature_name(k):v for k,v in module.items()} + module = {self.storage2name(k):v for k,v in module.items()} modules.append(module) self.put(path, modules) # modules = sorted(modules) @@ -2663,6 +2741,7 @@ def is_registered(self, key=None, subnet=0,max_age=60) -> bool: def get_modules(self, keys, subnet=0, max_age=60): futures = [ c.submit(self.get_module, kwargs=dict(module=k, subnet=subnet, max_age=max_age)) for k in keys] return c.wait(futures, timeout=30) + def get_module(self, module, subnet=0, @@ -2716,3 +2795,4 @@ def transform_stake_dmap(self, stake_storage: dict[tuple[Ss58Address, Ss58Addres return dict(transformed) + diff --git a/commune/server.py b/commune/server.py index 72db0f7e..0065b7c2 100644 --- a/commune/server.py +++ b/commune/server.py @@ -261,8 +261,8 @@ def wait_for_server(cls, time_waiting = 0 # rotating status thing - c.print(f'WAITING_FOR_SERVER(module{name})', color='cyan') - + c.print(f'waiting for {name} to start...', color='cyan') + while time_waiting < timeout: namespace = c.namespace(network=network, max_age=max_age) if name in namespace: @@ -275,11 +275,9 @@ def wait_for_server(cls, c.print(f'Error getting info for {name} --> {e}', color='red') c.sleep(sleep_interval) time_waiting += sleep_interval - # c.logs(name) - # c.kill(name) raise TimeoutError(f'Waited for {timeout} seconds for {name} to start') - def info(self): + def info(self, crypto_type: str = 'sr25519', **kwargs) -> dict: info = {} module = self.module info['schema'] = module.schema @@ -288,7 +286,6 @@ def info(self): info['key'] = module.key.ss58_address info['crypto_type'] = module.key.crypto_type return info - def add_endpoint(self, name, fn): setattr(self, name, fn) diff --git a/commune/utils/git.py b/commune/utils/git.py deleted file mode 100644 index b2461fc4..00000000 --- a/commune/utils/git.py +++ /dev/null @@ -1,33 +0,0 @@ -import requests -import json - -def get_folder_contents_advanced(url='commune-ai/commune.git', - host_url = 'https://github.com/', - auth_token=None): - try: - headers = { - 'Accept': 'application/json', - 'User-Agent': 'Python Script' - } - if not url.startswith(host_url): - url = host_url + url - - if auth_token: - headers['Authorization'] = f'token {auth_token}' - - response = requests.get(url, headers=headers) - response.raise_for_status() - - # Parse JSON response - content = response.json() - - # If it's a GitHub API response, it will be a list of files/folders - if isinstance(content, list): - return json.dumps(content, indent=2) - return response.text - - except Exception as e: - print(f"Error: {e}") - return None - - diff --git a/commune/utils/misc.py b/commune/utils/misc.py index 8bf8292a..1a2ebea1 100644 --- a/commune/utils/misc.py +++ b/commune/utils/misc.py @@ -4,6 +4,48 @@ import sys from typing import * import glob +import requests +import json + + + +def get_glob( path =None, recursive:bool = True, files_only:bool = True): + import glob + path = os.path.abspath(path) + if os.path.isdir(path): + path = os.path.join(path, '**') + paths = glob.glob(path, recursive=recursive) + if files_only: + paths = list(filter(lambda f:os.path.isfile(f), paths)) + return paths + +def file2text(path = './', + avoid_folders = ['__pycache__', + '.git', + '.ipynb_checkpoints', + 'package.lock', + 'egg-info', + 'Cargo.lock', + 'target/debug', + 'node_modules'], + relative=True, **kwargs): + path = os.path.abspath(os.path.expanduser(path)) + file2text = {} + for file in get_glob(path, recursive=True): + if os.path.isdir(file): + continue + if any([af in file for af in avoid_folders]): + continue + try: + with open(file, 'r') as f: + content = f.read() + file2text[file] = content + except Exception as e: + continue + if relative: + return {k[len(path)+1:]:v for k,v in file2text.items()} + return file2text + def abspath( path:str): return os.path.abspath(os.path.expanduser(path)) @@ -682,47 +724,14 @@ def get_line(module, idx): line = lines[max(idx, 0)] return line -def get_glob( path =None, recursive:bool = True, files_only:bool = True): - import glob - path = os.path.abspath(path) - if os.path.isdir(path): - path = os.path.join(path, '**') - paths = glob.glob(path, recursive=recursive) - if files_only: - paths = list(filter(lambda f:os.path.isfile(f), paths)) - return paths - -def file2text(path = './', - avoid_folders = ['__pycache__', - '.git', - '.ipynb_checkpoints', - 'package.lock', - 'egg-info', - 'Cargo.lock', - 'target/debug', - 'node_modules'], - relative=True, **kwargs): - path = os.path.abspath(os.path.expanduser(path)) - file2text = {} - for file in get_glob(path, recursive=True): - if os.path.isdir(file): - continue - if any([af in file for af in avoid_folders]): - continue - try: - with open(file, 'r') as f: - content = f.read() - file2text[file] = content - except Exception as e: - continue - if relative: - return {k[len(path)+1:]:v for k,v in file2text.items()} - return file2text def file2lines(path:str='./')-> List[str]: - file2text = file2text(path) - file2lines = {f: text.split('\n') for f, text in file2text.items()} - return file2lines + result = file2text(path) + return {f: text.split('\n') for f, text in result.items()} + +def file2n(path:str='./')-> List[str]: + result = file2text(path) + return {f: len(text.split('\n')) for f, text in result.items()} def munch( x:dict, recursive:bool=True)-> 'Munch': from munch import Munch @@ -1127,6 +1136,12 @@ def is_mnemonic(s: str) -> bool: # Match 12 or 24 words separated by spaces return bool(re.match(r'^(\w+ ){11}\w+$', s)) or bool(re.match(r'^(\w+ ){23}\w+$', s)) + +def file2functions(self, path): + path = os.path.abspath(path) + + return functions + def is_private_key(s: str) -> bool: import re # Match a 64-character hexadecimal string @@ -1134,3 +1149,32 @@ def is_private_key(s: str) -> bool: return bool(re.match(pattern, s)) +def get_folder_contents_advanced(url='commune-ai/commune.git', + host_url = 'https://github.com/', + auth_token=None): + try: + headers = { + 'Accept': 'application/json', + 'User-Agent': 'Python Script' + } + if not url.startswith(host_url): + url = host_url + url + + if auth_token: + headers['Authorization'] = f'token {auth_token}' + + response = requests.get(url, headers=headers) + response.raise_for_status() + + # Parse JSON response + content = response.json() + + # If it's a GitHub API response, it will be a list of files/folders + if isinstance(content, list): + return json.dumps(content, indent=2) + return response.text + + except Exception as e: + print(f"Error: {e}") + return None + diff --git a/commune/utils/os.py b/commune/utils/os.py index f60a7d0a..e8a745d0 100644 --- a/commune/utils/os.py +++ b/commune/utils/os.py @@ -1149,9 +1149,6 @@ async def check_port(port, ip): return used_ports - - - def resolve_port(port:int=None, **kwargs): ''' Resolves the port and finds one that is available @@ -1164,11 +1161,9 @@ def resolve_port(port:int=None, **kwargs): get_used_ports = used_ports - def has_free_ports(n:int = 1, **kwargs): return len(free_ports(n=n, **kwargs)) > 0 - def used_ports(ports:List[int] = None, ip:str = '0.0.0.0', port_range:Tuple[int, int] = None): import commune as c ''' diff --git a/commune/utils/utils.py b/commune/utils/utils.py deleted file mode 100644 index 1bf0ff84..00000000 --- a/commune/utils/utils.py +++ /dev/null @@ -1,21 +0,0 @@ - -import commune as c -class Utils(c.Module): - - @classmethod - def utils_paths(cls, search=None): - utils = c.find_functions(c.root_path + '/utils') - if search != None: - utils = [u for u in utils if search in u] - return sorted(utils) - - - @classmethod - def util2code(cls, search=None): - utils = cls.utils() - util2code = {} - for f in utils: - if search != None: - if search in f: - util2code[f] = c.fn_code(f) - return util2code \ No newline at end of file diff --git a/commune/vali.py b/commune/vali.py index d60e62c3..3060c624 100644 --- a/commune/vali.py +++ b/commune/vali.py @@ -231,6 +231,7 @@ def next_result(self, features=['score', 'name', 'key', 'i']): return result except Exception as e: result = c.detailed_error(e) + result.pop('success') c.print(f'ERROR({result})', color='red') return result