refactoring

commune/__init__.py

@@ -2,13 +2,12 @@
 
 
 from .module import Module # the module module
+M = c = Block = Agent  = Module # alias c.Module as c.Block, c.Lego, c.M
 from .vali import Vali # the vali module
 from .server import Server # the server module
 from .client import Client # the client module
 from .key import Key # the key module
 # set the module functions as globalsw
-M = IDONTGIVEAFUCKWHATYOUCALLTHIS = c = Block = Lego = M  = Agent  = Module # alias c.Module as c.Block, c.Lego, c.M
 c.add_to_globals(globals())
 key = c.get_key # override key function with file key in commune/key.py TODO: remove this line with a better solution
-utils = c.utils # set utils as a global
 

commune/client.py

@@ -6,8 +6,6 @@
 import requests
 import commune as c
 
-# from .pool import ClientPool
-
 class Client(c.Module):
     network2namespace = {}
     stream_prefix = 'data: '
@@ -16,7 +14,7 @@ def __init__(
             self,
             module : str = 'module',
             network: Optional[bool] = 'local',
-            key : Optional[str]= None,
+            key : Optional[str]= None ,
             **kwargs
         ):
         self.serializer = c.module('serializer')()
@@ -39,15 +37,13 @@ def call(cls,
                 timeout=40,
                 **extra_kwargs) -> None:
 
-
         if '/' in str(fn):
             module = '.'.join(fn.split('/')[:-1])
             fn = fn.split('/')[-1]
         else:
             module = fn
             fn = 'info'
         client =  cls(module=module, network=network)
-        # c.print(f'Client({module}/{fn}, key={key.key_address})') 
         return client.forward(fn=fn, args=args, kwargs=kwargs, timeout=timeout, **extra_kwargs)
 
     @classmethod
@@ -100,7 +96,8 @@ def get_url(self, fn, mode='http'):
         if ip in module_address:
             module_address = module_address.replace(ip, '0.0.0.0')
         url = f"{module_address}/{fn}/"
-        return url        
+        return url   
+
     def request(self, url: str,
                  data: dict, 
                 headers: dict, 

commune/module.py

@@ -390,7 +390,7 @@ def get_key(cls,key:str = None , **kwargs) -> None:
 
     @classmethod
     def files(cls, path='./', search:str = None, 
-              avoid_terms = ['__pycache__', '.git', '.ipynb_checkpoints', 'node_modules'], **kwargs) -> List[str]:
+              avoid_terms = ['__pycache__', '.git', '.ipynb_checkpoints', 'node_modules', 'artifacts'], **kwargs) -> List[str]:
         files =c.glob(path, **kwargs)
         files = [f for f in files if not any([at in f for at in avoid_terms])]
         if search != None:

commune/modules/agent/agent.py

@@ -31,8 +31,7 @@ def find_text(self, *args, **kwargs):
 
     def get_size(self, x):
         return len(str(x))
-
-
+
     def modules(self, 
                    query='', 
                    output_format="DICT(data:list[str])" , 

commune/modules/chat/chat.py

@@ -5,84 +5,35 @@ class Chat(c.Module):
 
     def __init__(self, 
                  max_tokens=420000, 
-                 password = None,
                  prompt = 'The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.',
-                 name='chat',
                  model = None,
                  history_path='history',
                 **kwargs):
 
         self.max_tokens = max_tokens
-        self.set_module(model, 
-                        password = password,
-                        name = name,
-                        history_path=history_path, 
-                        prompt=prompt,
-                        **kwargs)
-
-    def set_module(self,
-                    model, 
-                   history_path='history', 
-                   name='chat',
-                   password=None,
-                   prompt = 'The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.',
-                   key=None,
-                    **kwargs):
         self.prompt = prompt
-        self.admin_key = c.pwd2key(password) if password else self.key
         self.model = c.module('model.openrouter')(model=model, **kwargs)
-        self.models = self.model.models()
         self.history_path = self.resolve_path(history_path)
-        return {'success':True, 'msg':'set_module passed'}
 
-    def call(self, 
-            input = 'whats 2+2?' ,
-            temperature= 0.5,
-            max_tokens= 1000000,
-            model= 'anthropic/claude-3.5-sonnet', 
-            key = None,
-            stream=True, 
-            ):
-        # key = self.resolve_key(key)
-        data = c.locals2kwargs(locals())
-        signature = self.key.ticket(c.hash(data))
-        return signature
+    def generate(self,  text = 'whats 2+2?' , model= 'anthropic/claude-3.5-sonnet',  temperature= 0.5, max_tokens= 1000000,stream=True,  ):
+        text = self.process_text(text)
+        return self.model.generate(text, stream=stream, model=model, max_tokens=max_tokens,temperature=temperature )
 
-
-    def generate(self,  
-            text = 'whats 2+2?' ,
-            model= 'anthropic/claude-3.5-sonnet', 
-            temperature= 0.5,
-            max_tokens= 1000000,
-            context = None,
-            path = None,
-            stream=True, 
-            ):
-        context = context or path
-        # text = self.process_text(text, context=context)
-        output =  self.model.generate(text, stream=stream, model=model, max_tokens=max_tokens,temperature=temperature )
-        for token in output:
-            yield token
     forward = generate
 
     def ask(self, *text, **kwargs): 
         text = ' '.join(list(map(str, text)))
         return self.generate(text, **kwargs)
 
-    def process_text(self, text, context=None):
-        if context != None:
-            context = str(context)
-            if c.exists(context):
-                context =  str(c.file2text(context))
-            elif c.module_exists(context):
-                context =  c.code(context)
-            text = context + text
-
-        return text
+    def process_text(self, text):
+        new_text = ''
+        for word in text.split(' '):
+            if any([word.startswith(ch) for ch in ['.', '~', '/']]) and os.path.exists(word):
+                word = c.file2text(word)
+                print(word.keys())
+            new_text += str(word)
+        return new_text
 
-    def save_data(self, data):
-        path = self.data2path(data)
-        return c.put(path, data)
 
     def summarize(self, path='./', max_chars=10000): 
         if c.module_exists(path):
@@ -96,7 +47,6 @@ def summarize(self, path='./', max_chars=10000):
         elif os.path.isfile(path):
             c.print(f'Summarizing File: {path}')
             text = c.file2text(path)
-
         prompt = f'''
         GOAL
         summarize the following into tupples 
@@ -106,64 +56,7 @@ def summarize(self, path='./', max_chars=10000):
         '''
         return c.ask(prompt)
 
-    def user_files(self):
-        return c.get(self.data['path'])
 
-    def save_data(self, address, data):
-        return c.put(self.history_path + '/' + address +'/data.json', data)
-
-    def get_data(self, address):
-        return c.get(self.history_path + '/' + address +'/data.json')
-
-
-    def clear_history(self, address):
-        return c.rm(self.history_path +  '/'+ address)
-
-    def history_paths(self, address:str=None):
-        paths = []
-        if address == None:
-            for user_address in self.user_addresses():
-                 paths += self.history_paths(user_address)
-        else:
-            paths = c.ls(self.history_path + '/'+ address)
-        return paths
-
-    def save_data(self, data):
-        path = self.history_path + '/'+ data['address'] + '/' + str(data['time']) + '.json'
-        return c.put(path, data)
-
-    def history(self, address:str=None, columns=['datetime', 
-                                                 'input', 
-                                                 'output', 
-                                                 'prompt',
-                                                 'model', 
-                                                 'temperature',
-                                                   'max_tokens'], df=False):
-        paths = self.history_paths(address)
-        history =  []
-        for i, path in enumerate(paths):
-            try:
-                print(paths)
-                h = c.get(path)
-                h.update(h.pop('data'))
-                h['datetime'] = c.time2datetime(h.pop('time'))
-                h = {k:v for k,v in h.items() if k in columns}
-                history.append(h)
-            except Exception as e:
-                print(e)
-        # sort by time
-
-        history = sorted(history, key=lambda x: x['datetime'], reverse=True)
-        if df:
-            history = c.df(history)
-        return history
-
-    def user_addresses(self, display_name=False):
-        users = [u.split('/')[-1] for u in c.ls(self.history_path)]
-        return users
 
     def models(self):
-        return self.model.models()
-
-    def pricing(self, *args, **kwargs):
-        return self.model.pricing(*args,  **kwargs)
+        return self.model.models()

commune/server.py

@@ -280,6 +280,8 @@ def wait_for_server(cls,
     def info(self, crypto_type: str = 'sr25519', **kwargs) -> dict:
         info = {}
         module = self.module
+        module.address = c.ip() + ':' + str(module.port)
+
         info['schema'] = module.schema
         info['name'] = module.name 
         info['address'] = module.address

commune/utils/misc.py

@@ -9,33 +9,59 @@
 
 
 
-def get_glob( path =None, recursive:bool = True, files_only:bool = True):
+
+def type2files( path:str='./', **kwargs):
+    files = get_files(path, **kwargs)
+    type2files = {}
+    for f in files:
+        if '.' in f:
+            f_type = f.split('.')[-1]
+            if f_type not in type2files:
+                type2files[f_type] = []
+            type2files[f_type].append(f)
+    return type2files
+
+def type2filecount( path:str='./', **kwargs):
+    return {k: len(v) for k,v in type2files(path, **kwargs).items()}
+
+def get_files( path ='./', 
+              search=None,
+              avoid_terms = None,
+              include_terms = None,
+               recursive:bool = True, files_only:bool = True):
     import glob
     path = os.path.abspath(path)
     if os.path.isdir(path):
         path = os.path.join(path, '**')
     paths = glob.glob(path, recursive=recursive)
     if files_only:
         paths =  list(filter(lambda f:os.path.isfile(f), paths))
+    if avoid_terms != None:
+        paths = [p for p in paths if not any([term in p for term in avoid_terms])]
+    if include_terms != None:
+        paths = [p for p in paths if any([term in p for term in include_terms])]
+    if search != None:
+        paths = [p for p in paths if search in p]
     return paths
 
 def file2text(path = './', 
-                avoid_folders = ['__pycache__', 
+                avoid_terms = ['__pycache__', 
                                 '.git', 
                                 '.ipynb_checkpoints', 
                                 'package.lock',
                                 'egg-info',
                                 'Cargo.lock',
+                                'artifacts',
+                                'yarn.lock',
+                                'cache/',
                                 'target/debug',
                                 'node_modules'],
                 relative=True,  **kwargs):
     path = os.path.abspath(os.path.expanduser(path))
     file2text = {}
-    for file in get_glob(path, recursive=True):
+    for file in get_files(path, recursive=True, avoid_terms=avoid_terms , **kwargs):
         if os.path.isdir(file):
             continue
-        if any([af in file for af in avoid_folders]):
-            continue
         try:
             with open(file, 'r') as f:
                 content = f.read()
@@ -214,7 +240,7 @@ def tqdm(*args, **kwargs):
 def find_word( word:str, path='./')-> str:
     import commune as c
     path = os.path.abspath(path)
-    files = get_glob(path)
+    files = get_files(path)
     progress = c.tqdm(len(files))
     found_files = {}
     for f in files:

commune/utils/os.py

@@ -411,8 +411,6 @@ def file2chars( path='./', fmt='b') -> int:
     import commune as c
     files = c.glob(path)
     file2size = {}
-    for file in files:
-        file2size[file] = format_data_size(len(c.get_text(file), fmt))
     file2size = dict(sorted(file2size.items(), key=lambda item: item[1]))
     return file2size