subspace refactors

commune/cli.py

@@ -1,8 +1,9 @@
-import commune as c
 import sys
 import time
 import sys
+import commune as c
 print = c.print
+
 def determine_type(x):
     x = str(x)
     if isinstance(x, str) :
@@ -48,14 +49,13 @@ def determine_type(x):
             except ValueError:
                 pass
     return x
-
-
 def forward(argv = None,
             sep = '--', 
             fn_splitters = [':', '/', '//', '::'],
             base = 'module', 
             helper_fns = ['code', 'schema', 'fn_schema', 'help', 'fn_info', 'fn_hash'], 
             default_fn = 'vs'):
+
     t0 = time.time()
     argv = argv or sys.argv[1:]
     if len(argv) == 0:
@@ -135,8 +135,6 @@ def forward(argv = None,
     else:
         c.print(output)
     return output
-
-
 def main():
     forward()
 

commune/module.py

@@ -1881,6 +1881,9 @@ def new_modules(self, *modules, **kwargs):
         for module in modules:
             self.new_module(module=module, **kwargs)
 
+    def net(self):
+        return c.network()
+
     @classmethod
     def new_module( cls,
                    path : str ,
@@ -2060,6 +2063,22 @@ def resolve_extension( filename:str, extension = '.py') -> str:
                 return filename
         return filename + extension
 
+
+    def repo2path(self, search=None):
+        repo2path = {}
+        for p in c.ls('~/'): 
+            if os.path.exists(p+'/.git'):
+                r = p.split('/')[-1]
+                if search == None or search in r:
+                    repo2path[r] = p
+        return dict(sorted(repo2path.items(), key=lambda x: x[0]))
+
+    def repos(self, search=None):
+        return list(self.repo2path(search=search).keys())
+
+    def is_repo(self, repo:str):
+        return repo in self.repos()
+
     @classmethod
     def help(cls, *text, module=None,  **kwargs):
         text = ' '.join(map(str, text))
@@ -2071,9 +2090,7 @@ def help(cls, *text, module=None,  **kwargs):
     def time(self):
         return time.time()
 
-    def repos(self):
-        return c.ls('~/')
-
+
     def clone(self, repo:str, path:str=None, **kwargs):
         path = '~/' + repo if path == None else path
         cmd =  f'git clone {repo}'
@@ -2323,5 +2340,3 @@ def epoch(self, *args, **kwargs):
 Module.run(__name__)
 
 
-
-

commune/network/network.py

@@ -22,6 +22,10 @@ def set_network(self, network:str, tempo:int=60, path=None, **kwargs):
     def params(self,*args,  **kwargs):
         return { 'network': self.network, 'tempo' : self.tempo,'n': self.n}
 
+
+    def net(self):
+        return c.network()
+
     def modules(self, 
                 search=None, 
                 max_age=tempo, 

commune/network/subspace/subspace.py

@@ -58,12 +58,14 @@ def __init__(
         test = False,
         ws_options = {},
         timeout: int | None = None,
+        net = None,
     ):
         """
         Args:
             url: The URL of the network node to connect to.
             num_connections: The number of websocket connections to be opened.
         """
+        network = net or network # add a little shortcut
         self.set_network(network=network,
                          mode=mode,
                          url=url,  
@@ -2000,6 +2002,8 @@ def subnet_map(self, max_age=10, update=False, **kwargs) -> dict[int, str]:
 
     def netuid2subnet(self, *args, **kwargs):
         return {v:k for k,v in self.subnet_map(*args, **kwargs).items()}
+
+
 
     def resolve_subnet(self, subnet: str) -> int:
         subnet_map = self.subnet_map()

commune/server.py

@@ -253,12 +253,24 @@ def forward(self, fn:str, request: Request, catch_exception:bool=True) -> dict:
             try:
                 return self.forward(fn, request, catch_exception=False)
             except Exception as e:
-                return c.detailed_error(e)
+                result =  c.detailed_error(e)
+                c.print(result, color='red')
+                return result
         module = self.module
+
         data = self.loop.run_until_complete(request.json())
         # data = self.serializer.deserialize(data) 
-        kwargs = dict(data.get('kwargs', data.get('params', {}))) 
-        args = list(data.get('args', []))
+        if isinstance(data, str):
+            data = json.loads(data)
+
+        if 'kwargs' in data or 'params' in data:
+            kwargs = dict(data.get('kwargs', data.get('params', {}))) 
+        else:
+            kwargs = data
+        if 'args' in data:
+            args = list(data.get('args', []))
+        else:
+            args = []
         data = {'args': args, 'kwargs': kwargs}
         headers = dict(request.headers.items())
         headers['key'] = headers.get('key', headers.get('address', None))

commune/vali.py

@@ -95,10 +95,7 @@ def nex_epoch(self):
     results = []
 
     def epoch(self):
-        if len(self.futures) > 0:
-            print('Cancelling futures from previous epoch')
-            [f.cancel() for f in self.futures]
-        self.futures = []
+        futures = []
         self.results = []
         next_epoch = self.nex_epoch
         progress = c.tqdm(total=next_epoch, desc='Next Epoch')
@@ -110,20 +107,22 @@ def epoch(self):
         progress = c.tqdm(total=self.n, desc='Evaluating Modules')
         # return self.modules
         n = len(self.modules)
+
         for i, module in enumerate(self.modules):
             module["i"] = i
             c.print(f'EVAL(i={i}/{n} key={module["key"]} name={module["name"]})', color='yellow')
-            if len(self.futures) < self.batch_size:
-                self.futures.append(self.executor.submit(self.score_module, [module], timeout=self.timeout))
+            if len(futures) < self.batch_size:
+                futures.append(self.executor.submit(self.score_module, [module], timeout=self.timeout))
             else: 
-                self.results.append(self.next_result())
+                self.results.append(self.next_result(futures))
             progress.update(1)
-        while len(self.futures) > 0:
-            self.results.append(self.next_result())
+        while len(futures) > 0:
+            self.results.append(self.next_result(futures))
         self.results = [r for r in self.results if r.get('score', 0) > 0]
         self.epochs += 1
         self.epoch_time = c.time()
         c.print(self.vote())
+        print(self.scoreboard())
         return self.results
 
     def sync(self, update = False):
@@ -221,14 +220,20 @@ def module_paths(self):
     def run_epoch(cls, network='local', run_loop=False, **kwargs):
         return  cls(network=network, run_loop=run_loop, **kwargs).epoch()
 
-    def next_result(self,  features=['score', 'name', 'key', 'i']):
+    def next_result(self, futures:list, features=['score', 'name', 'key', 'i']):
         try:
-            for future in c.as_completed(self.futures, timeout=self.timeout):
-                self.futures.remove(future) 
-                result = future.result()
-                if all([f in result for f in features]):
-                    c.print(f'RESULT(score={result["score"]} key={result["key"]} name={result["name"]} )', color='green')
-                    return result
+            for future in c.as_completed(futures, timeout=self.timeout):
+                    futures.remove(future) 
+                    result = future.result()
+                    if all([f in result for f in features]):
+                        v_result = {f: result[f] for f in features}
+
+                        c.print(f'RESULT({v_result})', color='red')
+                        return result
+                    else:
+                        v_result = {f: result[f] for f in result if f not in ['success']}
+                        c.print(f'ERROR({result["error"]})', color='red')
+
         except Exception as e:
             result = c.detailed_error(e)
             result.pop('success')

modules/chat/chat.py

@@ -35,11 +35,36 @@ def process_text(self, text, threshold=1000):
                 word = c.file2text(word)
             new_text += str(word)
         return new_text
+
 
-    def reduce(self, text, max_chars=10000 , timeout=5, max_age=30, model='openai/o1-mini'):
-
+    def reduce(self, text, max_chars=10000 , timeout=40, max_age=30, model='openai/o1-mini'):
         if os.path.exists(text): 
-            text = str(c.file2text(text))
+            path = text
+            if os.path.isdir(path):
+                print('REDUCING A DIRECTORY -->', path)
+                future2path = {}
+                path2result = {}
+                paths = c.files(path)
+                progress = c.tqdm(len(paths), desc='Reducing', leave=False)
+                while len(paths) > 0:
+                    for p in paths:
+                        future = c.submit(self.reduce, [p], timeout=timeout)
+                        future2path[future] = p
+                    try:
+                        for future in c.as_completed(future2path, timeout=timeout):
+                            p = future2path[future]
+                            r = future.result()
+                            paths.remove(p)
+                            path2result[p] = r
+                            print('REDUCING A FILE -->', r)
+                            progress.update(1)
+                    except Exception as e:
+                        print(e)
+                return path2result
+            else:
+                assert os.path.exists(path), f'Path {path} does not exist'
+                print('REDUCING A FILE -->', path)
+                text = str(c.get_text(path))
         elif c.module_exists(text):
             text = c.code(text)
 
@@ -55,38 +80,32 @@ def reduce(self, text, max_chars=10000 , timeout=5, max_age=30, model='openai/o1
         OUTPUT FORMAT ONLY BETWEEN THE TAGS SO WE CAN PARSE
         <OUTPUT>DICT(data=List[Dict[str, str]])</OUTPUT>
         '''
-        print(f"TEXTSIZE : {len(text)}")
-        compress_ratio = 0
-        text_size = len(text)
         if len(text) >= max_chars * 2 :
             batch_text = [text[i:i+max_chars] for i in range(0, len(text), max_chars)]
-            print(f"TEXTSIZE : {text_size} > {max_chars} BATCH SIZE: {len(batch_text)}")
             futures =  [c.submit(self.reduce, [batch], timeout=timeout) for batch in batch_text]
-            text = ''
-            cnt = 0
+            output = ''
             try:
-                n = len(batch_text)
-                progress = c.progress(n)
-
                 for future in c.as_completed(futures, timeout=timeout):
-                    text += str(future.result())
-                    cnt += 1
-                    progress.update(1)
-                    print(f"SUMMARIZED: {cnt}/{n} COMPRESSION_RATIO: {compress_ratio}")
-                return text
+                    output += str(future.result())
             except Exception as e:
                 print(e)
-
             final_length = len(text)
-            compress_ratio = final_length/original_length
-            result = { 'compress_ratio': compress_ratio, 'final_length': final_length, 'original_length': original_length}
-            print(result)
-            return text
+            result = { 'compress_ratio': final_length/original_length, 
+                      'final_length': final_length, 
+                      'original_length': original_length, 
+                      "data": text}
+            return result
         if "'''" in text:
             text = text.replace("'''", '"""')
 
         data =  c.ask(text, model=model, stream=0)
-        return data
+        def process_data(data):
+            try:
+                data = data.split('<OUTPUT>')[1].split('</OUTPUT>')[0]
+                return data
+            except:
+                return data
+        return {"data": process_data(data)}
 
     def models(self):
         return self.model.models()

modules/git/git.py

@@ -4,12 +4,10 @@
 
 class Git(c.Module):
 
-
     def is_repo(self, libpath:str ):
         # has the .git folder
         return c.cmd(f'ls -a {libpath}').count('.git') > 0
-
-
+
     @staticmethod
     def clone(repo_url:str, target_directory:str = None, branch=None):
         prefix = 'https://github.com/'

modules/openrouter/openrouter.py

@@ -30,6 +30,7 @@ def generate(
         model:str = 'claude-3-sonnet',
         max_tokens: int = 100000,
         temperature: float = 1.0,
+        verbose=False,
     ) -> str | Generator[str, None, None]:
         """
         Generates a response using the OpenAI language model.
@@ -52,7 +53,7 @@ def generate(
         model = self.resolve_model(model)
         model_info = self.get_model_info(model)
         num_tokens = len(message)
-        print(f'Sending {num_tokens} tokens -> {model}')
+        c.print(f'Sending {num_tokens} tokens -> {model}', verbose=verbose)
         max_tokens = min(max_tokens, model_info['context_length'] - num_tokens)
         messages = history.copy()
         messages.append({"role": "user", "content": message})