diff --git a/commune/client/client.py b/commune/client/client.py index 93e37059..d60ecdfc 100644 --- a/commune/client/client.py +++ b/commune/client/client.py @@ -119,8 +119,9 @@ def get_url(self, fn, mode='http', network=None): url = f"{module_address}/{fn}/" return url def request(self, url: str, data: dict, headers: dict, timeout: int = 10, stream: bool = True): - response = self.session.post(url, json=data, headers=headers, timeout=timeout, stream=stream) try: + response = self.session.post(url, json=data, headers=headers, timeout=timeout, stream=stream) + if 'text/event-stream' in response.headers.get('Content-Type', ''): return self.stream(response) if 'application/json' in response.headers.get('Content-Type', ''): @@ -190,8 +191,10 @@ def forward(self, url = self.get_url(fn=fn, mode=mode, network=network) data = data or self.get_data(args=args, kwargs=kwargs, params=params, **extra_kwargs) headers = headers or self.get_headers(data=data, key=key) - kwargs = {**(kwargs or {}), **extra_kwargs} - result = self.request( url=url,data=data,headers= headers,timeout= timeout) + result = self.request(url=url, + data=data, + headers=headers, + timeout=timeout) return result def __del__(self): diff --git a/commune/key/key.py b/commune/key/key.py index ebedc87f..de898ddd 100644 --- a/commune/key/key.py +++ b/commune/key/key.py @@ -67,7 +67,6 @@ class MnemonicLanguageCode: class Keypair(c.Module): - keys_path = c.data_path + '/keys.json' def __init__(self, ss58_address: str = None, public_key: Union[bytes, str] = None, diff --git a/commune/module/_config.py b/commune/module/_config.py index b5e48d67..09ea71f6 100644 --- a/commune/module/_config.py +++ b/commune/module/_config.py @@ -151,4 +151,12 @@ def config_path(cls) -> str: def update_config(self, config): self.config.update(config) - return self.config \ No newline at end of file + return self.config + + + @classmethod + def base_config(cls, cache=True): + if cache and hasattr(cls, '_base_config'): + return cls._base_config + cls._base_config = cls.get_yaml(cls.config_path()) + return cls._base_config \ No newline at end of file diff --git a/commune/module/_manager.py b/commune/module/_manager.py index b39a9636..860423aa 100644 --- a/commune/module/_manager.py +++ b/commune/module/_manager.py @@ -14,7 +14,14 @@ def resolve_extension(cls, filename:str, extension = '.py') -> str: def simple2path(cls, simple:str, extension = '.py', - avoid_dirnames = ['', 'src', 'commune', 'commune/module', 'commune/modules', 'modules', 'blocks', 'agents', 'commune/agents'], + avoid_dirnames = ['', 'src', + 'commune', + 'commune/module', + 'commune/modules', + 'modules', + 'blocks', + 'agents', + 'commune/agents'], **kwargs) -> bool: """ converts the module path to a file path @@ -52,24 +59,19 @@ def simple2path(cls, if os.path.isdir(module_dirpath): simple_filename = simple.replace('.', '_') filename_options = [simple_filename, simple_filename + '_module', 'module_'+ simple_filename] + ['module'] + simple.split('.') + ['__init__'] - path_options += [module_dirpath + '/' + cls.resolve_extension(f) for f in filename_options] + path_options += [module_dirpath + '/' + f for f in filename_options] else: - module_filepath = dir_path + '/' + cls.resolve_extension(simple.replace('.', '/'), extension=extension) + module_filepath = dir_path + '/' + simple.replace('.', '/') path_options += [module_filepath] - for p in path_options: + p = cls.resolve_extension(p) if os.path.exists(p): p_text = cls.get_text(p) - # gas class in text - is_class_text = 'commune' in p_text and 'class ' in p_text or ' def ' in p_text - if is_class_text: - path = p - break path = p - + if 'commune' in p_text and 'class ' in p_text or ' def ' in p_text: + return p if path != None: break - assert path != None, f'MODULE {simple} DOES NOT EXIST' return path @@ -119,7 +121,6 @@ def path2simple(cls, module_extension = '.'+module_extension if path.endswith(module_extension): path = path[:-len(module_extension)] - if compress_path: # we want to remove redundant chunks # for example if the path is 'module/module' we want to remove the redundant module @@ -173,7 +174,6 @@ def resolve_cache_path(self, path): if path.startswith('_'): path = path[1:] path = f'cached_path/{path}' - print(path) return path @classmethod @@ -187,7 +187,8 @@ def find_classes(cls, path='./', working=False): path = os.path.abspath(path) if os.path.isdir(path): classes = [] - for p in cls.glob(path+'/**/**.py', recursive=True): + generator = cls.glob(path+'/**/**.py', recursive=True) + for p in generator: if p.endswith('.py'): p_classes = cls.find_classes(p ) if working: @@ -223,14 +224,60 @@ def find_classes(cls, path='./', working=False): classes = [c.replace(libpath_objpath_prefix, '') for c in classes] return classes + + + + @classmethod + def find_class2functions(cls, path, working=False): + + path = os.path.abspath(path) + if os.path.isdir(path): + class2functions = {} + for p in cls.glob(path+'/**/**.py', recursive=True): + if p.endswith('.py'): + object_path = cls.path2objectpath(p) + response = cls.find_class2functions(p ) + for k,v in response.items(): + class2functions[object_path+ '.' +k] = v + return class2functions + + code = cls.get_text(path) + classes = [] + class2functions = {} + class_functions = [] + new_class = None + for line in code.split('\n'): + if all([s in line for s in ['class ', ':']]): + new_class = line.split('class ')[-1].split('(')[0].strip() + if new_class.endswith(':'): + new_class = new_class[:-1] + if ' ' in new_class: + continue + classes += [new_class] + if len(class_functions) > 0: + class2functions[new_class] = cls.copy(class_functions) + class_functions = [] + if all([s in line for s in [' def', '(']]): + fn = line.split(' def')[-1].split('(')[0].strip() + class_functions += [fn] + if new_class != None: + class2functions[new_class] = class_functions + + return class2functions + @classmethod def path2objectpath(cls, path:str, **kwargs) -> str: - libpath = cls.libpath + '/' + cls.libname + libpath = cls.libpath + path.replace if path.startswith(libpath): - return cls.libname + '.' + path.replace(libpath , '')[1:].replace('/', '.').replace('.py', '') - pwd = cls.pwd() - if path.startswith(pwd): - return path.replace(pwd, '')[1:].replace('/', '.').replace('.py', '') + path = path.replace(libpath , '')[1:].replace('/', '.').replace('.py', '') + else: + pwd = cls.pwd() + if path.startswith(pwd): + path = path.replace(pwd, '')[1:].replace('/', '.').replace('.py', '') + + return path.replace('__init__.', '.') + @classmethod def find_functions(cls, path = './', working=False): @@ -429,7 +476,7 @@ def simplify_paths(cls, paths): return paths @classmethod - def simplify_path(cls, p, avoid_terms=['modules']): + def simplify_path(cls, p, avoid_terms=['modules', 'agents']): chunks = p.split('.') if len(chunks) < 2: return None @@ -479,6 +526,52 @@ def get_tree(cls, path): simple_paths = cls.simplify_paths(class_paths) return dict(zip(simple_paths, class_paths)) + @classmethod + def get_module(cls, + path:str = 'module', + cache=True, + verbose = False, + update_tree_if_fail = True, + init_kwargs = None, + catch_error = False, + ) -> str: + import commune as c + path = path or 'module' + if catch_error: + try: + return cls.get_module(path=path, cache=cache, + verbose=verbose, + update_tree_if_fail=update_tree_if_fail, + init_kwargs=init_kwargs, + catch_error=False) + except Exception as e: + return c.detailed_error(e) + if path in ['module', 'c']: + return c + # if the module is a valid import path + shortcuts = c.shortcuts() + if path in shortcuts: + path = shortcuts[path] + module = None + cache_key = path + t0 = c.time() + if cache and cache_key in c.module_cache: + module = c.module_cache[cache_key] + return module + module = c.simple2object(path) + # ensure module + if verbose: + c.print(f'Loaded {path} in {c.time() - t0} seconds', color='green') + + if init_kwargs != None: + module = module(**init_kwargs) + is_module = c.is_module(module) + if not is_module: + module = cls.obj2module(module) + if cache: + c.module_cache[cache_key] = module + return module + _tree = None @classmethod @@ -530,4 +623,58 @@ def modules(cls, search=None, cache=True, **kwargs)-> List[str]: @classmethod def has_module(cls, module): - return module in cls.modules() \ No newline at end of file + return module in cls.modules() + + + + + + def new_modules(self, *modules, **kwargs): + for module in modules: + self.new_module(module=module, **kwargs) + + + + @classmethod + def new_module( cls, + module : str , + base_module : str = 'demo', + folder_module : bool = False, + update=1 + ): + + import commune as c + base_module = c.module(base_module) + module_class_name = ''.join([m[0].capitalize() + m[1:] for m in module.split('.')]) + base_module_class_name = base_module.class_name() + base_module_code = base_module.code().replace(base_module_class_name, module_class_name) + pwd = c.pwd() + path = os.path.join(pwd, module.replace('.', '/')) + if folder_module: + dirpath = path + filename = module.replace('.', '_') + path = os.path.join(path, filename) + + path = path + '.py' + dirpath = os.path.dirname(path) + if os.path.exists(path) and not update: + return {'success': True, 'msg': f'Module {module} already exists', 'path': path} + if not os.path.exists(dirpath): + os.makedirs(dirpath, exist_ok=True) + + c.put_text(path, base_module_code) + + return {'success': True, 'msg': f'Created module {module}', 'path': path} + + add_module = new_module + + + @classmethod + def has_local_module(cls, path=None): + import commune as c + path = '.' if path == None else path + if os.path.exists(f'{path}/module.py'): + text = c.get_text(f'{path}/module.py') + if 'class ' in text: + return True + return False diff --git a/commune/module/_misc.py b/commune/module/_misc.py index 139c8197..8dbdae1f 100644 --- a/commune/module/_misc.py +++ b/commune/module/_misc.py @@ -206,6 +206,37 @@ def init_nn(self): torch.nn.Module.__init__(self) + @classmethod + def check_word(cls, word:str)-> str: + import commune as c + files = c.glob('./') + progress = c.tqdm(len(files)) + for f in files: + try: + text = c.get_text(f) + except Exception as e: + continue + if word in text: + return True + progress.update(1) + return False + + @classmethod + def wordinfolder(cls, word:str, path:str='./')-> bool: + import commune as c + path = c.resolve_path(path) + files = c.glob(path) + progress = c.tqdm(len(files)) + for f in files: + try: + text = c.get_text(f) + except Exception as e: + continue + if word in text: + return True + progress.update(1) + return False + def locals2hash(self, kwargs:dict = {'a': 1}, keys=['kwargs']) -> str: kwargs.pop('cls', None) @@ -944,4 +975,55 @@ def folder_structure(cls, path:str='./', search='py', max_depth:int=5, depth:int def copy(cls, data: Any) -> Any: import copy return copy.deepcopy(data) - \ No newline at end of file + + + @classmethod + def find_word(cls, word:str, path='./')-> str: + import commune as c + path = c.resolve_path(path) + files = c.glob(path) + progress = c.tqdm(len(files)) + found_files = {} + for f in files: + try: + text = c.get_text(f) + if word not in text: + continue + lines = text.split('\n') + except Exception as e: + continue + + line2text = {i:line for i, line in enumerate(lines) if word in line} + found_files[f[len(path)+1:]] = line2text + progress.update(1) + return found_files + + + + @classmethod + def pip_install(cls, + lib:str= None, + upgrade:bool=True , + verbose:str=True, + ): + import commune as c + + if lib in c.modules(): + c.print(f'Installing {lib} Module from local directory') + lib = c.resolve_object(lib).dirpath() + if lib == None: + lib = c.libpath + + if c.exists(lib): + cmd = f'pip install -e' + else: + cmd = f'pip install' + if upgrade: + cmd += ' --upgrade' + return cls.cmd(cmd, verbose=verbose) + + + @classmethod + def pip_exists(cls, lib:str, verbose:str=True): + return bool(lib in cls.pip_libs()) + diff --git a/commune/module/_os.py b/commune/module/_os.py index 53145da0..4d0d1c82 100644 --- a/commune/module/_os.py +++ b/commune/module/_os.py @@ -493,3 +493,13 @@ def memory_usage(fmt='gb'): process = psutil.Process() scale = fmt2scale.get(fmt) return (process.memory_info().rss // 1024) / scale + + @classmethod + def get_env(cls, key:str)-> None: + ''' + Pay attention to this function. It sets the environment variable + ''' + return os.environ[key] + + env = get_env + \ No newline at end of file diff --git a/commune/module/_storage.py b/commune/module/_storage.py index 3d4ddf4a..5e757329 100644 --- a/commune/module/_storage.py +++ b/commune/module/_storage.py @@ -653,3 +653,55 @@ def jsonable( value): + + def file2text(self, path = './', relative=True, **kwargs): + path = os.path.abspath(path) + file2text = {} + for file in c.glob(path, recursive=True): + with open(file, 'r') as f: + content = f.read() + file2text[file] = content + if relative: + print(path) + return {k[len(path)+1:]:v for k,v in file2text.items()} + + return file2text + + def file2lines(self, path:str='./')-> List[str]: + file2text = self.file2text(path) + file2lines = {f: text.split('\n') for f, text in file2text.items()} + return file2lines + + def num_files(self, path:str='./')-> int: + import commune as c + return len(c.glob(path)) + + def hidden_files(self, path:str='./')-> List[str]: + import commune as c + path = self.resolve_path(path) + files = [f[len(path)+1:] for f in c.glob(path)] + print(files) + hidden_files = [f for f in files if f.startswith('.')] + return hidden_files + + + @staticmethod + def format_data_size(x: Union[int, float], fmt:str='b', prettify:bool=False): + assert type(x) in [int, float], f'x must be int or float, not {type(x)}' + fmt2scale = { + 'b': 1, + 'kb': 1000, + 'mb': 1000**2, + 'gb': 1000**3, + 'GiB': 1024**3, + 'tb': 1000**4, + } + + assert fmt in fmt2scale.keys(), f'fmt must be one of {fmt2scale.keys()}' + scale = fmt2scale[fmt] + x = x/scale + + if prettify: + return f'{x:.2f} {f}' + else: + return x diff --git a/commune/module/_task.py b/commune/module/_task.py index 66fa8af5..e21ff711 100644 --- a/commune/module/_task.py +++ b/commune/module/_task.py @@ -8,6 +8,9 @@ class Task: + + thread_map = {} + @classmethod def wait(cls, futures:list, timeout:int = None, generator:bool=False, return_dict:bool = True) -> list: is_singleton = bool(not isinstance(futures, list)) diff --git a/commune/module/module.py b/commune/module/module.py index 931cc50f..780fc5a6 100755 --- a/commune/module/module.py +++ b/commune/module/module.py @@ -42,8 +42,6 @@ class c(*CORE_MODULES): rootpath = root_path = root = '/'.join(__file__.split('/')[:-2]) # the path to the root of the library homepath = home_path = os.path.expanduser('~') # the home path libpath = lib_path = os.path.dirname(root_path) # the path to the library - datapath = data_path = os.path.join(root_path, 'data') # the path to the data folder - modules_path = os.path.join(libpath, 'modules') # the path to the modules folder repo_path = os.path.dirname(root_path) # the path to the repo blacklist = [] # blacklist of functions to not to access for outside use server_mode = 'http' # http, grpc, ws (websocket) @@ -116,7 +114,7 @@ def module_name(cls, obj=None): return cls.name obj = cls.resolve_object(obj) module_file = inspect.getfile(obj) - return cls.path2simple(module_file) + return c.path2simple(module_file) path = name = module_name @@ -143,51 +141,7 @@ def sandbox(cls): module_cache = {} _obj = None - @classmethod - def get_module(cls, - path:str = 'module', - cache=True, - verbose = False, - update_tree_if_fail = True, - init_kwargs = None, - catch_error = False, - ) -> str: - path = path or 'module' - if catch_error: - try: - return cls.get_module(path=path, cache=cache, - verbose=verbose, - update_tree_if_fail=update_tree_if_fail, - init_kwargs=init_kwargs, - catch_error=False) - except Exception as e: - return c.detailed_error(e) - if path in ['module', 'c']: - return c - # if the module is a valid import path - shortcuts = c.shortcuts() - if path in shortcuts: - path = shortcuts[path] - module = None - cache_key = path - t0 = c.time() - if cache and cache_key in c.module_cache: - module = c.module_cache[cache_key] - return module - module = c.simple2object(path) - # ensure module - if verbose: - c.print(f'Loaded {path} in {c.time() - t0} seconds', color='green') - - if init_kwargs != None: - module = module(**init_kwargs) - is_module = c.is_module(module) - if not is_module: - module = cls.obj2module(module) - if cache: - c.module_cache[cache_key] = module - return module - + @classmethod def obj2module(cls,obj): import commune as c @@ -285,8 +239,7 @@ def serialize(cls, *args, **kwargs): @classmethod def deserialize(cls, *args, **kwargs): return c.module('serializer')().deserialize(*args, **kwargs) - - + @property def server_name(self): if not hasattr(self, '_server_name'): @@ -297,36 +250,6 @@ def server_name(self): def server_name(self, name): self._server_name = name - @classmethod - def resolve_server_name(cls, - module:str = None, - tag:str=None, - name:str = None, - tag_seperator:str='::', - **kwargs): - """ - Resolves the server name - """ - # if name is not specified, use the module as the name such that module::tag - if name == None: - module = cls.module_name() if module == None else module - - # module::tag - if tag_seperator in module: - module, tag = module.split(tag_seperator) - if tag_seperator in module: - module, tag = module.split(tag_seperator) - name = module - if tag in ['None','null'] : - tag = None - if tag != None: - name = f'{name}{tag_seperator}{tag}' - - # ensure that the name is a string - assert isinstance(name, str), f'Invalid name {name}' - return name - resolve_name = resolve_server_name - @classmethod def resolve_object(cls, obj:str = None, **kwargs): if cls._obj != None: @@ -414,10 +337,6 @@ def run(cls, name:str = None) -> Any: module = cls return getattr(module, args.function)(*args.args, **args.kwargs) - @classmethod - def learn(cls, *args, **kwargs): - return c.module('model.hf').learn(*args, **kwargs) - @classmethod def commit_hash(cls, libpath:str = None): if libpath == None: @@ -460,60 +379,19 @@ def module(cls,module: Any = 'module' , verbose=False, **kwargs): latency = c.time() - t0 c.print(f'Loaded {module} in {latency} seconds', color='green', verbose=verbose) return module_class + + _module = m = mod = module # UNDER CONSTRUCTION (USE WITH CAUTION) def setattr(self, k, v): setattr(self, k, v) - - def setattributes(self, new_attributes:Dict[str, Any]) -> None: - ''' - Set a dictionary to the slf attributes - ''' - assert isinstance(new_attributes, dict), f'locals must be a dictionary but is a {type(locals)}' - self.__dict__.update(new_attributes) - - @classmethod - def pip_install(cls, - lib:str= None, - upgrade:bool=True , - verbose:str=True, - ): - - - if lib in c.modules(): - c.print(f'Installing {lib} Module from local directory') - lib = c.resolve_object(lib).dirpath() - if lib == None: - lib = c.libpath - if c.exists(lib): - cmd = f'pip install -e' - else: - cmd = f'pip install' - if upgrade: - cmd += ' --upgrade' - return cls.cmd(cmd, verbose=verbose) - - @classmethod def pip_exists(cls, lib:str, verbose:str=True): return bool(lib in cls.pip_libs()) - @classmethod - def lib2version(cls, lib:str = None) -> dict: - lib2version = {} - for l in cls.pip_list(): - name = l.split(' ')[0].strip() - version = l.split(' ')[-1].strip() - if len(name) > 0: - lib2version[name] = version - if lib != None and lib == name: - return version - - return lib2version - @classmethod def version(cls, lib:str=libname): lines = [l for l in cls.cmd(f'pip3 list', verbose=False).split('\n') if l.startswith(lib)] @@ -521,54 +399,10 @@ def version(cls, lib:str=libname): return lines[0].split(' ')[-1].strip() else: return f'No Library Found {lib}' - - - - - - @classmethod - def resolve_ip(cls, ip=None, external:bool=True) -> str: - if ip == None: - if external: - ip = cls.external_ip() - else: - ip = '0.0.0.0' - assert isinstance(ip, str) - return ip - - @classmethod - def get_env(cls, key:str)-> None: - ''' - Pay attention to this function. It sets the environment variable - ''' - return os.environ[key] - - env = get_env - def forward(self, a=1, b=2): return a+b - @staticmethod - def format_data_size(x: Union[int, float], fmt:str='b', prettify:bool=False): - assert type(x) in [int, float], f'x must be int or float, not {type(x)}' - fmt2scale = { - 'b': 1, - 'kb': 1000, - 'mb': 1000**2, - 'gb': 1000**3, - 'GiB': 1024**3, - 'tb': 1000**4, - } - - assert fmt in fmt2scale.keys(), f'fmt must be one of {fmt2scale.keys()}' - scale = fmt2scale[fmt] - x = x/scale - - if prettify: - return f'{x:.2f} {f}' - else: - return x ### DICT LAND ### @@ -683,7 +517,6 @@ def encrypt(cls, """ key = c.get_key(key) return key.encrypt(data, password=password,**kwargs) - @classmethod def decrypt(cls, @@ -694,16 +527,6 @@ def decrypt(cls, key = c.get_key(key) return key.decrypt(data, password=password, **kwargs) - - - def resolve_key(self, key: str = None) -> str: - if key == None: - if hasattr(self, 'key'): - key = self.key - key = self.resolve_keypath(key) - key = self.get_key(key) - return key - @classmethod def type_str(cls, x): return type(x).__name__ @@ -736,25 +559,25 @@ def resolve_keypath(cls, key = None): if key == None: key = cls.module_name() return key - - def sign(self, data:dict = None, key: str = None, **kwargs) -> bool: - key = self.resolve_key(key) - signature = key.sign(data, **kwargs) - return signature + def resolve_key(self, key: str = None) -> str: + if key == None: + if hasattr(self, 'key'): + key = self.key + key = self.resolve_keypath(key) + key = self.get_key(key) + return key + def sign(self, data:dict = None, key: str = None, **kwargs) -> bool: + return self.resolve_key(key).sign(data, **kwargs) @classmethod def verify(cls, auth, key=None, **kwargs ) -> bool: - key = c.get_key(key) - return key.verify(auth, **kwargs) + return c.get_key(key).verify(auth, **kwargs) @classmethod def verify_ticket(cls, auth, key=None, **kwargs ) -> bool: - key = c.get_key(key) - return key.verify_ticket(auth, **kwargs) - - + return c.get_key(key).verify_ticket(auth, **kwargs) @classmethod def start(cls, *args, **kwargs): @@ -765,10 +588,6 @@ def remove_user(self, key: str) -> None: self.users = [] self.users.pop(key, None) - @classmethod - def check_module(cls, module:str): - return c.connect(module) - @classmethod def is_pwd(cls, module:str = None): if module != None: @@ -776,264 +595,23 @@ def is_pwd(cls, module:str = None): else: module = cls return module.dirpath() == c.pwd() - - - def new_modules(self, *modules, **kwargs): - for module in modules: - self.new_module(module=module, **kwargs) - - - - @classmethod - def new_module( cls, - module : str , - base_module : str = 'demo', - folder_module : bool = False, - update=1 - ): - - base_module = c.module(base_module) - module_class_name = ''.join([m[0].capitalize() + m[1:] for m in module.split('.')]) - base_module_class_name = base_module.class_name() - base_module_code = base_module.code().replace(base_module_class_name, module_class_name) - pwd = c.pwd() - path = os.path.join(pwd, module.replace('.', '/')) - if folder_module: - dirpath = path - filename = module.replace('.', '_') - path = os.path.join(path, filename) - - path = path + '.py' - dirpath = os.path.dirname(path) - if os.path.exists(path) and not update: - return {'success': True, 'msg': f'Module {module} already exists', 'path': path} - if not os.path.exists(dirpath): - os.makedirs(dirpath, exist_ok=True) - - c.put_text(path, base_module_code) - - return {'success': True, 'msg': f'Created module {module}', 'path': path} - add_module = new_module - - - thread_map = {} - - - @classmethod - def launch(cls, - module:str = None, - fn: str = 'serve', - name:Optional[str]=None, - tag : str = None, - args : list = None, - kwargs: dict = None, - device:str=None, - interpreter:str='python3', - autorestart: bool = True, - verbose: bool = False , - force:bool = True, - meta_fn: str = 'module_fn', - tag_seperator:str = '::', - cwd = None, - refresh:bool=True ): - - if hasattr(module, 'module_name'): - module = module.module_name() - - # avoid these references fucking shit up - args = args if args else [] - kwargs = kwargs if kwargs else {} - - # convert args and kwargs to json strings - kwargs = { - 'module': module , - 'fn': fn, - 'args': args, - 'kwargs': kwargs - } - - kwargs_str = json.dumps(kwargs).replace('"', "'") - - name = name or module - if refresh: - cls.pm2_kill(name) - module = c.module() - # build command to run pm2 - filepath = c.filepath() - cwd = cwd or module.dirpath() - command = f"pm2 start {filepath} --name {name} --interpreter {interpreter}" - - if not autorestart: - command += ' --no-autorestart' - if force: - command += ' -f ' - command = command + f' -- --fn {meta_fn} --kwargs "{kwargs_str}"' - env = {} - if device != None: - if isinstance(device, int): - env['CUDA_VISIBLE_DEVICES']=str(device) - if isinstance(device, list): - env['CUDA_VISIBLE_DEVICES']=','.join(list(map(str, device))) - if refresh: - cls.pm2_kill(name) - - cwd = cwd or module.dirpath() - - stdout = c.cmd(command, env=env, verbose=verbose, cwd=cwd) - return {'success':True, 'message':f'Launched {module}', 'command': command, 'stdout':stdout} - - - - - - @classmethod - def remote_fn(cls, - fn: str='train', - module: str = None, - args : list = None, - kwargs : dict = None, - name : str =None, - tag: str = None, - refresh : bool =True, - mode = 'pm2', - tag_seperator : str = '::', - cwd = None, - **extra_launch_kwargs - ): - - kwargs = c.locals2kwargs(kwargs) - if 'remote' in kwargs: - kwargs['remote'] = False - if len(fn.split('.'))>1: - module = '.'.join(fn.split('.')[:-1]) - fn = fn.split('.')[-1] - - kwargs = kwargs if kwargs else {} - args = args if args else [] - if 'remote' in kwargs: - kwargs['remote'] = False - - cwd = cwd or cls.dirpath() - kwargs = kwargs or {} - args = args or [] - module = cls.resolve_object(module) - # resolve the name - if name == None: - # if the module has a module_path function, use that as the name - if hasattr(module, 'module_path'): - name = module.module_name() - else: - name = module.__name__.lower() - - c.print(f'[bold cyan]Launching --> <<[/bold cyan][bold yellow]class:{module.__name__}[/bold yellow] [bold white]name[/bold white]:{name} [bold white]fn[/bold white]:{fn} [bold white]mode[/bold white]:{mode}>>', color='green') - - launch_kwargs = dict( - module=module, - fn = fn, - name=name, - tag=tag, - args = args, - kwargs = kwargs, - refresh=refresh, - **extra_launch_kwargs - ) - assert fn != None, 'fn must be specified for pm2 launch' - - return cls.launch(**launch_kwargs) - @classmethod def shortcuts(cls, cache=True) -> Dict[str, str]: - - return cls.get_yaml(os.path.dirname(__file__)+ '/module.yaml').get('shortcuts') - + return cls.get_yaml(os.path.dirname(__file__)+ '/module.yaml' ).get('shortcuts') + def __repr__(self) -> str: return f'<{self.class_name()}' def __str__(self) -> str: return f'<{self.class_name()}' - - - def pytest(self): - test_path = c.root_path + '/tests' - return c.cmd(f'pytest {test_path}', verbose=True) - - - - def check_word(self, word:str)-> str: - files = c.glob('./') - progress = c.tqdm(len(files)) - for f in files: - try: - text = c.get_text(f) - except Exception as e: - continue - if word in text: - return True - progress.update(1) - return False - - def file2text(self, path = './', relative=True, **kwargs): - path = os.path.abspath(path) - file2text = {} - for file in c.glob(path, recursive=True): - with open(file, 'r') as f: - content = f.read() - file2text[file] = content - if relative: - print(path) - return {k[len(path)+1:]:v for k,v in file2text.items()} - return file2text - - - def file2lines(self, path:str='./')-> List[str]: - file2text = self.file2text(path) - file2lines = {f: text.split('\n') for f, text in file2text.items()} - return file2lines - + @classmethod + def get_commune(cls): + from commune import c + return c - def num_files(self, path:str='./')-> int: - return len(c.glob(path)) - - def hidden_files(self, path:str='./')-> List[str]: - path = self.resolve_path(path) - files = [f[len(path)+1:] for f in c.glob(path)] - print(files) - hidden_files = [f for f in files if f.startswith('.')] - return hidden_files - - def find_word(self, word:str, path='./')-> str: - path = c.resolve_path(path) - files = c.glob(path) - progress = c.tqdm(len(files)) - found_files = {} - for f in files: - try: - text = c.get_text(f) - if word not in text: - continue - lines = text.split('\n') - except Exception as e: - continue - - line2text = {i:line for i, line in enumerate(lines) if word in line} - found_files[f[len(path)+1:]] = line2text - progress.update(1) - return found_files - - - # def update(self): - # c.ip(update=1) - # return c.namespace(update=1, public=1) - - - # def update_loop(self, interval:int = 1): - # while True: - # self.update() - # c.namespace(public=1) - # time.sleep(interval) def pull(self): return c.cmd('git pull', verbose=True, cwd=c.libpath) @@ -1047,14 +625,6 @@ def base_config(cls, cache=True): return cls._base_config cls._base_config = cls.get_yaml(cls.config_path()) return cls._base_config - - @classmethod - def cfg(cls): - base_cfg = cls.base_config() - local_cfg = cls.local_config() - - return base_cfg - @classmethod def local_config(cls, filename_options = ['module', 'commune', 'config', 'cfg'], cache=True): @@ -1069,31 +639,6 @@ def local_config(cls, filename_options = ['module', 'commune', 'config', 'cfg'], cls._local_config = local_config return cls._local_config - @classmethod - def has_local_module(cls, path=None): - path = '.' if path == None else path - if os.path.exists(f'{path}/module.py'): - text = c.get_text(f'{path}/module.py') - if 'class ' in text: - return True - return False - - - def wordinfolder(self, word:str, path:str='./')-> bool: - path = c.resolve_path(path) - files = c.glob(path) - progress = c.tqdm(len(files)) - for f in files: - try: - text = c.get_text(f) - except Exception as e: - continue - if word in text: - return True - progress.update(1) - return False - - # local update @classmethod @@ -1132,6 +677,9 @@ def sign(self, data:dict = None, key: str = None, **kwargs) -> bool: def logs(self, name:str = None, verbose: bool = False): return c.pm2_logs(name, verbose=verbose) + + def hardware(self, *args, **kwargs): + return c.obj('commune.utils.os.hardware')(*args, **kwargs) c.enable_routes() Module = c # Module is alias of c diff --git a/commune/modules/model/model.py b/commune/modules/model/model.py index f4022f5b..1ae14706 100644 --- a/commune/modules/model/model.py +++ b/commune/modules/model/model.py @@ -368,10 +368,7 @@ def quantize(cls,model:str,dynamic_q_layer : set = {torch.nn.Linear}, dtype=torc dtype=torch.qint8, **kwargs) return self - @classmethod - def resolve_server_name(cls, *args, **kwargs): - return cls.base_model().resolve_server_name(*args, **kwargs) - + @staticmethod def get_trainable_params(model: 'nn.Module') -> int: """ diff --git a/commune/server/manager.py b/commune/server/manager.py index 656c2eb0..752c3838 100644 --- a/commune/server/manager.py +++ b/commune/server/manager.py @@ -235,3 +235,129 @@ def serve(cls, 'name':name, 'kwargs': kwargs, 'module':module} + + + + @classmethod + def launch(cls, + module:str = None, + fn: str = 'serve', + name:Optional[str]=None, + tag : str = None, + args : list = None, + kwargs: dict = None, + device:str=None, + interpreter:str='python3', + autorestart: bool = True, + verbose: bool = False , + force:bool = True, + meta_fn: str = 'module_fn', + tag_seperator:str = '::', + cwd = None, + refresh:bool=True ): + import commune as c + + if hasattr(module, 'module_name'): + module = module.module_name() + + # avoid these references fucking shit up + args = args if args else [] + kwargs = kwargs if kwargs else {} + + # convert args and kwargs to json strings + kwargs = { + 'module': module , + 'fn': fn, + 'args': args, + 'kwargs': kwargs + } + + kwargs_str = json.dumps(kwargs).replace('"', "'") + + name = name or module + if refresh: + cls.pm2_kill(name) + module = c.module() + # build command to run pm2 + filepath = c.filepath() + cwd = cwd or module.dirpath() + command = f"pm2 start {filepath} --name {name} --interpreter {interpreter}" + + if not autorestart: + command += ' --no-autorestart' + if force: + command += ' -f ' + command = command + f' -- --fn {meta_fn} --kwargs "{kwargs_str}"' + env = {} + if device != None: + if isinstance(device, int): + env['CUDA_VISIBLE_DEVICES']=str(device) + if isinstance(device, list): + env['CUDA_VISIBLE_DEVICES']=','.join(list(map(str, device))) + if refresh: + cls.pm2_kill(name) + + cwd = cwd or module.dirpath() + + stdout = c.cmd(command, env=env, verbose=verbose, cwd=cwd) + return {'success':True, 'message':f'Launched {module}', 'command': command, 'stdout':stdout} + + + + + + @classmethod + def remote_fn(cls, + fn: str='train', + module: str = None, + args : list = None, + kwargs : dict = None, + name : str =None, + tag: str = None, + refresh : bool =True, + mode = 'pm2', + tag_seperator : str = '::', + cwd = None, + **extra_launch_kwargs + ): + import commune as c + + kwargs = c.locals2kwargs(kwargs) + if 'remote' in kwargs: + kwargs['remote'] = False + if len(fn.split('.'))>1: + module = '.'.join(fn.split('.')[:-1]) + fn = fn.split('.')[-1] + + kwargs = kwargs if kwargs else {} + args = args if args else [] + if 'remote' in kwargs: + kwargs['remote'] = False + + cwd = cwd or cls.dirpath() + kwargs = kwargs or {} + args = args or [] + module = cls.resolve_object(module) + # resolve the name + if name == None: + # if the module has a module_path function, use that as the name + if hasattr(module, 'module_path'): + name = module.module_name() + else: + name = module.__name__.lower() + + c.print(f'[bold cyan]Launching --> <<[/bold cyan][bold yellow]class:{module.__name__}[/bold yellow] [bold white]name[/bold white]:{name} [bold white]fn[/bold white]:{fn} [bold white]mode[/bold white]:{mode}>>', color='green') + + launch_kwargs = dict( + module=module, + fn = fn, + name=name, + tag=tag, + args = args, + kwargs = kwargs, + refresh=refresh, + **extra_launch_kwargs + ) + assert fn != None, 'fn must be specified for pm2 launch' + + return cls.launch(**launch_kwargs) diff --git a/commune/server/middleware.py b/commune/server/middleware.py index aa5bacf5..dc7cc6b6 100644 --- a/commune/server/middleware.py +++ b/commune/server/middleware.py @@ -6,16 +6,13 @@ class ServerMiddleware(BaseHTTPMiddleware): def __init__(self, app, max_bytes: int): super().__init__(app) self.max_bytes = max_bytes - async def dispatch(self, request: Request, call_next): content_length = request.headers.get('content-length') if content_length: if int(content_length) > self.max_bytes: return JSONResponse(status_code=413, content={"error": "Request too large"}) - body = await request.body() if len(body) > self.max_bytes: return JSONResponse(status_code=413, content={"error": "Request too large"}) - response = await call_next(request) return response \ No newline at end of file diff --git a/commune/server/server.py b/commune/server/server.py index f591f752..27c43a3e 100644 --- a/commune/server/server.py +++ b/commune/server/server.py @@ -67,7 +67,6 @@ def add_fn(self, name:str, fn: str): assert callable(fn), 'fn not callable' setattr(self.module, name, fn) return {'success':True, 'message':f'Added {name} to {self.name} module'} - def forward(self, fn, request: Request): headers = dict(request.headers.items()) # STEP 1 : VERIFY THE SIGNATURE AND STALENESS OF THE REQUEST TO MAKE SURE IT IS AUTHENTIC @@ -80,7 +79,6 @@ def forward(self, fn, request: Request): signature_data = {'data': headers['hash'], 'timestamp': headers['timestamp']} assert c.verify(auth=signature_data, signature=headers['signature'], address=key_address) self.access_module.verify(fn=fn, address=key_address) - # STEP 2 : PREPARE THE DATA FOR THE FUNCTION CALL if 'params' in data: data['kwargs'] = data['params'] diff --git a/commune/user/user.py b/commune/user/user.py index e33131ab..d8fb5294 100644 --- a/commune/user/user.py +++ b/commune/user/user.py @@ -18,30 +18,29 @@ def role2users(self): role2users[role] = [] role2users[role].append(user) return role2users - @classmethod - def add_user(cls, address, role='user', name=None, **kwargs): + + def add_user(self, address, role='user', name=None, **kwargs): assert c.valid_ss58_address(address), f'{address} is not a valid address' - users = cls.get('users', {}) + users = self.get('users', {}) info = {'role': role, 'name': name, **kwargs} users[address] = info - cls.put('users', users) + self.put('users', users) return {'success': True, 'user': address,'info':info} - @classmethod - def users(cls, role=None): - users = cls.get('users', {}) + def users(self, role=None): + users = self.get('users', {}) root_key_address = c.root_key().ss58_address if root_key_address not in users: - cls.add_admin(root_key_address) + self.add_admin(root_key_address) if role is not None: return {k:v for k,v in users.items() if v['role'] == role} - return cls.get('users', {}) + return self.get('users', {}) def roles(self): return list(set([v['role'] for k,v in self.users().items()])) - @classmethod + def is_user(self, address): return address in self.users() @@ -66,59 +65,59 @@ def blacklisted(self): return self.get('blacklist', []) - @classmethod - def get_user(cls, address): - users = cls.users() + + def get_user(self, address): + users = self.users() return users.get(address, None) - @classmethod - def update_user(cls, address, **kwargs): - info = cls.get_user(address) + + def update_user(self, address, **kwargs): + info = self.get_user(address) info.update(kwargs) - return cls.add_user(address, **info) - @classmethod - def get_role(cls, address:str, verbose:bool=False): + return self.add_user(address, **info) + + def get_role(self, address:str, verbose:bool=False): try: - return cls.get_user(address)['role'] + return self.get_user(address)['role'] except Exception as e: c.print(e, color='red', verbose=verbose) return None - @classmethod - def refresh_users(cls): - cls.put('users', {}) - assert len(cls.users()) == 0, 'users not refreshed' + + def refresh_users(self): + self.put('users', {}) + assert len(self.users()) == 0, 'users not refreshed' return {'success': True, 'msg': 'refreshed users'} - @classmethod - def user_exists(cls, address:str): - return address in cls.get('users', {}) + + def user_exists(self, address:str): + return address in self.get('users', {}) - @classmethod - def is_root_key(cls, address:str)-> str: + + def is_root_key(self, address:str)-> str: return address == c.root_key().ss58_address - @classmethod - def is_admin(cls, address:str): - return cls.get_role(address) == 'admin' - - @classmethod - def admins(cls): - return [k for k,v in cls.users().items() if v['role'] == 'admin'] - @classmethod - def add_admin(cls, address): - return cls.add_user(address, role='admin') - @classmethod - def rm_admin(cls, address): - return cls.rm_user(address) - @classmethod - def num_roles(cls, role:str): - return len([k for k,v in cls.users().items() if v['role'] == role]) - - @classmethod - def rm_user(cls, address): - users = cls.get('users', {}) + + def is_admin(self, address:str): + return self.get_role(address) == 'admin' + + + def admins(self): + return [k for k,v in self.users().items() if v['role'] == 'admin'] + + def add_admin(self, address): + return self.add_user(address, role='admin') + + def rm_admin(self, address): + return self.rm_user(address) + + def num_roles(self, role:str): + return len([k for k,v in self.users().items() if v['role'] == role]) + + + def rm_user(self, address): + users = self.get('users', {}) users.pop(address) - cls.put('users', users) + self.put('users', users) - assert not cls.user_exists(address), f'{address} still in users' - return {'success': True, 'msg': f'removed {address} from users', 'users': cls.users()} + assert not self.user_exists(address), f'{address} still in users' + return {'success': True, 'msg': f'removed {address} from users', 'users': self.users()} def df(self): @@ -132,11 +131,11 @@ def df(self): df = pd.DataFrame(df) return df - @classmethod - def app(cls): + + def app(self): import streamlit as st st.write('### Users') - self = cls() + self = self() users = self.users() @@ -162,32 +161,9 @@ def app(cls): [cols[1].write('\n') for i in range(2)] add_user = cols[1].button(f'Remove {rm_user_address[:4]}...') if add_user: - response = getattr(cls, f'rm_user')(add_user_address) + response = getattr(self, f'rm_user')(add_user_address) st.write(response) - def test_blacklisting(self): - blacklist = self.blacklisted() - key = c.get_key('test') - assert key.ss58_address not in self.blacklisted(), 'key already blacklisted' - self.blacklist_user(key.ss58_address) - assert key.ss58_address in self.blacklisted(), 'key not blacklisted' - self.whitelist_user(key.ss58_address) - assert key.ss58_address not in self.blacklisted(), 'key not whitelisted' - return {'success': True, 'msg': 'blacklist test passed'} - - def test_blacklisting(self): - blacklist = self.blacklisted() - key = c.get_key('test') - assert key.ss58_address not in self.blacklisted(), 'key already blacklisted' - self.blacklist_user(key.ss58_address) - assert key.ss58_address in self.blacklisted(), 'key not blacklisted' - self.whitelist_user(key.ss58_address) - assert key.ss58_address not in self.blacklisted(), 'key not whitelisted' - return {'success': True, 'msg': 'blacklist test passed'} - - - - User.run(__name__) diff --git a/commune/utils/asyncio.py b/commune/utils/asyncio.py index a539a197..7d9808cd 100755 --- a/commune/utils/asyncio.py +++ b/commune/utils/asyncio.py @@ -1,6 +1,7 @@ import asyncio -import aiofiles + async def async_read(path, mode='r'): + import aiofiles async with aiofiles.open(path, mode=mode) as f: data = await f.read() return data diff --git a/commune/utils/os.py b/commune/utils/os.py index edad806c..eabe2502 100644 --- a/commune/utils/os.py +++ b/commune/utils/os.py @@ -1,12 +1,15 @@ import os -from typing import * import shutil import subprocess import shlex +import sys +from typing import * def resolve_path(path): - return os.path.abspath(os.path.expanduser(path)) + path = os.path.expanduser(path) + path = os.path.abspath(path) + return path def check_pid(pid): """ Check For the existence of a unix pid. """ @@ -176,27 +179,27 @@ def memory_info(fmt='gb'): return response -def virtual_memory_available(cls): +def virtual_memory_available(): import psutil return psutil.virtual_memory().available -def virtual_memory_total(cls): +def virtual_memory_total(): import psutil return psutil.virtual_memory().total -def virtual_memory_percent(cls): +def virtual_memory_percent(): import psutil return psutil.virtual_memory().percent -def cpu_type(cls): +def cpu_type(): import platform return platform.processor() -def cpu_info(cls): +def cpu_info(): return { 'cpu_count': cpu_count(), @@ -211,20 +214,15 @@ def cpu_usage(self): return cpu_usage - - -def gpu_memory(cls): +def gpu_memory(): import torch return torch.cuda.memory_allocated() - -def num_gpus(cls): +def num_gpus(): import torch return torch.cuda.device_count() - - -def gpus(cls): +def gpus(): return list(range(num_gpus())) def add_rsa_key(b=2048, t='rsa'): @@ -390,12 +388,12 @@ def cp(path1:str, path2:str, refresh:bool = False): -def cuda_available(cls) -> bool: +def cuda_available() -> bool: import torch return torch.cuda.is_available() -def free_gpu_memory(cls): +def free_gpu_memory(): gpu_info = gpu_info() return {gpu_id: gpu_info['free'] for gpu_id, gpu_info in gpu_info.items()} @@ -446,7 +444,6 @@ def gpu_info(fmt='gb') -> Dict[int, Dict[str, float]]: gpu_map =gpu_info - def hardware(fmt:str='gb'): return { 'cpu': cpu_info(), @@ -455,8 +452,6 @@ def hardware(fmt:str='gb'): 'gpu': gpu_info(fmt=fmt), } - - def get_folder_size(folder_path:str='/'): folder_path = resolve_path(folder_path) """Calculate the total size of all files in the folder.""" @@ -468,7 +463,6 @@ def get_folder_size(folder_path:str='/'): total_size += os.path.getsize(file_path) return total_size - def find_largest_folder(directory: str = '~/'): directory = resolve_path(directory) """Find the largest folder in the given directory.""" @@ -485,12 +479,9 @@ def find_largest_folder(directory: str = '~/'): return largest_folder, largest_size - - def getcwd(*args, **kwargs): return os.getcwd(*args, **kwargs) - def argv(include_script:bool = False): import sys args = sys.argv @@ -499,7 +490,6 @@ def argv(include_script:bool = False): else: return args[1:] - def mv(path1, path2): assert os.path.exists(path1), path1 if not os.path.isdir(path2): @@ -511,21 +501,18 @@ def mv(path1, path2): assert not os.path.exists(path1), path1 return {'success': True, 'msg': f'Moved {path1} to {path2}'} - -def sys_path(cls): +def sys_path(): return sys.path - -def gc(cls): +def gc(): gc.collect() return {'success': True, 'msg': 'garbage collected'} - def get_pid(): return os.getpid() -def nest_asyncio(cls): +def nest_asyncio(): import nest_asyncio nest_asyncio.apply() diff --git a/commune/utils/torch.py b/commune/utils/torch.py index b59c3816..307dba34 100644 --- a/commune/utils/torch.py +++ b/commune/utils/torch.py @@ -130,8 +130,6 @@ def confuse_gradients(model): p.grad.data = torch.randn(p.grad.data.shape).to(p.grad.data.device) - - def get_device_memory(): import nvidia_smi diff --git a/commune/vali/parity/vali_parity.py b/commune/vali/parity/vali_parity.py deleted file mode 100644 index 45050569..00000000 --- a/commune/vali/parity/vali_parity.py +++ /dev/null @@ -1,103 +0,0 @@ -import commune as c - -class ValiParity(c.Module): - def __init__(self, run=True): - self.subspace = c.module('subspace')() - self.subnet = self.subspace.subnet() - if run: - c.thread(self.run) - self.seconds_per_epoch = self.subnet['tempo'] * 8 - - def votes(self, max_trust = 25) -> int: - modules = self.subspace.modules() - voted_modules = c.shuffle([m for m in modules if m['trust'] < max_trust])[:self.subnet['max_allowed_weights']] - uids = [m['uid'] for m in voted_modules] - max_trust = max([m['trust'] for m in voted_modules]) - weights = [max_trust - m['trust'] for m in voted_modules] - return {'uids': uids, 'weights': weights} - - - def run(self): - while True: - c.print('voting...') - r = self.vote() - c.print(r) - self.sleep(self.seconds_per_epoch) - def vote(self, key=None): - key = self.resolve_key(key) - try: - votes = self.votes() - response = self.subspace.vote(**votes, key=key) - except Exception as e: - self.subspace = c.module('subspace')() - e = c.detailed_error(e) - c.print(e) - return response - - @classmethod - def regloop(cls, n=100, tag='commie', remote: str = True, key=None, timeout=30): - - if remote: - kwargs = c.locals2kwargs(locals()) - kwargs['remote'] = False - return cls.remote_fn('regloop', kwargs=kwargs) - - - - cnt = 0 - self = cls(run=False) - - while True: - registered_servers = [] - subspace = c.module('subspace')() - namespace = subspace.namespace(search=self.module_path()) - - c.print('registered servers', namespace) - ip = c.ip() - i = 0 - name2futures = {} - while cnt < n: - - name = cls.resolve_server_name(tag=tag+str(i)) - module_key = c.get_key(name) - - futures = name2futures.get(name, []) - - address = ip + ':' + str(30333 + cnt) - - if name in namespace: - i += 1 - c.print('already registered', name) - continue - - try: - c.print('registering', name) - response = self.subspace.register(name=name, address=address, module_key=module_key.ss58_address, key=key) - if not response['success']: - if response['error']['name'] == 'NameAlreadyRegistered': - i += 1 - c.print(response) - except Exception as e: - e = c.detailed_error(e) - c.print(e) - - @classmethod - def vote_loop(cls, remote: str = True, min_staleness=200, search='vali', namespace=None, network=None): - if remote: - kwargs = c.locals2kwargs(locals()) - kwargs['remote'] = False - return cls.remote_fn('voteloop', kwargs=kwargs) - - self = cls(run=False) - stats = c.stats(search=search, df=False) - for module in stats: - c.print('voting for', module['name']) - if module['last_update'] > min_staleness: - c.print('skipping', module['name'], 'because it is was voted recently') - c.vote(key=module['name']) - vootloop = vote_loop - - - - - \ No newline at end of file