Merge pull request #13 from baidu/on-dev-1.7.0

commit 1.6.1 version

cup/ChangeLog

@@ -1,4 +1,21 @@
+Version 1.6.1 - starting from 2018.2.5 ~ 2018.6.1
+	* [New] cup.shell.is_proc_alive - Add optional to abandon vim|less|vi|tail|cat|more or custom filter	
+	* [Bug] cup.shell.get_pid - Fix grep to surely abandon vim|less|vi|tail|cat|more 
+	* [New] cup.log - Add support for stack manipulation, which can pop out function calls.
+	* [New] cup.err - Add UnImplemented exception class.
+	* [New] cup.exfile - Support temp files which will be removed immediately after the variable life ends.
+	* [Enhancement] cup.util.conf - support $ in a conf key
+	* [Doc] cup.shell - Fix doc bug. 
+	* [New] cup.shell - Add grep support string with space
+	* [New] cup.storage.obj - Support common object storage apis including ftp, s3
+	* [Bug] cup.res.linux - Getting cpuinfo has bugs (new kernel 3.10)
+	* [Enhancement] - cup.util.threadpool, add daemon_threads as the parameter
+		that you can use to let the threadpool threads behave like daemon-thread
+		(when the main thread exits, it exits as well)
+	* [Enhancement] - cup.util.conf - support conf line "[.test] # comments" 
+
 Version 1.6.0 - starting from 2017.9.6 ~ 2017.12.29
+	* [New] cup.bidu.icafe - interact with baidu icafe.
 	* [New] MsgBroker - Add a broker for handling system failures
 	* [New] cup.
 	* [Bug] Linux Resource Query Bug - related to data columuns

cup/cache.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*
 # #############################################################################
 #
-#  Copyright (c) 2014 Baidu.com,  Inc. All Rights Reserved
+#  Copyright (c) Baidu.com,  Inc. All Rights Reserved
 #
 # #############################################################################
 """

cup/err.py

@@ -16,7 +16,8 @@
 __all__ = [
     'BaseCupException', 'DecoratorException', 'LoggerException',
     'ResException', 'NoSuchProcess', 'AccessDenied', 'NetException',
-    'AsyncMsgError', 'ThreadTermException', 'LockFileError'
+    'AsyncMsgError', 'ThreadTermException', 'LockFileError',
+    'NotImplementedYet'
 ]
 
 
@@ -154,4 +155,23 @@ def __init__(self, expect, got):
         msg = 'expect failure, expect {0}, got {1}'.format(expect, got)
         BaseCupException.__init__(self, msg)
 
+
+class NotImplementedYet(BaseCupException):
+    """
+    Not implemented yet
+    """
+    def __init__(self, msg=''):
+        msg = 'The functionality is not implemented yet, {0}'.format(msg)
+        BaseCupException.__init__(self, msg)
+
+
+class ConfigError(BaseCupException):
+    """
+    ConfigError
+    """
+    def __init__(self, msg=''):
+        msg = 'Configuration Error: {0}'.format(msg)
+        BaseCupException.__init__(self, msg)
+
+
 # vi:set tw=0 ts=4 sw=4 nowrap fdm=indent

cup/exfile.py

@@ -17,10 +17,14 @@
 
 from cup import err
 from cup import decorators
+from cup import platforms
+
+if platforms.is_linux():
+    import tempfile
 
 __all__ = [
     'LockFile', 'FILELOCK_SHARED', 'FILELOCK_EXCLUSIVE',
-    'FILELOCK_NONBLOCKING', 'FILELOCK_UNLOCK'
+    'FILELOCK_NONBLOCKING', 'FILELOCK_UNLOCK', 'TempFile'
 ]
 
 
@@ -116,9 +120,51 @@ def unlock(self):
             raise err.LockFileError(error)
 
 
-# TODO tempfile
-# class TempFile(object):
-#     """
-#     tempfile
-#     """
+class TempFile(object):
+    """
+    tempfile, the temp file will be deleted immediately after the lifetime.
+
+    You can use TempFile like the original Python File Object.
+
+    ::
+
+        tmp = TempFile('./')
+        tmp.write / read /seek / etc
+        tmp.close()
+    """
+    def __init__(self, filedir, prefix='', suffix=''):
+        """
+        :param filedir:
+            temp file dir which contains the temp file
+
+        :prefix:
+            prefix of the temp filename
+
+        :suffix:
+            suffix of the file name. e.g. '.tmp'.
+        """
+        self._fdir = filedir
+        self._fp = None
+        self._prefix = prefix
+        self._suffix = suffix
+
+    def __enter__(self):
+        """enter"""
+        if platforms.is_linux():
+            self._fp = tempfile.TemporaryFile(
+                dir=self._fdir,
+                prefix=self._prefix,
+                suffix=self._suffix
+            )
+            return self._fp
+        else:
+            raise err.NotImplementedYet('TemporaryFile')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        exit
+        """
+        if platforms.is_linux():
+            self._fp.close()
+
 # vi:set tw=0 ts=4 sw=4 nowrap fdm=indent

cup/log.py

@@ -17,7 +17,9 @@
     'debug', 'info', 'warn', 'critical',
     'init_comlog', 'setloglevel',
     'ROTATION', 'INFINITE',
-    'reinit_comlog', 'get_inited_loggername', 'parse'
+    'reinit_comlog', 'get_inited_loggername', 'parse',
+    'backtrace_info', 'backtrace_debug', 'backtrace_error',
+    'backtrace_critical'
 ]
 
 
@@ -336,78 +338,78 @@ def _fail_handle(msg, e):
     print '%s\nerror:%s' % (msg, e)
 
 
-# def info(msg, back_trace_len=0):
-#     """
-#     logging.INFO的日志打印
-#     """
-#     try:
-#         msg = _log_file_func_info(msg, back_trace_len)
-#         loggerman = _LoggerMan()
-#         loggerman._getlogger().info(msg)
-#     except err.LoggerException:
-#         return
-#     except Exception as e:
-#         _fail_handle(msg, e)
-#
-#
-# def debug(msg, back_trace_len=0):
-#     """
-#     :param msg:
-#         logging.DEBUG级别的日志打印。
-#     :param back_trace_len:
-#         为扩展预留的参数， 正常使用可忽略。
-#
-#     """
-#     try:
-#         msg = _log_file_func_info(msg, back_trace_len)
-#         loggerman = _LoggerMan()
-#         loggerman._getlogger().debug(msg)
-#     except err.LoggerException:
-#         return
-#     except Exception as e:
-#         _fail_handle(msg, e)
-#
-#
-# def warn(msg, back_trace_len=0):
-#     """
-#     logging.WARN级别的日志打印
-#     """
-#     try:
-#         msg = _log_file_func_info(msg, back_trace_len)
-#         loggerman = _LoggerMan()
-#         loggerman._getlogger().warn(msg)
-#     except err.LoggerException:
-#         return
-#     except Exception as e:
-#         _fail_handle(msg, e)
-#
-#
-# def error(msg, back_trace_len=0):
-#     """
-#     logging.ERROR级别的日志打印
-#     """
-#     try:
-#         msg = _log_file_func_info(msg, back_trace_len)
-#         loggerman = _LoggerMan()
-#         loggerman._getlogger().error(msg)
-#     except err.LoggerException:
-#         return
-#     except Exception as error:
-#         _fail_handle(msg, error)
-#
-#
-# def critical(msg, back_trace_len=0):
-#     """
-#     logging.CRITICAL级别的日志打印
-#     """
-#     try:
-#         msg = _log_file_func_info(msg, back_trace_len)
-#         loggerman = _LoggerMan()
-#         loggerman._getlogger().critical(msg)
-#     except err.LoggerException:
-#         return
-#     except Exception as e:
-#         _fail_handle(msg, e)
+def backtrace_info(msg, back_trace_len=0):
+    """
+    logging.INFO的日志打印
+    """
+    try:
+        msg = _log_file_func_info(msg, back_trace_len)
+        loggerman = _LoggerMan()
+        loggerman._getlogger().info(msg)
+    except err.LoggerException:
+        return
+    except Exception as e:
+        _fail_handle(msg, e)
+
+
+def backtrace_debug(msg, back_trace_len=0):
+    """
+    :param msg:
+        logging.DEBUG级别的日志打印。
+    :param back_trace_len:
+        为扩展预留的参数， 正常使用可忽略。
+
+    """
+    try:
+        msg = _log_file_func_info(msg, back_trace_len)
+        loggerman = _LoggerMan()
+        loggerman._getlogger().debug(msg)
+    except err.LoggerException:
+        return
+    except Exception as e:
+        _fail_handle(msg, e)
+
+
+def backtrace_warn(msg, back_trace_len=0):
+    """
+    logging.WARN级别的日志打印
+    """
+    try:
+        msg = _log_file_func_info(msg, back_trace_len)
+        loggerman = _LoggerMan()
+        loggerman._getlogger().warn(msg)
+    except err.LoggerException:
+        return
+    except Exception as e:
+        _fail_handle(msg, e)
+
+
+def backtrace_error(msg, back_trace_len=0):
+    """
+    logging.ERROR级别的日志打印
+    """
+    try:
+        msg = _log_file_func_info(msg, back_trace_len)
+        loggerman = _LoggerMan()
+        loggerman._getlogger().error(msg)
+    except err.LoggerException:
+        return
+    except Exception as error:
+        _fail_handle(msg, error)
+
+
+def backtrace_critical(msg, back_trace_len=0):
+    """
+    logging.CRITICAL级别的日志打印
+    """
+    try:
+        msg = _log_file_func_info(msg, back_trace_len)
+        loggerman = _LoggerMan()
+        loggerman._getlogger().critical(msg)
+    except err.LoggerException:
+        return
+    except Exception as e:
+        _fail_handle(msg, e)
 
 
 def setloglevel(logginglevel):

cup/net/async/conn.py

@@ -35,6 +35,7 @@
 
 import cup
 from cup import log
+from cup import err as cuperr
 from cup.util import misc
 from cup.util import threadpool
 from cup.services import executor
@@ -917,7 +918,7 @@ def _do_write(self, context):
                 succ_len = sock.send(data)
                 log.debug('succeed to send length:%d' % succ_len)
                 msg.seek_write(succ_len)
-            except err.AsyncMsgError as error:
+            except cuperr.AsyncMsgError as error:
                 log.debug('has seek out of msg len, continue')
             except socket.error as error:
                 err = error.args[0]
@@ -981,7 +982,7 @@ def add_write_job(self, context):
             log.info('failed to get peerinfo, return')
             return
         if not context.try_writelock():
-            log.info(
+            log.debug(
                 'Another thread is writing the context, return. '
                 'Peerinfo:%s:%s' %
                 (peerinfo[0], peerinfo[1])

cup/res/linux.py

@@ -15,15 +15,16 @@
 """
 
 import os
-import errno
-import sys
 import re
-import collections
-import warnings
+import sys
 import time
+import errno
 import socket
 import base64
 import struct
+import threading
+import warnings
+import collections
 
 import cup
 
@@ -81,11 +82,8 @@ def get_kernel_version():
     e.g.('2', '6', '32'):
 
     """
-    reg = re.compile(r'\d{1}\.\d{1,2}\.\d{1,3}')
-    cmd = '/bin/uname -a'
-    versions = cup.shell.execshell_withpipe_str(cmd, False)
-    version = reg.findall(versions)[0]
-
+    versions = os.uname()[2]
+    version = versions[0: versions.find('_')]
     return tuple([int(info) for info in version.split('.')])
 
 
@@ -193,10 +191,17 @@ class MemInfo(collections.namedtuple('vmem', ' '.join([
 ]
 
 
-def _get_cpu_columns():
-    version = get_kernel_version()
-    if version >= (2, 6, 33):
-        _CPU_COLUMNS.append('guest_nice')
+# def _get_cpu_columns():
+#     version = get_kernel_version()
+#     if version >= (2, 6, 33):
+#         _CPU_COLUMNS.append('guest_nice')
+
+_COLUMN_LOCK = threading.Lock()
+
+_COLUMN_LOCK.acquire()
+if get_kernel_version() >= (2, 6, 33) and _CPU_COLUMNS.count('guest_nice') <= 0:
+    _CPU_COLUMNS.append('guest_nice')
+_COLUMN_LOCK.release()
 
 
 class CPUInfo(collections.namedtuple('CPUInfo', _CPU_COLUMNS)):