Adding multiprocessing

.gitignore

@@ -61,4 +61,5 @@ target/
 
 # virtual env
 venv/
+.venv/
 

requirements.txt

@@ -4,3 +4,4 @@ requests>=2.10.0
 six
 xmltodict
 lxml
+pathos

rtcclient/base.py

@@ -1,3 +1,5 @@
+from typing import Tuple
+from pathos.pools import ThreadPool
 import abc
 import logging
 from rtcclient import requests
@@ -20,8 +22,7 @@ def __init__(self, url):
         self.url = self.validate_url(url)
 
     def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__,
-                            str(self))
+        return "<%s %s>" % (self.__class__.__name__, str(self))
 
     @abc.abstractmethod
     def __str__(self):
@@ -52,9 +53,7 @@ def get_rtc_obj(self):
         pass
 
     @token_expire_handler
-    def get(self, url,
-            verify=False, headers=None, proxies=None,
-            timeout=60, **kwargs):
+    def get(self, url, verify=False, headers=None, proxies=None, timeout=60, **kwargs):
         """Sends a GET request. Refactor from requests module
 
         :param url: URL for the new :class:`Request` object.
@@ -74,19 +73,33 @@ def get(self, url,
         """
 
         self.log.debug("Get response from %s", url)
-        response = requests.get(url, verify=verify, headers=headers,
-                                proxies=proxies, timeout=timeout, **kwargs)
+        response = requests.get(
+            url,
+            verify=verify,
+            headers=headers,
+            proxies=proxies,
+            timeout=timeout,
+            **kwargs
+        )
         if response.status_code != 200:
-            self.log.error('Failed GET request at <%s> with response: %s',
-                           url,
-                           response.content)
+            self.log.error(
+                "Failed GET request at <%s> with response: %s", url, response.content
+            )
             response.raise_for_status()
         return response
 
     @token_expire_handler
-    def post(self, url, data=None, json=None,
-             verify=False, headers=None, proxies=None,
-             timeout=60, **kwargs):
+    def post(
+        self,
+        url,
+        data=None,
+        json=None,
+        verify=False,
+        headers=None,
+        proxies=None,
+        timeout=60,
+        **kwargs
+    ):
         """Sends a POST request. Refactor from requests module
 
         :param url: URL for the new :class:`Request` object.
@@ -109,23 +122,39 @@ def post(self, url, data=None, json=None,
         :rtype: requests.Response
         """
 
-        self.log.debug("Post a request to %s with data: %s and json: %s",
-                       url, data, json)
-        response = requests.post(url, data=data, json=json,
-                                 verify=verify, headers=headers,
-                                 proxies=proxies, timeout=timeout, **kwargs)
+        self.log.debug(
+            "Post a request to %s with data: %s and json: %s", url, data, json
+        )
+        response = requests.post(
+            url,
+            data=data,
+            json=json,
+            verify=verify,
+            headers=headers,
+            proxies=proxies,
+            timeout=timeout,
+            **kwargs
+        )
 
         if response.status_code not in [200, 201]:
-            self.log.error('Failed POST request at <%s> with response: %s',
-                           url,
-                           response.content)
+            self.log.error(
+                "Failed POST request at <%s> with response: %s", url, response.content
+            )
             self.log.info(response.status_code)
             response.raise_for_status()
         return response
 
     @token_expire_handler
-    def put(self, url, data=None, verify=False,
-            headers=None, proxies=None, timeout=60, **kwargs):
+    def put(
+        self,
+        url,
+        data=None,
+        verify=False,
+        headers=None,
+        proxies=None,
+        timeout=60,
+        **kwargs
+    ):
         """Sends a PUT request. Refactor from requests module
 
         :param url: URL for the new :class:`Request` object.
@@ -146,21 +175,27 @@ def put(self, url, data=None, verify=False,
         :rtype: requests.Response
         """
 
-        self.log.debug("Put a request to %s with data: %s",
-                       url, data)
-        response = requests.put(url, data=data,
-                                verify=verify, headers=headers,
-                                proxies=proxies, timeout=timeout, **kwargs)
+        self.log.debug("Put a request to %s with data: %s", url, data)
+        response = requests.put(
+            url,
+            data=data,
+            verify=verify,
+            headers=headers,
+            proxies=proxies,
+            timeout=timeout,
+            **kwargs
+        )
         if response.status_code not in [200, 201]:
-            self.log.error('Failed PUT request at <%s> with response: %s',
-                           url,
-                           response.content)
+            self.log.error(
+                "Failed PUT request at <%s> with response: %s", url, response.content
+            )
             response.raise_for_status()
         return response
 
     @token_expire_handler
-    def delete(self, url, headers=None, verify=False, proxies=None,
-               timeout=60, **kwargs):
+    def delete(
+        self, url, headers=None, verify=False, proxies=None, timeout=60, **kwargs
+    ):
         """Sends a DELETE request. Refactor from requests module
 
         :param url: URL for the new :class:`Request` object.
@@ -179,18 +214,19 @@ def delete(self, url, headers=None, verify=False, proxies=None,
         :rtype: requests.Response
         """
 
-        self.log.debug("Delete a request to %s",
-                       url)
-        response = requests.delete(url,
-                                   headers=headers,
-                                   verify=verify,
-                                   proxies=proxies,
-                                   timeout=timeout,
-                                   **kwargs)
+        self.log.debug("Delete a request to %s", url)
+        response = requests.delete(
+            url,
+            headers=headers,
+            verify=verify,
+            proxies=proxies,
+            timeout=timeout,
+            **kwargs
+        )
         if response.status_code not in [200, 201]:
-            self.log.error('Failed DELETE request at <%s> with response: %s',
-                           url,
-                           response.content)
+            self.log.error(
+                "Failed DELETE request at <%s> with response: %s", url, response.content
+            )
             response.raise_for_status()
         return response
 
@@ -207,7 +243,7 @@ def validate_url(cls, url):
             return None
 
         url = url.strip()
-        while url.endswith('/'):
+        while url.endswith("/"):
             url = url[:-1]
         return url
 
@@ -236,15 +272,17 @@ def get_rtc_obj(self):
     def _initialize(self):
         """Initialize the object from the request"""
 
-        self.log.debug("Start initializing data from %s",
-                       self.url)
-        resp = self.get(self.url,
-                        verify=False,
-                        proxies=self.rtc_obj.proxies,
-                        headers=self.rtc_obj.headers)
+        self.log.debug("Start initializing data from %s", self.url)
+        resp = self.get(
+            self.url,
+            verify=False,
+            proxies=self.rtc_obj.proxies,
+            headers=self.rtc_obj.headers,
+        )
         self.__initialize(resp)
-        self.log.info("Finish the initialization for <%s %s>",
-                      self.__class__.__name__, self)
+        self.log.info(
+            "Finish the initialization for <%s %s>", self.__class__.__name__, self
+        )
 
     def __initialize(self, resp):
         """Initialize from the response"""
@@ -257,11 +295,12 @@ def __initialize(self, resp):
     def __initializeFromRaw(self):
         """Initialze from raw data (OrderedDict)"""
 
-        for (key, value) in self.raw_data.items():
+        def process_items(item) -> Tuple:
+            key, value = item
             if key.startswith("@"):
                 # be compatible with IncludedInBuild
                 if "@oslc_cm:label" != key:
-                    continue
+                    return None
 
             attr = key.split(":")[-1].replace("-", "_")
             attr_list = attr.split(".")
@@ -270,9 +309,7 @@ def __initializeFromRaw(self):
             if len(attr_list) > 1:
                 # attr = "_".join([attr_list[-2],
                 #                  attr_list[-1]])
-                continue
-
-            self.field_alias[attr] = key
+                return None
 
             if isinstance(value, OrderedDict):
                 value_text = value.get("#text")
@@ -286,7 +323,15 @@ def __initializeFromRaw(self):
                         value = self.__get_rdf_resource_title(value)
                     except (exception.RTCException, Exception):
                         self.log.error("Unable to handle %s", value)
-            self.setattr(attr, value)
+            return key, attr, value
+
+        with ThreadPool() as pool:
+            for processed in pool.map(process_items, self.raw_data.items()):
+                if processed is None:
+                    continue
+                key, attr, value = processed
+                self.field_alias[attr] = key
+                self.setattr(attr, value)
 
     def __get_rdf_resource_title(self, rdf_url):
         # handle for /jts/users
@@ -301,10 +346,12 @@ def __get_rdf_resource_title(self, rdf_url):
         if "/resource/content/" in rdf_url:
             return rdf_url
 
-        resp = self.get(rdf_url,
-                        verify=False,
-                        proxies=self.rtc_obj.proxies,
-                        headers=self.rtc_obj.headers)
+        resp = self.get(
+            rdf_url,
+            verify=False,
+            proxies=self.rtc_obj.proxies,
+            headers=self.rtc_obj.headers,
+        )
         raw_data = xmltodict.parse(resp.content)
 
         root_key = list(raw_data.keys())[0]
@@ -313,13 +360,17 @@ def __get_rdf_resource_title(self, rdf_url):
             # no total count
             # only single resource
             # compatible with IncludedInBuild
-            return (raw_data[root_key].get("dc:title") or
-                    raw_data[root_key].get("foaf:nick"))
+            return raw_data[root_key].get("dc:title") or raw_data[root_key].get(
+                "foaf:nick"
+            )
         else:
             # multiple resource
             result_list = list()
-            entry_keys = [entry_key for entry_key in raw_data[root_key].keys()
-                          if not entry_key.startswith("@")]
+            entry_keys = [
+                entry_key
+                for entry_key in raw_data[root_key].keys()
+                if not entry_key.startswith("@")
+            ]
             for entry_key in entry_keys:
                 entries = raw_data[root_key][entry_key]
                 if isinstance(entries, OrderedDict):
@@ -336,9 +387,7 @@ def __get_rdf_resource_title(self, rdf_url):
 
     def __handle_rdf_entry(self, entry):
         # only return useful info instead of the whole object
-        return_fields = ["rtc_cm:userId",
-                         "dc:title",
-                         "dc:description"]
+        return_fields = ["rtc_cm:userId", "dc:title", "dc:description"]
         subkeys = entry.keys()
         for return_field in return_fields:
             if return_field in subkeys: