#13 support for endpoint includ files

WeBankPartners · Dec 7, 2020 · ed01035 · ed01035
1 parent 0b15fa4
commit ed01035
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 4 deletions.
diff --git a/init.sql b/init.sql
@@ -238,3 +238,28 @@ UNLOCK TABLES;
 
 SET FOREIGN_KEY_CHECKS = 1;
 -- Dump completed on 2020-11-23 17:08:28
+
+#@v0.1.0.1-begin@;
+UPDATE `rule` SET match_value='{force set ''''}{recursive set ''''}{help notset ''''}{version notset ''''}{path like ''*''}'
+  WHERE id=1;
+UPDATE `rule` SET match_value='{help notset ''''}'
+  WHERE id=2;
+UPDATE `rule` SET match_value='{force set ''''}{recursive set ''''}{help notset ''''}{version notset ''''}{path eq ''/''}'
+  WHERE id=6;
+UPDATE `rule` SET match_value='{name in [''KILL'', ''TERM'']}{pid set ''''}'
+  WHERE id=12;
+UPDATE `rule` SET match_value='{number in [''9'', ''15'']}{pid set ''''}'
+  WHERE id=13;
+UPDATE `rule` SET match_value='{s_kill set ''''}{pid set ''''}'
+  WHERE id=14;
+UPDATE `rule` SET match_value=''
+  WHERE id=15;
+UPDATE `rule` SET match_value=''
+  WHERE id=16;
+UPDATE `rule` SET match_value='{interact set ''''}{command ilike ''/dev/tcp/''}'
+  WHERE id=17;
+
+ALTER TABLE `service_script` ADD endpoint_include varchar(255) NULL;
+ALTER TABLE `service_script` CHANGE endpoint_include endpoint_include varchar(255) NULL AFTER endpoint_field;
+
+#@v0.1.0.1-end@;
diff --git a/wecube_plugins_itsdangerous/apps/processor/api.py b/wecube_plugins_itsdangerous/apps/processor/api.py
@@ -2,12 +2,14 @@
 
 from __future__ import absolute_import
 
+import fnmatch
 import glob
 import hashlib
 import json
 import logging
 import os
 import os.path
+import re
 import tempfile
 
 import requests
@@ -165,7 +167,16 @@ def _file_extension_supported(extensions, filename):
             if content:
                 scripts.append({'type': content_type, 'content': content, 'name': None})
         # if endpoint_field present, download from endpoint(s3/artifacts url from platform)
+        common_spliter = r',|\||;'
         if script_location['endpoint_field']:
+            user_include = script_location['endpoint_include']
+            user_include_files = None
+            if user_include:
+                user_include_files = set()
+                user_include_fields = re.split(common_spliter, user_include)
+                for field in user_include_fields:
+                    field_content = utils.get_item(plugin_param, field)
+                    user_include_files = user_include_files | set(re.split(common_spliter, field_content))
             endpoint_url = utils.get_item(plugin_param, script_location['endpoint_field'])
             filepath, filename = ensure_url_cached(endpoint_url)
             packed_extensions = [
@@ -179,19 +190,41 @@ def _file_extension_supported(extensions, filename):
                 for name in glob.glob(os.path.join(unzip_path, '**/*' + shell_extension), recursive=True):
                     if not os.path.isfile(name):
                         continue
-                    with open(name, 'r') as f:
-                        scripts.append({'type': 'shell', 'content': f.read(), 'name': name[len(unzip_path) + 1:]})
+                    shortpath = name[len(unzip_path) + 1:]
+                    included = False
+                    if user_include_files is not None:
+                        for user_include_file in user_include_files:
+                            if fnmatch.fnmatch(shortpath, user_include_file):
+                                included = True
+                                break
+                    else:
+                        included = True
+                    if included:
+                        with open(name, 'r') as f:
+                            scripts.append({'type': 'shell', 'content': f.read(), 'name': shortpath})
                 for name in glob.glob(os.path.join(unzip_path, '**/*' + sql_extension), recursive=True):
                     if not os.path.isfile(name):
                         continue
-                    with open(name, 'r') as f:
-                        scripts.append({'type': 'sql', 'content': f.read(), 'name': name[len(unzip_path) + 1:]})
+                    shortpath = name[len(unzip_path) + 1:]
+                    included = False
+                    if user_include_files is not None:
+                        for user_include_file in user_include_files:
+                            if fnmatch.fnmatch(shortpath, user_include_file):
+                                included = True
+                                break
+                    else:
+                        included = True
+                    if included:
+                        with open(name, 'r') as f:
+                            scripts.append({'type': 'sql', 'content': f.read(), 'name': shortpath})
             elif filepath.endswith(shell_extension):
                 with open(filepath, 'r') as f:
                     scripts.append({'type': 'shell', 'content': f.read(), 'name': filename})
             elif filepath.endswith(sql_extension):
                 with open(filepath, 'r') as f:
                     scripts.append({'type': 'sql', 'content': f.read(), 'name': filename})
+            else:
+                LOG.warn('the type of this file[%s] could not be determined', filename)
         return scripts
 
 

diff --git a/wecube_plugins_itsdangerous/db/models_manage.py b/wecube_plugins_itsdangerous/db/models_manage.py
@@ -186,6 +186,7 @@ class ServiceScript(Base, DictBase):
     content_type = Column(String(36))
     content_field = Column(String(63))
     endpoint_field = Column(String(63))
+    endpoint_include = Column(String(255))
 
     created_by = Column(String(36), nullable=True)
     created_time = Column(DateTime, nullable=True)