update changes to reflect https authentication for opensearch

opl/http.py

@@ -8,6 +8,12 @@
 session = requests.Session()
 
 
+def insecure():
+    session.verify = False
+    logging.debug("Disabling SSL verifications for this session")
+    disable_insecure_request_warnings(True)
+
+
 def disable_insecure_request_warnings(disable_it):
     if disable_it:
         logging.debug("Disabling insecure request warnings")

opl/investigator/config.py

@@ -75,6 +75,15 @@ def load_config(conf, fp):
         assert not conf.history_es_server.endswith("/")
         conf.history_es_index = data["history"]["es_index"]
         conf.history_es_query = data["history"]["es_query"]
+        if "es_server_user" in data["history"]:
+            conf.history_es_server_user = data["history"]["es_server_user"]
+            conf.history_es_server_pass_env_var = data["history"][
+                "es_server_pass_env_var"
+            ]
+        if "es_server_verify" in data["history"]:
+            conf.history_es_server_verify = data["history"]["es_server_verify"]
+        else:
+            conf.history_es_server_verify = True
 
     if conf.history_type == "sd_dir":
         conf.history_dir = data["history"]["dir"]
@@ -88,6 +97,16 @@ def load_config(conf, fp):
         conf.decisions_es_server = data["decisions"]["es_server"]
         assert not conf.decisions_es_server.endswith("/")
         conf.decisions_es_index = data["decisions"]["es_index"]
+        if "es_server_user" in data["decisions"]:
+            conf.decisions_es_server_user = data["decisions"]["es_server_user"]
+            conf.decisions_es_server_pass_env_var = data["decisions"][
+                "es_server_pass_env_var"
+            ]
+        if "es_server_verify" in data["decisions"]:
+            conf.decisions_es_server_verify = data["decisions"]["es_server_verify"]
+        else:
+            conf.decisions_es_server_verify = True
+
     if conf.decisions_type == "csv":
         conf.decisions_filename = data["decisions"]["filename"]
 

opl/investigator/elasticsearch_decisions.py

@@ -6,7 +6,9 @@
 import requests
 
 
-def store(server, index, decisions):
+def store(server, index, decisions, **kwargs):
+    es_server_user = kwargs.get("es_server_user")
+    decisions_es_server_pass_env_var = kwargs.get("es_server_pass_env_var")
     # This is our workaround on how to add additional metadata about the decision
     job_name = os.environ.get("JOB_NAME", "")
     build_url = os.environ.get("BUILD_URL", "")
@@ -26,7 +28,17 @@ def store(server, index, decisions):
             f"Storing decision to ES url={url}, headers={headers} and json={json.dumps(decision)}"
         )
 
-        response = requests.post(url, headers=headers, json=decision)
+        if es_server_user and decisions_es_server_pass_env_var:
+            # fetch the password from Jenkins credentials
+            open_search_password = os.environ.get(decisions_es_server_pass_env_var)
+            response = requests.post(
+                url,
+                auth=requests.auth.HTTPBasicAuth(es_server_user, open_search_password),
+                headers=headers,
+                json=decision,
+            )
+        else:
+            response = requests.post(url, headers=headers, json=decision)
 
         if not response.ok:
             logging.warning(f"Failed to store decision to ES: {response.text}")

opl/investigator/elasticsearch_loader.py

@@ -2,11 +2,16 @@
 import logging
 import tempfile
 
+import os
 import opl.http
 import opl.status_data
+from requests.auth import HTTPBasicAuth
 
 
-def load(server, index, query, paths):
+def load(server, index, query, paths, **kwargs):
+    es_server_user = kwargs.get("es_server_user")
+    es_server_pass_env_var = kwargs.get("es_server_pass_env_var")
+
     out = {}
 
     for path in paths:
@@ -21,7 +26,17 @@ def load(server, index, query, paths):
         f"Querying ES with url={url}, headers={headers} and json={json.dumps(data)}"
     )
 
-    response = opl.http.get(url, headers=headers, json=data)
+    if es_server_user and es_server_pass_env_var:
+        # fetch the password from Jenkins credentials
+        open_search_password = os.environ.get(es_server_pass_env_var)
+        response = opl.http.get(
+            url,
+            auth=HTTPBasicAuth(es_server_user, open_search_password),
+            headers=headers,
+            json=data,
+        )
+    else:
+        response = opl.http.get(url, headers=headers, json=data)
 
     for item in response["hits"]["hits"]:
         logging.debug(

opl/pass_or_fail.py

@@ -114,12 +114,23 @@ def main():
     if args.history_type == "csv":
         history = opl.investigator.csv_loader.load(args.history_file, args.sets)
     elif args.history_type == "elasticsearch":
+        if (
+            hasattr(args, "history_es_server_verify")
+            and not args.history_es_server_verify
+        ):
+            # SSL verification is disabled by default
+            opl.http.insecure()
         history = opl.investigator.elasticsearch_loader.load(
             args.history_es_server,
             args.history_es_index,
             args.history_es_query,
             args.sets,
+            es_server_user=getattr(args, "history_es_server_user", None),
+            es_server_pass_env_var=getattr(
+                args, "history_es_server_pass_env_var", None
+            ),
         )
+
     elif args.history_type == "sd_dir":
         history = opl.investigator.sd_dir_loader.load(
             args.history_dir, args.history_matchers, args.sets
@@ -200,8 +211,17 @@ def main():
     if not args.dry_run:
         for d_type in args.decisions_type:
             if d_type == "elasticsearch":
+                if hasattr(args, "es_server_verify") and not args.es_server_verify:
+                    # disable SSL verification
+                    opl.http.insecure()
                 opl.investigator.elasticsearch_decisions.store(
-                    args.decisions_es_server, args.decisions_es_index, info_all
+                    args.decisions_es_server,
+                    args.decisions_es_index,
+                    info_all,
+                    es_server_user=getattr(args, "decisions_es_server_user", None),
+                    es_server_pass_env_var=getattr(
+                        args, "decisions_es_server_pass_env_var", None
+                    ),
                 )
             if d_type == "csv":
                 opl.investigator.csv_decisions.store(args.decisions_filename, info_all)