From 33be079ac39ea96d23a1c3f9125ae62b38a4ebdf Mon Sep 17 00:00:00 2001 From: namrata-metron <157002078+namrata-metron@users.noreply.github.com> Date: Sun, 10 Mar 2024 15:13:36 +0530 Subject: [PATCH] Devo v2 dev (#32989) * write_to_table_command & write_to_lookup_table_command changes * updated test cases & bug fixes * updated devo_write_to_table test case * prevoius code reverted * diff changes * Bug fixes * Validation checks & Unit test cases added * code reverted & working on readme file * readme file changes updated * optional parameters check added * code reverted * updated test cases & implemented the requested suggestions * reverted changes * added additional unit test cases * Docker image tag updated * Updated the release note for latest docker image * Docker Image updated --- Packs/Devo/Integrations/Devo_v2/Devo_v2.py | 172 +++++++---- Packs/Devo/Integrations/Devo_v2/Devo_v2.yml | 7 +- .../Devo/Integrations/Devo_v2/Devo_v2_test.py | 279 +++++++++++++++++- Packs/Devo/Integrations/Devo_v2/README.md | 83 ++++-- Packs/Devo/ReleaseNotes/1_3_1.md | 7 + Packs/Devo/pack_metadata.json | 2 +- 6 files changed, 452 insertions(+), 98 deletions(-) create mode 100644 Packs/Devo/ReleaseNotes/1_3_1.md diff --git a/Packs/Devo/Integrations/Devo_v2/Devo_v2.py b/Packs/Devo/Integrations/Devo_v2/Devo_v2.py index 9036295bf52b..8ef908034d3e 100644 --- a/Packs/Devo/Integrations/Devo_v2/Devo_v2.py +++ b/Packs/Devo/Integrations/Devo_v2/Devo_v2.py @@ -33,6 +33,7 @@ TIMEOUT = demisto.params().get("timeout", "60") PORT = arg_to_number(demisto.params().get("port", "443") or "443") ITEMS_PER_PAGE = 50 +LIMIT = 100 HEALTHCHECK_WRITER_RECORD = [{"hello": "world", "from": "demisto-integration"}] HEALTHCHECK_WRITER_TABLE = "test.keep.free" RANGE_PATTERN = re.compile("^[0-9]+ [a-zA-Z]+") @@ -823,13 +824,42 @@ def multi_table_query_command(offset, items): return entry +def convert_to_str(value): + if isinstance(value, list) and not value: + return_warning("Empty list encountered.") + return '[]' + elif isinstance(value, dict) and not value: + return_warning("Empty dictionary encountered.") + return '{}' + elif isinstance(value, list | dict): + return json.dumps(value) + return str(value) + + def write_to_table_command(): - table_name = demisto.args()["tableName"] - records = check_type(demisto.args()["records"], list) + tag = demisto.args().get("tag", None) + tableName = demisto.args()["tableName"] + records_str = demisto.args()["records"] linq_base = demisto.args().get("linqLinkBase", None) + final_tag = tag or tableName + + # Use json.loads to parse the records string + try: + records = json.loads(records_str) + except json.JSONDecodeError: + return_error("Error decoding JSON. Please ensure the records are valid JSON.") + + # Ensure records is a list + if not isinstance(records, list): + return_error("The 'records' parameter must be a list.") + + # Check if all records are empty + if all(not record for record in records): + return_error("All records are empty.") + creds = get_writer_creds() - linq = f"from {table_name}" + linq = f"from {tableName}" sender = Sender( SenderConfigSSL( @@ -840,28 +870,37 @@ def write_to_table_command(): ) ) + total_events = 0 + total_bytes_sent = 0 + for r in records: - try: - sender.send(tag=table_name, msg=json.dumps(r)) - except TypeError: - sender.send(tag=table_name, msg=f"{r}") + # Convert each record to a JSON string or string + formatted_record = convert_to_str(r) + + # If the record is empty, skip sending it + if not formatted_record.strip(): + continue + + # Send each record to Devo with the specified tag + sender.send(tag=final_tag, msg=formatted_record) + + # Update totals + total_events += 1 + total_bytes_sent += len(formatted_record.encode("utf-8")) + + current_ts = int(time.time()) + start_ts = (current_ts - 30) * 1000 + end_ts = (current_ts + 30) * 1000 querylink = { "DevoTableLink": build_link( linq, - int(1000 * time.time()) - 3600000, - int(1000 * time.time()), + start_ts, + end_ts, linq_base=linq_base, ) } - entry = { - "Type": entryTypes["note"], - "Contents": {"recordsWritten": records}, - "ContentsFormat": formats["json"], - "ReadableContentsFormat": formats["markdown"], - "EntryContext": {"Devo.RecordsWritten": records, "Devo.LinqQuery": linq}, - } entry_linq = { "Type": entryTypes["note"], "Contents": querylink, @@ -869,24 +908,17 @@ def write_to_table_command(): "ReadableContentsFormat": formats["markdown"], "EntryContext": {"Devo.QueryLink": createContext(querylink)}, } - headers: list = [] - resultRecords: list = [] - innerDict: dict = {} - for obj in records: - record = json.loads(obj) - currKey = list(record.keys()) - currValue = list(record.values()) - headers.extend(currKey) - - innerDict.update(dict(zip(currKey, currValue))) # Create a dictionary using zip - resultRecords.append(innerDict) # Append the dictionary to the list - - demisto.debug("final array :") - demisto.debug(resultRecords) + entry = { + "Type": entryTypes["note"], + "Contents": {"TotalRecords": total_events, "TotalBytesSent": total_bytes_sent}, + "ContentsFormat": formats["json"], + "ReadableContentsFormat": formats["markdown"], + "EntryContext": {"Devo.TotalRecords": total_events, "Devo.TotalBytesSent": total_bytes_sent, "Devo.LinqQuery": linq}, + } - md = tableToMarkdown("Entries to load into Devo", resultRecords, headers) - entry["HumanReadable"] = md + md_message = f"Total Records Sent: {total_events}.\nTotal Bytes Sent: {total_bytes_sent}." + entry["HumanReadable"] = md_message md_linq = tableToMarkdown( "Link to Devo Query", @@ -898,9 +930,19 @@ def write_to_table_command(): def write_to_lookup_table_command(): - lookup_table_name = demisto.args()["lookupTableName"] - headers = check_type(demisto.args()["headers"], list) - records = check_type(demisto.args()["records"], list) + lookup_table_name = demisto.args().get("lookupTableName") + headers_param = demisto.args().get("headers") + records_param = demisto.args().get("records") + + if not lookup_table_name or not headers_param or not records_param: + return_error("Missing required arguments. Please provide lookupTableName, headers, and records.") + + # Parse JSON strings to Python objects + try: + headers = json.loads(headers_param) + records = json.loads(records_param) + except json.JSONDecodeError as e: + return_error(f"Failed to parse JSON: {str(e)}") creds = get_writer_creds() @@ -911,35 +953,59 @@ def write_to_lookup_table_command(): chain=creds["chain"].name, ) + con = None + total_events = 0 + total_bytes = 0 + try: - con = Sender(config=engine_config, timeout=60) + # Validate headers + if not isinstance(headers, dict) or "headers" not in headers or not isinstance(headers["headers"], list): + raise ValueError("Invalid headers format. 'headers' must be a list.") + + columns = headers["headers"] - lookup = Lookup(name=lookup_table_name, historic_tag=None, con=con) - # Order sensitive list - pHeaders = json.dumps(headers) + # Set default values for optional parameters + key_index = int(headers.get("key_index", 0)) # Ensure it's casted to integer + action = headers.get("action", "INC") # Set default value to 'INC' - lookup.send_control("START", pHeaders, "INC") + # Validate key_index + if key_index < 0: + raise ValueError("key_index must be a non-negative integer value.") + + # Validate action + if action not in {"INC", "FULL"}: + raise ValueError("action must be either 'INC' or 'FULL'.") + + con = Sender(config=engine_config, timeout=60) + lookup = Lookup(name=lookup_table_name, con=con) + lookup.send_headers(headers=columns, key_index=key_index, event="START", action=action) + + # Send data lines for r in records: - lookup.send_data_line(key_index=0, fields=r["values"]) + fields = r.get("fields", []) + delete = r.get("delete", False) + lookup.send_data_line(key_index=key_index, fields=fields, delete=delete) + total_events += 1 + total_bytes += len(json.dumps(r)) - lookup.send_control("END", pHeaders, "INC") - finally: + # Send end event + lookup.send_headers(headers=columns, key_index=key_index, event="END", action=action) + + # Flush buffer and shutdown connection con.flush_buffer() con.socket.shutdown(0) - entry = { - "Type": entryTypes["note"], - "Contents": {"recordsWritten": records}, - "ContentsFormat": formats["json"], - "ReadableContentsFormat": formats["markdown"], - "EntryContext": {"Devo.RecordsWritten": records}, - } + # Return three lines as output + return f"Lookup Table Name: {lookup_table_name}.\nTotal Records Sent: {total_events}.\nTotal Bytes Sent: {total_bytes}." - md = tableToMarkdown("Entries to load into Devo", records) - entry["HumanReadable"] = md + except Exception as e: + return_error(f"Failed to execute command write-to-lookup-table. Error: {str(e)}") - return [entry] + finally: + if con: + con.flush_buffer() + con.socket.shutdown(0) def main(): diff --git a/Packs/Devo/Integrations/Devo_v2/Devo_v2.yml b/Packs/Devo/Integrations/Devo_v2/Devo_v2.yml index 43e81a780924..b45992a27bdc 100644 --- a/Packs/Devo/Integrations/Devo_v2/Devo_v2.yml +++ b/Packs/Devo/Integrations/Devo_v2/Devo_v2.yml @@ -194,12 +194,14 @@ script: description: Queries multiple tables for a given token and returns relevant results. - name: devo-write-to-table arguments: + - name: tag + description: The tag to assign to the records. - name: tableName required: true description: The name of the table to write to. - name: records - required: true description: Records to write to the specified table. + required: true isArray: true - name: linqLinkBase description: Overrides the global Devo base domain for linq linking. @@ -221,7 +223,6 @@ script: - name: headers required: true description: Headers for lookup table control. - isArray: true - name: records required: true description: Records to write to the specified table. @@ -232,7 +233,7 @@ script: type: unknown description: Writes lookup table entry records to a specified Devo table. execution: true - dockerimage: demisto/devo:1.0.0.86778 + dockerimage: demisto/devo:1.0.0.89201 isfetch: true subtype: python3 tests: diff --git a/Packs/Devo/Integrations/Devo_v2/Devo_v2_test.py b/Packs/Devo/Integrations/Devo_v2/Devo_v2_test.py index 9de20fbadad4..60c2ba7f9839 100644 --- a/Packs/Devo/Integrations/Devo_v2/Devo_v2_test.py +++ b/Packs/Devo/Integrations/Devo_v2/Devo_v2_test.py @@ -140,22 +140,54 @@ "writeToContext": "true", "filtered_columns": "alertId,extraData,context" } +MOCK_MULTI_ARGUMENTS = { + "tables": ["app", "charlie", "test"], + "searchToken": "searching", + "from": time.time() - 60, + "to": time.time(), + "writeToContext": "true", + "items": -10 +} MOCK_WRITER_ARGS = { "tableName": "whatever.table", - "records": [{"foo": "hello"}, {"foo": "world"}, {"foo": "demisto"}], + "records": '[{"foo": "hello"}, {"foo": "world"}, {"foo": "demisto"}]', +} +MOCK_WRITER_ARGS_LIST = { + "tableName": "whatever.table", + "records": '[["a", "b", "c"], ["1", "2", "3"]]', +} +MOCK_WRITER_ARGS_EMPTY = { + "tableName": "whatever.table", + "records": '[1234, true]', +} +MOCK_WRITER_ARGS_STR = { + "tableName": "whatever.table", + "records": '["This is my first event", "This is my second log"]', } MOCK_WRITE_TO_TABLE_RECORDS = { "tableName": "whatever.table", - "records": ['{"foo": "hello"}', '{"foo": "world"}', '{"foo": "demisto"}'], + "records": '[{"hello": "world"}, {"abc": "xyz"}, {"data": "test"}]', } MOCK_LOOKUP_WRITER_ARGS = { "lookupTableName": "hello.world.lookup", - "headers": ["foo", "bar", "baz"], - "records": [ - {"key": "fookey", "values": ["fookey", "bar0", "baz0"]}, - {"key": "keyfoo", "values": ["keyfoo", "bar1", "baz1"]}, - {"key": "keykey", "values": ["keykey", "bar5", "baz6"]}, - ], + "headers": '{"headers": ["foo", "bar", "baz"], "key_index": 0, "action": "FULL"}', + "records": ('[{"fields": ["foo1", "bar1", "baz1"], "delete": false}, ' + '{"fields": ["foo2", "bar2", "baz2"]}, ' + '{"fields": ["foo3", "bar3", "baz3"]}]') +} +MOCK_LOOKUP_WRITER_ARGS_key = { + "lookupTableName": "hello.world.lookup", + "headers": '{"headers": ["foo", "bar", "baz"], "key_index": 0, "action": "FULL"}', + "records": ('[{"fields": ["foo1", "bar1", "baz1"], "delete": false}, ' + '{"fields": ["foo2", "bar2", "baz2"]}, ' + '{"fields": ["foo3", "bar3", "baz3"]}]') +} +MOCK_LOOKUP_WRITER_ARGS_action = { + "lookupTableName": "hello.world.lookup", + "headers": '{"headers": ["foo", "bar", "baz"], "key_index": 0, "action": "INC"}', + "records": ('[{"fields": ["foo1", "bar1", "baz1"], "delete": false}, ' + '{"fields": ["foo2", "bar2", "baz2"]}, ' + '{"fields": ["foo3", "bar3", "baz3"]}]') } MOCK_KEYS = {"foo": "bar", "baz": "bug"} OFFSET = 0 @@ -235,6 +267,9 @@ class MOCK_LOOKUP: + def send_headers(*args, **kw): + pass + def send_control(*args, **kw): pass @@ -453,6 +488,35 @@ def test_multi_query( assert results["HumanReadable"] == "No results found" +@patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True) +@patch("Devo_v2.READER_OAUTH_TOKEN", MOCK_READER_OAUTH_TOKEN, create=True) +@patch("Devo_v2.concurrent.futures.wait") +@patch("Devo_v2.concurrent.futures.ThreadPoolExecutor.submit") +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.ds.Reader.query") +@patch("Devo_v2.ds.Reader") +@patch("Devo_v2.get_types") +def test_multi_query_negative_items( + mock_query_types, + mock_query_reader, + mock_query_results, + mock_args_results, + mock_submit_results, + mock_wait_results, +): + mock_query_types.return_value = MOCK_KEYS + mock_query_reader.return_value = MOCK_READER + mock_query_results.return_value = copy.deepcopy(MOCK_QUERY_RESULTS) + mock_args_results.return_value = MOCK_MULTI_ARGUMENTS + mock_submit_results.return_value = None + mock_wait_results.return_value = (None, None) + try: + multi_table_query_command(OFFSET, ITEMS_PER_PAGE) + except ValueError as exc: + error_msg = str(exc) + assert "The 'limit' parameter cannot be negative." in error_msg + + @patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) @patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) @patch("Devo_v2.demisto.args") @@ -461,10 +525,66 @@ def test_write_devo(mock_load_results, mock_write_args): mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN mock_write_args.return_value = MOCK_WRITE_TO_TABLE_RECORDS results = write_to_table_command() - assert len(results[0]["EntryContext"]["Devo.RecordsWritten"]) == 3 + assert len(results) == 2 # We expect two entries in the results list assert results[0]["EntryContext"]["Devo.LinqQuery"] == "from whatever.table" +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_str(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS_STR + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert 'Failed to execute command devo-write-to-table.' in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_data(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "Error decoding JSON. Please ensure the records are valid JSON." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_list(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS_LIST + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "The 'records' parameter must be a list." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_no_data(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS_EMPTY + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "All records are empty." in error_msg + + @patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) @patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) @patch("Devo_v2.demisto.args") @@ -477,7 +597,146 @@ def test_write_lookup_devo( mock_lookup_writer_sender.return_value = MOCK_SENDER() mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() results = write_to_lookup_table_command() - assert len(results[0]["EntryContext"]["Devo.RecordsWritten"]) == 3 + assert isinstance(results, str) # We expect a string result + assert "Lookup Table Name: hello.world.lookup." in results + assert "Total Records Sent: 3." in results + assert "Total Bytes Sent: 125." in results + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +@patch("Devo_v2.Lookup") +def test_write_lookup_devo_header( + mock_lookup_writer_lookup, mock_lookup_writer_sender, mock_lookup_write_args +): + mock_lookup_write_args.return_value = MOCK_LOOKUP_WRITER_ARGS + mock_lookup_writer_sender.return_value = MOCK_SENDER() + mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() + try: + write_to_lookup_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "Invalid headers format. 'headers' must be a list." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +@patch("Devo_v2.Lookup") +def test_write_lookup_devo_invalid( + mock_lookup_writer_lookup, mock_lookup_writer_sender, mock_lookup_write_args +): + mock_lookup_write_args.return_value = MOCK_LOOKUP_WRITER_ARGS_key + mock_lookup_writer_sender.return_value = MOCK_SENDER() + mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() + try: + write_to_lookup_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "key_index must be a non-negative integer value." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +@patch("Devo_v2.Lookup") +def test_write_lookup_devo_invalid_action( + mock_lookup_writer_lookup, mock_lookup_writer_sender, mock_lookup_write_args +): + mock_lookup_write_args.return_value = MOCK_LOOKUP_WRITER_ARGS_action + mock_lookup_writer_sender.return_value = MOCK_SENDER() + mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() + try: + write_to_lookup_table_command() + except ValueError as err: + error = str(err) + assert "action must be either 'INC' or 'FULL'." in error + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_empty_records(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "Error decoding JSON. Please ensure the records are valid JSON." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_invalid_json(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = MOCK_WRITER_ARGS + try: + write_to_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "Error decoding JSON. Please ensure the records are valid JSON." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +@patch("Devo_v2.Lookup") +def test_write_lookup_devo_invalid_headers_format( + mock_lookup_writer_lookup, mock_lookup_writer_sender, mock_lookup_write_args +): + mock_lookup_write_args.return_value = MOCK_LOOKUP_WRITER_ARGS + mock_lookup_writer_sender.return_value = MOCK_SENDER() + mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() + try: + write_to_lookup_table_command() + except ValueError as exc: + error_msg = str(exc) + assert "Invalid headers format. 'headers' must be a list." in error_msg + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +def test_write_devo_empty_records_param(mock_load_results, mock_write_args): + mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN + mock_write_args.return_value = {"tag": "test_tag", "tableName": "test_table", "records": "{}"} + try: + write_to_table_command() + except SystemExit: + pass # Handle SystemExit gracefully in tests + + +@patch("Devo_v2.WRITER_RELAY", MOCK_WRITER_RELAY, create=True) +@patch("Devo_v2.WRITER_CREDENTIALS", MOCK_WRITER_CREDENTIALS, create=True) +@patch("Devo_v2.demisto.args") +@patch("Devo_v2.Sender") +@patch("Devo_v2.Lookup") +def test_write_lookup_missing_args( + mock_lookup_writer_lookup, mock_lookup_writer_sender, mock_lookup_write_args +): + # Ensure that headers and records are properly formatted JSON strings + mock_lookup_write_args.return_value = { + "lookupTableName": "test_table", + "headers": '["header1", "header2"]', + "records": '["record1", "record2"]' + } + mock_lookup_writer_sender.return_value = MOCK_SENDER() + mock_lookup_writer_lookup.return_value = MOCK_LOOKUP() + # Provide all required arguments + try: + write_to_lookup_table_command() + except SystemExit: + pass # Handle SystemExit gracefully in tests @patch("Devo_v2.demisto_ISO", return_value="2022-03-15T15:01:23.456Z") diff --git a/Packs/Devo/Integrations/Devo_v2/README.md b/Packs/Devo/Integrations/Devo_v2/README.md index 79969ca69081..8b19ae610950 100644 --- a/Packs/Devo/Integrations/Devo_v2/README.md +++ b/Packs/Devo/Integrations/Devo_v2/README.md @@ -353,10 +353,9 @@ Using unsupported formats will result in an error. | Devo.MultiResults | unknown | A list of dictionary results. | -#### Command Example - -```text -!devo-multi-table-query tables='["siem.logtrust.web.activity", "siem.logtrust.web.navigation"]' searchToken="john@doe.com" from=1576845233.193244 to=1576845293.193244 items_per_page=1000 +##### Command Example +``` +!devo-multi-table-query tables="[siem.logtrust.alert.info, siem.logtrust.web.navigation]" searchToken="parag@metronlabs.com" from=1707416980 to=1707805927 ``` #### Human Readable Output @@ -391,11 +390,11 @@ A Cortex XSOAR instance configured with the correct write JSON credentials #### Input -| **Argument Name** | **Description** | **Required** | -| --- | --- | --- | -| tableName | The name of the table to write to. | Required | -| records | Records to write to the specified table. | Required | -| linqLinkBase | Overrides the global Devo base domain for linq linking. | Optional | +| **Argument Name** | **Description** | **Required** | +|-------------------|-----------------------------------------------------------------|--------------| +| tableName | Table name to write to | Required | +| records | Records written to specified Devo table. | Required | +| linqLinkBase | Overrides the global link base so is able to be set at run time | Optional | #### Context Output @@ -406,24 +405,34 @@ A Cortex XSOAR instance configured with the correct write JSON credentials | Devo.QueryLink | unknown | The link to the Devo table for the executed query. | -#### Command Example - -```text -!devo-write-to-table tableName="my.app.test.test" records='[{"hello": "world"}, {"hello": "world"}]' +##### Command Example +``` +!devo-write-to-table tableName="my.app.test.test" records=`[ "This is my first event", "This is my second log", {"hello": "world"}, {"hello": "friend"}, ["a", "b", "c"], ["1", "2", "3"], 1234, true ]` ``` -#### Human Readable Output +##### Human Readable Output +Total Records Sent: 8. +Total Bytes Sent: 196. ->### Entries to load into Devo ->| hello | ->| --- | ->| world | ->| world | +##### Entries to load into Devo ->### Link to Devo Query ->|DevoTableLink| ->|---| ->|[Devo Direct Link](https://us.devo.com/welcome#/verticalApp?path=apps/custom/dsQueryForwarder&targetQuery=blah==)| +|eventdate|format|cluster|instance|message| +|---|---|---|---|---| +|2024-02-12 17:51:51.277|test|-|-|This is my first event| +|2024-02-12 17:51:51.277|test|-|-|This is my second log| +|2024-02-12 17:51:51.277|test|-|-|{"hello": "world"}| +|2024-02-12 17:51:51.277|test|-|-|{"hello": "friend"}| +|2024-02-12 17:51:51.277|test|-|-|["a", "b", "c"]| +|2024-02-12 17:51:51.277|test|-|-|["1", "2", "3"]| +|2024-02-12 17:51:51.277|test|-|-|1234| +|2024-02-12 17:51:51.277|test|-|-|True| + + +##### Link to Devo Query + +|DevoTableLink| +|---| +|[Devo Direct Link](https://us.devo.com/welcome#/verticalApp?path=apps/custom/dsQueryForwarder&targetQuery=blah==)| ### 5. devo-write-to-lookup-table @@ -459,19 +468,31 @@ A Cortex XSOAR instance configured with the correct write JSON credentials. | Devo.RecordsWritten | unknown | Lookup records written to the lookup table. | -#### Command Example - -```text -!devo-write-to-lookup-table lookupTableName="lookup123" headers='["foo", "bar", "baz"]' records='[{"key": "foo1", "values": ["foo1", "bar1", "baz1"]}]' +##### Command Example +``` +!devo-write-to-lookup-table lookupTableName="lookup123" headers=`{"headers": ["foo", "bar", "baz"], "key_index": 0, "action": "FULL"}` records=`[{"fields": ["foo1", "bar1", "baz1"], "delete": false}, {"fields": ["foo2", "bar2", "baz2"]}, {"fields": ["foo3", "bar3", "baz3"]}]` ``` -#### Human Readable Output +##### Human Readable Output +Lookup Table Name: lookup123. +Total Records Sent: 3. +Total Bytes Sent: 125. + +##### Entries to load into Devo +The headers of headers array is written into the my.lookup.control table. ->| foo | bar | baz | ->| ---- | ---- | ---- | ->| foo1 | bar1 | baz1 | +|eventdate|lookup|lookupId|lookupOp|type|lookupFields| +|---|---|---|---|---|---| +|2024-02-13 10:57:14.238|lookup123|1707802034.0032315_lookup123|FULL|START|[{"foo":{"type":"str","key":true}},{"bar":{"type":"str"}},{"baz":{"type":"str"}}]| +|2024-02-13 10:57:24.246|lookup123|1707802034.0032315_lookup123|FULL|END|[{"foo":{"type":"str","key":true}},{"bar":{"type":"str"}},{"baz":{"type":"str"}}]| +The fields of records array is written into the my.lookup.data table. +|eventdate|lookup|lookupId|lookupOp|rawData| +|---|---|---|---|---| +|2024-02-13 10:57:19.239|lookup123|1707802034.0032315_lookup123|null|"foo1", "bar1", "baz1"| +|2024-02-13 10:57:19.240|lookup123|1707802034.0032315_lookup123|null|"foo2", "bar2", "baz2"| +|2024-02-13 10:57:19.240|lookup123|1707802034.0032315_lookup123|null|"foo3", "bar3", "baz3"| #### Youtube Video Demo (Click Image, Will redirect to youtube) diff --git a/Packs/Devo/ReleaseNotes/1_3_1.md b/Packs/Devo/ReleaseNotes/1_3_1.md new file mode 100644 index 000000000000..1adf062ebffb --- /dev/null +++ b/Packs/Devo/ReleaseNotes/1_3_1.md @@ -0,0 +1,7 @@ +#### Integrations + +##### Devo v2 +- Updated the Docker image to: *demisto/devo:1.0.0.89201*. + + - Improved functionality for **devo-write-to-table** and **devo-write-to-lookup-table** commands. + diff --git a/Packs/Devo/pack_metadata.json b/Packs/Devo/pack_metadata.json index 99ef4a952261..9aa11a256d7f 100644 --- a/Packs/Devo/pack_metadata.json +++ b/Packs/Devo/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Devo", "description": "Use the Devo integration to query Devo for alerts, lookup tables, and to write to lookup tables.", "support": "partner", - "currentVersion": "1.3.0", + "currentVersion": "1.3.1", "author": "Devo", "url": "https://www.devo.com", "email": "support@devo.com",