From 4056810e59cc2d15e9c2fc0cae163035efc72ce3 Mon Sep 17 00:00:00 2001 From: Dan Halperin Date: Wed, 20 Jan 2021 11:18:56 -0800 Subject: [PATCH 01/11] add pre-commit, run it (#154) Have to ignore flake8 E203 and W503 - they disagree with black, and the Internet says that black is right. https://github.com/psf/black/issues/113 https://github.com/psf/black/issues/315 --- .pre-commit-config.yaml | 21 + netconan/anonymize_files.py | 116 ++-- netconan/default_pwd_regexes.py | 116 ++-- netconan/ip_anonymization.py | 118 ++-- netconan/netconan.py | 235 ++++--- netconan/sensitive_item_removal.py | 168 +++-- setup.cfg | 2 +- setup.py | 81 +-- tests/end_to_end/test_end_to_end.py | 54 +- tests/unit/test_anonymize_files.py | 123 ++-- tests/unit/test_as_number_anonymization.py | 112 +-- tests/unit/test_ip_anonymization.py | 414 ++++++------ tests/unit/test_parse_args.py | 46 +- tests/unit/test_sensitive_item_removal.py | 748 +++++++++++---------- tools/generate_reserved_tokens.py | 15 +- 15 files changed, 1367 insertions(+), 1002 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a4ab42f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + exclude: netconan/default_reserved_words.py +- repo: https://github.com/pre-commit/mirrors-isort + rev: v5.7.0 + hooks: + - id: isort + # args from https://black.readthedocs.io/en/stable/compatible_configs.html#isort + args: ["--multi-line=3", "--trailing-comma", "--force-grid-wrap=0", "--use-parentheses", "--ensure-newline-before-comments", "--line-length=88"] + exclude: netconan/default_reserved_words.py +- repo: git@github.com:humitos/mirrors-autoflake.git + rev: v1.3 + hooks: + - id: autoflake + args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variables"] + exclude: netconan/default_reserved_words.py diff --git a/netconan/anonymize_files.py b/netconan/anonymize_files.py index 61e5dc3..c66eff4 100644 --- a/netconan/anonymize_files.py +++ b/netconan/anonymize_files.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + import errno import logging import os @@ -21,21 +22,35 @@ import string from .default_reserved_words import default_reserved_words -from .ip_anonymization import ( - IpAnonymizer, IpV6Anonymizer, anonymize_ip_addr) +from .ip_anonymization import IpAnonymizer, IpV6Anonymizer, anonymize_ip_addr from .sensitive_item_removal import ( - anonymize_as_numbers, AsNumberAnonymizer, replace_matching_item, - SensitiveWordAnonymizer, generate_default_sensitive_item_regexes) + AsNumberAnonymizer, + SensitiveWordAnonymizer, + anonymize_as_numbers, + generate_default_sensitive_item_regexes, + replace_matching_item, +) _DEFAULT_SALT_LENGTH = 16 _CHAR_CHOICES = string.ascii_letters + string.digits -def anonymize_files(input_path, output_path, anon_pwd, anon_ip, - salt=None, dumpfile=None, sensitive_words=None, - undo_ip_anon=False, as_numbers=None, reserved_words=None, - preserve_prefixes=None, preserve_networks=None, - preserve_suffix_v4=None, preserve_suffix_v6=None): +def anonymize_files( + input_path, + output_path, + anon_pwd, + anon_ip, + salt=None, + dumpfile=None, + sensitive_words=None, + undo_ip_anon=False, + as_numbers=None, + reserved_words=None, + preserve_prefixes=None, + preserve_networks=None, + preserve_suffix_v4=None, + preserve_suffix_v6=None, +): """Anonymize each file in input and save to output.""" anonymizer4 = None anonymizer6 = None @@ -45,7 +60,9 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip, pwd_lookup = None # The salt is only used for IP and sensitive word anonymization: if salt is None: - salt = ''.join(random.choice(_CHAR_CHOICES) for _ in range(_DEFAULT_SALT_LENGTH)) + salt = "".join( + random.choice(_CHAR_CHOICES) for _ in range(_DEFAULT_SALT_LENGTH) + ) logging.warning('No salt was provided; using randomly generated "%s"', salt) logging.debug('Using salt: "%s"', salt) if anon_pwd: @@ -57,7 +74,12 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip, if sensitive_words is not None: anonymizer_sensitive_word = SensitiveWordAnonymizer(sensitive_words, salt) if anon_ip or undo_ip_anon: - anonymizer4 = IpAnonymizer(salt, preserve_prefixes, preserve_networks, preserve_suffix=preserve_suffix_v4) + anonymizer4 = IpAnonymizer( + salt, + preserve_prefixes, + preserve_networks, + preserve_suffix=preserve_suffix_v4, + ) anonymizer6 = IpV6Anonymizer(salt, preserve_suffix=preserve_suffix_v6) if as_numbers is not None: anonymizer_as_num = AsNumberAnonymizer(as_numbers, salt) @@ -73,40 +95,56 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip, if not os.listdir(input_path): raise ValueError("Input directory is empty") if os.path.isfile(output_path): - raise ValueError("Output path must be a directory if input path is " - "a directory") + raise ValueError( + "Output path must be a directory if input path is a directory" + ) for root, dirs, files in os.walk(input_path): rel_root = os.path.relpath(root, input_path) - file_list.extend([( - os.path.join(input_path, rel_root, f), - os.path.join(output_path, rel_root, f) - ) for f in files if not f.startswith('.')]) + file_list.extend( + [ + ( + os.path.join(input_path, rel_root, f), + os.path.join(output_path, rel_root, f), + ) + for f in files + if not f.startswith(".") + ] + ) for in_path, out_path in file_list: try: - anonymize_file(in_path, - out_path, - compiled_regexes=compiled_regexes, - pwd_lookup=pwd_lookup, - anonymizer_sensitive_word=anonymizer_sensitive_word, - anonymizer_as_num=anonymizer_as_num, - undo_ip_anon=undo_ip_anon, - anonymizer4=anonymizer4, - anonymizer6=anonymizer6) + anonymize_file( + in_path, + out_path, + compiled_regexes=compiled_regexes, + pwd_lookup=pwd_lookup, + anonymizer_sensitive_word=anonymizer_sensitive_word, + anonymizer_as_num=anonymizer_as_num, + undo_ip_anon=undo_ip_anon, + anonymizer4=anonymizer4, + anonymizer6=anonymizer6, + ) except Exception: - logging.error('Failed to anonymize file %s', in_path, exc_info=True) + logging.error("Failed to anonymize file %s", in_path, exc_info=True) if dumpfile is not None: - with open(dumpfile, 'w') as f_out: + with open(dumpfile, "w") as f_out: anonymizer4.dump_to_file(f_out) anonymizer6.dump_to_file(f_out) -def anonymize_file(filename_in, filename_out, compiled_regexes=None, - anonymizer4=None, anonymizer6=None, pwd_lookup=None, - anonymizer_sensitive_word=None, anonymizer_as_num=None, - undo_ip_anon=False): +def anonymize_file( + filename_in, + filename_out, + compiled_regexes=None, + anonymizer4=None, + anonymizer6=None, + pwd_lookup=None, + anonymizer_sensitive_word=None, + anonymizer_as_num=None, + undo_ip_anon=False, +): """Anonymize contents of input file and save to the output file. This only applies sensitive line removal if compiled_regexes and pwd_lookup @@ -119,16 +157,18 @@ def anonymize_file(filename_in, filename_out, compiled_regexes=None, _mkdirs(filename_out) if os.path.isdir(filename_out): - raise ValueError('Cannot write output file; ' - 'output file is a directory ({})' - .format(filename_out)) + raise ValueError( + "Cannot write output file; " + "output file is a directory ({})".format(filename_out) + ) - with open(filename_out, 'w') as f_out, open(filename_in, 'r') as f_in: + with open(filename_out, "w") as f_out, open(filename_in, "r") as f_in: for line in f_in: output_line = line if compiled_regexes is not None and pwd_lookup is not None: - output_line = replace_matching_item(compiled_regexes, - output_line, pwd_lookup) + output_line = replace_matching_item( + compiled_regexes, output_line, pwd_lookup + ) if anonymizer6 is not None: output_line = anonymize_ip_addr(anonymizer6, output_line, undo_ip_anon) diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index 2effdcb..fdc9848 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -49,47 +49,68 @@ # Some of these regexes need to be updated to support quote enclosed passwords # which is allowed for at least some syntax on Juniper devices default_pwd_line_regexes = [ - [(r'(?P(password|passwd)( level \d+)?( \d+)?( ENC)? )(\S+)', 6)], - [(r'(?Pusername( \S+)+ (password|secret)( \d| sha512)? )(\S+)', 5)], - [(r'(?P(enable )?secret( \d)? )(\S+)', 4)], - [(r'(?Pip ftp password( \d)? )(\S+)', 3)], - [(r'(?Pip ospf authentication-key( \d)? )(\S+)', 3)], - [(r'(?Pisis password )(\S+)(?=( level-\d)?)', 2)], - [(r'(?P(domain-password|area-password) )(\S+)', 3)], - [(r'(?Pip ospf message-digest-key \d+ md5( \d)? )(\S+)', 3)], - [(r'(?Pstandby( \d*)? authentication( text| md5 key-string( \d)?)? )(\S+)', 5)], - [(r'(?Pl2tp tunnel( \S+)? password( \d)? )(\S+)', 4)], - [(r'(?Pdigest secret( \d)? )(\S+)', 3)], - [(r'(?Pppp .* hostname )(\S+)', 2)], - [(r'(?Pppp .* password( \d)? )(\S+)', 3)], - [(r'(?P(ikev2 )?(local|remote)-authentication pre-shared-key )(\S+)', 4)], - [(r'(?P(\S )*pre-shared-key( remote| local)?( hex| hexadecimal| ascii-text| \d)? )(\S+)', 5)], - [(r'(?P(tacacs|radius)-server (\S+ )*key( \d)? )(\S+)', 5)], - [(r'(?Pkey( \d| hexadecimal)? )(\S+)', 3)], - [(r'(?Pntp authentication-key \d+ md5 )(\S+)', 2)], - [(r'(?Psyscon( password| address \S+) )(\S+)', 3)], - [(r'(?Psnmp-server user( \S+)+ (auth (md5|sha)) )(\S+)', 5), - (r'(?Psnmp-server user( \S+)+ priv( 3des| aes( \d+)?| des)? )(\S+)', 5)], - [(r'(?P(crypto )?isakmp key( \d)? )(\S+)', 4)], - [(r'(?Pset session-key (in|out)bound ah \d+ )(\S+)', 3)], - [(r'(?Pset session-key (in|out)bound esp \d+ cipher? )(\S+)', 3), - (r'(?Pset session-key (in|out)bound esp \d+(( cipher \S+)? authenticator) )(\S+)', 5)], - [(r'(?P(hello-)?authentication-key )([^;]+)', 3)], + [(r"(?P(password|passwd)( level \d+)?( \d+)?( ENC)? )(\S+)", 6)], + [(r"(?Pusername( \S+)+ (password|secret)( \d| sha512)? )(\S+)", 5)], + [(r"(?P(enable )?secret( \d)? )(\S+)", 4)], + [(r"(?Pip ftp password( \d)? )(\S+)", 3)], + [(r"(?Pip ospf authentication-key( \d)? )(\S+)", 3)], + [(r"(?Pisis password )(\S+)(?=( level-\d)?)", 2)], + [(r"(?P(domain-password|area-password) )(\S+)", 3)], + [(r"(?Pip ospf message-digest-key \d+ md5( \d)? )(\S+)", 3)], + [ + ( + r"(?Pstandby( \d*)? authentication( text| md5 key-string( \d)?)? )(\S+)", + 5, + ) + ], + [(r"(?Pl2tp tunnel( \S+)? password( \d)? )(\S+)", 4)], + [(r"(?Pdigest secret( \d)? )(\S+)", 3)], + [(r"(?Pppp .* hostname )(\S+)", 2)], + [(r"(?Pppp .* password( \d)? )(\S+)", 3)], + [(r"(?P(ikev2 )?(local|remote)-authentication pre-shared-key )(\S+)", 4)], + [ + ( + r"(?P(\S )*pre-shared-key( remote| local)?( hex| hexadecimal| ascii-text| \d)? )(\S+)", + 5, + ) + ], + [(r"(?P(tacacs|radius)-server (\S+ )*key( \d)? )(\S+)", 5)], + [(r"(?Pkey( \d| hexadecimal)? )(\S+)", 3)], + [(r"(?Pntp authentication-key \d+ md5 )(\S+)", 2)], + [(r"(?Psyscon( password| address \S+) )(\S+)", 3)], + [ + (r"(?Psnmp-server user( \S+)+ (auth (md5|sha)) )(\S+)", 5), + (r"(?Psnmp-server user( \S+)+ priv( 3des| aes( \d+)?| des)? )(\S+)", 5), + ], + [(r"(?P(crypto )?isakmp key( \d)? )(\S+)", 4)], + [(r"(?Pset session-key (in|out)bound ah \d+ )(\S+)", 3)], + [ + (r"(?Pset session-key (in|out)bound esp \d+ cipher? )(\S+)", 3), + ( + r"(?Pset session-key (in|out)bound esp \d+(( cipher \S+)? authenticator) )(\S+)", + 5, + ), + ], + [(r"(?P(hello-)?authentication-key )([^;]+)", 3)], # TODO(https://github.com/intentionet/netconan/issues/3): # Follow-up on these. They were just copied from RANCID so currently: # They are untested in general and need cases added for unit tests # They do not specifically capture sensitive info # They just identify lines where sensitive info exists - [(r'(cable shared-secret) (.*)', None)], - [(r'(wpa-psk ascii|hex \d) (.*)', None)], - [(r'(ldap-login-password) \S+(.*)', None)], - [(r'((ikev1 )?(pre-shared-key |key |failover key )(ascii-text |hexadecimal )?).*(.*)', None)], - [(r'(vpdn username (\S+) password)(.*)', None)], - [(r'(key-string \d?)(.*)', None)], - [(r'(message-digest-key \d+ md5 (7|encrypted)) (.*)', None)], - [(r'(.*?neighbor.*?) (\S*) password (.*)', None)], - [(r'(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)', None)], - + [(r"(cable shared-secret) (.*)", None)], + [(r"(wpa-psk ascii|hex \d) (.*)", None)], + [(r"(ldap-login-password) \S+(.*)", None)], + [ + ( + r"((ikev1 )?(pre-shared-key |key |failover key )(ascii-text |hexadecimal )?).*(.*)", + None, + ) + ], + [(r"(vpdn username (\S+) password)(.*)", None)], + [(r"(key-string \d?)(.*)", None)], + [(r"(message-digest-key \d+ md5 (7|encrypted)) (.*)", None)], + [(r"(.*?neighbor.*?) (\S*) password (.*)", None)], + [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", None)], # These are regexes for JUNOS # TODO(https://github.com/intentionet/netconan/issues/4): # Follow-up on these. They were modified from RANCID's regexes and currently: @@ -97,23 +118,28 @@ # They just identify lines where sensitive info exists # They need to be tested against config lines generated on a JUNOS router # (to make sure the regex handles different syntaxes allowed in the line) - [(r'(\S* )*md5 \d+ key [^ ;]+(.*)', None)], - [(r'(\S* )*(secret|simple-password) [^ ;]+(.*)', None)], - [(r'(\S* )*encrypted-password [^ ;]+(.*)', None)], - [(r'(\S* )*ssh-(rsa|dsa) \"(.*)', None)], - [(r'(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)', None)] + [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)], + [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)], + [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)], + [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)], + [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)], ] # Taken from RANCID community scrubbing regexes default_com_line_regexes = [ - [(r'(?P(snmp-server (\S+ )*community)( [08])? )(\S+)', 5)], + [(r"(?P(snmp-server (\S+ )*community)( [08])? )(\S+)", 5)], # TODO(https://github.com/intentionet/netconan/issues/5): # Confirm this catches all community possibilities for snmp-server - [(r'(?Psnmp-server host (\S+)( informs| traps| version ' - r'(?:1|2c|3 \S+)| vrf \S+)* )(\S+)', 4)], + [ + ( + r"(?Psnmp-server host (\S+)( informs| traps| version " + r"(?:1|2c|3 \S+)| vrf \S+)* )(\S+)", + 4, + ) + ], # This is from JUNOS # TODO(https://github.com/intentionet/netconan/issues/4): # See if we need to make the snmp keyword optional for Juniper # Also, this needs to be tested against config lines generated on a JUNOS router # (to make sure the regex handles different syntaxes allowed in the line) - [(r'(?P(\S* )*snmp( \S+)* (community|trap-group) )([^ ;]+)', 5)] + [(r"(?P(\S* )*snmp( \S+)* (community|trap-group) )([^ ;]+)", 5)], ] diff --git a/netconan/ip_anonymization.py b/netconan/ip_anonymization.py index df61529..721db0d 100644 --- a/netconan/ip_anonymization.py +++ b/netconan/ip_anonymization.py @@ -14,51 +14,52 @@ # limitations under the License. from __future__ import unicode_literals -from abc import ABCMeta, abstractmethod -from bidict import bidict import ipaddress import logging import re - +from abc import ABCMeta, abstractmethod from hashlib import md5 -from six import add_metaclass, iteritems, text_type, u +from bidict import bidict +from six import add_metaclass, iteritems, text_type, u -_IPv4_OCTET_PATTERN = r'(25[0-5]|(2[0-4]|1?[0-9])?[0-9])' +_IPv4_OCTET_PATTERN = r"(25[0-5]|(2[0-4]|1?[0-9])?[0-9])" # Match address starting at beginning of line or surrounded by these, appropriate enclosing chars -_IPv4_ENCLOSING = r'[^\w.]' # Match anything but "word" chars or `.` -_IPv6_ENCLOSING = r'[^\w:]' # Match anything but "word" chars or `:` +_IPv4_ENCLOSING = r"[^\w.]" # Match anything but "word" chars or `.` +_IPv6_ENCLOSING = r"[^\w:]" # Match anything but "word" chars or `:` # Deliberately allowing leading zeros and will remove them later -IPv4_PATTERN = re.compile(( - r'(?:(?<=^)|(?<={enclosing}))' - r'((0*{octet}\.){{3}}0*{octet})' - r'(?=/(\d{{1,3}}))?(?={enclosing}|$)').format( +IPv4_PATTERN = re.compile( + ( + r"(?:(?<=^)|(?<={enclosing}))" + r"((0*{octet}\.){{3}}0*{octet})" + r"(?=/(\d{{1,3}}))?(?={enclosing}|$)" + ).format( enclosing=_IPv4_ENCLOSING, octet=_IPv4_OCTET_PATTERN, - )) + ) +) # Modified from https://stackoverflow.com/a/17871737/1715495 IPv6_PATTERN = re.compile( - r'(?:(?<=^)|(?<={enclosing}))'.format(enclosing=_IPv6_ENCLOSING) + - r'(([0-9a-f]{1,4}:){7,7}[0-9a-f]{1,4}' - r'|([0-9a-f]{1,4}:){1,7}:' - r'|([0-9a-f]{1,4}:){1,6}:[0-9a-f]{1,4}' - r'|([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}' - r'|([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}' - r'|([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}' - r'|([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}' - r'|[0-9a-f]{1,4}:((:[0-9a-f]{1,4}){1,6})' - r'|:((:[0-9a-f]{1,4}){1,7}|:)' - r'|fe80:(:[0-9a-f]{0,4}){0,4}%[0-9a-z]{1,}' + - r'|::(ffff(:0{{1,4}})?:)?({octet}\.){{3}}{octet}' - r'|([0-9a-f]{{1,4}}:){{1,4}}:({octet}\.){{3}}{octet})' - r'(?={enclosing}|$)'.format( - enclosing=_IPv6_ENCLOSING, - octet=_IPv4_OCTET_PATTERN), - re.IGNORECASE) + r"(?:(?<=^)|(?<={enclosing}))".format(enclosing=_IPv6_ENCLOSING) + + r"(([0-9a-f]{1,4}:){7,7}[0-9a-f]{1,4}" + r"|([0-9a-f]{1,4}:){1,7}:" + r"|([0-9a-f]{1,4}:){1,6}:[0-9a-f]{1,4}" + r"|([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}" + r"|([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}" + r"|([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}" + r"|([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}" + r"|[0-9a-f]{1,4}:((:[0-9a-f]{1,4}){1,6})" + r"|:((:[0-9a-f]{1,4}){1,7}|:)" + r"|fe80:(:[0-9a-f]{0,4}){0,4}%[0-9a-z]{1,}" + + r"|::(ffff(:0{{1,4}})?:)?({octet}\.){{3}}{octet}" + r"|([0-9a-f]{{1,4}}:){{1,4}}:({octet}\.){{3}}{octet})" + r"(?={enclosing}|$)".format(enclosing=_IPv6_ENCLOSING, octet=_IPv4_OCTET_PATTERN), + re.IGNORECASE, +) def _generate_bit_from_hash(salt, string): @@ -75,11 +76,13 @@ def _ensure_unicode(str): @add_metaclass(ABCMeta) class _BaseIpAnonymizer(object): - def __init__(self, salt, length, salter=_generate_bit_from_hash, preserve_suffix=None): + def __init__( + self, salt, length, salter=_generate_bit_from_hash, preserve_suffix=None + ): self.salt = salt - self.cache = bidict({'': ''}) + self.cache = bidict({"": ""}) self.length = length - self.fmt = '{{:0{length}b}}'.format(length=length) + self.fmt = "{{:0{length}b}}".format(length=length) self.salter = salter self.preserve_suffix = 0 if preserve_suffix is None else preserve_suffix @@ -88,7 +91,10 @@ def anonymize(self, ip_int): if self.preserve_suffix == 0: anon_bits = self._anonymize_bits(bits) else: - to_anon, to_preserve = bits[:-self.preserve_suffix], bits[-self.preserve_suffix:] + to_anon, to_preserve = ( + bits[: -self.preserve_suffix], + bits[-self.preserve_suffix :], + ) anon_bits = self._anonymize_bits(to_anon) + to_preserve return int(anon_bits, 2) @@ -125,13 +131,15 @@ def _deanonymize_bits(self, bits): return ret def dump_to_file(self, file_out): - ips = ((bits, anon_bits) - for bits, anon_bits in iteritems(self.cache) - if len(bits) == self.length) + ips = ( + (bits, anon_bits) + for bits, anon_bits in iteritems(self.cache) + if len(bits) == self.length + ) for bits, anon_bits in ips: ip = self._ip_to_str(bits) anon = self._ip_to_str(anon_bits) - file_out.write('{}\t{}\n'.format(ip, anon)) + file_out.write("{}\t{}\n".format(ip, anon)) @classmethod def _ip_to_str(cls, bits): @@ -161,21 +169,21 @@ class IpAnonymizer(_BaseIpAnonymizer): """An anonymizer for IPv4 addresses.""" IPV4_CLASSES = ( - '0.0.0.0/1', # Class A - '128.0.0.0/2', # Class B - '192.0.0.0/3', # Class C - '224.0.0.0/4', # Class D (implies class E) + "0.0.0.0/1", # Class A + "128.0.0.0/2", # Class B + "192.0.0.0/3", # Class C + "224.0.0.0/4", # Class D (implies class E) ) RFC_1918_NETWORKS = ( - '10.0.0.0/8', # Private-use subnet - '172.16.0.0/12', # Private-use subnet - '192.168.0.0/16', # Private-use subnet + "10.0.0.0/8", # Private-use subnet + "172.16.0.0/12", # Private-use subnet + "192.168.0.0/16", # Private-use subnet ) DEFAULT_PRESERVED_PREFIXES = IPV4_CLASSES + RFC_1918_NETWORKS - _DROP_ZEROS_PATTERN = re.compile(r'0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)') + _DROP_ZEROS_PATTERN = re.compile(r"0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)") def __init__(self, salt, preserve_prefixes=None, preserve_addresses=None, **kwargs): """Create an anonymizer using the specified salt.""" @@ -187,8 +195,7 @@ def __init__(self, salt, preserve_prefixes=None, preserve_addresses=None, **kwar self._preserve_addresses = [] if preserve_addresses is not None: self._preserve_addresses = [ - ipaddress.ip_network(_ensure_unicode(n)) - for n in preserve_addresses + ipaddress.ip_network(_ensure_unicode(n)) for n in preserve_addresses ] # Make sure the prefixes are also preserved for preserved blocks, so # anonymized addresses outside the block don't accidentally collide @@ -197,11 +204,13 @@ def __init__(self, salt, preserve_prefixes=None, preserve_addresses=None, **kwar # Preserve relevant prefixes for subnet_str in preserve_prefixes: subnet = ipaddress.ip_network(_ensure_unicode(subnet_str)) - prefix_bits = self.fmt.format(int(subnet.network_address))[:subnet.prefixlen] + prefix_bits = self.fmt.format(int(subnet.network_address))[ + : subnet.prefixlen + ] for position in range(len(prefix_bits)): value = prefix_bits[:position] - self.cache[value + '0'] = value + '0' - self.cache[value + '1'] = value + '1' + self.cache[value + "0"] = value + "0" + self.cache[value + "1"] = value + "1" def _is_mask(self, possible_mask_int): """Return True if the input int can be used as a 32-bit prefix mask. @@ -231,7 +240,7 @@ def make_addr(cls, addr_str): those zeros will be ignored (1.2.3.40) -- they will NOT be interpreted as octal (1.2.3.32). """ - addr_str = IpAnonymizer._DROP_ZEROS_PATTERN.sub(r'\1.\2.\3.\4', addr_str) + addr_str = IpAnonymizer._DROP_ZEROS_PATTERN.sub(r"\1.\2.\3.\4", addr_str) return ipaddress.IPv4Address(_ensure_unicode(addr_str)) @classmethod @@ -243,8 +252,7 @@ def should_anonymize(self, ip_int): """Check if a given address should be anonymized (e.g. is it a mask or address?).""" ip = ipaddress.ip_address(ip_int) return not ( - self._is_mask(ip_int) or - any([ip in n for n in self._preserve_addresses]) + self._is_mask(ip_int) or any([ip in n for n in self._preserve_addresses]) ) @@ -303,4 +311,6 @@ def anonymize_ip_addr(anonymizer, line, undo_ip_anon=False): will be replaced with the unanonymized address. """ pattern = anonymizer.get_addr_pattern() - return pattern.sub(lambda match: _anonymize_match(anonymizer, match.group(0), undo_ip_anon), line) + return pattern.sub( + lambda match: _anonymize_match(anonymizer, match.group(0), undo_ip_anon), line + ) diff --git a/netconan/netconan.py b/netconan/netconan.py index 7d001c6..cc586a4 100644 --- a/netconan/netconan.py +++ b/netconan/netconan.py @@ -16,14 +16,14 @@ from __future__ import absolute_import import argparse - -import configargparse import logging import sys -from .ip_anonymization import IpAnonymizer -from .anonymize_files import anonymize_files +import configargparse + from . import __version__ +from .anonymize_files import anonymize_files +from .ip_anonymization import IpAnonymizer def host_bits(x): @@ -31,7 +31,7 @@ def host_bits(x): # the name of this function is used for error message, apparently. val = int(x) if val < 0 or val > 32: - raise argparse.ArgumentError('valid values are [0, 32]') + raise argparse.ArgumentError("valid values are [0, 32]") return val @@ -47,47 +47,113 @@ def _parse_args(argv): values override config file values which override defaults. Config file syntax allows: key=value, flag=true, stuff=[a,b,c] (for more details, see here https://goo.gl/R74nmi). - """ - ) - - parser.add_argument('--version', action='version', version=__version__, - help='Print version number and exit') - parser.add_argument('-a', '--anonymize-ips', action='store_true', default=False, - help='Anonymize IP addresses') - parser.add_argument('-c', '--config', is_config_file=True, - help='Netconan configuration file with defaults for these CLI parameters') - parser.add_argument('-d', '--dump-ip-map', default=None, - help='Dump IP address anonymization map to specified file') - parser.add_argument('-i', '--input', required=True, - help='Input file or directory containing files to anonymize') - parser.add_argument('-l', '--log-level', default='INFO', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - help='Determines what level of logs to display') - parser.add_argument('-n', '--as-numbers', default=None, - help='List of comma separated AS numbers to anonymize') - parser.add_argument('-o', '--output', required=True, - help='Output file or directory where anonymized files are placed') - parser.add_argument('-p', '--anonymize-passwords', action='store_true', default=False, - help='Anonymize password and snmp community lines') - parser.add_argument('-r', '--reserved-words', default=None, - help='List of comma separated words that should not be anonymized') - parser.add_argument('-s', '--salt', default=None, - help='Salt for IP and sensitive keyword anonymization') - parser.add_argument('-u', '--undo', action='store_true', default=False, - help='Undo reversible anonymization (must specify salt)') - parser.add_argument('-w', '--sensitive-words', default=None, - help='List of comma separated keywords to anonymize') - parser.add_argument('--preserve-prefixes', - default=','.join(IpAnonymizer.DEFAULT_PRESERVED_PREFIXES), - help='List of comma separated IP prefixes to preserve. Specified prefixes are preserved, but the host bits within those prefixes are still anonymized. To preserve prefixes and host bits in specified blocks, use --preserve-addresses instead') - parser.add_argument('--preserve-addresses', default=None, - help='List of comma separated IP addresses or networks to preserve. Prefixes and host bits within those networks are preserved. To preserve just prefixes and anonymize host bits, use --preserve-prefixes') - parser.add_argument('--preserve-private-addresses', - action='store_true', default=False, - help='Preserve private-use IP addresses. Prefixes and host bits within the private-use IP networks are preserved. To preserve specific addresses or networks, use --preserve-addresses instead. To preserve just prefixes and anonymize host bits, use --preserve-prefixes') - parser.add_argument('--preserve-host-bits', - type=host_bits, default=8, - help='Preserve the trailing bits of IP addresses, aka the host bits of a network. Set this value large enough to represent the largest interface network (e.g., 8 for a /24 or 12 for a /20) or NAT pool.') + """, + ) + + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print version number and exit", + ) + parser.add_argument( + "-a", + "--anonymize-ips", + action="store_true", + default=False, + help="Anonymize IP addresses", + ) + parser.add_argument( + "-c", + "--config", + is_config_file=True, + help="Netconan configuration file with defaults for these CLI parameters", + ) + parser.add_argument( + "-d", + "--dump-ip-map", + default=None, + help="Dump IP address anonymization map to specified file", + ) + parser.add_argument( + "-i", + "--input", + required=True, + help="Input file or directory containing files to anonymize", + ) + parser.add_argument( + "-l", + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Determines what level of logs to display", + ) + parser.add_argument( + "-n", + "--as-numbers", + default=None, + help="List of comma separated AS numbers to anonymize", + ) + parser.add_argument( + "-o", + "--output", + required=True, + help="Output file or directory where anonymized files are placed", + ) + parser.add_argument( + "-p", + "--anonymize-passwords", + action="store_true", + default=False, + help="Anonymize password and snmp community lines", + ) + parser.add_argument( + "-r", + "--reserved-words", + default=None, + help="List of comma separated words that should not be anonymized", + ) + parser.add_argument( + "-s", + "--salt", + default=None, + help="Salt for IP and sensitive keyword anonymization", + ) + parser.add_argument( + "-u", + "--undo", + action="store_true", + default=False, + help="Undo reversible anonymization (must specify salt)", + ) + parser.add_argument( + "-w", + "--sensitive-words", + default=None, + help="List of comma separated keywords to anonymize", + ) + parser.add_argument( + "--preserve-prefixes", + default=",".join(IpAnonymizer.DEFAULT_PRESERVED_PREFIXES), + help="List of comma separated IP prefixes to preserve. Specified prefixes are preserved, but the host bits within those prefixes are still anonymized. To preserve prefixes and host bits in specified blocks, use --preserve-addresses instead", + ) + parser.add_argument( + "--preserve-addresses", + default=None, + help="List of comma separated IP addresses or networks to preserve. Prefixes and host bits within those networks are preserved. To preserve just prefixes and anonymize host bits, use --preserve-prefixes", + ) + parser.add_argument( + "--preserve-private-addresses", + action="store_true", + default=False, + help="Preserve private-use IP addresses. Prefixes and host bits within the private-use IP networks are preserved. To preserve specific addresses or networks, use --preserve-addresses instead. To preserve just prefixes and anonymize host bits, use --preserve-prefixes", + ) + parser.add_argument( + "--preserve-host-bits", + type=host_bits, + default=8, + help="Preserve the trailing bits of IP addresses, aka the host bits of a network. Set this value large enough to represent the largest interface network (e.g., 8 for a /24 or 12 for a /20) or NAT pool.", + ) return parser.parse_args(argv) @@ -99,68 +165,85 @@ def main(argv=sys.argv[1:]): raise ValueError("Input must be specified") log_level = logging.getLevelName(args.log_level) - logging.basicConfig(format='%(levelname)s %(message)s', level=log_level) + logging.basicConfig(format="%(levelname)s %(message)s", level=log_level) if not args.output: raise ValueError("Output must be specified") if args.undo: if args.anonymize_ips: - raise ValueError('Cannot anonymize and undo anonymization, select ' - 'only one.') + raise ValueError( + "Cannot anonymize and undo anonymization, select only one." + ) if args.salt is None: - raise ValueError('Salt used for anonymization must be specified in ' - 'order to undo anonymization.') + raise ValueError( + "Salt used for anonymization must be specified in order to undo anonymization." + ) if args.dump_ip_map is not None: if not args.anonymize_ips: - raise ValueError('Can only dump IP address map when anonymizing IP ' - 'addresses.') + raise ValueError( + "Can only dump IP address map when anonymizing IP addresses." + ) as_numbers = None if args.as_numbers is not None: - as_numbers = args.as_numbers.split(',') + as_numbers = args.as_numbers.split(",") reserved_words = None if args.reserved_words is not None: - reserved_words = args.reserved_words.split(',') + reserved_words = args.reserved_words.split(",") sensitive_words = None if args.sensitive_words is not None: - sensitive_words = args.sensitive_words.split(',') + sensitive_words = args.sensitive_words.split(",") preserve_prefixes = None if args.preserve_prefixes is not None: - preserve_prefixes = args.preserve_prefixes.split(',') + preserve_prefixes = args.preserve_prefixes.split(",") preserve_addresses = None if args.preserve_addresses is not None: - preserve_addresses = args.preserve_addresses.split(',') + preserve_addresses = args.preserve_addresses.split(",") if args.preserve_private_addresses: addrs = list(IpAnonymizer.RFC_1918_NETWORKS) # Merge private addresses with explicitly preserved addresses - preserve_addresses = addrs if preserve_addresses is None else ( - preserve_addresses + addrs + preserve_addresses = ( + addrs if preserve_addresses is None else (preserve_addresses + addrs) ) - if not any([ - as_numbers, - sensitive_words, - args.anonymize_passwords, - args.anonymize_ips, - args.undo - ]): - logging.warning('No anonymization options turned on, ' - 'no output file(s) will be generated.') + if not any( + [ + as_numbers, + sensitive_words, + args.anonymize_passwords, + args.anonymize_ips, + args.undo, + ] + ): + logging.warning( + "No anonymization options turned on, " + "no output file(s) will be generated." + ) else: - anonymize_files(args.input, args.output, args.anonymize_passwords, - args.anonymize_ips, args.salt, args.dump_ip_map, - sensitive_words, args.undo, as_numbers, reserved_words, - preserve_prefixes, preserve_addresses, - preserve_suffix_v4=args.preserve_host_bits, - preserve_suffix_v6=args.preserve_host_bits) + anonymize_files( + args.input, + args.output, + args.anonymize_passwords, + args.anonymize_ips, + args.salt, + args.dump_ip_map, + sensitive_words, + args.undo, + as_numbers, + reserved_words, + preserve_prefixes, + preserve_addresses, + preserve_suffix_v4=args.preserve_host_bits, + preserve_suffix_v6=args.preserve_host_bits, + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/netconan/sensitive_item_removal.py b/netconan/sensitive_item_removal.py index 19a911f..19f1096 100644 --- a/netconan/sensitive_item_removal.py +++ b/netconan/sensitive_item_removal.py @@ -14,25 +14,27 @@ # limitations under the License. from __future__ import absolute_import -import re -import logging +import logging +import re from binascii import b2a_hex from enum import Enum from hashlib import md5 -from .default_pwd_regexes import default_pwd_line_regexes, default_com_line_regexes -from .default_reserved_words import default_reserved_words + # Using passlib for digests not supported by hashlib from passlib.hash import cisco_type7, md5_crypt, sha512_crypt from six import b +from .default_pwd_regexes import default_com_line_regexes, default_pwd_line_regexes +from .default_reserved_words import default_reserved_words # A regex matching any of the characters that are allowed to precede a password # regex (e.g. sensitive line is allowed to be in quotes or after a colon) # This is an ignored group, so it does not muck with the password regex indicies # And the ?<= is a lookbehind, not part of regex match text/sub text -_ALLOWED_REGEX_PREFIX = (r'(?:(?<={prefix})|(?<={prefix} )|(?<=^)|(?<=^ ))' - .format(prefix=r'[^-_a-zA-Z\d]')) +_ALLOWED_REGEX_PREFIX = r"(?:(?<={prefix})|(?<={prefix} )|(?<=^)|(?<=^ ))".format( + prefix=r"[^-_a-zA-Z\d]" +) # Number of digits to extract from hash for sensitive keyword replacement _ANON_SENSITIVE_WORD_LEN = 6 @@ -41,39 +43,39 @@ # following patterns, mostly extracted from examples at # https://www.cisco.com/c/en/us/td/docs/routers/crs/software/crs_r4-1/routing/command/reference/b_routing_cr41crs/b_routing_cr41crs_chapter_01000.html # Text followed by the word 'additive' -_IGNORED_COMM_ADDITIVE = r'\S+ additive' +_IGNORED_COMM_ADDITIVE = r"\S+ additive" # Numeric, colon separated, and parameter ($) communities -_IGNORED_COMM_COLON = r'(peeras|\$\w+|\d+)\:(peeras|\$\w+|\d+)' +_IGNORED_COMM_COLON = r"(peeras|\$\w+|\d+)\:(peeras|\$\w+|\d+)" # List of communities enclosed in parenthesis, being permissive here for the # content inside the parenthesis for simplicity -_IGNORED_COMM_LIST = r'\([\S ]+\)' +_IGNORED_COMM_LIST = r"\([\S ]+\)" # Well-known BPG communities -_IGNORED_COMM_WELL_KNOWN = 'gshut|internet|local-AS|no-advertise|no-export|none' -_IGNORED_COMMUNITIES = (r'((\d+|{additive}|{colon}|{list}|{well_known})(?!\S))' - .format(additive=_IGNORED_COMM_ADDITIVE, - colon=_IGNORED_COMM_COLON, - list=_IGNORED_COMM_LIST, - well_known=_IGNORED_COMM_WELL_KNOWN)) +_IGNORED_COMM_WELL_KNOWN = "gshut|internet|local-AS|no-advertise|no-export|none" +_IGNORED_COMMUNITIES = r"((\d+|{additive}|{colon}|{list}|{well_known})(?!\S))".format( + additive=_IGNORED_COMM_ADDITIVE, + colon=_IGNORED_COMM_COLON, + list=_IGNORED_COMM_LIST, + well_known=_IGNORED_COMM_WELL_KNOWN, +) -_LINE_SCRUBBED_MESSAGE = '! Sensitive line SCRUBBED by netconan' +_LINE_SCRUBBED_MESSAGE = "! Sensitive line SCRUBBED by netconan" # Text that is allowed to surround passwords, to be preserved -_PASSWORD_ENCLOSING_TEXT = ['\\\'', '\\"', '\'', '"', ' '] -_PASSWORD_ENCLOSING_HEAD_TEXT = _PASSWORD_ENCLOSING_TEXT + ['[', '{'] -_PASSWORD_ENCLOSING_TAIL_TEXT = _PASSWORD_ENCLOSING_TEXT + [']', '}', ';', ','] +_PASSWORD_ENCLOSING_TEXT = ["\\'", '\\"', "'", '"', " "] +_PASSWORD_ENCLOSING_HEAD_TEXT = _PASSWORD_ENCLOSING_TEXT + ["[", "{"] +_PASSWORD_ENCLOSING_TAIL_TEXT = _PASSWORD_ENCLOSING_TEXT + ["]", "}", ";", ","] # These are extra regexes to find lines that seem like they might contain # sensitive info (these are not already caught by RANCID default regexes) extra_password_regexes = [ - [(r'(?<=encrypted-password )(\S+)', None)], + [(r"(?<=encrypted-password )(\S+)", None)], [(r'(?<=key ")([^"]+)', 1)], - [(r'(?<=key-hash sha256 )(\S+)', 1)], + [(r"(?<=key-hash sha256 )(\S+)", 1)], # Replace communities that do not look like well-known BGP communities # i.e. snmp communities - [(r'(?<=set community )((?!{ignore})\S+)' - .format(ignore=_IGNORED_COMMUNITIES), 1)], - [(r'(?<=snmp-server mib community-map )([^ :]+)', 1)], - [(r'(?<=snmp-community )(\S+)', 1)], + [(r"(?<=set community )((?!{ignore})\S+)".format(ignore=_IGNORED_COMMUNITIES), 1)], + [(r"(?<=snmp-server mib community-map )([^ :]+)", 1)], + [(r"(?<=snmp-community )(\S+)", 1)], # Catch-all's matching what looks like hashed passwords [(r'("?\$9\$[^\s;"]+)', 1)], [(r'("?\$1\$[^\s;"]+)', 1)], @@ -101,15 +103,18 @@ def _generate_as_number_regex(self, as_numbers): """Generate regex for finding AS number.""" # Match a non-digit, any of the AS numbers and another non-digit # Using lookahead and lookbehind to match on context but not include that context in the match - self.as_num_regex = re.compile(r'(?:(?<=\D)|(?<=^))({})(?=\D|$)'.format( - '|'.join(as_numbers))) + self.as_num_regex = re.compile( + r"(?:(?<=\D)|(?<=^))({})(?=\D|$)".format("|".join(as_numbers)) + ) def _generate_as_number_replacement(self, as_number): """Generate a replacement AS number for the given AS number and salt.""" hash_val = int(md5((self.salt + as_number).encode()).hexdigest(), 16) as_number = int(as_number) if as_number < 0 or as_number > 4294967295: - raise ValueError('AS number provided was outside accepted range (0-4294967295)') + raise ValueError( + "AS number provided was outside accepted range (0-4294967295)" + ) block_begin = 0 for next_block_begin in self._AS_NUM_BOUNDARIES: @@ -119,7 +124,10 @@ def _generate_as_number_replacement(self, as_number): def _generate_as_number_replacement_map(self, as_numbers): """Generate map of AS numbers and their replacements.""" - self.as_num_map = {as_num: self._generate_as_number_replacement(as_num) for as_num in as_numbers} + self.as_num_map = { + as_num: self._generate_as_number_replacement(as_num) + for as_num in as_numbers + } def get_as_number_pattern(self): """Return the compiled regex to find AS numbers.""" @@ -140,7 +148,9 @@ def __init__(self, sensitive_words, salt, reserved_words=default_reserved_words) self.sens_regex = self._generate_sensitive_word_regex(sensitive_words_) self.sens_word_replacements = {} # Figure out which reserved words may clash with sensitive words, so they can be preserved in anonymization - self.conflicting_words = self._generate_conflicting_reserved_word_list(sensitive_words_) + self.conflicting_words = self._generate_conflicting_reserved_word_list( + sensitive_words_ + ) def anonymize(self, line): """Anonymize sensitive words from the input line.""" @@ -148,26 +158,34 @@ def anonymize(self, line): leading, words, trailing = _split_line(line) # Anonymize only words that do not match the conflicting (reserved) words words = [ - w if w in self.conflicting_words else self.sens_regex.sub(self._lookup_anon_word, w) for w in words + w + if w in self.conflicting_words + else self.sens_regex.sub(self._lookup_anon_word, w) + for w in words ] # Restore leading and trailing whitespace since those were removed when splitting into words - line = leading + ' '.join(words) + trailing + line = leading + " ".join(words) + trailing return line def _generate_conflicting_reserved_word_list(self, sensitive_words): """Return a list of reserved words that may conflict with the specified sensitive words.""" conflicting_words = set() for sensitive_word in sensitive_words: - conflicting_words.update(set([w for w in self.reserved_words if sensitive_word in w])) + conflicting_words.update( + set([w for w in self.reserved_words if sensitive_word in w]) + ) if conflicting_words: - logging.warning('Specified sensitive words overlap with reserved words. ' - 'The following reserved words will be preserved: %s', conflicting_words) + logging.warning( + "Specified sensitive words overlap with reserved words. " + "The following reserved words will be preserved: %s", + conflicting_words, + ) return conflicting_words @classmethod def _generate_sensitive_word_regex(cls, sensitive_words): """Compile and return regex for the specified list of sensitive words.""" - return re.compile('({})'.format('|'.join(sensitive_words)), re.IGNORECASE) + return re.compile("({})".format("|".join(sensitive_words)), re.IGNORECASE) def _get_or_generate_sensitive_word_replacement(self, sensitive_word): """Return the replacement string for the given sensitive word. @@ -178,7 +196,9 @@ def _get_or_generate_sensitive_word_replacement(self, sensitive_word): if replacement is None: # Only using part of the md5 hash result as the anonymized replacement # to cut down on the size of the replacements - replacement = md5((self.salt + sensitive_word).encode()).hexdigest()[:_ANON_SENSITIVE_WORD_LEN] + replacement = md5((self.salt + sensitive_word).encode()).hexdigest()[ + :_ANON_SENSITIVE_WORD_LEN + ] self.sens_word_replacements[sensitive_word] = replacement return replacement @@ -216,20 +236,18 @@ def _anonymize_value(raw_val, lookup, reserved_words): # Separate enclosing text (e.g. quotes) from the underlying value sens_head, val, sens_tail = _extract_enclosing_text(raw_val) if val in reserved_words: - logging.debug('Skipping anonymization of reserved word: "%s"', - val) + logging.debug('Skipping anonymization of reserved word: "%s"', val) return raw_val if not val: - logging.debug('Nothing to anonymize after removing special characters') + logging.debug("Nothing to anonymize after removing special characters") return raw_val if val in lookup: anon_val = lookup[val] - logging.debug( - 'Anonymized input "%s" to "%s" (via lookup)', val, anon_val) + logging.debug('Anonymized input "%s" to "%s" (via lookup)', val, anon_val) return sens_head + anon_val + sens_tail - anon_val = 'netconanRemoved{}'.format(len(lookup)) + anon_val = "netconanRemoved{}".format(len(lookup)) item_format = _check_sensitive_item_format(val) if item_format == _sensitive_item_formats.cisco_type7: # Not salting sensitive data, using static salt here to more easily @@ -245,10 +263,10 @@ def _anonymize_value(raw_val, lookup, reserved_words): anon_val = b2a_hex(b(anon_val)).decode() if item_format == _sensitive_item_formats.md5: - old_salt_size = len(val.split('$')[2]) + old_salt_size = len(val.split("$")[2]) # Not salting sensitive data, using static salt here to more easily # identify anonymized lines - anon_val = md5_crypt.using(salt='0' * old_salt_size).hash(anon_val) + anon_val = md5_crypt.using(salt="0" * old_salt_size).hash(anon_val) if item_format == _sensitive_item_formats.sha512: # Hash anon_val w/standard rounds=5000 to omit rounds parameter from hash output @@ -258,7 +276,7 @@ def _anonymize_value(raw_val, lookup, reserved_words): # TODO(https://github.com/intentionet/netconan/issues/16) # Encode base anon_val instead of just returning a constant here # This value corresponds to encoding: Conan812183 - anon_val = '$9$0000IRc-dsJGirewg4JDj9At0RhSreK8Xhc' + anon_val = "$9$0000IRc-dsJGirewg4JDj9At0RhSreK8Xhc" lookup[val] = anon_val logging.debug('Anonymized input "%s" to "%s"', val, anon_val) @@ -271,32 +289,32 @@ def _check_sensitive_item_format(val): # Order is important here (e.g. type 7 looks like hex or text, but has a # specific format so it should override hex or text) - if re.match(r'^\$9\$[\S]+$', val): + if re.match(r"^\$9\$[\S]+$", val): item_format = _sensitive_item_formats.juniper_type9 - if re.match(r'^\$6\$[\S]+$', val): + if re.match(r"^\$6\$[\S]+$", val): item_format = _sensitive_item_formats.sha512 - if re.match(r'^\$1\$[\S]+\$[\S]+$', val): + if re.match(r"^\$1\$[\S]+\$[\S]+$", val): item_format = _sensitive_item_formats.md5 - if re.match(r'^[0-9a-fA-F]+$', val): + if re.match(r"^[0-9a-fA-F]+$", val): item_format = _sensitive_item_formats.hexadecimal - if re.match(r'^[01][0-9]([0-9a-fA-F]{2})+$', val): + if re.match(r"^[01][0-9]([0-9a-fA-F]{2})+$", val): item_format = _sensitive_item_formats.cisco_type7 - if re.match(r'^[0-9]+$', val): + if re.match(r"^[0-9]+$", val): item_format = _sensitive_item_formats.numeric return item_format -def _extract_enclosing_text(in_val, head='', tail=''): +def _extract_enclosing_text(in_val, head="", tail=""): """Extract allowed enclosing text from input and return the enclosing and enclosed text.""" val = in_val for head_text in _PASSWORD_ENCLOSING_HEAD_TEXT: if val.startswith(head_text): head += head_text - val = val[len(head_text):] + val = val[len(head_text) :] for tail_text in _PASSWORD_ENCLOSING_TAIL_TEXT: if val.endswith(tail_text): tail = tail_text + tail - val = val[:-len(tail_text)] + val = val[: -len(tail_text)] if val != in_val: return _extract_enclosing_text(val, head, tail) @@ -305,19 +323,25 @@ def _extract_enclosing_text(in_val, head='', tail=''): def generate_default_sensitive_item_regexes(): """Compile and return the default password and community line regexes.""" - combined_regexes = default_pwd_line_regexes + default_com_line_regexes + \ - extra_password_regexes - return [[(re.compile(_ALLOWED_REGEX_PREFIX + regex_), num) for regex_, num in group] - for group in combined_regexes] - - -def replace_matching_item(compiled_regexes, input_line, pwd_lookup, reserved_words=default_reserved_words): + combined_regexes = ( + default_pwd_line_regexes + default_com_line_regexes + extra_password_regexes + ) + return [ + [(re.compile(_ALLOWED_REGEX_PREFIX + regex_), num) for regex_, num in group] + for group in combined_regexes + ] + + +def replace_matching_item( + compiled_regexes, input_line, pwd_lookup, reserved_words=default_reserved_words +): """If line matches a regex, anonymize or remove the line.""" # Collapse whitespace to simplify regexes, also preserve leading and trailing whitespace leading, words, trailing = _split_line(input_line) # Save enclosing text (like quotes) to avoid removing during anonymization leading, output_line, trailing = _extract_enclosing_text( - ' '.join(words), leading, trailing) + " ".join(words), leading, trailing + ) # Note: compiled_regexes is a list of lists; the inner list is a group of # related regexes @@ -331,24 +355,26 @@ def replace_matching_item(compiled_regexes, input_line, pwd_lookup, reserved_wor if match is None: continue match_found = True - logging.debug('Match found on %s', output_line.rstrip()) + logging.debug("Match found on %s", output_line.rstrip()) # If this regex cannot preserve text around sensitive info, # then just remove the whole line if sensitive_item_num is None: logging.warning( 'Anonymizing sensitive info in lines like "%s" is currently' - ' unsupported, so removing this line completely', - compiled_re.pattern) - output_line = compiled_re.sub( - _LINE_SCRUBBED_MESSAGE, output_line) + " unsupported, so removing this line completely", + compiled_re.pattern, + ) + output_line = compiled_re.sub(_LINE_SCRUBBED_MESSAGE, output_line) break # This is text preceding the password and shouldn't be anonymized - prefix = match.group('prefix') if 'prefix' in match.groupdict() else "" + prefix = match.group("prefix") if "prefix" in match.groupdict() else "" # re.sub replaces the entire matching string, which includes prefix # Therefore, anon_val should have prefix prepended if applicable - anon_val = prefix + _anonymize_value(match.group(sensitive_item_num), pwd_lookup, reserved_words) + anon_val = prefix + _anonymize_value( + match.group(sensitive_item_num), pwd_lookup, reserved_words + ) output_line = compiled_re.sub(anon_val, output_line) # If any matches existed in this regex group, stop processing more regexes @@ -361,4 +387,4 @@ def replace_matching_item(compiled_regexes, input_line, pwd_lookup, reserved_wor def _split_line(line): """Split line into leading whitespace, list of words, and trailing whitespace.""" - return line[:-len(line.lstrip())], line.split(), line[len(line.rstrip()):] + return line[: -len(line.lstrip())], line.split(), line[len(line.rstrip()) :] diff --git a/setup.cfg b/setup.cfg index 97d0174..7d0cfe0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,5 +23,5 @@ universal=1 [flake8] filename=*.py, -extend-ignore=E501,D401 +extend-ignore=E203,E501,D401,W503 exclude=docs,__pychache__,.eggs,*.egg,build,virtualEnv,virtualEnv3 diff --git a/setup.py b/setup.py index c221dee..94e6819 100644 --- a/setup.py +++ b/setup.py @@ -18,112 +18,99 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Always prefer setuptools over distutils -from setuptools import setup, find_packages from os import path +# Always prefer setuptools over distutils +from setuptools import find_packages, setup + here = path.abspath(path.dirname(__file__)) about = {} -with open(path.join(here, 'netconan', '__init__.py'), 'r') as f: +with open(path.join(here, "netconan", "__init__.py"), "r") as f: exec(f.read(), about) -with open(path.join(here, 'README.rst')) as f: +with open(path.join(here, "README.rst")) as f: readme = f.read() setup( - name=about['__name__'], + name=about["__name__"], # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version=about['__version__'], - description=about['__desc__'], + version=about["__version__"], + description=about["__desc__"], long_description=readme, # The project's main homepage. - url=about['__url__'], + url=about["__url__"], # Author details - author='Intentionet', - author_email='netconan-dev@intentionet.com', - + author="Intentionet", + author_email="netconan-dev@intentionet.com", # Choose your license - license='Apache License 2.0', - + license="Apache License 2.0", # What does your project relate to? - keywords='network configuration anonymizer', - + keywords="network configuration anonymizer", # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - + packages=find_packages(exclude=["contrib", "docs", "tests"]), # Alternatively, if you want to distribute just a my_module.py, uncomment # this: # py_modules=["my_module"], - # List run-time dependencies here. These will be installed by pip when # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=[ - 'configargparse<1.0.0', - 'bidict<1.0.0', + "configargparse<1.0.0", + "bidict<1.0.0", # Only use enum34 for Python older than 3.4 'enum34<2.0.0; python_version < "3.4"', - 'ipaddress<2.0.0', - 'passlib<2.0.0', - 'six<2.0.0' + "ipaddress<2.0.0", + "passlib<2.0.0", + "six<2.0.0", ], - # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: # $ pip install -e .[dev,test] extras_require={ - 'dev': [ - 'flake8<4.0.0', - 'flake8-docstrings<2.0.0', - 'pydocstyle<4.0.0' - ], + "dev": ["flake8<4.0.0", "flake8-docstrings<2.0.0", "pydocstyle<4.0.0"], # Duplicated test deps here for now, since dependency resolution is # failing for python2.7 in CI - 'test': [ - 'pytest>=4.6.0,<5.0.0', - 'pytest-cov<3.0.0', - 'requests_mock<2.0.0', - 'testfixtures<7.0.0', + "test": [ + "pytest>=4.6.0,<5.0.0", + "pytest-cov<3.0.0", + "requests_mock<2.0.0", + "testfixtures<7.0.0", # zipp 2.2 does not work w/ Python < 3.6 - 'zipp<2.2', + "zipp<2.2", ], }, - # List pytest requirements for running unit tests - setup_requires=['pytest-runner<6.0'], + setup_requires=["pytest-runner<6.0"], # pytest 5+ does not support Python 2 tests_require=[ - 'pytest>=4.6.0,<5.0.0', - 'pytest-cov<3.0.0', - 'requests_mock<2.0.0', - 'testfixtures<7.0.0', + "pytest>=4.6.0,<5.0.0", + "pytest-cov<3.0.0", + "requests_mock<2.0.0", + "testfixtures<7.0.0", # zipp 2.2 does not work w/ Python < 3.6 - 'zipp<2.2', + "zipp<2.2", ], - # If there are data files included in your packages that need to be # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. package_data={}, - # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa # In this case, 'data_file' will be installed into '/my_data' data_files=[], - # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow # pip to create the appropriate form of executable for the target platform. entry_points={ - 'console_scripts': [ - 'netconan = netconan.netconan:main', + "console_scripts": [ + "netconan = netconan.netconan:main", ], }, ) diff --git a/tests/end_to_end/test_end_to_end.py b/tests/end_to_end/test_end_to_end.py index dc05c52..af4f5cf 100644 --- a/tests/end_to_end/test_end_to_end.py +++ b/tests/end_to_end/test_end_to_end.py @@ -14,11 +14,12 @@ # limitations under the License. import os.path + import pytest import six -from netconan.netconan import main from netconan import __version__ +from netconan.netconan import main INPUT_CONTENTS = """ # Intentionet's sensitive test file @@ -64,24 +65,33 @@ def test_end_to_end(tmpdir): ref_file.write(REF_CONTENTS) args = [ - '-i', str(input_dir), - '-o', str(output_dir), - '-s', 'TESTSALT', - '-a', - '-p', - '-w', 'intentionet,sensitive,ADDR', - '-r', 'reservedword', - '-n', '65432,12345', - '--preserve-addresses', '11.11.0.0/16,111.111.111.111', - '--preserve-prefixes', '192.168.2.0/24', - '--preserve-host-bits', '17', + "-i", + str(input_dir), + "-o", + str(output_dir), + "-s", + "TESTSALT", + "-a", + "-p", + "-w", + "intentionet,sensitive,ADDR", + "-r", + "reservedword", + "-n", + "65432,12345", + "--preserve-addresses", + "11.11.0.0/16,111.111.111.111", + "--preserve-prefixes", + "192.168.2.0/24", + "--preserve-host-bits", + "17", ] main(args) with open(str(ref_file)) as f_ref, open(str(output_file)) as f_out: # Compare lines for more readable failed assertion message - t_ref = f_ref.read().split('\n') - t_out = f_out.read().split('\n') + t_ref = f_ref.read().split("\n") + t_out = f_out.read().split("\n") # Make sure output file lines match ref lines assert t_ref == t_out @@ -97,22 +107,26 @@ def test_end_to_end_no_anonymization(tmpdir): output_file = output_dir.join(filename) args = [ - '-i', str(input_dir), - '-o', str(output_dir), - '-s', 'TESTSALT', - '-r', 'reservedword', + "-i", + str(input_dir), + "-o", + str(output_dir), + "-s", + "TESTSALT", + "-r", + "reservedword", ] main(args) # Make sure no output file was generated # when no anonymization args are supplied - assert(not os.path.exists(str(output_file))) + assert not os.path.exists(str(output_file)) def test_version(capsys): """Test that version info is printed.""" with pytest.raises(SystemExit): - main(['--version']) + main(["--version"]) captured = capsys.readouterr() # Python2 prints version info in err instead of out if six.PY2: diff --git a/tests/unit/test_anonymize_files.py b/tests/unit/test_anonymize_files.py index faa1ba5..ce587a8 100644 --- a/tests/unit/test_anonymize_files.py +++ b/tests/unit/test_anonymize_files.py @@ -16,7 +16,6 @@ import os import pytest - from testfixtures import LogCapture from netconan.anonymize_files import anonymize_file, anonymize_files @@ -47,9 +46,15 @@ def test_anonymize_files_bad_input_empty(tmpdir): input_dir = tmpdir.mkdir("input") output_dir = tmpdir.mkdir("output") - with pytest.raises(ValueError, match='Input directory is empty'): - anonymize_files(str(input_dir), str(output_dir), True, True, salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + with pytest.raises(ValueError, match="Input directory is empty"): + anonymize_files( + str(input_dir), + str(output_dir), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) def test_anonymize_files_bad_input_missing(tmpdir): @@ -59,10 +64,15 @@ def test_anonymize_files_bad_input_missing(tmpdir): output_file = tmpdir.mkdir("out").join(filename) - with pytest.raises(ValueError, match='Input does not exist'): - anonymize_files(str(input_file), str(output_file), True, True, - salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + with pytest.raises(ValueError, match="Input does not exist"): + anonymize_files( + str(input_file), + str(output_file), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) def test_anonymize_files_bad_output_file(tmpdir): @@ -73,21 +83,27 @@ def test_anonymize_files_bad_output_file(tmpdir): output_file = tmpdir.mkdir("out").mkdir(filename) - with pytest.raises(ValueError, match='Cannot write output file.*'): + with pytest.raises(ValueError, match="Cannot write output file.*"): anonymize_file(str(input_file), str(output_file)) # Anonymizing files should complete okay, because it skips the errored file with LogCapture() as log_capture: - anonymize_files(str(input_file), str(output_file), True, True, - salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + anonymize_files( + str(input_file), + str(output_file), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) # Confirm the correct message is logged log_capture.check_present( - ('root', 'ERROR', 'Failed to anonymize file {}'.format(str(input_file))) + ("root", "ERROR", "Failed to anonymize file {}".format(str(input_file))) ) # Confirm the exception info was also logged - assert ('Cannot write output file; output file is a directory' - in str(log_capture.records[-1].exc_info[1])) + assert "Cannot write output file; output file is a directory" in str( + log_capture.records[-1].exc_info[1] + ) def test_anonymize_files_bad_output_dir(tmpdir): @@ -97,12 +113,17 @@ def test_anonymize_files_bad_output_dir(tmpdir): input_dir.join(filename).write(_INPUT_CONTENTS) output_file = tmpdir.join("out") - output_file.write('blah') - - with pytest.raises(ValueError, match='Output path must be a directory.*'): - anonymize_files(str(input_dir), str(output_file), True, True, - salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + output_file.write("blah") + + with pytest.raises(ValueError, match="Output path must be a directory.*"): + anonymize_files( + str(input_dir), + str(output_file), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) def test_anonymize_files_dir(tmpdir): @@ -114,12 +135,18 @@ def test_anonymize_files_dir(tmpdir): output_dir = tmpdir.mkdir("output") output_file = output_dir.join(filename) - anonymize_files(str(input_dir), str(output_dir), True, True, salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + anonymize_files( + str(input_dir), + str(output_dir), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) # Make sure output file exists and matches the ref - assert(os.path.isfile(str(output_file))) - assert(read_file(str(output_file)) == _REF_CONTENTS) + assert os.path.isfile(str(output_file)) + assert read_file(str(output_file)) == _REF_CONTENTS def test_anonymize_files_dir_skip_hidden(tmpdir): @@ -132,11 +159,17 @@ def test_anonymize_files_dir_skip_hidden(tmpdir): output_dir = tmpdir.mkdir("output") output_file = output_dir.join(filename) - anonymize_files(str(input_dir), str(output_dir), True, True, salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + anonymize_files( + str(input_dir), + str(output_dir), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) # Make sure output file does not exist - assert(not os.path.exists(str(output_file))) + assert not os.path.exists(str(output_file)) def test_anonymize_files_dir_nested(tmpdir): @@ -150,15 +183,21 @@ def test_anonymize_files_dir_nested(tmpdir): output_file_1 = output_dir.join("subdir1").join(filename) output_file_2 = output_dir.join("subdir2").join("subsubdir").join(filename) - anonymize_files(str(input_dir), str(output_dir), True, True, salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + anonymize_files( + str(input_dir), + str(output_dir), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) # Make sure both output files exists and match the ref - assert(os.path.isfile(str(output_file_1))) - assert(read_file(str(output_file_1)) == _REF_CONTENTS) + assert os.path.isfile(str(output_file_1)) + assert read_file(str(output_file_1)) == _REF_CONTENTS - assert(os.path.isfile(str(output_file_2))) - assert(read_file(str(output_file_2)) == _REF_CONTENTS) + assert os.path.isfile(str(output_file_2)) + assert read_file(str(output_file_2)) == _REF_CONTENTS def test_anonymize_files_file(tmpdir): @@ -169,15 +208,21 @@ def test_anonymize_files_file(tmpdir): output_file = tmpdir.mkdir("out").join(filename) - anonymize_files(str(input_file), str(output_file), True, True, salt=_SALT, - sensitive_words=_SENSITIVE_WORDS) + anonymize_files( + str(input_file), + str(output_file), + True, + True, + salt=_SALT, + sensitive_words=_SENSITIVE_WORDS, + ) # Make sure output file exists and matches the ref - assert(os.path.isfile(str(output_file))) - assert(read_file(str(output_file)) == _REF_CONTENTS) + assert os.path.isfile(str(output_file)) + assert read_file(str(output_file)) == _REF_CONTENTS def read_file(file_path): """Read and return contents of file at specified path.""" - with open(file_path, 'r') as f: + with open(file_path, "r") as f: return f.read() diff --git a/tests/unit/test_as_number_anonymization.py b/tests/unit/test_as_number_anonymization.py index 90b98a3..95a1a2f 100644 --- a/tests/unit/test_as_number_anonymization.py +++ b/tests/unit/test_as_number_anonymization.py @@ -13,23 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from netconan.sensitive_item_removal import ( - anonymize_as_numbers, AsNumberAnonymizer) import pytest +from netconan.sensitive_item_removal import AsNumberAnonymizer, anonymize_as_numbers -SALT = 'saltForTest' +SALT = "saltForTest" -@pytest.mark.parametrize('raw_line, sensitive_as_numbers', [ - ('something {} something', ['123']), - ('123-{}abc', ['65530']), - ('asdf{0}_asdf{0}asdf', ['65530']), - ('anonymize.{} and {}?', ['4567', '1234567']), - ('{}', ['12345']), - ('{} and other text', ['4567']), - ('other text and {}', ['4567']) -]) +@pytest.mark.parametrize( + "raw_line, sensitive_as_numbers", + [ + ("something {} something", ["123"]), + ("123-{}abc", ["65530"]), + ("asdf{0}_asdf{0}asdf", ["65530"]), + ("anonymize.{} and {}?", ["4567", "1234567"]), + ("{}", ["12345"]), + ("{} and other text", ["4567"]), + ("other text and {}", ["4567"]), + ], +) def test_anonymize_as_numbers(raw_line, sensitive_as_numbers): """Test anonymization of lines with AS numbers.""" anonymizer_as_number = AsNumberAnonymizer(sensitive_as_numbers, SALT) @@ -38,23 +40,29 @@ def test_anonymize_as_numbers(raw_line, sensitive_as_numbers): anon_line = anonymize_as_numbers(anonymizer_as_number, line) # Anonymize each AS number individually & build another anon line - anon_numbers = [anonymize_as_numbers(anonymizer_as_number, number) for number in sensitive_as_numbers] + anon_numbers = [ + anonymize_as_numbers(anonymizer_as_number, number) + for number in sensitive_as_numbers + ] individually_anon_line = raw_line.format(*anon_numbers) # Make sure anonymizing each number individually gives the same result as anonymizing all at once - assert(anon_line == individually_anon_line) + assert anon_line == individually_anon_line for as_number in sensitive_as_numbers: # Make sure all AS numbers are removed from the line - assert(as_number not in anon_line) - - -@pytest.mark.parametrize('raw_line, sensitive_as_numbers', [ - ('123{}890', ['65530']), - ('{}{}', ['1234', '5678']), - ('{}000', ['1234']), - ('000{}', ['1234']) -]) + assert as_number not in anon_line + + +@pytest.mark.parametrize( + "raw_line, sensitive_as_numbers", + [ + ("123{}890", ["65530"]), + ("{}{}", ["1234", "5678"]), + ("{}000", ["1234"]), + ("000{}", ["1234"]), + ], +) def test_anonymize_as_numbers_ignore_sub_numbers(raw_line, sensitive_as_numbers): """Test that matching 'AS numbers' within other numbers are not replaced.""" anonymizer_as_number = AsNumberAnonymizer(sensitive_as_numbers, SALT) @@ -63,24 +71,27 @@ def test_anonymize_as_numbers_ignore_sub_numbers(raw_line, sensitive_as_numbers) anon_line = anonymize_as_numbers(anonymizer_as_number, line) # Make sure substrings of other numbers are not affected - assert(anon_line == line) - - -@pytest.mark.parametrize('as_number', [ - '0', - '1234', - '65534', - '65535', - '70000', - '123456789', - '4199999999', - '4230000000', - '4294967295' -]) + assert anon_line == line + + +@pytest.mark.parametrize( + "as_number", + [ + "0", + "1234", + "65534", + "65535", + "70000", + "123456789", + "4199999999", + "4230000000", + "4294967295", + ], +) def test_anonymize_as_num(as_number): """Test anonymization of AS numbers.""" anonymizer = AsNumberAnonymizer([as_number], SALT) - assert(anonymizer.anonymize(as_number) != as_number) + assert anonymizer.anonymize(as_number) != as_number def get_as_number_block(as_number): @@ -93,22 +104,27 @@ def get_as_number_block(as_number): block += 1 -@pytest.mark.parametrize('as_number', [ - '0', '64511', # Original public block - '64512', '65535', # Original private block - '65536', '4199999999', # Expanded public block - '4200000000', '4294967295' # Expanded private block -]) +@pytest.mark.parametrize( + "as_number", + [ + "0", + "64511", # Original public block + "64512", + "65535", # Original private block + "65536", + "4199999999", # Expanded public block + "4200000000", + "4294967295", # Expanded private block + ], +) def test_preserve_as_block(as_number): """Test that original AS number block is preserved after anonymization.""" anonymizer = AsNumberAnonymizer([as_number], SALT) new_as_number = anonymizer.anonymize(as_number) - assert(get_as_number_block(new_as_number) == get_as_number_block(as_number)) + assert get_as_number_block(new_as_number) == get_as_number_block(as_number) -@pytest.mark.parametrize('invalid_as_number', [ - '-1', '4294967296' -]) +@pytest.mark.parametrize("invalid_as_number", ["-1", "4294967296"]) def test_as_number_invalid(invalid_as_number): """Test that exception is thrown with invalid AS number.""" with pytest.raises(ValueError): diff --git a/tests/unit/test_ip_anonymization.py b/tests/unit/test_ip_anonymization.py index e39faa2..db04880 100644 --- a/tests/unit/test_ip_anonymization.py +++ b/tests/unit/test_ip_anonymization.py @@ -14,86 +14,92 @@ # limitations under the License. from __future__ import unicode_literals + import ipaddress -import pytest import re +import pytest + from netconan.ip_anonymization import ( - IpAnonymizer, IpV6Anonymizer, anonymize_ip_addr, _ensure_unicode) + IpAnonymizer, + IpV6Anonymizer, + _ensure_unicode, + anonymize_ip_addr, +) ip_v4_classes = [ - '0.0.0.0/1', # Class A - '128.0.0.0/2', # Class B - '192.0.0.0/3', # Class C - '224.0.0.0/4', # Class D (implies class E) + "0.0.0.0/1", # Class A + "128.0.0.0/2", # Class B + "192.0.0.0/3", # Class C + "224.0.0.0/4", # Class D (implies class E) ] ip_v4_list = [ - ('12.13.14.15'), - ('237.73.212.5'), - ('123.45.67.89'), - ('92.210.0.255'), - ('128.7.55.12'), - ('223.123.21.99'), - ('193.99.99.99'), - ('225.99.99.99'), - ('241.99.99.99'), - ('249.99.99.99'), - ('254.254.254.254'), - ('009.010.011.012'), - ('1.2.3.0000014'), + ("12.13.14.15"), + ("237.73.212.5"), + ("123.45.67.89"), + ("92.210.0.255"), + ("128.7.55.12"), + ("223.123.21.99"), + ("193.99.99.99"), + ("225.99.99.99"), + ("241.99.99.99"), + ("249.99.99.99"), + ("254.254.254.254"), + ("009.010.011.012"), + ("1.2.3.0000014"), ] ip_v6_list = [ - ('1234::5678'), - ('::1'), - ('1::'), - ('1::1'), - ('2001:db8:85a3:7:8:8a2e:370:7334'), - ('2001:db8:a0b:12f0::1'), - ('ffff:ffff::ffff:ffff'), - ('a:b:c:d:e:f:1:2'), - ('aAaA:bBbB:cCcC:dDdD:eEeE:fFfF:1010:2929'), - ('ffff:eeee:dddd:cccc:bbbb:AaAa:9999:8888'), + ("1234::5678"), + ("::1"), + ("1::"), + ("1::1"), + ("2001:db8:85a3:7:8:8a2e:370:7334"), + ("2001:db8:a0b:12f0::1"), + ("ffff:ffff::ffff:ffff"), + ("a:b:c:d:e:f:1:2"), + ("aAaA:bBbB:cCcC:dDdD:eEeE:fFfF:1010:2929"), + ("ffff:eeee:dddd:cccc:bbbb:AaAa:9999:8888"), ] # Private-use blocks defined at https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml # Tuples consist of: start of block, end of block, block subnet private_blocks = [ - ('10.0.0.0', '10.255.255.255', '10.0.0.0/8'), - ('172.16.0.0', '172.31.255.255', '172.16.0.0/12'), - ('192.168.0.0', '192.168.255.255', '192.168.0.0/16'), + ("10.0.0.0", "10.255.255.255", "10.0.0.0/8"), + ("172.16.0.0", "172.31.255.255", "172.16.0.0/12"), + ("192.168.0.0", "192.168.255.255", "192.168.0.0/16"), ] -SALT = 'saltForTest' +SALT = "saltForTest" -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def anonymizer_v4(): """Most tests in this module use a single IPv4 anonymizer.""" return IpAnonymizer(SALT) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def anonymizer_v6(): """All tests in this module use a single IPv6 anonymizer.""" return IpV6Anonymizer(SALT) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def anonymizer(request): """Create a generic fixture for different types of anonymizers.""" - if request.param == 'v4': + if request.param == "v4": return IpAnonymizer(SALT) - elif request.param == 'v6': + elif request.param == "v6": return IpV6Anonymizer(SALT) - elif request.param == 'flipv4': + elif request.param == "flipv4": return IpAnonymizer(SALT, salter=lambda a, b: 1) else: - raise ValueError('Invalid anonymizer type {}'.format(request.param)) + raise ValueError("Invalid anonymizer type {}".format(request.param)) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def flip_anonymizer_v4(): """Create an anonymizer that flips every bit except for class bits.""" # Don't preserve private blocks, because that reduces the number of bits flipped @@ -111,81 +117,87 @@ def anonymize_line_general(anonymizer, line, ip_addrs): # Make sure anonymizing each address individually is the same as # anonymizing all at once - assert(anon_line == individually_anon_line) + assert anon_line == individually_anon_line for ip_addr in ip_addrs: # Make sure the original ip address(es) are removed from the anonymized line - assert(ip_addr not in anon_line) - - -@pytest.mark.parametrize('line, ip_addrs', [ - ('ip address {} 255.255.255.254', ['123.45.67.89']), - ('ip address {} 255.0.0.0', ['10.0.0.0']), - ('ip address {}/16', ['10.0.0.0']), - ('tacacs-server host {}', ['10.1.1.17']), - ('tacacs-server host {}', ['001.021.201.012']), - ('syscon address {} Password', ['10.73.212.5']), - ('1 permit tcp host {} host {} eq 2', ['1.2.3.4', '1.2.3.45']), - ('1 permit tcp host {} host {} eq 2', ['1.2.123.4', '11.2.123.4']), - ('1 permit tcp host {} host {} eq 2', ['1.2.30.45', '1.2.30.4']), - ('1 permit tcp host {} host {} eq 2', ['11.20.3.4', '1.20.3.4']), - ('something host {} host {} host {}', ['1.2.3.4', '1.2.3.5', '1.2.3.45']), - # These formats may occur in Batfish output - ('"{}"', ['1.2.3.45']), - ('({})', ['1.2.3.45']), - ('[IP addresses:{},{}]', ['1.2.3.45', '1.2.3.5']), - ('flow:{}->{}', ['1.2.3.45', '1.2.3.5']), - ('something={}', ['1.2.3.45']), - ('something <{}>', ['1.2.3.45']), - ('something \'{}\'', ['1.2.3.45']), - ]) + assert ip_addr not in anon_line + + +@pytest.mark.parametrize( + "line, ip_addrs", + [ + ("ip address {} 255.255.255.254", ["123.45.67.89"]), + ("ip address {} 255.0.0.0", ["10.0.0.0"]), + ("ip address {}/16", ["10.0.0.0"]), + ("tacacs-server host {}", ["10.1.1.17"]), + ("tacacs-server host {}", ["001.021.201.012"]), + ("syscon address {} Password", ["10.73.212.5"]), + ("1 permit tcp host {} host {} eq 2", ["1.2.3.4", "1.2.3.45"]), + ("1 permit tcp host {} host {} eq 2", ["1.2.123.4", "11.2.123.4"]), + ("1 permit tcp host {} host {} eq 2", ["1.2.30.45", "1.2.30.4"]), + ("1 permit tcp host {} host {} eq 2", ["11.20.3.4", "1.20.3.4"]), + ("something host {} host {} host {}", ["1.2.3.4", "1.2.3.5", "1.2.3.45"]), + # These formats may occur in Batfish output + ('"{}"', ["1.2.3.45"]), + ("({})", ["1.2.3.45"]), + ("[IP addresses:{},{}]", ["1.2.3.45", "1.2.3.5"]), + ("flow:{}->{}", ["1.2.3.45", "1.2.3.5"]), + ("something={}", ["1.2.3.45"]), + ("something <{}>", ["1.2.3.45"]), + ("something '{}'", ["1.2.3.45"]), + ], +) def test_v4_anonymize_line(anonymizer_v4, line, ip_addrs): """Test IPv4 address removal from config lines.""" anonymize_line_general(anonymizer_v4, line, ip_addrs) -@pytest.mark.parametrize('enclosing', ":;[]$~!@#$%^&*()-+=[]|<>?") +@pytest.mark.parametrize("enclosing", ":;[]$~!@#$%^&*()-+=[]|<>?") def test_v4_anonymize_enclosed_addr(anonymizer_v4, enclosing): """Test IPv4 address removal from config lines with different enclosing characters.""" - ip_addr = '1.2.3.4' + ip_addr = "1.2.3.4" line = enclosing + "{}" + enclosing anonymize_line_general(anonymizer_v4, line, [ip_addr]) -@pytest.mark.parametrize('line, ip_addrs', [ - ('ip address {} something::something', ['1234::5678']), - ('ip address {} blah {}', ['1234::', '1234:5678::9abc:def0']), - ('ip address {} blah {} blah', ['::1', '1234:5678:abcd:dcba::9abc:def0']), - ('ip address {}/16 blah', ['::1']), - ('ip address {}/16 blah', ['1::']), - ('ip address {}/16 blah', ['1::1']), - ('ip address {}/16 blah', ['ffff:ffff::ffff:ffff']), - ]) +@pytest.mark.parametrize( + "line, ip_addrs", + [ + ("ip address {} something::something", ["1234::5678"]), + ("ip address {} blah {}", ["1234::", "1234:5678::9abc:def0"]), + ("ip address {} blah {} blah", ["::1", "1234:5678:abcd:dcba::9abc:def0"]), + ("ip address {}/16 blah", ["::1"]), + ("ip address {}/16 blah", ["1::"]), + ("ip address {}/16 blah", ["1::1"]), + ("ip address {}/16 blah", ["ffff:ffff::ffff:ffff"]), + ], +) def test_v6_anonymize_line(anonymizer_v6, line, ip_addrs): """Test IPv6 address removal from config lines.""" anonymize_line_general(anonymizer_v6, line, ip_addrs) -@pytest.mark.parametrize('enclosing', ".;[]$~!@#$%^&*()-+=[]|<>?") +@pytest.mark.parametrize("enclosing", ".;[]$~!@#$%^&*()-+=[]|<>?") def test_v6_anonymize_enclosed_addr(anonymizer_v6, enclosing): """Test IPv6 address removal from config lines with different enclosing characters.""" - ip_addr = '1::1' + ip_addr = "1::1" line = enclosing + "{}" + enclosing anonymize_line_general(anonymizer_v6, line, [ip_addr]) def get_ip_v4_class(ip_int): """Return the letter corresponding to the IP class the ip_int is in.""" - if ((ip_int & 0x80000000) == 0x00000000): - return 'A' - elif ((ip_int & 0xC0000000) == 0x80000000): - return 'B' - elif ((ip_int & 0xE0000000) == 0xC0000000): - return 'C' - elif ((ip_int & 0xF0000000) == 0xE0000000): - return 'D' + if (ip_int & 0x80000000) == 0x00000000: + return "A" + elif (ip_int & 0xC0000000) == 0x80000000: + return "B" + elif (ip_int & 0xE0000000) == 0xC0000000: + return "C" + elif (ip_int & 0xF0000000) == 0xE0000000: + return "D" else: - return 'E' + return "E" def get_ip_v4_class_mask(ip_int): @@ -200,63 +212,71 @@ def get_ip_v4_class_mask(ip_int): return 0x80000000 -@pytest.mark.parametrize('ip_addr', [ - '0.0.0.0', '127.255.255.255', # Class A - '128.0.0.0', '191.255.255.255', # Class B - '192.0.0.0', '223.255.255.255', # Class C - '224.0.0.0', '239.255.255.255', # Class D - '240.0.0.0', '247.255.255.255', # Class E -]) +@pytest.mark.parametrize( + "ip_addr", + [ + "0.0.0.0", + "127.255.255.255", # Class A + "128.0.0.0", + "191.255.255.255", # Class B + "192.0.0.0", + "223.255.255.255", # Class C + "224.0.0.0", + "239.255.255.255", # Class D + "240.0.0.0", + "247.255.255.255", # Class E + ], +) def test_v4_class_preserved(flip_anonymizer_v4, ip_addr): """Test that IPv4 classes are preserved.""" ip_int = int(flip_anonymizer_v4.make_addr(ip_addr)) ip_int_anon = flip_anonymizer_v4.anonymize(ip_int) # IP v4 class should match after anonymization - assert(get_ip_v4_class(ip_int) == get_ip_v4_class(ip_int_anon)) + assert get_ip_v4_class(ip_int) == get_ip_v4_class(ip_int_anon) # Anonymized ip address should not match the original ip address - assert(ip_int != ip_int_anon) + assert ip_int != ip_int_anon # All bits that are not forced to be preserved are flipped class_mask = get_ip_v4_class_mask(ip_int) - assert(0xFFFFFFFF ^ class_mask == ip_int ^ ip_int_anon) + assert 0xFFFFFFFF ^ class_mask == ip_int ^ ip_int_anon def test_preserve_custom_prefixes(): """Test that a custom prefix is preserved correctly.""" - subnet = '170.0.0.0/8' + subnet = "170.0.0.0/8" anonymizer = IpAnonymizer(SALT, [subnet]) - ip_start = int(anonymizer.make_addr('170.0.0.0')) + ip_start = int(anonymizer.make_addr("170.0.0.0")) ip_start_anon = anonymizer.anonymize(ip_start) - ip_end = int(anonymizer.make_addr('170.255.255.255')) + ip_end = int(anonymizer.make_addr("170.255.255.255")) ip_end_anon = anonymizer.anonymize(ip_end) network = ipaddress.ip_network(_ensure_unicode(subnet)) # Make sure the anonymized addresses are different from the originals - assert (ip_start_anon != ip_start) - assert (ip_end_anon != ip_end) + assert ip_start_anon != ip_start + assert ip_end_anon != ip_end # Make sure the anonymized addresses have the same prefix as the originals - assert (ipaddress.ip_address(ip_start_anon) in network) - assert (ipaddress.ip_address(ip_end_anon) in network) + assert ipaddress.ip_address(ip_start_anon) in network + assert ipaddress.ip_address(ip_end_anon) in network def test_preserve_custom_addresses(): """Test that addresses within a preserved block are flagged correctly as NOT needing anonymization.""" addresses = [ - '170.0.0.0/8', - '11.11.11.11', + "170.0.0.0/8", + "11.11.11.11", ] anonymizer = IpAnonymizer(SALT, preserve_addresses=addresses) - ip_start = int(anonymizer.make_addr('170.0.0.0')) - ip_end = int(anonymizer.make_addr('170.255.255.255')) - ip_other = int(anonymizer.make_addr('11.11.11.11')) - ip_outside = int(anonymizer.make_addr('10.11.12.13')) + ip_start = int(anonymizer.make_addr("170.0.0.0")) + ip_end = int(anonymizer.make_addr("170.255.255.255")) + ip_other = int(anonymizer.make_addr("11.11.11.11")) + ip_outside = int(anonymizer.make_addr("10.11.12.13")) # Make sure anonymizer indicates addresses within preserved network blocks should not be anonymized assert not anonymizer.should_anonymize(ip_start) @@ -269,7 +289,7 @@ def test_preserve_custom_addresses(): def test_preserve_address_preserves_prefix(): """Test that common prefixes are still preserved when addresses are preserved.""" - addr_str = '11.11.11.11' + addr_str = "11.11.11.11" addr = ipaddress.ip_address(addr_str) addr_int = int(addr) @@ -279,9 +299,7 @@ def test_preserve_address_preserves_prefix(): for i in range(addr_len): # Anonymize an address similar to the original, with 1 bit flipped similar_addr = ipaddress.ip_address(addr_int ^ (1 << i)) - similar_anon = ipaddress.ip_address( - anonymizer.anonymize(int(similar_addr)) - ) + similar_anon = ipaddress.ip_address(anonymizer.anonymize(int(similar_addr))) # Confirm cpl before anonymization matches cpl after anonymization assert _cpl_v4(similar_anon, addr) == _cpl_v4(similar_addr, addr) @@ -303,7 +321,7 @@ def _cpl_v4(left, right): return max_shift -@pytest.mark.parametrize('start, end, subnet', private_blocks) +@pytest.mark.parametrize("start, end, subnet", private_blocks) def test_preserve_private_prefixes(anonymizer_v4, start, end, subnet): """Test that private-use prefixes are preserved by default.""" ip_int_start = int(anonymizer_v4.make_addr(start)) @@ -315,39 +333,42 @@ def test_preserve_private_prefixes(anonymizer_v4, start, end, subnet): network = ipaddress.ip_network(_ensure_unicode(subnet)) # Make sure addresses in the block stay in the block - assert (ipaddress.ip_address(ip_int_start_anon) in network) - assert (ipaddress.ip_address(ip_int_end_anon) in network) + assert ipaddress.ip_address(ip_int_start_anon) in network + assert ipaddress.ip_address(ip_int_end_anon) in network -@pytest.mark.parametrize('length', range(0, 33)) +@pytest.mark.parametrize("length", range(0, 33)) def test_preserve_host_bits(length): """Test that host bits are preserved, for every length.""" - anonymizer = IpAnonymizer(salt=SALT, salter=lambda a, b: 1, preserve_suffix=length, preserve_prefixes=[]) + anonymizer = IpAnonymizer( + salt=SALT, salter=lambda a, b: 1, preserve_suffix=length, preserve_prefixes=[] + ) randomized = anonymizer.anonymize(0) # The first <32-length> bits are 1, the last bits are all 0 - expected = '1' * (32 - length) + '0' * length - assert '{:032b}'.format(randomized) == expected + expected = "1" * (32 - length) + "0" * length + assert "{:032b}".format(randomized) == expected -@pytest.mark.parametrize('anonymizer,ip_addr', - [('v4', s) for s in ip_v4_list] + - [('v6', s) for s in ip_v6_list], - indirect=['anonymizer']) +@pytest.mark.parametrize( + "anonymizer,ip_addr", + [("v4", s) for s in ip_v4_list] + [("v6", s) for s in ip_v6_list], + indirect=["anonymizer"], +) def test_anonymize_addr(anonymizer, ip_addr): """Test conversion from original to anonymized IP address.""" ip_int = int(anonymizer.make_addr(ip_addr)) ip_int_anon = anonymizer.anonymize(ip_int) # Anonymized ip address should not match the original address - assert(ip_int != ip_int_anon) + assert ip_int != ip_int_anon full_bit_mask = (1 << anonymizer.length) - 1 # Confirm prefixes for similar addresses are preserved after anonymization for i in range(0, anonymizer.length): # Flip the ith bit of the org address and use that as the similar address - diff_mask = (1 << i) + diff_mask = 1 << i ip_int_similar = ip_int ^ diff_mask ip_int_similar_anon = anonymizer.anonymize(ip_int_similar) @@ -355,10 +376,10 @@ def test_anonymize_addr(anonymizer, ip_addr): same_mask = full_bit_mask & (full_bit_mask << (i + 1)) # Common prefix for addresses should match after anonymization - assert(ip_int_similar_anon & same_mask == ip_int_anon & same_mask) + assert ip_int_similar_anon & same_mask == ip_int_anon & same_mask # Confirm the bit that is different in the original addresses is different in the anonymized addresses - assert(ip_int_similar_anon & diff_mask != ip_int_anon & diff_mask) + assert ip_int_similar_anon & diff_mask != ip_int_anon & diff_mask def test_anonymize_ip_order_independent(): @@ -376,7 +397,7 @@ def test_anonymize_ip_order_independent(): ip_int_anon_reverse = anonymizer_v4_reverse.anonymize(ip_int_reverse) # Confirm anonymizing in reverse order does not affect # anonymization results - assert(ip_int_anon_reverse == ip_lookup_forward[ip_int_reverse]) + assert ip_int_anon_reverse == ip_lookup_forward[ip_int_reverse] anonymizer_v4_extras = IpAnonymizer(SALT) for ip_addr in ip_v4_list: @@ -386,10 +407,10 @@ def test_anonymize_ip_order_independent(): anonymizer_v4_extras.anonymize(ip_int_inverted) # Confirm anonymizing with extra addresses in-between does not # affect anonymization results - assert(ip_int_anon_extras == ip_lookup_forward[ip_int_extras]) + assert ip_int_anon_extras == ip_lookup_forward[ip_int_extras] -@pytest.mark.parametrize('ip_addr', ip_v4_list) +@pytest.mark.parametrize("ip_addr", ip_v4_list) def test_deanonymize_ip(anonymizer_v4, ip_addr): """Test reversing IP anonymization.""" ip_int = int(anonymizer_v4.make_addr(ip_addr)) @@ -397,7 +418,7 @@ def test_deanonymize_ip(anonymizer_v4, ip_addr): ip_int_unanon = anonymizer_v4.deanonymize(ip_int_anon) # Make sure unanonymizing an anonymized address produces the original address - assert(ip_int == ip_int_unanon) + assert ip_int == ip_int_unanon def test_dump_iptree(tmpdir, anonymizer_v4): @@ -414,81 +435,90 @@ def test_dump_iptree(tmpdir, anonymizer_v4): ip_mapping[str(ip_addr)] = ip_addr_anon filename = str(tmpdir.mkdir("test").join("test_dump_iptree.txt")) - with open(filename, 'w') as f_tmp: + with open(filename, "w") as f_tmp: anonymizer_v4.dump_to_file(f_tmp) - with open(filename, 'r') as f_tmp: + with open(filename, "r") as f_tmp: # Build mapping dict from the output of the ip_tree dump for line in f_tmp.readlines(): - m = re.match(r'\s*(\d+\.\d+.\d+.\d+)\s+(\d+\.\d+.\d+.\d+)\s*', line) + m = re.match(r"\s*(\d+\.\d+.\d+.\d+)\s+(\d+\.\d+.\d+.\d+)\s*", line) ip_addr = m.group(1) ip_addr_anon = m.group(2) ip_mapping_from_dump[ip_addr] = ip_addr_anon for ip_addr in ip_mapping: # Confirm anon addresses from ip_tree dump match anon addresses from _convert_to_anon_ip - assert(ip_mapping[ip_addr] == ip_mapping_from_dump[ip_addr]) - - -@pytest.mark.parametrize('line', [ - '01:23:45:67:89:ab', - '01:02:03:04:05:06:07:08:09', - '01:02:03:04::05:06:07:08', - '1.2.3.4.example.net', - '1.2.3.4something.example.net', - 'a.1.2.3.4', - '1.2.3', - '1.2.3.4.5', - 'something1::abc', - '123::ABsomething', - '1.2.333.4', - '1.2.0333.4', - '1.256.3.4', - ]) + assert ip_mapping[ip_addr] == ip_mapping_from_dump[ip_addr] + + +@pytest.mark.parametrize( + "line", + [ + "01:23:45:67:89:ab", + "01:02:03:04:05:06:07:08:09", + "01:02:03:04::05:06:07:08", + "1.2.3.4.example.net", + "1.2.3.4something.example.net", + "a.1.2.3.4", + "1.2.3", + "1.2.3.4.5", + "something1::abc", + "123::ABsomething", + "1.2.333.4", + "1.2.0333.4", + "1.256.3.4", + ], +) def test_false_positives(anonymizer_v4, anonymizer_v6, line): """Test that text without a valid address is not anonymized.""" anon_line = anonymize_ip_addr(anonymizer_v4, line) anon_line = anonymize_ip_addr(anonymizer_v6, anon_line) # Confirm the anonymized line is unchanged - assert(line == anon_line) - - -@pytest.mark.parametrize('zeros, no_zeros', [ - ('0.0.0.0', '0.0.0.0'), - ('0.0.0.3', '0.0.0.3'), - ('128.0.0.0', '128.0.0.0'), - ('0.127.0.0', '0.127.0.0'), - ('10.73.212.5', '10.73.212.5'), - ('010.73.212.05', '10.73.212.5'), - ('255.255.255.255', '255.255.255.255'), - ('170.255.85.1', '170.255.85.1'), - ('10.11.12.13', '10.11.12.13'), - ('010.11.12.13', '10.11.12.13'), - ('10.011.12.13', '10.11.12.13'), - ('10.11.012.13', '10.11.12.13'), - ('10.11.12.013', '10.11.12.13'), - ('010.0011.00000012.000', '10.11.12.0'), - ]) + assert line == anon_line + + +@pytest.mark.parametrize( + "zeros, no_zeros", + [ + ("0.0.0.0", "0.0.0.0"), + ("0.0.0.3", "0.0.0.3"), + ("128.0.0.0", "128.0.0.0"), + ("0.127.0.0", "0.127.0.0"), + ("10.73.212.5", "10.73.212.5"), + ("010.73.212.05", "10.73.212.5"), + ("255.255.255.255", "255.255.255.255"), + ("170.255.85.1", "170.255.85.1"), + ("10.11.12.13", "10.11.12.13"), + ("010.11.12.13", "10.11.12.13"), + ("10.011.12.13", "10.11.12.13"), + ("10.11.012.13", "10.11.12.13"), + ("10.11.12.013", "10.11.12.13"), + ("010.0011.00000012.000", "10.11.12.0"), + ], +) def test_v4_anonymizer_ignores_leading_zeros(anonymizer_v4, zeros, no_zeros): """Test that v4 IP address ignore leading zeros & don't interpret octal.""" - assert(ipaddress.IPv4Address(no_zeros) == anonymizer_v4.make_addr(zeros)) - - -@pytest.mark.parametrize('ip_int, expected', [ - (0b00000000000000000000000000000000, False), - (0b00000000000000000000000000000001, False), - (0b00000000000000000000000000001111, False), - (0b11110000000000000000000000000000, False), - (0b10000000000000000000000000000000, False), - (0b01111111111000000000000000000000, True), - (0b00000011111000000000000000000000, True), - (0b00000000000100000000000000000000, True), - (0b00010101001001000000000000000000, True), - (0b00000000000000000010000000000000, True), - (0b00000000000000000011111111111110, True), - (0b00000000010000000100000000000000, True), - ]) + assert ipaddress.IPv4Address(no_zeros) == anonymizer_v4.make_addr(zeros) + + +@pytest.mark.parametrize( + "ip_int, expected", + [ + (0b00000000000000000000000000000000, False), + (0b00000000000000000000000000000001, False), + (0b00000000000000000000000000001111, False), + (0b11110000000000000000000000000000, False), + (0b10000000000000000000000000000000, False), + (0b01111111111000000000000000000000, True), + (0b00000011111000000000000000000000, True), + (0b00000000000100000000000000000000, True), + (0b00010101001001000000000000000000, True), + (0b00000000000000000010000000000000, True), + (0b00000000000000000011111111111110, True), + (0b00000000010000000100000000000000, True), + ], +) def test_v4_should_anonymize(anonymizer_v4, ip_int, expected): """Test that the IpV4 anonymizer does not anonymize masks.""" - assert(expected == anonymizer_v4.should_anonymize(ip_int)) + assert expected == anonymizer_v4.should_anonymize(ip_int) diff --git a/tests/unit/test_parse_args.py b/tests/unit/test_parse_args.py index 14ca031..a168304 100644 --- a/tests/unit/test_parse_args.py +++ b/tests/unit/test_parse_args.py @@ -26,7 +26,7 @@ def test_defaults(): assert not args.anonymize_passwords assert not args.anonymize_ips assert args.dump_ip_map is None - assert 'INFO' == args.log_level + assert "INFO" == args.log_level assert args.salt is None assert not args.undo assert args.sensitive_words is None @@ -34,17 +34,19 @@ def test_defaults(): def test_no_config_file(): """Test command line args are parsed.""" - args = _parse_args([ - "--input=in", - "--output=out", - "--anonymize-passwords", - "--anonymize-ips", - "--dump-ip-map=dump", - "--log-level=CRITICAL", - "--salt=salty", - "--undo", - "--sensitive-words=secret,password", - ]) + args = _parse_args( + [ + "--input=in", + "--output=out", + "--anonymize-passwords", + "--anonymize-ips", + "--dump-ip-map=dump", + "--log-level=CRITICAL", + "--salt=salty", + "--undo", + "--sensitive-words=secret,password", + ] + ) assert "in" == args.input assert "out" == args.output @@ -59,12 +61,14 @@ def test_no_config_file(): def test_config_file(tmpdir): """Test config file args are parsed.""" - cfg_file = str(tmpdir.mkdir('config_file').join('config.cfg')) - with open(cfg_file, 'w') as f: - f.write("""[Defaults] + cfg_file = str(tmpdir.mkdir("config_file").join("config.cfg")) + with open(cfg_file, "w") as f: + f.write( + """[Defaults] input=in output=out - log-level=CRITICAL""") + log-level=CRITICAL""" + ) args = _parse_args(["-c={}".format(cfg_file)]) assert "in" == args.input @@ -80,12 +84,14 @@ def test_config_file(tmpdir): def test_config_file_and_override(tmpdir): """Test command line args override config file args.""" - cfg_file = str(tmpdir.mkdir('config_file').join('config.cfg')) - with open(cfg_file, 'w') as f: - f.write("""[Defaults] + cfg_file = str(tmpdir.mkdir("config_file").join("config.cfg")) + with open(cfg_file, "w") as f: + f.write( + """[Defaults] input=in output=out - log-level=CRITICAL""") + log-level=CRITICAL""" + ) args = _parse_args(["-c={}".format(cfg_file), "--input=override"]) assert "override" == args.input diff --git a/tests/unit/test_sensitive_item_removal.py b/tests/unit/test_sensitive_item_removal.py index c42d039..3692215 100644 --- a/tests/unit/test_sensitive_item_removal.py +++ b/tests/unit/test_sensitive_item_removal.py @@ -13,217 +13,262 @@ # See the License for the specific language governing permissions and # limitations under the License. -from netconan.sensitive_item_removal import ( - _anonymize_value, _check_sensitive_item_format, _extract_enclosing_text, - _LINE_SCRUBBED_MESSAGE, _sensitive_item_formats, - generate_default_sensitive_item_regexes, replace_matching_item, - SensitiveWordAnonymizer) import pytest +from netconan.sensitive_item_removal import ( + _LINE_SCRUBBED_MESSAGE, + SensitiveWordAnonymizer, + _anonymize_value, + _check_sensitive_item_format, + _extract_enclosing_text, + _sensitive_item_formats, + generate_default_sensitive_item_regexes, + replace_matching_item, +) + # Tuple format is config_line, sensitive_text (should not be in output line) # TODO(https://github.com/intentionet/netconan/issues/3): # Add more Arista config lines arista_password_lines = [ - ('username noc secret sha512 {}', '$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0') + ( + "username noc secret sha512 {}", + "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0", + ) ] # TODO(https://github.com/intentionet/netconan/issues/3): # Add in additional test lines (these are just first pass from IOS) cisco_password_lines = [ - (' password 7 {}', '122A00190102180D3C2E'), - ('username Someone password 0 {}', 'RemoveMe'), - ('username Someone password {}', 'RemoveMe'), - ('username Someone password 7 {}', '122A00190102180D3C2E'), - ('enable password level 12 {}', 'RemoveMe'), - ('enable password 7 {}', '122A00190102180D3C2E'), - ('enable password level 3 5 {}', '$1$wtHI$0rN7R8PKwC30AsCGA77vy.'), - ('enable secret 5 {}', '$1$wtHI$0rN7R8PKwC30AsCGA77vy.'), - ('username Someone view Someview password 7 {}', '122A00190102180D3C2E'), - ('username Someone password {}', 'RemoveMe'), - ('username Someone secret 5 {}', '$1$wtHI$0rN7R8PKwC30AsCGA77vy.'), - ('username Someone secret {}', 'RemoveMe'), - ('username Someone view Someview secret {}', 'RemoveMe'), - ('ip ftp password {}', 'RemoveMe'), - ('ip ftp password 0 {}', 'RemoveMe'), - ('ip ftp password 7 {}', '122A00190102180D3C2E'), - (' ip ospf authentication-key {}', 'RemoveMe'), - (' ip ospf authentication-key 0 {}', 'RemoveMe'), - ('isis password {}', 'RemoveMe'), - ('domain-password {}', 'RemoveMe'), - ('domain-password {} authenticate snp validate', 'RemoveMe'), - ('area-password {} authenticate snp send-only', 'RemoveMe'), - ('ip ospf message-digest-key 123 md5 {}', 'RemoveMe'), - ('ip ospf message-digest-key 124 md5 7 {}', '122A00190102180D3C2E'), - ('standby authentication {}', 'RemoveMe'), - ('standby authentication md5 key-string {} timeout 123', 'RemoveMe'), - ('standby authentication md5 key-string 7 {}', 'RemoveMe'), - ('standby authentication text {}', 'RemoveMe'), - ('l2tp tunnel password 0 {}', 'RemoveMe'), - ('l2tp tunnel password {}', 'RemoveMe'), - ('digest secret {} hash MD5', 'RemoveMe'), - ('digest secret {}', 'RemoveMe'), - ('digest secret 0 {}', 'RemoveMe'), - ('ppp chap password {}', 'RemoveMe'), - ('ppp chap password 0 {}', 'RemoveMe'), - ('ppp chap hostname {}', 'RemoveMe'), - ('pre-shared-key {}', 'RemoveMe'), - ('pre-shared-key 0 {}', 'RemoveMe'), - ('pre-shared-key local 0 {}', 'RemoveMe'), - ('pre-shared-key remote hex {}', '1234a'), - ('pre-shared-key remote 6 {}', 'FLgBaJHXdYY_AcHZZMgQ_RhTDJXHUBAAB'), - ('tacacs-server host 1.1.1.1 key {}', 'RemoveMe'), - ('radius-server host 1.1.1.1 key 0 {}', 'RemoveMe'), - ('tacacs-server key 7 {}', '122A00190102180D3C2E'), - (' key 0 {}', 'RemoveMe'), - ('ntp authentication-key 4294967295 md5 {}', 'RemoveMe'), - ('ntp authentication-key 123 md5 {} 1', 'RemoveMe'), - ('syscon address 1.1.1.1 {}', 'RemoveMe'), - ('snmp-server user Someone Somegroup remote Crap v3 auth md5 {}', 'RemoveMe'), - ('snmp-server user Someone Somegroup v3 auth sha {0} priv 3des {0}', 'RemoveMe'), - ('snmp-server user Someone Somegroup v3 auth sha {0} priv {0}', 'RemoveMe'), - ('snmp-server user Someone Somegroup auth md5 {0} priv aes 128 {0}', 'RemoveMe'), - ('snmp-server user Someone Somegroup auth md5 {0} priv {0} something', 'RemoveMe'), + (" password 7 {}", "122A00190102180D3C2E"), + ("username Someone password 0 {}", "RemoveMe"), + ("username Someone password {}", "RemoveMe"), + ("username Someone password 7 {}", "122A00190102180D3C2E"), + ("enable password level 12 {}", "RemoveMe"), + ("enable password 7 {}", "122A00190102180D3C2E"), + ("enable password level 3 5 {}", "$1$wtHI$0rN7R8PKwC30AsCGA77vy."), + ("enable secret 5 {}", "$1$wtHI$0rN7R8PKwC30AsCGA77vy."), + ("username Someone view Someview password 7 {}", "122A00190102180D3C2E"), + ("username Someone password {}", "RemoveMe"), + ("username Someone secret 5 {}", "$1$wtHI$0rN7R8PKwC30AsCGA77vy."), + ("username Someone secret {}", "RemoveMe"), + ("username Someone view Someview secret {}", "RemoveMe"), + ("ip ftp password {}", "RemoveMe"), + ("ip ftp password 0 {}", "RemoveMe"), + ("ip ftp password 7 {}", "122A00190102180D3C2E"), + (" ip ospf authentication-key {}", "RemoveMe"), + (" ip ospf authentication-key 0 {}", "RemoveMe"), + ("isis password {}", "RemoveMe"), + ("domain-password {}", "RemoveMe"), + ("domain-password {} authenticate snp validate", "RemoveMe"), + ("area-password {} authenticate snp send-only", "RemoveMe"), + ("ip ospf message-digest-key 123 md5 {}", "RemoveMe"), + ("ip ospf message-digest-key 124 md5 7 {}", "122A00190102180D3C2E"), + ("standby authentication {}", "RemoveMe"), + ("standby authentication md5 key-string {} timeout 123", "RemoveMe"), + ("standby authentication md5 key-string 7 {}", "RemoveMe"), + ("standby authentication text {}", "RemoveMe"), + ("l2tp tunnel password 0 {}", "RemoveMe"), + ("l2tp tunnel password {}", "RemoveMe"), + ("digest secret {} hash MD5", "RemoveMe"), + ("digest secret {}", "RemoveMe"), + ("digest secret 0 {}", "RemoveMe"), + ("ppp chap password {}", "RemoveMe"), + ("ppp chap password 0 {}", "RemoveMe"), + ("ppp chap hostname {}", "RemoveMe"), + ("pre-shared-key {}", "RemoveMe"), + ("pre-shared-key 0 {}", "RemoveMe"), + ("pre-shared-key local 0 {}", "RemoveMe"), + ("pre-shared-key remote hex {}", "1234a"), + ("pre-shared-key remote 6 {}", "FLgBaJHXdYY_AcHZZMgQ_RhTDJXHUBAAB"), + ("tacacs-server host 1.1.1.1 key {}", "RemoveMe"), + ("radius-server host 1.1.1.1 key 0 {}", "RemoveMe"), + ("tacacs-server key 7 {}", "122A00190102180D3C2E"), + (" key 0 {}", "RemoveMe"), + ("ntp authentication-key 4294967295 md5 {}", "RemoveMe"), + ("ntp authentication-key 123 md5 {} 1", "RemoveMe"), + ("syscon address 1.1.1.1 {}", "RemoveMe"), + ("snmp-server user Someone Somegroup remote Crap v3 auth md5 {}", "RemoveMe"), + ("snmp-server user Someone Somegroup v3 auth sha {0} priv 3des {0}", "RemoveMe"), + ("snmp-server user Someone Somegroup v3 auth sha {0} priv {0}", "RemoveMe"), + ("snmp-server user Someone Somegroup auth md5 {0} priv aes 128 {0}", "RemoveMe"), + ("snmp-server user Someone Somegroup auth md5 {0} priv {0} something", "RemoveMe"), # TODO: Figure out SHA format, this line throws: Error in Auth password - ('snmp-server user Someone Somegroup v3 encrypted auth sha {}', 'RemoveMe'), - ('crypto isakmp key {} address 1.1.1.1 255.255.255.0', 'RemoveMe'), - ('crypto isakmp key 6 {} hostname Something', 'RemoveMe'), - ('set session-key inbound ah 4294967295 {}', '1234abcdef'), - ('set session-key outbound esp 256 authenticator {}', '1234abcdef'), - ('set session-key outbound esp 256 cipher {0} authenticator {0}', '1234abcdef'), - ('key-hash sha256 {}', 'RemoveMe') + ("snmp-server user Someone Somegroup v3 encrypted auth sha {}", "RemoveMe"), + ("crypto isakmp key {} address 1.1.1.1 255.255.255.0", "RemoveMe"), + ("crypto isakmp key 6 {} hostname Something", "RemoveMe"), + ("set session-key inbound ah 4294967295 {}", "1234abcdef"), + ("set session-key outbound esp 256 authenticator {}", "1234abcdef"), + ("set session-key outbound esp 256 cipher {0} authenticator {0}", "1234abcdef"), + ("key-hash sha256 {}", "RemoveMe"), ] cisco_snmp_community_lines = [ - ('snmp-server community {} ro 1', 'RemoveMe'), - ('snmp-server community {} Something', 'RemoveMe'), - ('snmp-server host 1.1.1.1 vrf Something informs {} config', 'RemoveMe'), - ('snmp-server host 1.1.1.1 informs version 1 {} ipsec', 'RemoveMe'), - ('snmp-server host 1.1.1.1 traps version 2c {}', 'RemoveMe'), - ('snmp-server host 1.1.1.1 informs version 3 auth {} ipsec', 'RemoveMe'), - ('snmp-server host 1.1.1.1 traps version 3 noauth {}', 'RemoveMe'), - ('snmp-server host 1.1.1.1 informs version 3 priv {} memory', 'RemoveMe'), - ('snmp-server host 1.1.1.1 version 2c {}', 'RemoveMe'), - ('snmp-server host 1.1.1.1 {} vrrp', 'RemoveMe'), - ('snmp-server mib community-map {}:100 context public1', 'RemoveMe'), - ('snmp-server community {} RW 2', 'secretcommunity'), - ('rf-switch snmp-community {}', 'RemoveMe') + ("snmp-server community {} ro 1", "RemoveMe"), + ("snmp-server community {} Something", "RemoveMe"), + ("snmp-server host 1.1.1.1 vrf Something informs {} config", "RemoveMe"), + ("snmp-server host 1.1.1.1 informs version 1 {} ipsec", "RemoveMe"), + ("snmp-server host 1.1.1.1 traps version 2c {}", "RemoveMe"), + ("snmp-server host 1.1.1.1 informs version 3 auth {} ipsec", "RemoveMe"), + ("snmp-server host 1.1.1.1 traps version 3 noauth {}", "RemoveMe"), + ("snmp-server host 1.1.1.1 informs version 3 priv {} memory", "RemoveMe"), + ("snmp-server host 1.1.1.1 version 2c {}", "RemoveMe"), + ("snmp-server host 1.1.1.1 {} vrrp", "RemoveMe"), + ("snmp-server mib community-map {}:100 context public1", "RemoveMe"), + ("snmp-server community {} RW 2", "secretcommunity"), + ("rf-switch snmp-community {}", "RemoveMe"), ] # TODO(https://github.com/intentionet/netconan/issues/3): fortinet_password_lines = [ - ('password ENC {}', 'SH2nlSm9QL9tapcHPXIqAXvX7vBJuuqu22hpa0JX0sBuKIo7z2g0Kz/+0KyH4E=') + ( + "password ENC {}", + "SH2nlSm9QL9tapcHPXIqAXvX7vBJuuqu22hpa0JX0sBuKIo7z2g0Kz/+0KyH4E=", + ) ] # TODO(https://github.com/intentionet/netconan/issues/4): # Add more Juniper config lines juniper_password_lines = [ - ('secret "{}"', '$9$Be4EhyVb2GDkevYo'), - ('set interfaces irb unit 5 family inet address 1.2.3.0/24 vrrp-group 5 authentication-key "{}"', '$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p'), - ('set system tacplus-server 1.2.3.4 secret "{}"', '$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t'), - ('set system tacplus-server 1.2.3.4 secret "{}"', '$9$YVgoZk.5n6AHq9tORlegoJGDkPfQCtOP5Qn9pRE'), - ('set security ike policy test-ike-policy pre-shared-key ascii-text "{}"', '$9$/E6g9tO1IcSrvfTCu1hKv-VwgJD'), - ('set system root-authentication encrypted-password "{}"', '$1$CXKwIUfL$6vLSvatE2TCaM25U4u9Bh1'), - ('set system login user admin authentication encrypted-password "{}"', '$1$67Q0XA3z$YqiBW/xxKWr74oHPXEkIv1'), - ('set system login user someone authenitcation "{}"', '$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/'), - ('set system license keys key "{}"', 'SOMETHING'), + ('secret "{}"', "$9$Be4EhyVb2GDkevYo"), + ( + 'set interfaces irb unit 5 family inet address 1.2.3.0/24 vrrp-group 5 authentication-key "{}"', + "$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p", + ), + ('set system tacplus-server 1.2.3.4 secret "{}"', "$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t"), + ( + 'set system tacplus-server 1.2.3.4 secret "{}"', + "$9$YVgoZk.5n6AHq9tORlegoJGDkPfQCtOP5Qn9pRE", + ), + ( + 'set security ike policy test-ike-policy pre-shared-key ascii-text "{}"', + "$9$/E6g9tO1IcSrvfTCu1hKv-VwgJD", + ), + ( + 'set system root-authentication encrypted-password "{}"', + "$1$CXKwIUfL$6vLSvatE2TCaM25U4u9Bh1", + ), + ( + 'set system login user admin authentication encrypted-password "{}"', + "$1$67Q0XA3z$YqiBW/xxKWr74oHPXEkIv1", + ), + ( + 'set system login user someone authenitcation "{}"', + "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", + ), + ('set system license keys key "{}"', "SOMETHING"), # Does not pass yet, see TODO(https://github.com/intentionet/netconan/issues/107) - pytest.param('set system license keys key "{}"', 'SOMETHING sensitive', marks=pytest.mark.skip()), - ('set snmp community {} authorization read-only', 'SECRETTEXT'), - ('set snmp trap-group {} otherstuff', 'SECRETTEXT'), - ('key hexadecimal {}', 'ABCDEF123456'), - ('authentication-key "{}";', '$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p'), - ('hello-authentication-key {}', '$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p'), + pytest.param( + 'set system license keys key "{}"', + "SOMETHING sensitive", + marks=pytest.mark.skip(), + ), + ("set snmp community {} authorization read-only", "SECRETTEXT"), + ("set snmp trap-group {} otherstuff", "SECRETTEXT"), + ("key hexadecimal {}", "ABCDEF123456"), + ('authentication-key "{}";', "$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p"), + ("hello-authentication-key {}", "$9$i.m5OBEevLz3RSevx7-VwgZj5TFCA0Tz9p"), ] misc_password_lines = [ - ('my password is ', '$1$salt$abcdefghijklmnopqrs'), - ('set community {} trailing text', 'RemoveMe'), - ('set community {}', '1234a'), - ('set community {}', 'a1234') + ("my password is ", "$1$salt$abcdefghijklmnopqrs"), + ("set community {} trailing text", "RemoveMe"), + ("set community {}", "1234a"), + ("set community {}", "a1234"), ] -sensitive_lines = (arista_password_lines + - cisco_password_lines + - cisco_snmp_community_lines + - fortinet_password_lines + - juniper_password_lines + - misc_password_lines) +sensitive_lines = ( + arista_password_lines + + cisco_password_lines + + cisco_snmp_community_lines + + fortinet_password_lines + + juniper_password_lines + + misc_password_lines +) sensitive_items_and_formats = [ - ('094F4107180B', _sensitive_item_formats.cisco_type7), - ('00071C080555', _sensitive_item_formats.cisco_type7), - ('1608030A2B25', _sensitive_item_formats.cisco_type7), - ('070C2E424F072E04043A0E1E01', _sensitive_item_formats.cisco_type7), - ('01999999', _sensitive_item_formats.numeric), - ('987654321', _sensitive_item_formats.numeric), - ('0000000000000000', _sensitive_item_formats.numeric), - ('1234567890', _sensitive_item_formats.numeric), - ('7', _sensitive_item_formats.numeric), - ('A', _sensitive_item_formats.hexadecimal), - ('0FFFFFFFFF', _sensitive_item_formats.hexadecimal), - ('ABCDEF', _sensitive_item_formats.hexadecimal), - ('7ab34c2fe31', _sensitive_item_formats.hexadecimal), - ('deadBEEF', _sensitive_item_formats.hexadecimal), - ('27a', _sensitive_item_formats.hexadecimal), - ('$1$SALT$mutX1.3APXbr8JdR/Xi6t.', _sensitive_item_formats.md5), - ('$1$SALT$X8i6w2OOpAaEMNBGfSoZC0', _sensitive_item_formats.md5), - ('$1$SALT$ddio24/QfJatZkSKGuB4Z/', _sensitive_item_formats.md5), - ('$1$salt$rwny14pmwbMjy1WTfxf4h/', _sensitive_item_formats.md5), - ('$1$salt$BFdHEr6MVYydPmpY3FPXV/', _sensitive_item_formats.md5), - ('$1$salt$jp6JinwkFEV.2OCDaXrmO1', _sensitive_item_formats.md5), - ('$1$./4k$OVkG7VKh5GKt1/XjSO78.0', _sensitive_item_formats.md5), - ('$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/', _sensitive_item_formats.md5), - ('$1$67Q0XA3z$YqiBW/xxKWr74oHPXEkIv1', _sensitive_item_formats.md5), - ('thisIsATest', _sensitive_item_formats.text), - ('netconan', _sensitive_item_formats.text), - ('STRING', _sensitive_item_formats.text), - ('text_here', _sensitive_item_formats.text), - ('more-text-here0', _sensitive_item_formats.text), - ('ABCDEFG', _sensitive_item_formats.text), - ('$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t', _sensitive_item_formats.juniper_type9), - ('$9$YVgoZk.5n6AHq9tORlegoJGDkPfQCtOP5Qn9pRE', _sensitive_item_formats.juniper_type9), - ('$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0', _sensitive_item_formats.sha512) + ("094F4107180B", _sensitive_item_formats.cisco_type7), + ("00071C080555", _sensitive_item_formats.cisco_type7), + ("1608030A2B25", _sensitive_item_formats.cisco_type7), + ("070C2E424F072E04043A0E1E01", _sensitive_item_formats.cisco_type7), + ("01999999", _sensitive_item_formats.numeric), + ("987654321", _sensitive_item_formats.numeric), + ("0000000000000000", _sensitive_item_formats.numeric), + ("1234567890", _sensitive_item_formats.numeric), + ("7", _sensitive_item_formats.numeric), + ("A", _sensitive_item_formats.hexadecimal), + ("0FFFFFFFFF", _sensitive_item_formats.hexadecimal), + ("ABCDEF", _sensitive_item_formats.hexadecimal), + ("7ab34c2fe31", _sensitive_item_formats.hexadecimal), + ("deadBEEF", _sensitive_item_formats.hexadecimal), + ("27a", _sensitive_item_formats.hexadecimal), + ("$1$SALT$mutX1.3APXbr8JdR/Xi6t.", _sensitive_item_formats.md5), + ("$1$SALT$X8i6w2OOpAaEMNBGfSoZC0", _sensitive_item_formats.md5), + ("$1$SALT$ddio24/QfJatZkSKGuB4Z/", _sensitive_item_formats.md5), + ("$1$salt$rwny14pmwbMjy1WTfxf4h/", _sensitive_item_formats.md5), + ("$1$salt$BFdHEr6MVYydPmpY3FPXV/", _sensitive_item_formats.md5), + ("$1$salt$jp6JinwkFEV.2OCDaXrmO1", _sensitive_item_formats.md5), + ("$1$./4k$OVkG7VKh5GKt1/XjSO78.0", _sensitive_item_formats.md5), + ("$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", _sensitive_item_formats.md5), + ("$1$67Q0XA3z$YqiBW/xxKWr74oHPXEkIv1", _sensitive_item_formats.md5), + ("thisIsATest", _sensitive_item_formats.text), + ("netconan", _sensitive_item_formats.text), + ("STRING", _sensitive_item_formats.text), + ("text_here", _sensitive_item_formats.text), + ("more-text-here0", _sensitive_item_formats.text), + ("ABCDEFG", _sensitive_item_formats.text), + ("$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t", _sensitive_item_formats.juniper_type9), + ( + "$9$YVgoZk.5n6AHq9tORlegoJGDkPfQCtOP5Qn9pRE", + _sensitive_item_formats.juniper_type9, + ), + ( + "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0", + _sensitive_item_formats.sha512, + ), ] unique_passwords = [ - '12345ABCDEF', - 'ABCDEF123456789', - 'F', - 'FF', - '1A2B3C4D5E6F', - '0000000A0000000', - 'DEADBEEF', - '15260305170338051C362636', - 'ThisIsATest', - 'FLgBaJHXdYY_AcHZZMgQ_RhTDJXHUBAAB', - '122A00190102180D3C2E', - '$1$wtHI$0rN7R8PKwC30AsCGA77vy.', - 'JDYkqyIFWeBvzpljSfWmRZrmRSRE8syxKlOSjP9RCCkFinZbJI3GD5c6rckJR/Qju2PKLmOewbheAA==', - 'Password', - '2ndPassword', - 'PasswordThree', - '$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t', - '$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/', - '$6$NQJRTiqxZiNR0aWI$hU1EPleWl6wGcMtDxaMEqNhN8WnxEqmeFjWC5h8oh5USSn5P9ZgFXbf2giO8nEtM.yBXO3O6b.76LQ1zlmG3B0' + "12345ABCDEF", + "ABCDEF123456789", + "F", + "FF", + "1A2B3C4D5E6F", + "0000000A0000000", + "DEADBEEF", + "15260305170338051C362636", + "ThisIsATest", + "FLgBaJHXdYY_AcHZZMgQ_RhTDJXHUBAAB", + "122A00190102180D3C2E", + "$1$wtHI$0rN7R8PKwC30AsCGA77vy.", + "JDYkqyIFWeBvzpljSfWmRZrmRSRE8syxKlOSjP9RCCkFinZbJI3GD5c6rckJR/Qju2PKLmOewbheAA==", + "Password", + "2ndPassword", + "PasswordThree", + "$9$HqfQ1IcrK8n/t0IcvM24aZGi6/t", + "$1$CNANTest$xAfu6Am1d5D/.6OVICuOu/", + "$6$NQJRTiqxZiNR0aWI$hU1EPleWl6wGcMtDxaMEqNhN8WnxEqmeFjWC5h8oh5USSn5P9ZgFXbf2giO8nEtM.yBXO3O6b.76LQ1zlmG3B0", ] -SALT = 'saltForTest' +SALT = "saltForTest" -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def regexes(): """Compile regexes once for all tests in this module.""" return generate_default_sensitive_item_regexes() -@pytest.mark.parametrize('raw_line, sensitive_words', [ - ('something {} something', ['secret']), - ('something{}something', ['secret']), - ('{}', ['secret']), - ('a{0}b{0}c{0}d', ['secret']), - ('testing {} and {}.', ['SECRET', 'blah']), - ('testing {}{}.', ['secret', 'blah']) -]) +@pytest.mark.parametrize( + "raw_line, sensitive_words", + [ + ("something {} something", ["secret"]), + ("something{}something", ["secret"]), + ("{}", ["secret"]), + ("a{0}b{0}c{0}d", ["secret"]), + ("testing {} and {}.", ["SECRET", "blah"]), + ("testing {}{}.", ["secret", "blah"]), + ], +) def test_anonymize_sensitive_words(raw_line, sensitive_words): """Test anonymization of specified sensitive words.""" sens_word_anonymizer = SensitiveWordAnonymizer(sensitive_words, SALT, []) @@ -239,41 +284,42 @@ def test_anonymize_sensitive_words(raw_line, sensitive_words): # Make sure reanonymizing each word individually gives the same result as # anonymizing all at once, the first time - assert(anon_line == individually_anon_line) + assert anon_line == individually_anon_line for sens_word in sensitive_words: # Make sure all sensitive words are removed from the anonymized line - assert(sens_word not in anon_line) + assert sens_word not in anon_line # Test for case insensitivity # Make sure all sensitive words are removed from the lowercase line - assert(sens_word.lower() not in anon_line_lower) + assert sens_word.lower() not in anon_line_lower # Make sure all sensitive words are removed from the uppercase line - assert(sens_word.upper() not in anon_line_upper) + assert sens_word.upper() not in anon_line_upper def test_anonymize_sensitive_words_preserve_reserved_word(): """Test preservation of reserved words when anonymizing sensitive words.""" - reserved_word = 'search' + reserved_word = "search" # Intentionally use different case than reserved word - keyword = 'SEA' - keyword_plural = 'seas' - line = '{reserved_word} {keyword} {keyword_plural}'\ - .format(reserved_word=reserved_word, keyword=keyword, keyword_plural=keyword_plural) + keyword = "SEA" + keyword_plural = "seas" + line = "{reserved_word} {keyword} {keyword_plural}".format( + reserved_word=reserved_word, keyword=keyword, keyword_plural=keyword_plural + ) anonymizer = SensitiveWordAnonymizer([keyword], SALT, [reserved_word]) anon_line = anonymizer.anonymize(line) # Confirm keyword and plural keyword are removed from the line - assert(keyword not in anon_line.split()) - assert(keyword_plural not in anon_line.split()) + assert keyword not in anon_line.split() + assert keyword_plural not in anon_line.split() # Confirm the reserved word was not replaced - assert(reserved_word in anon_line.split()) + assert reserved_word in anon_line.split() -@pytest.mark.parametrize('val', unique_passwords) +@pytest.mark.parametrize("val", unique_passwords) def test__anonymize_value(val): """Test sensitive item anonymization.""" pwd_lookup = {} @@ -282,20 +328,20 @@ def test__anonymize_value(val): anon_val_format = _check_sensitive_item_format(anon_val) # Confirm the anonymized value does not match the original value - assert(anon_val != val) + assert anon_val != val # Confirm format for anonmymized value matches format of the original value - assert(anon_val_format == val_format) + assert anon_val_format == val_format if val_format == _sensitive_item_formats.md5: - org_salt_size = len(val.split('$')[2]) - anon_salt_size = len(anon_val.split('$')[2]) + org_salt_size = len(val.split("$")[2]) + anon_salt_size = len(anon_val.split("$")[2]) # Make sure salt size is preserved for md5 sensitive items # (Cisco should stay 4 character, Juniper 8 character, etc) - assert(org_salt_size == anon_salt_size) + assert org_salt_size == anon_salt_size # Confirm reanonymizing same source value results in same anonymized value - assert(anon_val == _anonymize_value(val, pwd_lookup, {})) + assert anon_val == _anonymize_value(val, pwd_lookup, {}) def test__anonymize_value_unique(): @@ -306,241 +352,257 @@ def test__anonymize_value_unique(): for anon_val in anon_vals: # Confirm unique source values have unique anonymized values - assert(anon_val not in unique_anon_vals) + assert anon_val not in unique_anon_vals unique_anon_vals.add(anon_val) -@pytest.mark.parametrize('val, format_', sensitive_items_and_formats) +@pytest.mark.parametrize("val, format_", sensitive_items_and_formats) def test__check_sensitive_item_format(val, format_): """Test sensitive item format detection.""" item_format = _check_sensitive_item_format(val) - assert(item_format == format_) - - -@pytest.mark.parametrize('raw_val', unique_passwords) -@pytest.mark.parametrize('head_text', [ - '\'', - '"', - '\\\'', - '\\"', - '[', - '[ ', - '"[\'[', - '" [', - '{', -]) + assert item_format == format_ + + +@pytest.mark.parametrize("raw_val", unique_passwords) +@pytest.mark.parametrize( + "head_text", + [ + "'", + '"', + "\\'", + '\\"', + "[", + "[ ", + "\"['[", + '" [', + "{", + ], +) def test__extract_enclosing_text_head(raw_val, head_text): """Test extraction of leading text.""" val = head_text + raw_val head, extracted_text, tail = _extract_enclosing_text(val) # Confirm the extracted text matches the original text - assert(extracted_text == raw_val) + assert extracted_text == raw_val # Confirm the leading text matches the prepended text - assert (head == head_text) - assert (not tail) - - -@pytest.mark.parametrize('raw_val', unique_passwords) -@pytest.mark.parametrize('tail_text', [ - '\'', - '"', - '\\\'', - '\\"', - ']', - ' ]', - ';', - ',', - '"],', - '] ;', - '}', -]) + assert head == head_text + assert not tail + + +@pytest.mark.parametrize("raw_val", unique_passwords) +@pytest.mark.parametrize( + "tail_text", + [ + "'", + '"', + "\\'", + '\\"', + "]", + " ]", + ";", + ",", + '"],', + "] ;", + "}", + ], +) def test__extract_enclosing_text_tail(raw_val, tail_text): """Test extraction of trailing text.""" val = raw_val + tail_text head, extracted_text, tail = _extract_enclosing_text(val) # Confirm the extracted text matches the original text - assert (extracted_text == raw_val) + assert extracted_text == raw_val # Confirm the trailing text matches the appended text - assert (tail == tail_text) - assert (not head) + assert tail == tail_text + assert not head -@pytest.mark.parametrize('val', unique_passwords) -@pytest.mark.parametrize('quote', [ - '\'', - '"', - '\\\'', - '\\"' -]) +@pytest.mark.parametrize("val", unique_passwords) +@pytest.mark.parametrize("quote", ["'", '"', "\\'", '\\"']) def test__extract_enclosing_text(val, quote): """Test extraction of enclosing quotes.""" enclosed_val = quote + val + quote head, extracted_text, tail = _extract_enclosing_text(enclosed_val) # Confirm the extracted text matches the original text - assert(extracted_text == val) + assert extracted_text == val # Confirm the extracted enclosing text matches the original enclosing text - assert (head == tail) - assert (head == quote) + assert head == tail + assert head == quote -@pytest.mark.parametrize('raw_config_line,sensitive_text', sensitive_lines) +@pytest.mark.parametrize("raw_config_line,sensitive_text", sensitive_lines) def test_pwd_removal(regexes, raw_config_line, sensitive_text): """Test removal of passwords and communities from config lines.""" config_line = raw_config_line.format(sensitive_text) pwd_lookup = {} anon_line = replace_matching_item(regexes, config_line, pwd_lookup) # Make sure the output line does not contain the sensitive text - assert(sensitive_text not in anon_line) + assert sensitive_text not in anon_line if _LINE_SCRUBBED_MESSAGE not in anon_line: # If the line wasn't "completely scrubbed", # make sure context was preserved anon_val = _anonymize_value(sensitive_text, pwd_lookup, {}) - assert(anon_line == raw_config_line.format(anon_val)) + assert anon_line == raw_config_line.format(anon_val) def test_pwd_removal_with_whitespace(regexes): """Test removal of password when a sensitive line contains extra whitespace.""" - sensitive_text = 'RemoveMe' - sensitive_line = ' password 0 \t{}'.format(sensitive_text) - assert(sensitive_text not in replace_matching_item( - regexes, sensitive_line, {})) - - -@pytest.mark.parametrize('config_line, sensitive_text', [ - ('snmp-server user Someone Somegroup auth md5 ipaddress priv {0} something', 'RemoveMe'), - ('snmp-server user Someone Somegroup auth md5 {0} priv ipaddress something', 'RemoveMe') -]) + sensitive_text = "RemoveMe" + sensitive_line = " password 0 \t{}".format(sensitive_text) + assert sensitive_text not in replace_matching_item(regexes, sensitive_line, {}) + + +@pytest.mark.parametrize( + "config_line, sensitive_text", + [ + ( + "snmp-server user Someone Somegroup auth md5 ipaddress priv {0} something", + "RemoveMe", + ), + ( + "snmp-server user Someone Somegroup auth md5 {0} priv ipaddress something", + "RemoveMe", + ), + ], +) def test_pwd_removal_and_preserve_reserved_word(regexes, config_line, sensitive_text): """Test removal of passwords when reserved words must be skipped.""" config_line = config_line.format(sensitive_text) pwd_lookup = {} - assert(sensitive_text not in replace_matching_item( - regexes, config_line, pwd_lookup)) - - -@pytest.mark.parametrize('config_line', [ - 'password ipaddress', - 'set community p2p', - 'digest secret snmp', - 'password {', - 'password "ip"', -]) + assert sensitive_text not in replace_matching_item(regexes, config_line, pwd_lookup) + + +@pytest.mark.parametrize( + "config_line", + [ + "password ipaddress", + "set community p2p", + "digest secret snmp", + "password {", + 'password "ip"', + ], +) def test_pwd_removal_preserve_reserved_word(regexes, config_line): """Test that reserved words are preserved even if they appear in password lines.""" pwd_lookup = {} - assert (config_line == replace_matching_item( - regexes, config_line, pwd_lookup)) - - -@pytest.mark.parametrize('config_line, anon_line', [ - ('"key": "password FOOBAR",', '"key": "password netconanRemoved0",'), - ('{"key": "cable shared-secret FOOBAR"}', '{"key": "! Sensitive line SCRUBBED by netconan"}'), - ('password "FOOBAR";', 'password "netconanRemoved0";'), -]) + assert config_line == replace_matching_item(regexes, config_line, pwd_lookup) + + +@pytest.mark.parametrize( + "config_line, anon_line", + [ + ('"key": "password FOOBAR",', '"key": "password netconanRemoved0",'), + ( + '{"key": "cable shared-secret FOOBAR"}', + '{"key": "! Sensitive line SCRUBBED by netconan"}', + ), + ('password "FOOBAR";', 'password "netconanRemoved0";'), + ], +) def test_pwd_removal_preserve_context(regexes, config_line, anon_line): """Test that context is preserved replacing/removing passwords.""" pwd_lookup = {} - assert (anon_line == replace_matching_item( - regexes, config_line, pwd_lookup)) + assert anon_line == replace_matching_item(regexes, config_line, pwd_lookup) -@pytest.mark.parametrize('whitespace', [ - ' ', - '\t', - '\n', - ' \t\n' -]) +@pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"]) def test_pwd_removal_preserve_leading_whitespace(regexes, whitespace): """Test leading whitespace is preserved in config lines.""" - config_line = '{whitespace}{line}'.format(line='password secret', - whitespace=whitespace) + config_line = "{whitespace}{line}".format( + line="password secret", whitespace=whitespace + ) pwd_lookup = {} processed_line = replace_matching_item(regexes, config_line, pwd_lookup) - assert(processed_line.startswith(whitespace)) + assert processed_line.startswith(whitespace) -@pytest.mark.parametrize('whitespace', [ - ' ', - '\t', - '\n', - ' \t\n' -]) +@pytest.mark.parametrize("whitespace", [" ", "\t", "\n", " \t\n"]) def test_pwd_removal_preserve_trailing_whitespace(regexes, whitespace): """Test trailing whitespace is preserved in config lines.""" - config_line = '{line}{whitespace}'.format(line='password secret', - whitespace=whitespace) + config_line = "{line}{whitespace}".format( + line="password secret", whitespace=whitespace + ) pwd_lookup = {} processed_line = replace_matching_item(regexes, config_line, pwd_lookup) - assert(processed_line.endswith(whitespace)) - - -@pytest.mark.parametrize('config_line,sensitive_text', sensitive_lines) -@pytest.mark.parametrize('prepend_text', [ - '"', - '\'', - '{', - ':', - 'something " ', - 'something \' ', - 'something { ', - 'something : ' -]) + assert processed_line.endswith(whitespace) + + +@pytest.mark.parametrize("config_line,sensitive_text", sensitive_lines) +@pytest.mark.parametrize( + "prepend_text", + [ + '"', + "'", + "{", + ":", + 'something " ', + "something ' ", + "something { ", + "something : ", + ], +) def test_pwd_removal_prepend(regexes, config_line, sensitive_text, prepend_text): """Test that sensitive lines are still anonymized correctly if preceded by allowed text.""" config_line = prepend_text + config_line.format(sensitive_text) pwd_lookup = {} - assert(sensitive_text not in replace_matching_item(regexes, config_line, pwd_lookup)) - - -@pytest.mark.parametrize('config_line,sensitive_text', sensitive_lines) -@pytest.mark.parametrize('append_text', [ - '"', - '\'', - '}', - '" something', - '\' something', - '} something', -]) + assert sensitive_text not in replace_matching_item(regexes, config_line, pwd_lookup) + + +@pytest.mark.parametrize("config_line,sensitive_text", sensitive_lines) +@pytest.mark.parametrize( + "append_text", + [ + '"', + "'", + "}", + '" something', + "' something", + "} something", + ], +) def test_pwd_removal_append(regexes, config_line, sensitive_text, append_text): """Test that sensitive lines are still anonymized correctly if followed by allowed text.""" config_line = config_line.format(sensitive_text) + append_text pwd_lookup = {} - assert(sensitive_text not in replace_matching_item(regexes, config_line, pwd_lookup)) - - -@pytest.mark.parametrize('config_line', [ - 'nothing in this string should be replaced', - ' interface GigabitEthernet0/0', - 'ip address 1.2.3.4 255.255.255.0', - 'set community 12345', - 'set community 1234:5678', - 'set community (1234:5678)', - 'set community 1234:5678 additive', - 'set community (1234:5678) additive', - 'set community gshut', - 'set community internet', - 'set community local-AS', - 'set community no-advertise', - 'set community no-export', - 'set community (no-export)', - 'set community none', - 'set community (12345 123:456 $foo:$bar no-export)', - 'set community (12345 123:456 $foo:$bar no-export) additive', - 'set community $foo:123', - 'set community $foo:$bar', - 'set community 123:$bar', - 'set community blah additive', - 'set community no-export additive', - 'set community peeras:24', -]) + assert sensitive_text not in replace_matching_item(regexes, config_line, pwd_lookup) + + +@pytest.mark.parametrize( + "config_line", + [ + "nothing in this string should be replaced", + " interface GigabitEthernet0/0", + "ip address 1.2.3.4 255.255.255.0", + "set community 12345", + "set community 1234:5678", + "set community (1234:5678)", + "set community 1234:5678 additive", + "set community (1234:5678) additive", + "set community gshut", + "set community internet", + "set community local-AS", + "set community no-advertise", + "set community no-export", + "set community (no-export)", + "set community none", + "set community (12345 123:456 $foo:$bar no-export)", + "set community (12345 123:456 $foo:$bar no-export) additive", + "set community $foo:123", + "set community $foo:$bar", + "set community 123:$bar", + "set community blah additive", + "set community no-export additive", + "set community peeras:24", + ], +) def test_pwd_removal_insensitive_lines(regexes, config_line): """Make sure benign lines are not affected by sensitive_item_removal.""" pwd_lookup = {} # Collapse all whitespace in original config_line and add newline since # that will be done by replace_matching_item - config_line = '{}\n'.format(' '.join(config_line.split())) - assert(config_line == replace_matching_item(regexes, config_line, pwd_lookup)) + config_line = "{}\n".format(" ".join(config_line.split())) + assert config_line == replace_matching_item(regexes, config_line, pwd_lookup) diff --git a/tools/generate_reserved_tokens.py b/tools/generate_reserved_tokens.py index 81d5895..222848e 100755 --- a/tools/generate_reserved_tokens.py +++ b/tools/generate_reserved_tokens.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from rules_python.python.runfiles import runfiles -from pathlib import Path import os import re +from pathlib import Path + +from rules_python.python.runfiles import runfiles DEFAULT_PREFIX = " " TOKEN_REGEX = re.compile(r"'([^']+)'=\d+") @@ -56,10 +57,10 @@ def get_token(line): def write_reserved_words_file(filename, token_set): """Writes a reserved words file with the given token set.""" - with open(filename, 'w') as file: + with open(filename, "w") as file: file.write(PREAMBLE) for token in sorted(token_set): - file.write('{}\'{}\',\n'.format(DEFAULT_PREFIX, token)) + file.write("{}'{}',\n".format(DEFAULT_PREFIX, token)) file.write(POSTAMBLE) @@ -70,14 +71,12 @@ def write_reserved_words_file(filename, token_set): output_path = Path(workspace_dir) / "netconan" / "default_reserved_words.py" rf = runfiles.Create() - tokens_path = rf.Rlocation( - "netconan/tools/concatenated.tokens" - ) + tokens_path = rf.Rlocation("netconan/tools/concatenated.tokens") if not tokens_path: raise RuntimeError("Cannot resolve tokens file") tokens = set() - with open(tokens_path, 'r') as f_in: + with open(tokens_path, "r") as f_in: for line in f_in: token = get_token(line) if token: From 43baec014c58aadb8c0353c7a01631a961f71b12 Mon Sep 17 00:00:00 2001 From: Dan Halperin Date: Wed, 27 Jan 2021 09:57:40 -0800 Subject: [PATCH 02/11] fortinet: add detection for pksecret secrets (#158) Also expand password and pksecret to unencrypted passwords. Fix #157 --- netconan/default_pwd_regexes.py | 3 ++- tests/unit/test_sensitive_item_removal.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index fdc9848..9a5cdf9 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -49,7 +49,8 @@ # Some of these regexes need to be updated to support quote enclosed passwords # which is allowed for at least some syntax on Juniper devices default_pwd_line_regexes = [ - [(r"(?P(password|passwd)( level \d+)?( \d+)?( ENC)? )(\S+)", 6)], + [(r"(?Pset (password|pksecret)( ENC)? )(\S+)", 4)], + [(r"(?P(password|passwd)( level \d+)?( \d+)? )(\S+)", 5)], [(r"(?Pusername( \S+)+ (password|secret)( \d| sha512)? )(\S+)", 5)], [(r"(?P(enable )?secret( \d)? )(\S+)", 4)], [(r"(?Pip ftp password( \d)? )(\S+)", 3)], diff --git a/tests/unit/test_sensitive_item_removal.py b/tests/unit/test_sensitive_item_removal.py index 3692215..8ac1d3d 100644 --- a/tests/unit/test_sensitive_item_removal.py +++ b/tests/unit/test_sensitive_item_removal.py @@ -121,9 +121,21 @@ # TODO(https://github.com/intentionet/netconan/issues/3): fortinet_password_lines = [ ( - "password ENC {}", + "set password ENC {}", "SH2nlSm9QL9tapcHPXIqAXvX7vBJuuqu22hpa0JX0sBuKIo7z2g0Kz/+0KyH4E=", - ) + ), + ( + "set password {}", + "mysecret", + ), + ( + "set pksecret ENC {}", + "SH2nlSm9QL9tapcHPXIqAXvX7vBJuuqu22hpa0JX0sBuKIo7z2g0Kz/+0KyH4E=", + ), + ( + "set pksecret {}", + "mysecret", + ), ] # TODO(https://github.com/intentionet/netconan/issues/4): From 4cf12902c2ada79dd39bd83ca068c4b65d9642c0 Mon Sep 17 00:00:00 2001 From: Spencer Fraint Date: Thu, 28 Jan 2021 17:31:11 -0800 Subject: [PATCH 03/11] Better IP address detection (#159) --- netconan/ip_anonymization.py | 4 ++-- tests/unit/test_ip_anonymization.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/netconan/ip_anonymization.py b/netconan/ip_anonymization.py index 721db0d..39105e5 100644 --- a/netconan/ip_anonymization.py +++ b/netconan/ip_anonymization.py @@ -27,8 +27,8 @@ _IPv4_OCTET_PATTERN = r"(25[0-5]|(2[0-4]|1?[0-9])?[0-9])" # Match address starting at beginning of line or surrounded by these, appropriate enclosing chars -_IPv4_ENCLOSING = r"[^\w.]" # Match anything but "word" chars or `.` -_IPv6_ENCLOSING = r"[^\w:]" # Match anything but "word" chars or `:` +_IPv4_ENCLOSING = r"[^a-zA-Z0-9.]" # Match anything but "word" chars (minus underscore) or `.` +_IPv6_ENCLOSING = r"[^a-zA-Z0-9:]" # Match anything but "word" chars (minus underscore) or `:` # Deliberately allowing leading zeros and will remove them later IPv4_PATTERN = re.compile( diff --git a/tests/unit/test_ip_anonymization.py b/tests/unit/test_ip_anonymization.py index db04880..feef1d3 100644 --- a/tests/unit/test_ip_anonymization.py +++ b/tests/unit/test_ip_anonymization.py @@ -153,7 +153,7 @@ def test_v4_anonymize_line(anonymizer_v4, line, ip_addrs): anonymize_line_general(anonymizer_v4, line, ip_addrs) -@pytest.mark.parametrize("enclosing", ":;[]$~!@#$%^&*()-+=[]|<>?") +@pytest.mark.parametrize("enclosing", "_:;[]$~!@#$%^&*()-+=[]|<>?") def test_v4_anonymize_enclosed_addr(anonymizer_v4, enclosing): """Test IPv4 address removal from config lines with different enclosing characters.""" ip_addr = "1.2.3.4" From 99eae53f1c81cfb0fe26dd69d897e0717d776725 Mon Sep 17 00:00:00 2001 From: Daniel Halperin Date: Fri, 19 Feb 2021 11:20:03 -0800 Subject: [PATCH 04/11] netconan: fix deanonymize and test --- netconan/ip_anonymization.py | 20 ++++++++-- tests/end_to_end/test_end_to_end.py | 58 ++++++++++++++++++----------- tests/unit/test_ip_anonymization.py | 8 +++- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/netconan/ip_anonymization.py b/netconan/ip_anonymization.py index 39105e5..c79fd06 100644 --- a/netconan/ip_anonymization.py +++ b/netconan/ip_anonymization.py @@ -27,8 +27,12 @@ _IPv4_OCTET_PATTERN = r"(25[0-5]|(2[0-4]|1?[0-9])?[0-9])" # Match address starting at beginning of line or surrounded by these, appropriate enclosing chars -_IPv4_ENCLOSING = r"[^a-zA-Z0-9.]" # Match anything but "word" chars (minus underscore) or `.` -_IPv6_ENCLOSING = r"[^a-zA-Z0-9:]" # Match anything but "word" chars (minus underscore) or `:` +_IPv4_ENCLOSING = ( + r"[^a-zA-Z0-9.]" # Match anything but "word" chars (minus underscore) or `.` +) +_IPv6_ENCLOSING = ( + r"[^a-zA-Z0-9:]" # Match anything but "word" chars (minus underscore) or `:` +) # Deliberately allowing leading zeros and will remove them later IPv4_PATTERN = re.compile( @@ -113,8 +117,16 @@ def _anonymize_bits(self, bits): def deanonymize(self, ip_int): bits = self.fmt.format(ip_int) - anon_bits = self._deanonymize_bits(bits) - return int(anon_bits, 2) + if self.preserve_suffix == 0: + deanon_bits = self._deanonymize_bits(bits) + else: + to_deanon, to_preserve = ( + bits[: -self.preserve_suffix], + bits[-self.preserve_suffix :], + ) + deanon_bits = self._deanonymize_bits(to_deanon) + to_preserve + + return int(deanon_bits, 2) def _deanonymize_bits(self, bits): ret = self.cache.inv.get(bits) diff --git a/tests/end_to_end/test_end_to_end.py b/tests/end_to_end/test_end_to_end.py index af4f5cf..7face2c 100644 --- a/tests/end_to_end/test_end_to_end.py +++ b/tests/end_to_end/test_end_to_end.py @@ -36,7 +36,7 @@ """ -REF_CONTENTS = """ +ANON_REF_CONTENTS = """ # a4daba's fd8607 test file ip address 192.168.2.1 255.255.255.255 ip address 111.111.111.111 @@ -51,27 +51,41 @@ """ +DEANON_REF_CONTENTS = """ +# a4daba's fd8607 test file +ip address 192.168.2.1 255.255.255.255 +ip address 111.111.111.111 +ip address 1.2.3.4 0.0.0.0 +my hash is $1$0000$CxUUGIrqPb7GaB5midrQZ. +AS num 8625 and 64818 should be changed +password netconanRemoved1 +password reservedword +ip address 11.11.11.11 0.0.0.0 +ip address 11.11.197.79 0.0.0.0 +# 3b836f word 10b348 here -def test_end_to_end(tmpdir): - """Test Netconan main with simulated input file and commandline args.""" - filename = "test.txt" - input_dir = tmpdir.mkdir("input") - input_dir.join(filename).write(INPUT_CONTENTS) +""" - output_dir = tmpdir.mkdir("output") - output_file = output_dir.join(filename) - ref_file = tmpdir.join(filename) - ref_file.write(REF_CONTENTS) +def run_test(input_dir, output_dir, filename, ref, args): + used_args = args + ["-i", str(input_dir), "-o", str(output_dir)] + main(used_args) + # Compare lines for more readable failed assertion message + t_ref = ref.split("\n") + with open(str(output_dir.join(filename))) as f_out: + t_out = f_out.read().split("\n") + + # Make sure output file lines match ref lines + assert t_ref == t_out + + +def test_end_to_end(tmpdir): + """Test Netconan main with simulated input file and commandline args.""" + filename = "test.txt" args = [ - "-i", - str(input_dir), - "-o", - str(output_dir), "-s", "TESTSALT", - "-a", "-p", "-w", "intentionet,sensitive,ADDR", @@ -86,15 +100,15 @@ def test_end_to_end(tmpdir): "--preserve-host-bits", "17", ] - main(args) - with open(str(ref_file)) as f_ref, open(str(output_file)) as f_out: - # Compare lines for more readable failed assertion message - t_ref = f_ref.read().split("\n") - t_out = f_out.read().split("\n") + input_dir = tmpdir.mkdir("input") + input_dir.join(filename).write(INPUT_CONTENTS) - # Make sure output file lines match ref lines - assert t_ref == t_out + anon_dir = tmpdir.mkdir("anon") + run_test(input_dir, anon_dir, filename, ANON_REF_CONTENTS, args + ["-a"]) + + deanon_dir = tmpdir.mkdir("deanon") + run_test(anon_dir, deanon_dir, filename, DEANON_REF_CONTENTS, args + ["-u"]) def test_end_to_end_no_anonymization(tmpdir): diff --git a/tests/unit/test_ip_anonymization.py b/tests/unit/test_ip_anonymization.py index feef1d3..461438d 100644 --- a/tests/unit/test_ip_anonymization.py +++ b/tests/unit/test_ip_anonymization.py @@ -343,11 +343,15 @@ def test_preserve_host_bits(length): anonymizer = IpAnonymizer( salt=SALT, salter=lambda a, b: 1, preserve_suffix=length, preserve_prefixes=[] ) - randomized = anonymizer.anonymize(0) + anonymized = anonymizer.anonymize(0) # The first <32-length> bits are 1, the last bits are all 0 expected = "1" * (32 - length) + "0" * length - assert "{:032b}".format(randomized) == expected + assert "{:032b}".format(anonymized) == expected + + # Deanonymization flips the bits back + deanonymized = anonymizer.deanonymize(anonymized) + assert deanonymized == 0 @pytest.mark.parametrize( From 34e67beebd478b48e7009ac06d7bc3c775c800d3 Mon Sep 17 00:00:00 2001 From: Daniel Halperin Date: Fri, 19 Feb 2021 11:28:08 -0800 Subject: [PATCH 05/11] dockstring --- tests/end_to_end/test_end_to_end.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end_to_end/test_end_to_end.py b/tests/end_to_end/test_end_to_end.py index 7face2c..13b151c 100644 --- a/tests/end_to_end/test_end_to_end.py +++ b/tests/end_to_end/test_end_to_end.py @@ -68,6 +68,7 @@ def run_test(input_dir, output_dir, filename, ref, args): + """Executes a test that the given filename is netconan-ified to ref.""" used_args = args + ["-i", str(input_dir), "-o", str(output_dir)] main(used_args) From 5684747073c528fd7cba07dde44bdc5a0f33b8cf Mon Sep 17 00:00:00 2001 From: Dan Halperin Date: Sat, 20 Feb 2021 10:29:25 -0800 Subject: [PATCH 06/11] pre-commit: add flake8 (#161) --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4ab42f..8c8aabc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,3 +19,8 @@ repos: - id: autoflake args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variables"] exclude: netconan/default_reserved_words.py +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + additional_dependencies: [flake8-docstrings] From a3bb928589503de825cf03b9c0fa55032460194f Mon Sep 17 00:00:00 2001 From: Dan Halperin Date: Thu, 1 Apr 2021 15:57:17 -0700 Subject: [PATCH 07/11] update reserved words (#163) Including adding new Batfish grammars to the list --- netconan/default_reserved_words.py | 63 +++++++++++++++++++++++++++++- tools/BUILD.bazel | 2 + 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/netconan/default_reserved_words.py b/netconan/default_reserved_words.py index 25b3ca2..fea3db7 100644 --- a/netconan/default_reserved_words.py +++ b/netconan/default_reserved_words.py @@ -147,6 +147,7 @@ 'address-table', 'address-virtual', 'addrgroup', + 'addrgrp', 'adjacency', 'adjacency-check', 'adjacency-stale-timer', @@ -239,6 +240,7 @@ 'alert', 'alert-group', 'alert-timeout', + 'alertmail', 'alerts', 'alg', 'alg-based-cac', @@ -259,6 +261,7 @@ 'allow-fail-through', 'allow-non-ssl', 'allow-nopassword-remote-login', + 'allow-routing', 'allow-self-ping', 'allow-service', 'allow-snooped-clients', @@ -304,10 +307,12 @@ 'app', 'app-service', 'appcategory', + 'append', 'appletalk', 'application', 'application-filter', 'application-group', + 'application-override', 'application-protocol', 'application-set', 'application-tracking', @@ -358,6 +363,7 @@ 'assoc-retransmit', 'associate', 'associate-vrf', + 'associated-interface', 'association', 'async', 'async-bootp', @@ -574,6 +580,7 @@ 'bsr-border', 'bsr-candidate', 'buckets', + 'buffer', 'buffer-length', 'buffer-limit', 'buffer-size', @@ -584,6 +591,7 @@ 'building configuration', 'bundle', 'bundle-speed', + 'burst', 'burst-size', 'bypass', 'bytes', @@ -1144,8 +1152,11 @@ 'dspfarm', 'dsrwait', 'dss', + 'dst', 'dst-mac', 'dst-nat', + 'dstaddr', + 'dstintf', 'dstopts', 'dsu', 'dtcp-only', @@ -1193,6 +1204,7 @@ 'ecn', 'edca-parameters-profile', 'edge', + 'edit', 'edition', 'ef', 'effective-ip', @@ -1210,6 +1222,7 @@ 'eligible', 'else', 'elseif', + 'emac-vlan', 'email', 'email-addr', 'email-contact', @@ -1220,6 +1233,7 @@ 'enable-acl-counter', 'enable-authentication', 'enable-qos-statistics', + 'enable-sender-side-loop-detection', 'enable-welcome-page', 'enabled', 'encapsulation', @@ -1232,6 +1246,7 @@ 'encryption-algorithm', 'end', 'end-class-map', + 'end-ip', 'end-policy', 'end-policy-map', 'end-set', @@ -1311,12 +1326,14 @@ 'events', 'evpn', 'exact', + 'exact-match', 'exceed-action', 'except', 'exception', 'exception-slave', 'excessive-bandwidth-use', 'exclude', + 'exclude-member', 'excluded-address', 'exec', 'exec-timeout', @@ -1374,6 +1391,7 @@ 'external-router-id', 'fabric', 'fabric-mode', + 'fabric-object', 'fabric-options', 'fabricpath', 'facility', @@ -1471,6 +1489,7 @@ 'forced', 'forever', 'format', + 'fortiguard-wf', 'forward', 'forward-digits', 'forward-error-correction', @@ -1528,6 +1547,7 @@ 'general-profile', 'generate', 'generic-alert', + 'geography', 'get', 'gid', 'gig-default', @@ -1580,6 +1600,7 @@ 'group24', 'group5', 'groups', + 'gshut', 'gt', 'gtp', 'gtp-c', @@ -1760,6 +1781,8 @@ 'icmp6-time-exceeded', 'icmp6-type', 'icmp6-unreachable', + 'icmpcode', + 'icmptype', 'icmpv6', 'icmpv6-malformed', 'id', @@ -1786,6 +1809,7 @@ 'ietf-format', 'if', 'if-needed', + 'if-route-exists', 'iface', 'ifacl', 'ifdescr', @@ -1879,6 +1903,7 @@ 'input-list', 'input-vlan-map', 'insecure', + 'insert', 'inservice', 'inside', 'inspect', @@ -1906,6 +1931,7 @@ 'interface-mode', 'interface-routes', 'interface-specific', + 'interface-subnet', 'interface-switch', 'interface-transmit-statistics', 'interface-type', @@ -1958,6 +1984,7 @@ 'ipmask', 'ipother', 'ipp', + 'iprange', 'ipsec', 'ipsec-crypto-profiles', 'ipsec-interfaces', @@ -2535,6 +2562,7 @@ 'match-across-virtuals', 'match-all', 'match-any', + 'match-ip-address', 'match-map', 'match-none', 'matches-any', @@ -2741,6 +2769,7 @@ 'mtu-discovery', 'mtu-failure', 'mtu-ignore', + 'mtu-override', 'multi-chassis', 'multi-config', 'multi-topology', @@ -2762,6 +2791,7 @@ 'mvpn', 'mvr', 'mvrp', + 'nac-quar', 'name', 'name-lookup', 'name-resolution', @@ -2901,6 +2931,7 @@ 'no-ping-record-route', 'no-ping-time-stamp', 'no-prepend', + 'no-prepend-global-as', 'no-proxy-arp', 'no-readvertise', 'no-redirects', @@ -3179,6 +3210,7 @@ 'phone-number', 'phone-proxy', 'phy', + 'physical', 'physical-layer', 'physical-port', 'pickup', @@ -3253,7 +3285,7 @@ 'post-policy', 'post-rulebase', 'post-up', - 'postrotuing', + 'postrouting', 'power', 'power-level', 'power-mgr', @@ -3281,6 +3313,7 @@ 'preferred-path', 'prefix', 'prefix-export-limit', + 'prefix-len', 'prefix-len-range', 'prefix-length', 'prefix-length-range', @@ -3294,7 +3327,7 @@ 'prefix-priority', 'prefix-set', 'prepend', - 'prerotuing', + 'prerouting', 'preserve-attributes', 'prf', 'pri-group', @@ -3345,6 +3378,7 @@ 'protocol', 'protocol-discovery', 'protocol-http', + 'protocol-number', 'protocol-object', 'protocol-unreachable', 'protocol-version', @@ -3491,6 +3525,7 @@ 'redistributed-prefixes', 'redundancy', 'redundancy-group', + 'redundant', 'redundant-ether-options', 'redundant-parent', 'reed-solomon', @@ -3532,6 +3567,7 @@ 'remove-private', 'remove-private-as', 'removed', + 'rename', 'renegotiate-max-record-delay', 'renegotiate-period', 'renegotiate-size', @@ -3542,6 +3578,7 @@ 'replace', 'replace-as', 'replace:', + 'replacemsg', 'reply', 'reply-to', 'report-flood', @@ -3750,9 +3787,12 @@ 'scripting', 'scripts', 'sctp', + 'sctp-portrange', 'sdm', + 'sdn', 'sdr', 'sdrowner', + 'sdwan', 'secondary', 'secondary-dialtone', 'secondary-ntp-server', @@ -3935,6 +3975,7 @@ 'sni-require', 'snmp', 'snmp-authfail', + 'snmp-index', 'snmp-server', 'snmp-trap', 'snmpgetclient', @@ -3982,6 +4023,7 @@ 'source-threshold', 'source-translation', 'source-user', + 'spam', 'span', 'spanning-tree', 'sparse-dense-mode', @@ -4013,6 +4055,8 @@ 'src-ip', 'src-mac', 'src-nat', + 'srcaddr', + 'srcintf', 'srlg', 'srlg-cost', 'srlg-value', @@ -4031,6 +4075,7 @@ 'ssl-forward-proxy-bypass', 'ssl-profile', 'ssl-sign-hash', + 'sslvpn', 'ssm', 'stack-mac', 'stack-mib', @@ -4039,6 +4084,7 @@ 'stalepath-time', 'standard', 'standby', + 'start-ip', 'start-stop', 'start-time', 'startup-query-count', @@ -4100,6 +4146,7 @@ 'stub-prefix-lsa', 'sub-option', 'sub-route-map', + 'sub-type', 'subcategory', 'subinterface', 'subject-name', @@ -4211,6 +4258,7 @@ 'tag-switching', 'tag-type', 'tagged', + 'tagging', 'tail-drop', 'talk', 'tap', @@ -4278,6 +4326,7 @@ 'tcp-pim-auto-rp', 'tcp-pop2', 'tcp-pop3', + 'tcp-portrange', 'tcp-pptp', 'tcp-proxy-reassembly', 'tcp-rsh', @@ -4310,6 +4359,7 @@ 'tcp-uucp', 'tcp-whois', 'tcp-www', + 'tcp/udp/sctp', 'tcpmux', 'tcpnethaspsrv', 'tcs-load-balance', @@ -4398,6 +4448,7 @@ 'traffic-group', 'traffic-index', 'traffic-loopback', + 'traffic-quota', 'traffic-share', 'transceiver', 'transceiver-type-check', @@ -4505,6 +4556,7 @@ 'udp-pcanywhere-status', 'udp-pim-auto-rp', 'udp-port', + 'udp-portrange', 'udp-radius', 'udp-radius-acct', 'udp-rip', @@ -4545,6 +4597,7 @@ 'unnumbered', 'unreachable', 'unreachables', + 'unselect', 'unset', 'unsuppress-map', 'unsuppress-route', @@ -4606,6 +4659,7 @@ 'users', 'using', 'util-interval', + 'utm', 'uucp', 'uucp-path', 'uuid', @@ -4635,6 +4689,7 @@ 'vap-enable', 'variance', 'vdc', + 'vdom', 'vendor-option', 'ver', 'verification', @@ -4702,6 +4757,7 @@ 'vpls', 'vpn', 'vpn-dialer', + 'vpn-distinguisher', 'vpn-filter', 'vpn-group-policy', 'vpn-idle-timeout', @@ -4762,6 +4818,7 @@ 'web-server', 'webapi', 'webauth', + 'webproxy', 'websocket', 'webvpn', 'wed', @@ -4778,6 +4835,7 @@ 'wide-metric', 'wide-metrics-only', 'wideband', + 'wildcard', 'wildcard-address', 'window', 'window-size', @@ -4795,6 +4853,7 @@ 'wispr', 'withdraw', 'without-csd', + 'wl-mesh', 'wlan', 'wmm', 'wms-general-profile', diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 65d7230..7c5d65d 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -5,6 +5,7 @@ filegroup( srcs = [ "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/arista:AristaLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/cisco:CiscoLexer.tokens", + "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/cisco_asa:AsaLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/cisco_nxos:CiscoNxosLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/cisco_xr:CiscoXrLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/cumulus_concatenated:CumulusConcatenatedLexer.tokens", @@ -16,6 +17,7 @@ filegroup( "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/f5_bigip_structured:F5BigipStructuredLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/flatjuniper:FlatJuniperLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/flatvyos:FlatVyosLexer.tokens", + "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/fortios:FortiosLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/iptables:IptablesLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/juniper:JuniperLexer.tokens", "@batfish//projects/batfish/src/main/antlr4/org/batfish/grammar/mrv:MrvLexer.tokens", From 77e33f87161b671c3cd53a148b1418e5078e1f48 Mon Sep 17 00:00:00 2001 From: Dan Halperin Date: Thu, 1 Apr 2021 15:58:46 -0700 Subject: [PATCH 08/11] add more Arista and NX-OS password regexes (#162) --- netconan/default_pwd_regexes.py | 2 ++ tests/unit/test_sensitive_item_removal.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/netconan/default_pwd_regexes.py b/netconan/default_pwd_regexes.py index 9a5cdf9..2aa04c7 100644 --- a/netconan/default_pwd_regexes.py +++ b/netconan/default_pwd_regexes.py @@ -55,6 +55,8 @@ [(r"(?P(enable )?secret( \d)? )(\S+)", 4)], [(r"(?Pip ftp password( \d)? )(\S+)", 3)], [(r"(?Pip ospf authentication-key( \d)? )(\S+)", 3)], + [(r"(?Pip ospf message-digest-key \d+ md5( \d)? )(\S+)", 3)], + [(r"(?Pauthentication text )(\S+)", 2)], [(r"(?Pisis password )(\S+)(?=( level-\d)?)", 2)], [(r"(?P(domain-password|area-password) )(\S+)", 3)], [(r"(?Pip ospf message-digest-key \d+ md5( \d)? )(\S+)", 3)], diff --git a/tests/unit/test_sensitive_item_removal.py b/tests/unit/test_sensitive_item_removal.py index 8ac1d3d..588e34a 100644 --- a/tests/unit/test_sensitive_item_removal.py +++ b/tests/unit/test_sensitive_item_removal.py @@ -34,7 +34,8 @@ ( "username noc secret sha512 {}", "$6$RMxgK5ALGIf.nWEC$tHuKCyfNtJMCY561P52dTzHUmYMmLxb/Mxik.j3vMUs8lMCPocM00/NAS.SN6GCWx7d/vQIgxnClyQLAb7n3x0", - ) + ), + (" vrrp 2 authentication text {}", "RemoveMe"), ] # TODO(https://github.com/intentionet/netconan/issues/3): # Add in additional test lines (these are just first pass from IOS) @@ -57,6 +58,8 @@ ("ip ftp password 7 {}", "122A00190102180D3C2E"), (" ip ospf authentication-key {}", "RemoveMe"), (" ip ospf authentication-key 0 {}", "RemoveMe"), + (" ip ospf message-digest-key 1 md5 {}", "RemoveMe"), + (" ip ospf message-digest-key 1 md5 3 {}", "RemoveMe"), ("isis password {}", "RemoveMe"), ("domain-password {}", "RemoveMe"), ("domain-password {} authenticate snp validate", "RemoveMe"), @@ -100,6 +103,7 @@ ("set session-key outbound esp 256 authenticator {}", "1234abcdef"), ("set session-key outbound esp 256 cipher {0} authenticator {0}", "1234abcdef"), ("key-hash sha256 {}", "RemoveMe"), + (" authentication text {}", "RemoveMe"), # HSRP ] cisco_snmp_community_lines = [ From 8bfff3c31d9d729011c829bf960ffea9c8a30088 Mon Sep 17 00:00:00 2001 From: Spencer Fraint Date: Mon, 26 Apr 2021 14:39:20 -0700 Subject: [PATCH 09/11] Fix `-d` (ip-map-dump) when preserving host bits (#165) --- netconan/ip_anonymization.py | 3 +++ tests/unit/test_ip_anonymization.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/netconan/ip_anonymization.py b/netconan/ip_anonymization.py index c79fd06..a16516b 100644 --- a/netconan/ip_anonymization.py +++ b/netconan/ip_anonymization.py @@ -100,6 +100,9 @@ def anonymize(self, ip_int): bits[-self.preserve_suffix :], ) anon_bits = self._anonymize_bits(to_anon) + to_preserve + # Intentionally caching here in addition to _anonymize_bits caching + # To add full addr (including preserved host bits) to cache map + self.cache[bits] = anon_bits return int(anon_bits, 2) def _anonymize_bits(self, bits): diff --git a/tests/unit/test_ip_anonymization.py b/tests/unit/test_ip_anonymization.py index 461438d..563dee9 100644 --- a/tests/unit/test_ip_anonymization.py +++ b/tests/unit/test_ip_anonymization.py @@ -80,6 +80,12 @@ def anonymizer_v4(): return IpAnonymizer(SALT) +@pytest.fixture(scope="module") +def anonymizer_v4_preserving_host_bits(): + """IPv4 anonymizer which also preserves some host bits.""" + return IpAnonymizer(SALT, preserve_suffix=8) + + @pytest.fixture(scope="module") def anonymizer_v6(): """All tests in this module use a single IPv6 anonymizer.""" @@ -455,6 +461,34 @@ def test_dump_iptree(tmpdir, anonymizer_v4): assert ip_mapping[ip_addr] == ip_mapping_from_dump[ip_addr] +def test_dump_iptree_preserving_host_bits(tmpdir, anonymizer_v4_preserving_host_bits): + """Test IP address anonymization map-dump, when preserving host bits.""" + ip_map_ref = {} + ip_map_from_dump = {} + + # Build reference map + for ip_addr_raw in ip_v4_list: + ip_addr = anonymizer_v4_preserving_host_bits.make_addr(ip_addr_raw) + ip_int_anon = anonymizer_v4_preserving_host_bits.anonymize(int(ip_addr)) + ip_addr_anon = str(ipaddress.IPv4Address(ip_int_anon)) + ip_map_ref[str(ip_addr)] = ip_addr_anon + + # Build mapping dict from the output of the ip_tree dump text file + filename = str(tmpdir.mkdir("test").join("test_dump_iptree.txt")) + with open(filename, "w") as f_tmp: + anonymizer_v4_preserving_host_bits.dump_to_file(f_tmp) + with open(filename, "r") as f_tmp: + for line in f_tmp.readlines(): + m = re.match(r"\s*(\d+\.\d+.\d+.\d+)\s+(\d+\.\d+.\d+.\d+)\s*", line) + ip_addr = m.group(1) + ip_addr_anon = m.group(2) + ip_map_from_dump[ip_addr] = ip_addr_anon + + # Confirm dumped text map lines up with expected/reference map + for ip_addr in ip_map_ref: + assert ip_map_ref[ip_addr] == ip_map_from_dump[ip_addr] + + @pytest.mark.parametrize( "line", [ From a6f01d27689c07f5b400ce6e600d849ccb6a76f0 Mon Sep 17 00:00:00 2001 From: Spencer Fraint Date: Tue, 27 Apr 2021 11:24:22 -0700 Subject: [PATCH 10/11] Token update (#166) --- netconan/default_reserved_words.py | 151 ++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 4 deletions(-) diff --git a/netconan/default_reserved_words.py b/netconan/default_reserved_words.py index fea3db7..9c1537a 100644 --- a/netconan/default_reserved_words.py +++ b/netconan/default_reserved_words.py @@ -38,11 +38,17 @@ '/', '1', '10000full', + '10000half', '1000full', + '1000half', '100full', '100g', '100gfull', + '100ghalf', + '100half', + '10full', '10g-4x', + '10half', '2', '2-byte', '25gbase-cr', @@ -81,6 +87,7 @@ 'accept-dialin', 'accept-lifetime', 'accept-own', + 'accept-rate', 'accept-register', 'accept-rp', 'accept-summary', @@ -99,6 +106,7 @@ 'accounting-port', 'accounting-server-group', 'accounting-threshold', + 'accprofile', 'acct-port', 'acfe', 'ack', @@ -145,7 +153,9 @@ 'address-range', 'address-set', 'address-table', + 'address-unreachable', 'address-virtual', + 'address6', 'addrgroup', 'addrgrp', 'adjacency', @@ -177,6 +187,7 @@ 'advertise-map', 'advertise-only', 'advertise-peer-as', + 'advertise-to', 'advertisement', 'advertisement-interval', 'aes', @@ -261,7 +272,9 @@ 'allow-fail-through', 'allow-non-ssl', 'allow-nopassword-remote-login', + 'allow-override', 'allow-routing', + 'allow-rp', 'allow-self-ping', 'allow-service', 'allow-snooped-clients', @@ -304,6 +317,7 @@ 'ap-rule-matching', 'ap-system-profile', 'api', + 'api-user', 'app', 'app-service', 'appcategory', @@ -331,6 +345,7 @@ 'arm-rf-domain-profile', 'arns', 'arp', + 'arp-inspect', 'arp-nd-suppress', 'arp-resp', 'as', @@ -423,6 +438,7 @@ 'auto-cert-allow-all', 'auto-cert-allowed-addrs', 'auto-cert-prov', + 'auto-config', 'auto-cost', 'auto-discard', 'auto-export', @@ -442,9 +458,13 @@ 'autoconfig', 'autohang', 'autohangup', + 'automation-action', + 'automation-stitch', + 'automation-trigger', 'autonomous-system', 'autorecovery', 'autoroute', + 'autoroute-exclude', 'autorp', 'autoselect', 'autostate', @@ -487,6 +507,7 @@ 'bc', 'bcmc-optimization', 'bcn-rpt-req-profile', + 'be', 'beacon', 'before', 'bestpath', @@ -510,6 +531,7 @@ 'bidir-rp-limit', 'bidirectional', 'biff', + 'big', 'bind', 'bind-interface', 'bittorrent', @@ -595,6 +617,7 @@ 'burst-size', 'bypass', 'bytes', + 'c-multicast-routing', 'ca', 'ca-cert', 'ca-cert-bundle', @@ -605,6 +628,8 @@ 'cable-upstream', 'cablelength', 'cache', + 'cache-sa-holdtime', + 'cache-sa-state', 'cache-size', 'cache-timeout', 'calipso-option', @@ -617,6 +642,8 @@ 'callhome', 'cam-acl', 'cam-profile', + 'candidate-bsr', + 'candidate-rp', 'capability', 'capacity', 'captive', @@ -671,6 +698,7 @@ 'cir', 'circuit-id', 'circuit-type', + 'cisco', 'cisco_tdp', 'cisp', 'citadel', @@ -709,6 +737,7 @@ 'clns', 'clock', 'clock-period', + 'clone', 'closed', 'cluster', 'cluster-id', @@ -799,12 +828,14 @@ 'controller', 'controller-ip', 'convergence', + 'convergence-timeout', 'conversion-error', 'cookie', 'copp', 'cops', 'copy', 'copy-attributes', + 'core-tree-protocol', 'cos', 'cos-mapping', 'cos-next-hop-map', @@ -848,6 +879,7 @@ 'cts', 'current configuration', 'custom', + 'custom-language', 'customer-id', 'cvspserver', 'cvx', @@ -865,6 +897,7 @@ 'dampening-interval', 'dampening-profile', 'damping', + 'data', 'data-group', 'data-privacy', 'database', @@ -914,6 +947,7 @@ 'default-network', 'default-node-monitor', 'default-originate', + 'default-peer', 'default-policy', 'default-role', 'default-route', @@ -1039,6 +1073,7 @@ 'dhcp-giaddr', 'dhcp-local-server', 'dhcp-relay', + 'dhcp-snoop', 'dhcp-snooping-vlan', 'dhcpd', 'dhcprelay', @@ -1065,6 +1100,7 @@ 'directed-request', 'direction', 'directly-connected-sources', + 'directory', 'disable', 'disable-4byte-as', 'disable-advertisement', @@ -1078,6 +1114,8 @@ 'discovered-ap-cnt', 'discovery', 'discriminator', + 'display', + 'display-location', 'display-name', 'dispute', 'distance', @@ -1192,6 +1230,7 @@ 'early-offer', 'ebgp', 'ebgp-multihop', + 'ebgp-multipath', 'ece', 'echo', 'echo-cancel', @@ -1226,6 +1265,8 @@ 'email', 'email-addr', 'email-contact', + 'email-server', + 'embedded-rp', 'emergencies', 'empty', 'enable', @@ -1313,6 +1354,7 @@ 'ethernet-switching-options', 'ethertype', 'etype', + 'eui-64', 'evaluate', 'evasive', 'event', @@ -1357,12 +1399,14 @@ 'expire', 'explicit-null', 'explicit-priority', + 'explicit-rpf-vector', 'explicit-tracking', 'export', 'export-localpref', 'export-nexthop', 'export-protocol', 'export-rib', + 'export-rt', 'exporter', 'exporter-map', 'expression', @@ -1378,6 +1422,7 @@ 'extended-community', 'extended-counters', 'extended-delay', + 'extended-show-width', 'extended-vni-list', 'extensible-subscriber', 'extension-header', @@ -1499,6 +1544,7 @@ 'forwarding', 'forwarding-class', 'forwarding-class-accounting', + 'forwarding-latency', 'forwarding-options', 'forwarding-table', 'forwarding-threshold', @@ -1600,6 +1646,7 @@ 'group24', 'group5', 'groups', + 'groups-per-interface', 'gshut', 'gt', 'gtp', @@ -1645,6 +1692,7 @@ 'heartbeat-time', 'heartbeat-timeout', 'hello', + 'hello-adjacency', 'hello-authentication', 'hello-authentication-key', 'hello-authentication-type', @@ -1663,6 +1711,7 @@ 'high', 'high-availability', 'high-resolution', + 'highest-ip', 'hip-header', 'hip-profiles', 'history', @@ -1674,6 +1723,7 @@ 'hold-character', 'hold-queue', 'hold-time', + 'holdtime', 'home-address-option', 'homedir', 'hop-by-hop', @@ -1734,6 +1784,7 @@ 'hw-switch', 'hwaddress', 'ibgp', + 'ibgp-multipath', 'iburst', 'icap', 'iccp', @@ -1800,12 +1851,14 @@ 'idle-timeout-override', 'idletimeout', 'idp-cert', + 'idp-sync-update', 'ids', 'ids-option', 'ids-profile', 'iec', 'ieee', 'ieee-mms-ssl', + 'ietf', 'ietf-format', 'if', 'if-needed', @@ -1826,6 +1879,7 @@ 'ignore-l3-incompletes', 'igp', 'igp-cost', + 'igp-intact', 'igrp', 'ike', 'ike-crypto-profiles', @@ -1852,6 +1906,7 @@ 'import-nexthop', 'import-policy', 'import-rib', + 'import-rt', 'in', 'in-place', 'inactive', @@ -1928,9 +1983,11 @@ 'intercept', 'interconnect-device', 'interface', + 'interface-inheritance', 'interface-mode', 'interface-routes', 'interface-specific', + 'interface-statistics', 'interface-subnet', 'interface-switch', 'interface-transmit-statistics', @@ -1940,6 +1997,8 @@ 'internal', 'internet', 'internet-options', + 'internet-service-id', + 'internet-service-name', 'interval', 'interworking', 'interzone', @@ -1947,6 +2006,7 @@ 'intra-area', 'intra-interface', 'intrazone', + 'invalid', 'invalid-spi-recovery', 'invalid-username-log', 'invert', @@ -2035,6 +2095,7 @@ 'iso-tsap', 'iso-vpn', 'isolate', + 'isolation', 'ispf', 'issuer-name', 'iuc', @@ -2042,6 +2103,7 @@ 'join-prune', 'join-prune-count', 'join-prune-interval', + 'join-prune-mtu', 'jp-interval', 'jp-policy', 'jumbo', @@ -2366,6 +2428,7 @@ 'link-fail', 'link-fault-signaling', 'link-flap', + 'link-local', 'link-local-groups-suppression', 'link-mode', 'link-protection', @@ -2446,6 +2509,7 @@ 'log-settings', 'log-start', 'log-syslog', + 'log-traps', 'log-updown', 'logfile', 'logging', @@ -2459,6 +2523,7 @@ 'logout-warning', 'long', 'longer', + 'longest-prefix', 'lookup', 'loop', 'loop-inconsistency', @@ -2504,10 +2569,8 @@ 'm2.', 'm3.', 'm4.', - 'm4route-mem', 'm5.', 'm6.', - 'm6route-mem', 'm7.', 'm8-15', 'm8.', @@ -2548,6 +2611,7 @@ 'map-t', 'mapped-port', 'mapping', + 'mapping-agent', 'marketing-name', 'martians', 'mask', @@ -2565,6 +2629,9 @@ 'match-ip-address', 'match-map', 'match-none', + 'match1', + 'match2', + 'match3', 'matches-any', 'matches-every', 'matip-type-a', @@ -2599,6 +2666,7 @@ 'max-renegotiations-per-minute', 'max-reuse', 'max-route', + 'max-servers', 'max-session-number', 'max-sessions', 'max-sessions-per-connection', @@ -2628,6 +2696,7 @@ 'md5', 'mdix', 'mdt', + 'mdt-hello-interval', 'med', 'media', 'media-termination', @@ -2677,6 +2746,7 @@ 'mgt-config', 'mh', 'mib', + 'mibs', 'micro-bfd', 'microcode', 'microsoft-ds', @@ -2727,6 +2797,9 @@ 'module', 'module-type', 'modulus', + 'mofrr', + 'mofrr-lockout-timer', + 'mofrr-loss-detection-timer', 'mon', 'monitor', 'monitor-interface', @@ -2744,6 +2817,7 @@ 'mroute', 'mroute-cache', 'mrouter', + 'ms', 'ms-rpc', 'ms-sql-m', 'ms-sql-s', @@ -2774,8 +2848,10 @@ 'multi-config', 'multi-topology', 'multicast', + 'multicast-address', 'multicast-boundary', 'multicast-group', + 'multicast-intact', 'multicast-mac', 'multicast-mode', 'multicast-routing', @@ -2811,6 +2887,7 @@ 'native-vlan-id', 'nbar', 'nbma', + 'nbr-unconfig', 'ncp', 'nd', 'nd-na', @@ -2825,6 +2902,8 @@ 'negotiation', 'neighbor', 'neighbor-advertisement', + 'neighbor-check-on-recv', + 'neighbor-check-on-send', 'neighbor-discovery', 'neighbor-down', 'neighbor-filter', @@ -2920,6 +2999,7 @@ 'no-gateway-community', 'no-install', 'no-ipv4-routing', + 'no-limit', 'no-nat-traversal', 'no-neighbor-down-notification', 'no-neighbor-learn', @@ -2954,6 +3034,9 @@ 'node-link-protection', 'noe', 'nohangup', + 'nomatch1', + 'nomatch2', + 'nomatch3', 'non-broadcast', 'non-client', 'non-client-nrt', @@ -2964,6 +3047,7 @@ 'non-ect', 'non-exist-map', 'non-mlag', + 'non-revertive', 'non-silent', 'non500-isakmp', 'none', @@ -2990,8 +3074,11 @@ 'notifyservicename', 'notifyserviceprotocol', 'notifyserviceraw', + 'np6', 'nsf', 'nsr', + 'nsr-delay', + 'nsr-down', 'nssa', 'nssa-external', 'nsw-fe', @@ -3004,6 +3091,7 @@ 'ntpaltaddress', 'ntpsourceinterface', 'null', + 'num-thread', 'nv', 'nve', 'nxapi', @@ -3022,6 +3110,7 @@ 'off', 'offset', 'offset-list', + 'old-register-checksum', 'olsr', 'on', 'on-failure', @@ -3033,6 +3122,7 @@ 'one-connect', 'one-out-of', 'only-ofdm', + 'oom-handling', 'open', 'open-delay-time', 'openflow', @@ -3093,6 +3183,7 @@ 'overload-control', 'override', 'override-connection-limit', + 'override-interval', 'override-metric', 'overrides', 'ovsdb-shutdown', @@ -3151,6 +3242,7 @@ 'pbkdf2', 'pbr', 'pbr-statistics', + 'pbts', 'pcanywhere-data', 'pcanywhere-status', 'pcp', @@ -3179,11 +3271,14 @@ 'peer-vtep', 'peers', 'penalty-period', + 'per-ce', 'per-entry', 'per-link', 'per-packet', + 'per-prefix', 'per-unit-scheduler', 'per-vlan', + 'per-vrf', 'percent', 'percentage', 'perfect-forward-secrecy', @@ -3221,6 +3316,7 @@ 'ping', 'ping-death', 'pinning', + 'pir', 'pki', 'pkix-timestamp', 'pkt-krb-ipsec', @@ -3257,6 +3353,7 @@ 'port', 'port-channel', 'port-channel-protocol', + 'port-description', 'port-mirror', 'port-mirroring', 'port-mode', @@ -3356,6 +3453,7 @@ 'proactive', 'probe', 'process', + 'process-failures', 'process-max-time', 'processes', 'product', @@ -3366,6 +3464,7 @@ 'prompt', 'prone-to-misuse', 'propagate', + 'propagation-delay', 'proposal', 'proposal-set', 'proposals', @@ -3425,6 +3524,7 @@ 'qualified-next-hop', 'querier', 'querier-timeout', + 'query', 'query-count', 'query-interval', 'query-max-response-time', @@ -3466,6 +3566,7 @@ 'rate', 'rate-limit', 'rate-mode', + 'rate-per-route', 'rate-thresholds-profile', 'raw', 'rbacl', @@ -3473,6 +3574,7 @@ 'rcp', 'rcv-queue', 'rd', + 'rd-set', 're-mail-ck', 'reachability', 'reachable-time', @@ -3494,6 +3596,7 @@ 'reauthentication', 'receive', 'receive-only', + 'receive-queue', 'receive-window', 'received', 'recirculation', @@ -3541,6 +3644,7 @@ 'register-source', 'registered', 'regulatory-domain-profile', + 'reinit', 'reject', 'reject-default-route', 'rekey', @@ -3579,6 +3683,7 @@ 'replace-as', 'replace:', 'replacemsg', + 'replacemsg-image', 'reply', 'reply-to', 'report-flood', @@ -3594,6 +3699,7 @@ 'require-wpa', 'required', 'required-option-missing', + 'reserve', 'reset', 'reset-both', 'reset-client', @@ -3664,6 +3770,7 @@ 'rmon', 'rmonitor', 'robustness', + 'robustness-count', 'robustness-variable', 'rogue-ap-aware', 'role', @@ -3685,6 +3792,8 @@ 'route-lookup', 'route-map', 'route-map-cache', + 'route-map-in', + 'route-map-out', 'route-only', 'route-policy', 'route-preference', @@ -3693,6 +3802,7 @@ 'route-reflector-client', 'route-source', 'route-table', + 'route-tag', 'route-target', 'route-to-peer', 'route-type', @@ -3722,10 +3832,13 @@ 'rp-announce-filter', 'rp-candidate', 'rp-list', + 'rp-static-deny', 'rpc-program-number', 'rpc2portmap', + 'rpf', 'rpf-check', 'rpf-failure', + 'rpf-redirect', 'rpf-vector', 'rpl-option', 'rpm', @@ -3737,6 +3850,7 @@ 'rst', 'rstp', 'rsvp', + 'rsvp-te', 'rsync', 'rt', 'rtcp-inactivity', @@ -3795,7 +3909,9 @@ 'sdwan', 'secondary', 'secondary-dialtone', + 'secondary-ip', 'secondary-ntp-server', + 'secondaryip', 'seconds', 'secret', 'secure', @@ -3813,6 +3929,7 @@ 'select', 'selection', 'selective', + 'selective-ack', 'self', 'self-allow', 'self-device', @@ -3867,6 +3984,7 @@ 'session-authorization', 'session-disconnect-warning', 'session-group', + 'session-helper', 'session-id', 'session-key', 'session-limit', @@ -3891,6 +4009,7 @@ 'sftp', 'sftp-server', 'sg-expiry-timer', + 'sg-list', 'sg-rpf-failure', 'sgmp', 'sha', @@ -4077,6 +4196,7 @@ 'ssl-sign-hash', 'sslvpn', 'ssm', + 'sso-admin', 'stack-mac', 'stack-mib', 'stack-unit', @@ -4084,6 +4204,7 @@ 'stalepath-time', 'standard', 'standby', + 'start', 'start-ip', 'start-stop', 'start-time', @@ -4102,6 +4223,7 @@ 'static-ipv6', 'static-nat', 'static-route', + 'static-rpf', 'station', 'station-address', 'station-port', @@ -4120,6 +4242,7 @@ 'stop-record', 'stop-timer', 'stopbits', + 'storage', 'storm-control', 'storm-control-profiles', 'stp', @@ -4149,6 +4272,7 @@ 'sub-type', 'subcategory', 'subinterface', + 'subinterfaces', 'subject-name', 'submgmt', 'submission', @@ -4181,10 +4305,12 @@ 'supplementary-services', 'suppress', 'suppress-arp', + 'suppress-data-registers', 'suppress-fib-pending', 'suppress-inactive', 'suppress-map', 'suppress-ra', + 'suppress-rpf-change-prunes', 'suppressed', 'supress-fa', 'suspect-rogue-conf-level', @@ -4196,6 +4322,7 @@ 'svrloc', 'switch', 'switch-cert', + 'switch-controller', 'switch-options', 'switch-priority', 'switch-profile', @@ -4203,6 +4330,7 @@ 'switchback', 'switching-mode', 'switchname', + 'switchover', 'switchover-on-routing-crash', 'switchport', 'symmetric', @@ -4213,9 +4341,11 @@ 'syn-frag', 'sync', 'sync-failover', + 'sync-igp-shortcuts', 'sync-only', 'synchronization', 'synchronous', + 'synwait-time', 'sys', 'sys-mac', 'syscontact', @@ -4225,9 +4355,12 @@ 'sysopt', 'systat', 'system', + 'system-capabilities', 'system-connected', + 'system-description', 'system-init', 'system-max', + 'system-name', 'system-priority', 'system-profile', 'system-services', @@ -4266,6 +4399,7 @@ 'target-host', 'target-host-port', 'targeted-broadcast', + 'targeted-hello', 'targets', 'task', 'task execute', @@ -4417,6 +4551,7 @@ 'timing', 'tls', 'tls-proxy', + 'tlv-select', 'tlv-set', 'tm-voq-collection', 'to', @@ -4469,6 +4604,7 @@ 'transmitter', 'transparent-hw-flooding', 'transport', + 'transport-address', 'transport-method', 'transport-mode', 'trap', @@ -4527,8 +4663,6 @@ 'type-2', 'type-7', 'type7', - 'u4route-mem', - 'u6route-mem', 'uauth', 'uc-tx-queue', 'ucmp', @@ -4584,6 +4718,7 @@ 'unclean-shutdown', 'unicast', 'unicast-address', + 'unicast-qos-adjust', 'unicast-routing', 'unidirectional', 'unique', @@ -4629,6 +4764,7 @@ 'url-list', 'urpf', 'urpf-logging', + 'us', 'use', 'use-acl', 'use-bia', @@ -4676,6 +4812,7 @@ 'vacant-message', 'vacl', 'vad', + 'valid', 'valid-11a-40mhz-channel-pair', 'valid-11a-80mhz-channel-group', 'valid-11a-channel', @@ -4684,6 +4821,7 @@ 'valid-and-protected-ssid', 'valid-network-oui-profile', 'validate', + 'validation-state', 'validation-usage', 'value', 'vap-enable', @@ -4717,6 +4855,7 @@ 'virtual-switch', 'virtual-template', 'virtual-wire', + 'visibility', 'visible-vsys', 'vlan', 'vlan-aware', @@ -4729,6 +4868,7 @@ 'vlan-raw-device', 'vlan-tagging', 'vlan-tags', + 'vlanid', 'vlans', 'vlans-disabled', 'vlans-enabled', @@ -4801,6 +4941,8 @@ 'wait-igp-convergence', 'wait-install', 'wait-start', + 'wanopt', + 'warning', 'warning-limit', 'warning-only', 'warnings', @@ -4837,6 +4979,7 @@ 'wideband', 'wildcard', 'wildcard-address', + 'wildcard-fqdn', 'window', 'window-size', 'winnuke', From 9d1e17078375497bf1c910452a3804a66a8a8763 Mon Sep 17 00:00:00 2001 From: Spencer Fraint Date: Tue, 27 Apr 2021 14:00:32 -0700 Subject: [PATCH 11/11] Prepare for release 0.12.2: Updating version number --- netconan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netconan/__init__.py b/netconan/__init__.py index 765eb8c..cb51714 100644 --- a/netconan/__init__.py +++ b/netconan/__init__.py @@ -19,4 +19,4 @@ __url__ = "https://github.com/intentionet/netconan" -__version__ = "0.12.1" +__version__ = "0.12.2"