Merge pull request #167 from intentionet/patch-0.12.2

Patch 0.12.2

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+    -   id: black
+        exclude: netconan/default_reserved_words.py
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.7.0
+    hooks:
+    -   id: isort
+        # args from https://black.readthedocs.io/en/stable/compatible_configs.html#isort
+        args: ["--multi-line=3", "--trailing-comma", "--force-grid-wrap=0", "--use-parentheses", "--ensure-newline-before-comments", "--line-length=88"]
+        exclude: netconan/default_reserved_words.py
+-   repo: [email protected]:humitos/mirrors-autoflake.git
+    rev: v1.3
+    hooks:
+    -   id: autoflake
+        args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variables"]
+        exclude: netconan/default_reserved_words.py
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
+    hooks:
+    -   id: flake8
+        additional_dependencies: [flake8-docstrings]

netconan/__init__.py

@@ -19,4 +19,4 @@
 
 __url__ = "https://github.com/intentionet/netconan"
 
-__version__ = "0.12.1"
+__version__ = "0.12.2"

netconan/anonymize_files.py

@@ -14,28 +14,43 @@
 #   limitations under the License.
 
 from __future__ import absolute_import
+
 import errno
 import logging
 import os
 import random
 import string
 
 from .default_reserved_words import default_reserved_words
-from .ip_anonymization import (
-    IpAnonymizer, IpV6Anonymizer, anonymize_ip_addr)
+from .ip_anonymization import IpAnonymizer, IpV6Anonymizer, anonymize_ip_addr
 from .sensitive_item_removal import (
-    anonymize_as_numbers, AsNumberAnonymizer, replace_matching_item,
-    SensitiveWordAnonymizer, generate_default_sensitive_item_regexes)
+    AsNumberAnonymizer,
+    SensitiveWordAnonymizer,
+    anonymize_as_numbers,
+    generate_default_sensitive_item_regexes,
+    replace_matching_item,
+)
 
 _DEFAULT_SALT_LENGTH = 16
 _CHAR_CHOICES = string.ascii_letters + string.digits
 
 
-def anonymize_files(input_path, output_path, anon_pwd, anon_ip,
-                    salt=None, dumpfile=None, sensitive_words=None,
-                    undo_ip_anon=False, as_numbers=None, reserved_words=None,
-                    preserve_prefixes=None, preserve_networks=None,
-                    preserve_suffix_v4=None, preserve_suffix_v6=None):
+def anonymize_files(
+    input_path,
+    output_path,
+    anon_pwd,
+    anon_ip,
+    salt=None,
+    dumpfile=None,
+    sensitive_words=None,
+    undo_ip_anon=False,
+    as_numbers=None,
+    reserved_words=None,
+    preserve_prefixes=None,
+    preserve_networks=None,
+    preserve_suffix_v4=None,
+    preserve_suffix_v6=None,
+):
     """Anonymize each file in input and save to output."""
     anonymizer4 = None
     anonymizer6 = None
@@ -45,7 +60,9 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip,
     pwd_lookup = None
     # The salt is only used for IP and sensitive word anonymization:
     if salt is None:
-        salt = ''.join(random.choice(_CHAR_CHOICES) for _ in range(_DEFAULT_SALT_LENGTH))
+        salt = "".join(
+            random.choice(_CHAR_CHOICES) for _ in range(_DEFAULT_SALT_LENGTH)
+        )
         logging.warning('No salt was provided; using randomly generated "%s"', salt)
     logging.debug('Using salt: "%s"', salt)
     if anon_pwd:
@@ -57,7 +74,12 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip,
     if sensitive_words is not None:
         anonymizer_sensitive_word = SensitiveWordAnonymizer(sensitive_words, salt)
     if anon_ip or undo_ip_anon:
-        anonymizer4 = IpAnonymizer(salt, preserve_prefixes, preserve_networks, preserve_suffix=preserve_suffix_v4)
+        anonymizer4 = IpAnonymizer(
+            salt,
+            preserve_prefixes,
+            preserve_networks,
+            preserve_suffix=preserve_suffix_v4,
+        )
         anonymizer6 = IpV6Anonymizer(salt, preserve_suffix=preserve_suffix_v6)
     if as_numbers is not None:
         anonymizer_as_num = AsNumberAnonymizer(as_numbers, salt)
@@ -73,40 +95,56 @@ def anonymize_files(input_path, output_path, anon_pwd, anon_ip,
         if not os.listdir(input_path):
             raise ValueError("Input directory is empty")
         if os.path.isfile(output_path):
-            raise ValueError("Output path must be a directory if input path is "
-                             "a directory")
+            raise ValueError(
+                "Output path must be a directory if input path is a directory"
+            )
 
         for root, dirs, files in os.walk(input_path):
             rel_root = os.path.relpath(root, input_path)
-            file_list.extend([(
-                os.path.join(input_path, rel_root, f),
-                os.path.join(output_path, rel_root, f)
-            ) for f in files if not f.startswith('.')])
+            file_list.extend(
+                [
+                    (
+                        os.path.join(input_path, rel_root, f),
+                        os.path.join(output_path, rel_root, f),
+                    )
+                    for f in files
+                    if not f.startswith(".")
+                ]
+            )
 
     for in_path, out_path in file_list:
         try:
-            anonymize_file(in_path,
-                           out_path,
-                           compiled_regexes=compiled_regexes,
-                           pwd_lookup=pwd_lookup,
-                           anonymizer_sensitive_word=anonymizer_sensitive_word,
-                           anonymizer_as_num=anonymizer_as_num,
-                           undo_ip_anon=undo_ip_anon,
-                           anonymizer4=anonymizer4,
-                           anonymizer6=anonymizer6)
+            anonymize_file(
+                in_path,
+                out_path,
+                compiled_regexes=compiled_regexes,
+                pwd_lookup=pwd_lookup,
+                anonymizer_sensitive_word=anonymizer_sensitive_word,
+                anonymizer_as_num=anonymizer_as_num,
+                undo_ip_anon=undo_ip_anon,
+                anonymizer4=anonymizer4,
+                anonymizer6=anonymizer6,
+            )
         except Exception:
-            logging.error('Failed to anonymize file %s', in_path, exc_info=True)
+            logging.error("Failed to anonymize file %s", in_path, exc_info=True)
 
     if dumpfile is not None:
-        with open(dumpfile, 'w') as f_out:
+        with open(dumpfile, "w") as f_out:
             anonymizer4.dump_to_file(f_out)
             anonymizer6.dump_to_file(f_out)
 
 
-def anonymize_file(filename_in, filename_out, compiled_regexes=None,
-                   anonymizer4=None, anonymizer6=None, pwd_lookup=None,
-                   anonymizer_sensitive_word=None, anonymizer_as_num=None,
-                   undo_ip_anon=False):
+def anonymize_file(
+    filename_in,
+    filename_out,
+    compiled_regexes=None,
+    anonymizer4=None,
+    anonymizer6=None,
+    pwd_lookup=None,
+    anonymizer_sensitive_word=None,
+    anonymizer_as_num=None,
+    undo_ip_anon=False,
+):
     """Anonymize contents of input file and save to the output file.
 
     This only applies sensitive line removal if compiled_regexes and pwd_lookup
@@ -119,16 +157,18 @@ def anonymize_file(filename_in, filename_out, compiled_regexes=None,
     _mkdirs(filename_out)
 
     if os.path.isdir(filename_out):
-        raise ValueError('Cannot write output file; '
-                         'output file is a directory ({})'
-                         .format(filename_out))
+        raise ValueError(
+            "Cannot write output file; "
+            "output file is a directory ({})".format(filename_out)
+        )
 
-    with open(filename_out, 'w') as f_out, open(filename_in, 'r') as f_in:
+    with open(filename_out, "w") as f_out, open(filename_in, "r") as f_in:
         for line in f_in:
             output_line = line
             if compiled_regexes is not None and pwd_lookup is not None:
-                output_line = replace_matching_item(compiled_regexes,
-                                                    output_line, pwd_lookup)
+                output_line = replace_matching_item(
+                    compiled_regexes, output_line, pwd_lookup
+                )
 
             if anonymizer6 is not None:
                 output_line = anonymize_ip_addr(anonymizer6, output_line, undo_ip_anon)

netconan/default_pwd_regexes.py

@@ -49,71 +49,100 @@
 # Some of these regexes need to be updated to support quote enclosed passwords
 # which is allowed for at least some syntax on Juniper devices
 default_pwd_line_regexes = [
-    [(r'(?P<prefix>(password|passwd)( level \d+)?( \d+)?( ENC)? )(\S+)', 6)],
-    [(r'(?P<prefix>username( \S+)+ (password|secret)( \d| sha512)? )(\S+)', 5)],
-    [(r'(?P<prefix>(enable )?secret( \d)? )(\S+)', 4)],
-    [(r'(?P<prefix>ip ftp password( \d)? )(\S+)', 3)],
-    [(r'(?P<prefix>ip ospf authentication-key( \d)? )(\S+)', 3)],
-    [(r'(?P<prefix>isis password )(\S+)(?=( level-\d)?)', 2)],
-    [(r'(?P<prefix>(domain-password|area-password) )(\S+)', 3)],
-    [(r'(?P<prefix>ip ospf message-digest-key \d+ md5( \d)? )(\S+)', 3)],
-    [(r'(?P<prefix>standby( \d*)? authentication( text| md5 key-string( \d)?)? )(\S+)', 5)],
-    [(r'(?P<prefix>l2tp tunnel( \S+)? password( \d)? )(\S+)', 4)],
-    [(r'(?P<prefix>digest secret( \d)? )(\S+)', 3)],
-    [(r'(?P<prefix>ppp .* hostname )(\S+)', 2)],
-    [(r'(?P<prefix>ppp .* password( \d)? )(\S+)', 3)],
-    [(r'(?P<prefix>(ikev2 )?(local|remote)-authentication pre-shared-key )(\S+)', 4)],
-    [(r'(?P<prefix>(\S )*pre-shared-key( remote| local)?( hex| hexadecimal| ascii-text| \d)? )(\S+)', 5)],
-    [(r'(?P<prefix>(tacacs|radius)-server (\S+ )*key( \d)? )(\S+)', 5)],
-    [(r'(?P<prefix>key( \d| hexadecimal)? )(\S+)', 3)],
-    [(r'(?P<prefix>ntp authentication-key \d+ md5 )(\S+)', 2)],
-    [(r'(?P<prefix>syscon( password| address \S+) )(\S+)', 3)],
-    [(r'(?P<prefix>snmp-server user( \S+)+ (auth (md5|sha)) )(\S+)', 5),
-     (r'(?P<prefix>snmp-server user( \S+)+ priv( 3des| aes( \d+)?| des)? )(\S+)', 5)],
-    [(r'(?P<prefix>(crypto )?isakmp key( \d)? )(\S+)', 4)],
-    [(r'(?P<prefix>set session-key (in|out)bound ah \d+ )(\S+)', 3)],
-    [(r'(?P<prefix>set session-key (in|out)bound esp \d+ cipher? )(\S+)', 3),
-     (r'(?P<prefix>set session-key (in|out)bound esp \d+(( cipher \S+)? authenticator) )(\S+)', 5)],
-    [(r'(?P<prefix>(hello-)?authentication-key )([^;]+)', 3)],
+    [(r"(?P<prefix>set (password|pksecret)( ENC)? )(\S+)", 4)],
+    [(r"(?P<prefix>(password|passwd)( level \d+)?( \d+)? )(\S+)", 5)],
+    [(r"(?P<prefix>username( \S+)+ (password|secret)( \d| sha512)? )(\S+)", 5)],
+    [(r"(?P<prefix>(enable )?secret( \d)? )(\S+)", 4)],
+    [(r"(?P<prefix>ip ftp password( \d)? )(\S+)", 3)],
+    [(r"(?P<prefix>ip ospf authentication-key( \d)? )(\S+)", 3)],
+    [(r"(?P<prefix>ip ospf message-digest-key \d+ md5( \d)? )(\S+)", 3)],
+    [(r"(?P<prefix>authentication text )(\S+)", 2)],
+    [(r"(?P<prefix>isis password )(\S+)(?=( level-\d)?)", 2)],
+    [(r"(?P<prefix>(domain-password|area-password) )(\S+)", 3)],
+    [(r"(?P<prefix>ip ospf message-digest-key \d+ md5( \d)? )(\S+)", 3)],
+    [
+        (
+            r"(?P<prefix>standby( \d*)? authentication( text| md5 key-string( \d)?)? )(\S+)",
+            5,
+        )
+    ],
+    [(r"(?P<prefix>l2tp tunnel( \S+)? password( \d)? )(\S+)", 4)],
+    [(r"(?P<prefix>digest secret( \d)? )(\S+)", 3)],
+    [(r"(?P<prefix>ppp .* hostname )(\S+)", 2)],
+    [(r"(?P<prefix>ppp .* password( \d)? )(\S+)", 3)],
+    [(r"(?P<prefix>(ikev2 )?(local|remote)-authentication pre-shared-key )(\S+)", 4)],
+    [
+        (
+            r"(?P<prefix>(\S )*pre-shared-key( remote| local)?( hex| hexadecimal| ascii-text| \d)? )(\S+)",
+            5,
+        )
+    ],
+    [(r"(?P<prefix>(tacacs|radius)-server (\S+ )*key( \d)? )(\S+)", 5)],
+    [(r"(?P<prefix>key( \d| hexadecimal)? )(\S+)", 3)],
+    [(r"(?P<prefix>ntp authentication-key \d+ md5 )(\S+)", 2)],
+    [(r"(?P<prefix>syscon( password| address \S+) )(\S+)", 3)],
+    [
+        (r"(?P<prefix>snmp-server user( \S+)+ (auth (md5|sha)) )(\S+)", 5),
+        (r"(?P<prefix>snmp-server user( \S+)+ priv( 3des| aes( \d+)?| des)? )(\S+)", 5),
+    ],
+    [(r"(?P<prefix>(crypto )?isakmp key( \d)? )(\S+)", 4)],
+    [(r"(?P<prefix>set session-key (in|out)bound ah \d+ )(\S+)", 3)],
+    [
+        (r"(?P<prefix>set session-key (in|out)bound esp \d+ cipher? )(\S+)", 3),
+        (
+            r"(?P<prefix>set session-key (in|out)bound esp \d+(( cipher \S+)? authenticator) )(\S+)",
+            5,
+        ),
+    ],
+    [(r"(?P<prefix>(hello-)?authentication-key )([^;]+)", 3)],
     # TODO(https://github.com/intentionet/netconan/issues/3):
     # Follow-up on these.  They were just copied from RANCID so currently:
     #   They are untested in general and need cases added for unit tests
     #   They do not specifically capture sensitive info
     #   They just identify lines where sensitive info exists
-    [(r'(cable shared-secret) (.*)', None)],
-    [(r'(wpa-psk ascii|hex \d) (.*)', None)],
-    [(r'(ldap-login-password) \S+(.*)', None)],
-    [(r'((ikev1 )?(pre-shared-key |key |failover key )(ascii-text |hexadecimal )?).*(.*)', None)],
-    [(r'(vpdn username (\S+) password)(.*)', None)],
-    [(r'(key-string \d?)(.*)', None)],
-    [(r'(message-digest-key \d+ md5 (7|encrypted)) (.*)', None)],
-    [(r'(.*?neighbor.*?) (\S*) password (.*)', None)],
-    [(r'(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)', None)],
-
+    [(r"(cable shared-secret) (.*)", None)],
+    [(r"(wpa-psk ascii|hex \d) (.*)", None)],
+    [(r"(ldap-login-password) \S+(.*)", None)],
+    [
+        (
+            r"((ikev1 )?(pre-shared-key |key |failover key )(ascii-text |hexadecimal )?).*(.*)",
+            None,
+        )
+    ],
+    [(r"(vpdn username (\S+) password)(.*)", None)],
+    [(r"(key-string \d?)(.*)", None)],
+    [(r"(message-digest-key \d+ md5 (7|encrypted)) (.*)", None)],
+    [(r"(.*?neighbor.*?) (\S*) password (.*)", None)],
+    [(r"(wlccp \S+ username (\S+)( .*)? password( \d)?) (\S+)(.*)", None)],
     # These are regexes for JUNOS
     # TODO(https://github.com/intentionet/netconan/issues/4):
     # Follow-up on these.  They were modified from RANCID's regexes and currently:
     #   They do not have capture groups for sensitive info
     #   They just identify lines where sensitive info exists
     #   They need to be tested against config lines generated on a JUNOS router
     #     (to make sure the regex handles different syntaxes allowed in the line)
-    [(r'(\S* )*md5 \d+ key [^ ;]+(.*)', None)],
-    [(r'(\S* )*(secret|simple-password) [^ ;]+(.*)', None)],
-    [(r'(\S* )*encrypted-password [^ ;]+(.*)', None)],
-    [(r'(\S* )*ssh-(rsa|dsa) \"(.*)', None)],
-    [(r'(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)', None)]
+    [(r"(\S* )*md5 \d+ key [^ ;]+(.*)", None)],
+    [(r"(\S* )*(secret|simple-password) [^ ;]+(.*)", None)],
+    [(r"(\S* )*encrypted-password [^ ;]+(.*)", None)],
+    [(r"(\S* )*ssh-(rsa|dsa) \"(.*)", None)],
+    [(r"(\S* )*((pre-shared-|)key (ascii-text|hexadecimal)) [^ ;]+(.*)", None)],
 ]
 # Taken from RANCID community scrubbing regexes
 default_com_line_regexes = [
-    [(r'(?P<prefix>(snmp-server (\S+ )*community)( [08])? )(\S+)', 5)],
+    [(r"(?P<prefix>(snmp-server (\S+ )*community)( [08])? )(\S+)", 5)],
     # TODO(https://github.com/intentionet/netconan/issues/5):
     # Confirm this catches all community possibilities for snmp-server
-    [(r'(?P<prefix>snmp-server host (\S+)( informs| traps| version '
-     r'(?:1|2c|3 \S+)| vrf \S+)* )(\S+)', 4)],
+    [
+        (
+            r"(?P<prefix>snmp-server host (\S+)( informs| traps| version "
+            r"(?:1|2c|3 \S+)| vrf \S+)* )(\S+)",
+            4,
+        )
+    ],
     # This is from JUNOS
     # TODO(https://github.com/intentionet/netconan/issues/4):
     # See if we need to make the snmp keyword optional for Juniper
     # Also, this needs to be tested against config lines generated on a JUNOS router
     #     (to make sure the regex handles different syntaxes allowed in the line)
-    [(r'(?P<prefix>(\S* )*snmp( \S+)* (community|trap-group) )([^ ;]+)', 5)]
+    [(r"(?P<prefix>(\S* )*snmp( \S+)* (community|trap-group) )([^ ;]+)", 5)],
 ]