From 77a29b060e3cabae1337fa0fa4839a5768c683f8 Mon Sep 17 00:00:00 2001 From: James Harmison Date: Wed, 23 Dec 2020 14:14:09 -0500 Subject: [PATCH 01/10] Reorganized Dockerfile to shorten dev rebuilds --- Dockerfile | 28 +++++++++----------- {app/tmp => pre}/README.md | 0 {app/tmp => pre}/ilorest-3.0.1-7.x86_64.rpm | Bin 3 files changed, 13 insertions(+), 15 deletions(-) rename {app/tmp => pre}/README.md (100%) rename {app/tmp => pre}/ilorest-3.0.1-7.x86_64.rpm (100%) diff --git a/Dockerfile b/Dockerfile index 95d1e3d..5db21db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,28 +13,26 @@ RUN mkdir -p /deps/python /deps/ansible; \ COPY version.txt /version.txt COPY requirements.txt /deps/python_requirements.txt COPY requirements.yml /deps/ansible_requirements.yml -RUN microdnf update; \ +COPY home /root +COPY pre /pre +RUN set -ex; \ + microdnf update; \ microdnf install python3 jq openssh-clients tar sshpass findutils telnet less ncurses; \ - pip3 install --user -r /deps/python_requirements.txt; \ + pip3 install --user -r /deps/python_requirements.txt /pre/faros_config; \ ansible-galaxy collection install -r /deps/ansible_requirements.yml; \ microdnf clean all; \ - rm -rf /var/cache/yum /tmp/* /root/.cache /usr/lib/python3.8/site-packages /usr/lib64/python3.8/__pycache__; - -# Install application -WORKDIR /app -COPY app /app -COPY data.skel /data.skel -COPY home /root - -# Initialize application -RUN rpm -i /app/tmp/ilorest-3.0.1-7.x86_64.rpm; \ - chmod -Rv g-rwx /root/.ssh; chmod -Rv o-rwx /root/.ssh; \ - rm -rf /app/tmp; \ + rpm -i /pre/ilorest-3.0.1-7.x86_64.rpm; \ + rm -rf /pre /var/cache/dnf /var/cache/yum /tmp/* /root/.cache /usr/lib/python3.8/site-packages /usr/lib64/python3.8/__pycache__; \ + chmod -Rv g-rwx,o-rwx /root/.ssh; \ cd /usr/local/bin; \ curl https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz | tar xvzf -; \ curl https://raw.githubusercontent.com/project-faros/farosctl/master/bin/farosctl > farosctl; \ chmod 755 farosctl; +# Install application +COPY data.skel /data.skel +COPY app /app +WORKDIR /app + ENTRYPOINT ["/app/bin/entry.sh"] CMD ["/app/bin/run.sh"] - diff --git a/app/tmp/README.md b/pre/README.md similarity index 100% rename from app/tmp/README.md rename to pre/README.md diff --git a/app/tmp/ilorest-3.0.1-7.x86_64.rpm b/pre/ilorest-3.0.1-7.x86_64.rpm similarity index 100% rename from app/tmp/ilorest-3.0.1-7.x86_64.rpm rename to pre/ilorest-3.0.1-7.x86_64.rpm From 59c263b320f5f02f7370dca6300e867587099f08 Mon Sep 17 00:00:00 2001 From: James Harmison Date: Wed, 23 Dec 2020 13:57:08 -0500 Subject: [PATCH 02/10] Added `detect drives` command This new command will create `drives.yml` in the `/data` directory inside the container (mapping to user's config directory). This YAML file has parsed output from `lsblk -J` for each node stored in an array, which enables us to get a lot of information about disks (including that which would be necessary to hide the root filesystem onto which CoreOS is installed) --- app/cli/detect | 1 + .../detect.d/drives/detect-drives.yml | 31 +++++++++++++++++++ app/playbooks/detect.d/drives/main.sh | 6 ++++ 3 files changed, 38 insertions(+) create mode 120000 app/cli/detect create mode 100644 app/playbooks/detect.d/drives/detect-drives.yml create mode 100644 app/playbooks/detect.d/drives/main.sh diff --git a/app/cli/detect b/app/cli/detect new file mode 120000 index 0000000..1fa21ed --- /dev/null +++ b/app/cli/detect @@ -0,0 +1 @@ +.command \ No newline at end of file diff --git a/app/playbooks/detect.d/drives/detect-drives.yml b/app/playbooks/detect.d/drives/detect-drives.yml new file mode 100644 index 0000000..37f17b5 --- /dev/null +++ b/app/playbooks/detect.d/drives/detect-drives.yml @@ -0,0 +1,31 @@ +- name: query cluster facts + hosts: localhost + become: no + gather_facts: no + + tasks: + - name: lookup cluster nodes + set_fact: + cluster_nodes: "{{ lookup('k8s', api_version='v1', kind='Node') | json_query('[*].metadata.name')}}" + + - name: query cluster node drives + shell: oc debug -n default node/{{ item }} -- chroot /host lsblk -J + loop: "{{ cluster_nodes }}" + register: cluster_drives + ignore_errors: yes + changed_when: no + + - name: save discovered drives + set_stats: + data: + drive_data: "{{ drive_data|from_yaml }}" + vars: + drive_data: | + {% for result in cluster_drives.results %} + {{ result.item }}: + {% for blockdevice in (result.stdout|from_json).blockdevices %} + {% if blockdevice.type == "disk" %} + - {{ blockdevice|to_json }} + {% endif %} + {% endfor %} + {% endfor %} diff --git a/app/playbooks/detect.d/drives/main.sh b/app/playbooks/detect.d/drives/main.sh new file mode 100644 index 0000000..b18b138 --- /dev/null +++ b/app/playbooks/detect.d/drives/main.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +ME=$(dirname $0) + +STATS_FILE=/tmp/drives.yml ansible-playbook $ME/detect-drives.yml "${@}" || exit $? +cp /tmp/drives.yml /data/drives.yml From 75ebb94da2999b8bf5d188c1285cfa41f180ab9d Mon Sep 17 00:00:00 2001 From: James Harmison Date: Mon, 28 Dec 2020 16:01:04 -0500 Subject: [PATCH 03/10] Removed old configuration mechanism - Converted various shell calls to proxy only - Removed all TUI library elements - Removed references to config.sh in playbook scripts - Removed config playbook directory --- app/bin/entry.sh | 3 - app/bin/run.sh | 5 +- app/cli/shutdown | 1 - app/cli/ssh | 1 - app/cli/startup | 1 - app/lib/python/conftui.py | 336 ----------------- app/playbooks/config.d/all | 1 - app/playbooks/config.d/cluster/config.py | 59 --- app/playbooks/config.d/cluster/config.py.bkp | 366 ------------------- app/playbooks/config.d/cluster/main.sh | 7 - app/playbooks/config.d/proxy/config.py | 35 -- app/playbooks/config.d/proxy/main.sh | 9 - data.skel/config.sh | 14 - home/.bashrc | 22 +- requirements.txt | 1 - 15 files changed, 11 insertions(+), 850 deletions(-) delete mode 100644 app/lib/python/conftui.py delete mode 120000 app/playbooks/config.d/all delete mode 100644 app/playbooks/config.d/cluster/config.py delete mode 100644 app/playbooks/config.d/cluster/config.py.bkp delete mode 100755 app/playbooks/config.d/cluster/main.sh delete mode 100644 app/playbooks/config.d/proxy/config.py delete mode 100755 app/playbooks/config.d/proxy/main.sh delete mode 100644 data.skel/config.sh diff --git a/app/bin/entry.sh b/app/bin/entry.sh index a543dfb..365cf3b 100755 --- a/app/bin/entry.sh +++ b/app/bin/entry.sh @@ -4,9 +4,6 @@ if [ "$1" != "cat" ] && [ "$1" != "ls" ] && [ "$1" != "type" ]; then source /app/bin/shim-check.sh fi -if [ -e /data/config.sh ]; then - source /data/config.sh -fi if [ -e /data/proxy.sh ]; then source /data/proxy.sh fi diff --git a/app/bin/run.sh b/app/bin/run.sh index 5f4f4e1..f3dcb8b 100755 --- a/app/bin/run.sh +++ b/app/bin/run.sh @@ -1,8 +1,7 @@ #!/bin/bash # data directory initialization -if [ ! -e /data/config.sh ]; then - cp /data.skel/config.sh /data/config.sh +if [ ! -e /data/config.yml ]; then + cp /data.skel/config.yml /data/config.yml fi mkdir -p /data/ansible - diff --git a/app/cli/shutdown b/app/cli/shutdown index c330b62..ce48d56 100755 --- a/app/cli/shutdown +++ b/app/cli/shutdown @@ -2,7 +2,6 @@ ## PREPARE ENVIRONMENT cd /app -source /data/config.sh ## EXECUTE ansible-playbook /app/playbooks/shutdown.yml || exit 1 diff --git a/app/cli/ssh b/app/cli/ssh index d35b8d5..3ef02c7 100755 --- a/app/cli/ssh +++ b/app/cli/ssh @@ -5,7 +5,6 @@ if [ $# -ne 1 ]; then exit 1 fi -source /data/config.sh cd /app HOST_DATA=$(ansible-inventory --host $1) diff --git a/app/cli/startup b/app/cli/startup index 6df9db9..6cd90f2 100755 --- a/app/cli/startup +++ b/app/cli/startup @@ -2,7 +2,6 @@ ## PREPARE ENVIRONMENT cd /app -source /data/config.sh ## EXECUTE ansible-playbook /app/playbooks/startup.yml || exit 1 diff --git a/app/lib/python/conftui.py b/app/lib/python/conftui.py deleted file mode 100644 index 399f71d..0000000 --- a/app/lib/python/conftui.py +++ /dev/null @@ -1,336 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import json -from PyInquirer import prompt, Separator, default_style -from prompt_toolkit.shortcuts import Token, print_tokens - -STYLE = default_style - - -class Parameter(object): - disabled = False - _value_reprfun = str - - def __init__(self, name, prompt, disabled=False, default=None): - self._name = name - self._value = os.environ.get(self._name, default or '') - self._prompt = prompt - self.disabled = disabled - - @property - def name(self): - return self._name - - @property - def value(self): - return self._value - - @value.setter - def value(self, value): - self._value = value - - @property - def prompt(self): - return self._prompt - - def update(self): - question = [ - { - 'type': 'input', - 'name': 'newval', - 'message': self.prompt, - 'default': self.value, - } - ] - answer = prompt(question) - self.value = answer['newval'] - - def __repr__(self): - return '{}: {}'.format(self.prompt, self._value_reprfun(self.value)) - - def to_bash(self): - return "export {}={}".format(self.name, self.jsonify()) - - def jsonify(self): - try: - json.loads(self.value) - return "'" + self.value + "'" - except json.decoder.JSONDecodeError: - return json.dumps(self.value) - - -class PasswordParameter(Parameter): - - def __init__(self, name, prompt, disabled=False): - super().__init__(name, prompt, disabled) - - def _value_reprfun(self, password): - if not password: - return '' - return '*********' - - def update(self): - question = [ - { - 'type': 'password', - 'name': 'newval', - 'message': self.prompt - } - ] - answer = prompt(question) - self.value = answer['newval'] - -class ChoiceParameter(Parameter): - - def __init__(self, name, prompt, choices, value_reprfun=str, default=''): - self._name = name - self._value = os.environ.get(self._name, default) - self._prompt = prompt - self._choices = choices - self._value_reprfun = value_reprfun - - def update(self): - question = [ - { - 'type': 'list', - 'message': self._prompt, - 'name': 'choice', - 'default': self._value_reprfun(self._value), - 'choices': - [self._value_reprfun(item) for item in self._choices] - } - ] - answer = prompt(question) - self._value = answer['choice'] - - -class BooleanParameter(ChoiceParameter): - - def __init__(self, name, prompt, default="True"): - super().__init__(name, prompt, [True, False], default=default) - -class CheckParameter(Parameter): - - def __init__(self, name, prompt, choices): - self._name = name - self._value = json.loads(os.environ.get(self._name, '')) - self._prompt = prompt - self._choices = choices - self._choices = [{'name': f'{choice}', - 'checked': f'{choice}' in self._value} - for choice in choices] - - def update(self): - question = [ - { - 'type': 'checkbox', - 'message': self._prompt, - 'name': 'choice', - 'choices': self._choices - } - ] - answer = prompt(question) - self._value = answer['choice'] - - def to_bash(self): - return "export {}='{}'".format(self.name, json.dumps(self.value)) - - -class StaticParameter(Parameter): - - def __init__(self, name, prompt, value): - super().__init__(name, prompt, 'Static Value') - self._value = value - - -class ListDictParameter(Parameter): - - def __init__(self, name, prompt, keys, default=None): - self._name = name - self._value = json.loads(os.environ.get(self._name, default or '[]')) - self._prompt = prompt - self._primary_key = keys[0][0] - - # normalize keys - self._keys = [] - for key in keys: - if len(key) < 3: - self._keys += [(key[0], key[-1], "")] - continue - if len(key) == 3: - self._keys += [key] - - for val in self._value: - if not val.get(key[0]): - val[key[0]] = self._keys[-1][2] - - def _value_reprfun(self, value): - return '{} items'.format(len(self.value)) - - def print_status(self): - tokens = [] - tokens += [(Token.QuestionMark, '!'), - (Token.Question, f' Current {self._prompt}:\n')] - for entry in self._value: - for idx, key in enumerate(self._keys): - if idx == 0: - ptr = ' - ' - else: - ptr = ' ' - tokens += [(Token.Pointer, ptr), - (Token.Arboted, f'{key[1]}: {entry.get(key[0], key[2])}\n')] - print_tokens(tokens, style=STYLE) - sys.stdout.write('\n\n') - - def update(self): - done = False - - while not done: - self.print_status() - - question = [ - { - 'type': 'expand', - 'message': '{}: What would you like to do?'.format(self.prompt), - 'name': 'action', - 'default': 'a', - 'choices': [ - { - 'key': 'a', - 'name': 'Add Entry', - 'value': 'a' - }, - { - 'key': 'e', - 'name': 'Edit Entry', - 'value': 'e' - }, - { - 'key': 'r', - 'name': 'Remove Entry', - 'value': 'r' - }, - { - 'key': 'd', - 'name': 'Done', - 'value': 'd' - } - ] - } - ] - answer = prompt(question) - - if answer['action'] == 'd': - done = True - elif answer['action'] in 'er': - self._update_edit(answer['action']) - elif answer['action'] == 'a': - self._value.append(self._mkentry({})) - - def _update_edit(self, action_code): - if action_code == 'r': - action = 'remove' - else: - action = 'edit' - - question = [ - { - 'type': 'list', - 'name': 'index', - 'message': 'Which item would you like to {}?'.format(action), - 'choices': [{'name': item[self._primary_key], - 'value': index} for (index, item) in enumerate(self.value)] - } - ] - answer = prompt(question) - - if action_code == 'r': - self.value.pop(answer['index']) - else: - self.value[answer['index']] = self._mkentry(self.value[answer['index']]) - - def _mkentry(self, defaults): - questions = [ {'type': 'input', - 'message': item[1], - 'name': item[0], - 'default': defaults.get(item[0], item[2])} - for item in self._keys] - return prompt(questions) - - def to_bash(self): - return "export {}='{}'".format(self.name, json.dumps(self.value)) - - -class ParameterCollection(list): - - def __init__(self, name, prompt, values): - super().__init__(values) - self._name = name - self._prompt = prompt - - def to_choices(self): - out = [Separator(self._prompt)] - for item in self: - out += [{'name': repr(item), - 'description': str(item.value), - 'value': '{}|{}'.format(self._name, item.name)}] - if item.disabled: - out[-1].update({'disabled': item.disabled}) - return out - - def to_bash(self): - return ['# {}'.format(self._prompt.upper())] + [item.to_bash() for item in self] - - def get_param(self, pname): - - def filter_fun(val): - return val.name == pname - - return list(filter(filter_fun, self))[0] - - -class Configurator(object): - - def __init__(self, path, footer): - self._path = path - self._footer = footer - self.all = [] - - def _main_menu(self): - question = [ - { - 'type': 'checkbox', - 'message': 'Which items would you like to change?', - 'name': 'parameters', - 'choices': [val for section in self.all for val in section.to_choices()] - } - ] - return prompt(question) - - def _update_param(self, raw_param): - (collection, param_name) = raw_param.split('|') - try: - getattr(self, collection).get_param(param_name).update() - except AttributeError: - raise ValueError('{} not a valid collection'.format(collection)) - - def dump(self): - with open(self._path, 'w') as outfile: - _ = [outfile.write(line + '\n') for section in self.all for line in section.to_bash()] - outfile.write(self._footer) - - def configurate(self): - loop = True - - while loop: - to_update = self._main_menu() - print('') - - for parameter in to_update['parameters']: - self._update_param(parameter) - print('') - - loop = bool(to_update['parameters']) - - self.dump() diff --git a/app/playbooks/config.d/all b/app/playbooks/config.d/all deleted file mode 120000 index e89bbc8..0000000 --- a/app/playbooks/config.d/all +++ /dev/null @@ -1 +0,0 @@ -cluster \ No newline at end of file diff --git a/app/playbooks/config.d/cluster/config.py b/app/playbooks/config.d/cluster/config.py deleted file mode 100644 index f6e7fc6..0000000 --- a/app/playbooks/config.d/cluster/config.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -import sys -import os -from conftui import (Configurator, ParameterCollection, Parameter, - ListDictParameter, PasswordParameter, ChoiceParameter, - CheckParameter, StaticParameter) - -CONFIG_PATH = '/data/config.sh' -CONFIG_FOOTER = '' - -class ClusterConfigurator(Configurator): - - def __init__(self, path, footer, rtr_interfaces): - self._path = path - self._footer = footer - - self.router = ParameterCollection('router', 'Router Configuration', [ - CheckParameter('ROUTER_LAN_INT', 'LAN Interfaces', rtr_interfaces), - Parameter('SUBNET', 'Subnet'), - ChoiceParameter('SUBNET_MASK', 'Subnet Mask', ['20', '21', '22', '23', '24', '25', '26', '27']), - CheckParameter('ALLOWED_SERVICES', 'Permitted Ingress Traffic', ['SSH to Bastion', 'HTTPS to Cluster API', 'HTTP to Cluster Apps', 'HTTPS to Cluster Apps', 'HTTPS to Cockpit Panel', 'External to Internal Routing - DANGER']), - ListDictParameter('DNS_FORWARDERS', 'Upstream DNS Forwarders', - [('server', 'DNS Server')], - default='[{"server": "1.1.1.1"}]') - ]) - self.cluster = ParameterCollection('cluster', 'Cluster Configuration', [ - PasswordParameter('ADMIN_PASSWORD', 'Adminstrator Password'), - PasswordParameter('PULL_SECRET', 'Pull Secret') - ]) - self.architecture = ParameterCollection('architecture', 'Host Record Configuration', [ - StaticParameter('MGMT_PROVIDER', 'Machine Management Provider', 'ilo'), - Parameter('MGMT_USER', 'Machine Management User'), - PasswordParameter('MGMT_PASSWORD', 'Machine Management Password'), - ListDictParameter('CP_NODES', 'Control Plane Machines', - [('name', 'Node Name'), ('mac', 'MAC Address'), - ('mgmt_mac', 'Management MAC Address'), - ('install_drive', 'OS Install Drive', - os.environ.get('BOOT_DRIVE'))]) - ]) - self.extra = ParameterCollection('extra', 'Extra DNS/DHCP Records', [ - ListDictParameter('EXTRA_NODES', 'Static IP Reservations', - [('name', 'Node Name'), ('mac', 'MAC Address'), ('ip', 'Requested IP Address')]), - ListDictParameter('IGNORE_MACS', 'DHCP Ignored MAC Addresses', - [('name', 'Entry Name'), ('mac', 'MAC Address')]) - ]) - - self.all = [self.router, self.cluster, self.architecture, self.extra] - - -def main(): - rtr_interfaces = os.environ['BASTION_INTERFACES'].split() - return ClusterConfigurator( - CONFIG_PATH, - CONFIG_FOOTER, - rtr_interfaces).configurate() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/app/playbooks/config.d/cluster/config.py.bkp b/app/playbooks/config.d/cluster/config.py.bkp deleted file mode 100644 index ff89015..0000000 --- a/app/playbooks/config.d/cluster/config.py.bkp +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env python -import sys -import os -import json -from PyInquirer import prompt, Separator, default_style -from prompt_toolkit.shortcuts import Token, print_tokens - - -CONFIG_PATH = '/data/config.sh' -CONFIG_FOOTER = '' -STYLE = default_style - - - -class Parameter(object): - disabled = False - _value_reprfun = str - - def __init__(self, name, prompt, disabled=False, default=None): - self._name = name - self._value = os.environ.get(self._name, default or '') - self._prompt = prompt - self.disabled = disabled - - @property - def name(self): - return self._name - - @property - def value(self): - return self._value - - @value.setter - def value(self, value): - self._value = value - - @property - def prompt(self): - return self._prompt - - def update(self): - question = [ - { - 'type': 'input', - 'name': 'newval', - 'message': self.prompt, - 'default': self.value, - } - ] - answer = prompt(question) - self.value = answer['newval'] - - def __repr__(self): - return '{}: {}'.format(self.prompt, self._value_reprfun(self.value)) - - def to_bash(self): - return "export {}='{}'".format(self.name, self.value) - - -class PasswordParameter(Parameter): - - def __init__(self, name, prompt, disabled=False): - super().__init__(name, prompt, disabled) - - def _value_reprfun(self, password): - if not password: - return '' - return '*********' - - def update(self): - question = [ - { - 'type': 'password', - 'name': 'newval', - 'message': self.prompt - } - ] - answer = prompt(question) - self.value = answer['newval'] - -class ChoiceParameter(Parameter): - - def __init__(self, name, prompt, choices, value_reprfun=str): - self._name = name - self._value = os.environ.get(self._name, '') - self._prompt = prompt - self._choices = choices - self._value_reprfun = value_reprfun - - def update(self): - question = [ - { - 'type': 'list', - 'message': self._prompt, - 'name': 'choice', - 'default': self._value, - 'choices': self._choices - } - ] - answer = prompt(question) - self._value = answer['choice'] - - -class CheckParameter(Parameter): - - def __init__(self, name, prompt, choices): - self._name = name - self._value = json.loads(os.environ.get(self._name, '')) - self._prompt = prompt - self._choices = choices - self._choices = [{'name': f'{choice}', - 'checked': f'{choice}' in self._value} - for choice in choices] - - def update(self): - question = [ - { - 'type': 'checkbox', - 'message': self._prompt, - 'name': 'choice', - 'choices': self._choices - } - ] - answer = prompt(question) - self._value = answer['choice'] - - def to_bash(self): - return "export {}='{}'".format(self.name, json.dumps(self.value)) - - -class StaticParameter(Parameter): - - def __init__(self, name, prompt, value): - super().__init__(name, prompt, 'Static Value') - self._value = value - - -class ListDictParameter(Parameter): - - def __init__(self, name, prompt, keys, default=None): - self._name = name - self._value = json.loads(os.environ.get(self._name, default or '[]')) - self._prompt = prompt - self._primary_key = keys[0][0] - - # normalize keys - self._keys = [] - for key in keys: - if len(key) < 3: - self._keys += [(key[0], key[-1], "")] - continue - if len(key) == 3: - self._keys += [key] - - for val in self._value: - if not val.get(key[0]): - val[key[0]] = self._keys[-1][2] - - def _value_reprfun(self, value): - return '{} items'.format(len(self.value)) - - def print_status(self): - tokens = [] - tokens += [(Token.QuestionMark, '!'), - (Token.Question, f' Current {self._prompt}:\n')] - for entry in self._value: - for idx, key in enumerate(self._keys): - if idx == 0: - ptr = ' - ' - else: - ptr = ' ' - tokens += [(Token.Pointer, ptr), - (Token.Arboted, f'{key[1]}: {entry.get(key[0], key[2])}\n')] - print_tokens(tokens, style=STYLE) - sys.stdout.write('\n\n') - - def update(self): - done = False - - while not done: - self.print_status() - - question = [ - { - 'type': 'expand', - 'message': '{}: What would you like to do?'.format(self.prompt), - 'name': 'action', - 'default': 'a', - 'choices': [ - { - 'key': 'a', - 'name': 'Add Entry', - 'value': 'a' - }, - { - 'key': 'e', - 'name': 'Edit Entry', - 'value': 'e' - }, - { - 'key': 'r', - 'name': 'Remove Entry', - 'value': 'r' - }, - { - 'key': 'd', - 'name': 'Done', - 'value': 'd' - } - ] - } - ] - answer = prompt(question) - - if answer['action'] == 'd': - done = True - elif answer['action'] in 'er': - self._update_edit(answer['action']) - elif answer['action'] == 'a': - self._value.append(self._mkentry({})) - - def _update_edit(self, action_code): - if action_code == 'r': - action = 'remove' - else: - action = 'edit' - - question = [ - { - 'type': 'list', - 'name': 'index', - 'message': 'Which item would you like to {}?'.format(action), - 'choices': [{'name': item[self._primary_key], - 'value': index} for (index, item) in enumerate(self.value)] - } - ] - answer = prompt(question) - - if action_code == 'r': - self.value.pop(answer['index']) - else: - self.value[answer['index']] = self._mkentry(self.value[answer['index']]) - - def _mkentry(self, defaults): - questions = [ {'type': 'input', - 'message': item[1], - 'name': item[0], - 'default': defaults.get(item[0], item[2])} - for item in self._keys] - return prompt(questions) - - def to_bash(self): - return "export {}='{}'".format(self.name, json.dumps(self.value)) - - -class ParameterCollection(list): - - def __init__(self, name, prompt, values): - super().__init__(values) - self._name = name - self._prompt = prompt - - def to_choices(self): - out = [Separator(self._prompt)] - for item in self: - out += [{'name': repr(item), - 'description': str(item.value), - 'value': '{}|{}'.format(self._name, item.name)}] - if item.disabled: - out[-1].update({'disabled': item.disabled}) - return out - - def to_bash(self): - return ['# {}'.format(self._prompt.upper())] + [item.to_bash() for item in self] - - def get_param(self, pname): - - def filter_fun(val): - return val.name == pname - - return list(filter(filter_fun, self))[0] - - -class configurator(object): - - def __init__(self, path, footer, rtr_interfaces): - self._path = path - self._footer = footer - - self.router = ParameterCollection('router', 'Router Configuration', [ - CheckParameter('ROUTER_LAN_INT', 'LAN Interfaces', rtr_interfaces), - Parameter('SUBNET', 'Subnet'), - ChoiceParameter('SUBNET_MASK', 'Subnet Mask', ['20', '21', '22', '23', '24', '25', '26', '27']), - CheckParameter('ALLOWED_SERVICES', 'Permitted Ingress Traffic', ['SSH to Bastion', 'HTTPS to Cluster API', 'HTTP to Cluster Apps', 'HTTPS to Cluster Apps', 'HTTPS to Cockpit Panel', 'External to Internal Routing - DANGER']), - ListDictParameter('DNS_FORWARDERS', 'Upstream DNS Forwarders', - [('server', 'DNS Server')], - default='[{"server": "1.1.1.1"}]')]) - self.cluster = ParameterCollection('cluster', 'Cluster Configuration', [ - PasswordParameter('ADMIN_PASSWORD', 'Adminstrator Password'), - PasswordParameter('PULL_SECRET', 'Pull Secret')]) - self.architecture = ParameterCollection('architecture', 'Host Record Configuration', [ - StaticParameter('MGMT_PROVIDER', 'Machine Management Provider', 'ilo'), - Parameter('MGMT_USER', 'Machine Management User'), - PasswordParameter('MGMT_PASSWORD', 'Machine Management Password'), - ListDictParameter('CP_NODES', 'Control Plane Machines', - [('name', 'Node Name'), ('mac', 'MAC Address'), - ('mgmt_mac', 'Management MAC Address'), - ('install_drive', 'OS Install Drive', - os.environ.get('BOOT_DRIVE'))])]) - self.extra = ParameterCollection('extra', 'Extra DNS/DHCP Records', [ - ListDictParameter('EXTRA_NODES', 'Static IP Reservations', - [('name', 'Node Name'), ('mac', 'MAC Address'), ('ip', 'Requested IP Address')]), - ListDictParameter('IGNORE_MACS', 'DHCP Ignored MAC Addresses', - [('name', 'Entry Name'), ('mac', 'MAC Address')])]) - - self.all = [self.router, self.cluster, self.architecture, self.extra] - - def _main_menu(self): - question = [ - { - 'type': 'checkbox', - 'message': 'Which items would you like to change?', - 'name': 'parameters', - 'choices': [val for section in self.all for val in section.to_choices()] - } - ] - return prompt(question) - - def _update_param(self, raw_param): - (collection, param_name) = raw_param.split('|') - try: - getattr(self, collection).get_param(param_name).update() - except AttributeError: - raise ValueError('{} not a valid collection'.format(collection)) - - def dump(self): - with open(self._path, 'w') as outfile: - _ = [outfile.write(line + '\n') for section in self.all for line in section.to_bash()] - outfile.write(self._footer) - - def configurate(self): - loop = True - - while loop: - to_update = self._main_menu() - print('') - - for parameter in to_update['parameters']: - self._update_param(parameter) - print('') - - loop = bool(to_update['parameters']) - - self.dump() - - -def main(): - rtr_interfaces = os.environ['BASTION_INTERFACES'].split() - return configurator( - CONFIG_PATH, - CONFIG_FOOTER, - rtr_interfaces).configurate() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/app/playbooks/config.d/cluster/main.sh b/app/playbooks/config.d/cluster/main.sh deleted file mode 100755 index f6547bc..0000000 --- a/app/playbooks/config.d/cluster/main.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd /app -source /data/config.sh 2> /dev/null -python3 /app/playbooks/config.d/cluster/config.py -source /data/config.sh 2> /dev/null -python3 /app/inventory.py --verify > /dev/null diff --git a/app/playbooks/config.d/proxy/config.py b/app/playbooks/config.d/proxy/config.py deleted file mode 100644 index 1cd0286..0000000 --- a/app/playbooks/config.d/proxy/config.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -import sys -from conftui import (Configurator, ParameterCollection, Parameter, - ListDictParameter, PasswordParameter, - BooleanParameter) - -CONFIG_PATH = '/data/proxy.sh' -CONFIG_FOOTER = '' - -class ProxyConfigurator(Configurator): - - def __init__(self, path, footer): - self._path = path - self._footer = footer - - self.proxy = ParameterCollection('proxy', 'Proxy Configuration', [ - BooleanParameter('PROXY', 'Setup cluster proxy'), - Parameter('PROXY_HTTP', 'HTTP Proxy'), - Parameter('PROXY_HTTPS', 'HTTPS Proxy'), - ListDictParameter('PROXY_NOPROXY', 'No Proxy Destinations', - [('dest', 'Destination')]), - PasswordParameter('PROXY_CA', 'Additional CA Bundle') - ]) - - self.all = [self.proxy] - - -def main(): - return ProxyConfigurator( - CONFIG_PATH, - CONFIG_FOOTER).configurate() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/app/playbooks/config.d/proxy/main.sh b/app/playbooks/config.d/proxy/main.sh deleted file mode 100755 index 7326014..0000000 --- a/app/playbooks/config.d/proxy/main.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -cd /app -source /data/config.sh 2> /dev/null -source /data/proxy.sh 2> /dev/null -python3 /app/playbooks/config.d/proxy/config.py -source /data/config.sh 2> /dev/null -source /data/proxy.sh 2> /dev/null -python3 /app/inventory.py --verify > /dev/null diff --git a/data.skel/config.sh b/data.skel/config.sh deleted file mode 100644 index 7de94e5..0000000 --- a/data.skel/config.sh +++ /dev/null @@ -1,14 +0,0 @@ -# NETWORK ROUTER CONFIGURATION -export ROUTER_LAN_INT='[]' -export SUBNET=192.168.8.0 -export SUBNET_MASK=24 -export ALLOWED_SERVICES='["SSH to Bastion"]' -# CLUSTER CONFIGURATION -export ADMIN_PASSWORD='admin' -export PULL_SECRET='' -# CLUSTER HARDWARE -export MGMT_PROVIDER='ilo' -export MGMT_USER='Administrator' -export MGMT_PASSWORD='ilo-pass' -export BASTION_MGMT_MAC='ff:ff:ff:ff:ff:ff' -export CP_NODES='[{"name": "node-0", "mac": "ff:ff:ff:ff:ff:ff", "mgmt_mac": "ff:ff:ff:ff:ff:ff"}, {"name": "node-1", "mac": "ff:ff:ff:ff:ff:ff", "mgmt_mac": "ff:ff:ff:ff:ff:ff"}, {"name": "node-2", "mac": "ff:ff:ff:ff:ff:ff", "mgmt_mac": "ff:ff:ff:ff:ff:ff"}]' diff --git a/home/.bashrc b/home/.bashrc index 64f769d..008c8a8 100644 --- a/home/.bashrc +++ b/home/.bashrc @@ -1,6 +1,6 @@ # Get the aliases and functions if [ -f /etc/bashrc ]; then - . /etc/bashrc + . /etc/bashrc fi # Update proxy settings in container @@ -23,17 +23,15 @@ function set_proxy() { # User specific environment and startup programs function ps1() { - _CONFIG_LAST_MODIFY=$(stat -c %Z /data/config.sh) - if [[ $_CONFIG_LAST_MODIFY -gt $_CONFIG_LAST_LOAD ]]; then - echo " -- Configuration Reloaded -- " - source /data/config.sh 2> /dev/null - source /data/proxy.sh 2> /dev/null - export _CONFIG_LAST_LOAD=$(date +%s) - set_proxy - fi - export PS1="[\u@${CLUSTER_NAME} \W]\$ " + _CONFIG_LAST_MODIFY=$(stat -c %Z /data/proxy.sh) + if [[ $_CONFIG_LAST_MODIFY -gt $_CONFIG_LAST_LOAD ]]; then + echo " -- Proxy Configuration Reloaded -- " + source /data/proxy.sh 2> /dev/null + export _CONFIG_LAST_LOAD=$(date +%s) + set_proxy + fi + export PS1="[\u@${CLUSTER_NAME} \W]\$ " } -export _CONFIG_LAST_LOAD="0" export PROMPT_COMMAND=ps1 export KUBECONFIG=/data/openshift-installer/auth/kubeconfig @@ -42,6 +40,4 @@ PYTHONUSERBASE=/deps/python ANSIBLE_COLLECTIONS_PATH=/deps/ansible PATH=/deps/python/bin:$PATH - - alias ll='ls -la' diff --git a/requirements.txt b/requirements.txt index bf23cef..24533cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -pyinquirer==1.0.3 ansible<2.11 # for management/ilo provider From d62ab4519188ca68d2d135f5f862c5cb3c060b0d Mon Sep 17 00:00:00 2001 From: James Harmison Date: Mon, 28 Dec 2020 16:01:23 -0500 Subject: [PATCH 04/10] Added config parsing library --- data.skel/config.yml | 24 +++++++++ pre/faros_config/README.md | 7 +++ pre/faros_config/VERSION.txt | 1 + pre/faros_config/example_config.yml | 65 +++++++++++++++++++++++ pre/faros_config/faros_config/__init__.py | 22 ++++++++ pre/faros_config/faros_config/bastion.py | 5 ++ pre/faros_config/faros_config/cluster.py | 29 ++++++++++ pre/faros_config/faros_config/common.py | 6 +++ pre/faros_config/faros_config/network.py | 43 +++++++++++++++ pre/faros_config/faros_config/proxy.py | 9 ++++ pre/faros_config/setup.cfg | 13 +++++ pre/faros_config/setup.py | 4 ++ pre/faros_config/test_config.py | 11 ++++ 13 files changed, 239 insertions(+) create mode 100644 data.skel/config.yml create mode 100644 pre/faros_config/README.md create mode 100644 pre/faros_config/VERSION.txt create mode 100644 pre/faros_config/example_config.yml create mode 100644 pre/faros_config/faros_config/__init__.py create mode 100644 pre/faros_config/faros_config/bastion.py create mode 100644 pre/faros_config/faros_config/cluster.py create mode 100644 pre/faros_config/faros_config/common.py create mode 100644 pre/faros_config/faros_config/network.py create mode 100644 pre/faros_config/faros_config/proxy.py create mode 100644 pre/faros_config/setup.cfg create mode 100644 pre/faros_config/setup.py create mode 100644 pre/faros_config/test_config.py diff --git a/data.skel/config.yml b/data.skel/config.yml new file mode 100644 index 0000000..75cf923 --- /dev/null +++ b/data.skel/config.yml @@ -0,0 +1,24 @@ +network: + port_forward: + - SSH to Bastion + lan: + subnet: 192.168.8.0/24 + interfaces: [] +bastion: + become_pass: admin +cluster: + pull_secret: '' + management: + provider: ilo + user: Administrator + password: ilo-pass + nodes: + - name: node-0 + mac: ff:ff:ff:ff:ff:ff + mgmt_mac: ff:ff:ff:ff:ff:ff + - name: node-1 + mac: ff:ff:ff:ff:ff:ff + mgmt_mac: ff:ff:ff:ff:ff:ff + - name: node-2 + mac: ff:ff:ff:ff:ff:ff + mgmt_mac: ff:ff:ff:ff:ff:ff diff --git a/pre/faros_config/README.md b/pre/faros_config/README.md new file mode 100644 index 0000000..5d87dcf --- /dev/null +++ b/pre/faros_config/README.md @@ -0,0 +1,7 @@ +# Faros Config + +This small library is used to validate configuration provided to Project Faros. It contains the necessary parts to load a configuration from the environment, mix it with configuration from a templated YAML file, and provide a single Python object that is easy for the Inventory to work with while generating variables for hosts. + +## NOTE + +This library is not intended to be used outside of Project Faros, or indeed outside of the Project Faros cluster-manager container. diff --git a/pre/faros_config/VERSION.txt b/pre/faros_config/VERSION.txt new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/pre/faros_config/VERSION.txt @@ -0,0 +1 @@ +0.1.0 diff --git a/pre/faros_config/example_config.yml b/pre/faros_config/example_config.yml new file mode 100644 index 0000000..bc37659 --- /dev/null +++ b/pre/faros_config/example_config.yml @@ -0,0 +1,65 @@ +network: + port_forward: + - SSH to Bastion + - HTTPS to Cluster API + - HTTP to Cluster Apps + - HTTPS to Cluster Apps + - HTTPS to Cockpit Panel + lan: + subnet: 192.168.8.0/24 + interfaces: + - eno1 + - eno5 + dns_forward_resolvers: + - 10.1.1.1 + dhcp: + ignore_macs: + - name: node-0-eno1 + mac: da:d5:de:ad:be:ef + - name: node-0-eno2 + mac: da:d5:de:ad:be:ef + - name: node-0-eno3 + mac: da:d5:de:ad:be:ef + extra_reservations: + - name: wifi + mac: da:d5:de:ad:be:ef + ip: 192.168.8.127 + - name: chassis + mac: da:d5:de:ad:be:ef + ip: 192.168.8.50 + - name: bastion-ilo + mac: da:d5:de:ad:be:ef + ip: 192.168.8.51 + - name: client + mac: da:d5:de:ad:be:ef + ip: 192.168.8.52 +bastion: + become_pass: +cluster: + pull_secret: '{"auths":{"cloud.openshift.com":{"auth":"sometoken","email":"some@example.com"}}}' # etc + management: + provider: ilo + user: Administrator + password: + nodes: + - name: node-0 + mac: da:d5:de:ad:be:ef + mgmt_mac: fe:eb:da:ed:5d:ad + - name: node-1 + mac: da:d5:de:ad:be:ef + mgmt_mac: fe:eb:da:ed:5d:ad + - name: node-2 + mac: da:d5:de:ad:be:ef + mgmt_mac: fe:eb:da:ed:5d:ad +proxy: + http: proxy.example.com + https: secure-proxy.example.com + noproxy: + - registry.access.redhat.com + ca: | + # Proxy Server Certificate + -----BEGIN CERTIFICATE----- + MIIGlTCCBH2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjjELMAkGA1UEBhMCVVMx + ... + epmW5U8YK4yf + -----END CERTIFICATE---- diff --git a/pre/faros_config/faros_config/__init__.py b/pre/faros_config/faros_config/__init__.py new file mode 100644 index 0000000..6bd9245 --- /dev/null +++ b/pre/faros_config/faros_config/__init__.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel +from typing import Optional +import yaml + +from .network import NetworkConfig +from .bastion import BastionConfig +from .cluster import ClusterConfig +from .proxy import ProxyConfig + + +class FarosConfig(BaseModel): + network: NetworkConfig + bastion: BastionConfig + cluster: ClusterConfig + proxy: Optional[ProxyConfig] + + @classmethod + def from_yaml(cls, yaml_file: str) -> 'FarosConfig': + with open(yaml_file) as f: + config = yaml.safe_load(f) + + return cls.parse_obj(config) diff --git a/pre/faros_config/faros_config/bastion.py b/pre/faros_config/faros_config/bastion.py new file mode 100644 index 0000000..b342bc9 --- /dev/null +++ b/pre/faros_config/faros_config/bastion.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class BastionConfig(BaseModel): + become_pass: str diff --git a/pre/faros_config/faros_config/cluster.py b/pre/faros_config/faros_config/cluster.py new file mode 100644 index 0000000..0aa46f6 --- /dev/null +++ b/pre/faros_config/faros_config/cluster.py @@ -0,0 +1,29 @@ +from pydantic import BaseModel, constr +from typing import List, Optional + +from .common import StrEnum + +MacAddress = constr(regex=r'(([0-9A-Fa-f]{2}[-:]){5}[0-9A-Fa-f]{2})|(([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})') # noqa: E501 + + +class ManagementProviderItem(StrEnum): + ILO = "ilo" + + +class ManagementConfig(BaseModel): + provider: ManagementProviderItem + user: str + password: str + + +class NodeConfig(BaseModel): + name: str + mac: MacAddress + mgmt_mac: MacAddress + install_drive: Optional[str] + + +class ClusterConfig(BaseModel): + pull_secret: str + management: ManagementConfig + nodes: List[NodeConfig] diff --git a/pre/faros_config/faros_config/common.py b/pre/faros_config/faros_config/common.py new file mode 100644 index 0000000..6ba3d8c --- /dev/null +++ b/pre/faros_config/faros_config/common.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class StrEnum(str, Enum): + def __str__(self): + return self.value diff --git a/pre/faros_config/faros_config/network.py b/pre/faros_config/faros_config/network.py new file mode 100644 index 0000000..15e5e0e --- /dev/null +++ b/pre/faros_config/faros_config/network.py @@ -0,0 +1,43 @@ +from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network +from pydantic import BaseModel, constr +from typing import List, Union + +from .common import StrEnum + +MacAddress = constr(regex=r'(([0-9A-Fa-f]{2}[-:]){5}[0-9A-Fa-f]{2})|(([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})') # noqa: E501 + + +class PortForwardConfigItem(StrEnum): + SSH_TO_BASTION = "SSH to Bastion" + HTTPS_TO_CLUSTER_API = "HTTPS to Cluster API" + HTTP_TO_CLUSTER_APPS = "HTTP to Cluster Apps" + HTTPS_TO_CLUSTER_APPS = "HTTPS to Cluster Apps" + HTTPS_TO_COCKPIT_PANEL = "HTTPS to Cockpit Panel" + + +class NameMacPair(BaseModel): + name: str + mac: MacAddress + + +class NameMacIpSet(BaseModel): + name: str + mac: MacAddress + ip: Union[IPv4Address, IPv6Address] + + +class DhcpConfig(BaseModel): + ignore_macs: List[NameMacPair] + extra_reservations: List[NameMacIpSet] + + +class LanConfig(BaseModel): + subnet: Union[IPv4Network, IPv6Network] + interfaces: List[str] + dns_forward_resolvers: List[Union[IPv4Address, IPv6Address]] + dhcp: DhcpConfig + + +class NetworkConfig(BaseModel): + port_forward: List[PortForwardConfigItem] + lan: LanConfig diff --git a/pre/faros_config/faros_config/proxy.py b/pre/faros_config/faros_config/proxy.py new file mode 100644 index 0000000..0a11369 --- /dev/null +++ b/pre/faros_config/faros_config/proxy.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel +from typing import List + + +class ProxyConfig(BaseModel): + http: str + https: str + noproxy: List[str] + ca: str diff --git a/pre/faros_config/setup.cfg b/pre/faros_config/setup.cfg new file mode 100644 index 0000000..0236cdf --- /dev/null +++ b/pre/faros_config/setup.cfg @@ -0,0 +1,13 @@ +[metadata] +name = faros-config +version = file: VERSION.txt +description = A configuration handler for Project Faros +long_description = file: README.md +long_description_content_type = text/markdown +license = GNU GPL v3 + +[options] +packages = find: +install_requires = + PyYAML==5.3.1 + pydantic==1.7.3 diff --git a/pre/faros_config/setup.py b/pre/faros_config/setup.py new file mode 100644 index 0000000..c823345 --- /dev/null +++ b/pre/faros_config/setup.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +from setuptools import setup + +setup() diff --git a/pre/faros_config/test_config.py b/pre/faros_config/test_config.py new file mode 100644 index 0000000..a559f2d --- /dev/null +++ b/pre/faros_config/test_config.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# This should grow to be a proper library with actual tests at some point + +from faros_config import FarosConfig + +config = FarosConfig.from_yaml('example_config.yml') + +print(config) +print() +for port in config.network.port_forward: + print(port) From 136073b2329eb9b0fd653941aabf811329c77435 Mon Sep 17 00:00:00 2001 From: James Harmison Date: Tue, 29 Dec 2020 11:31:25 -0500 Subject: [PATCH 05/10] Implement new config library in inventory --- app/inventory.py | 253 +++++++++++++++++++++++++++-------------------- 1 file changed, 146 insertions(+), 107 deletions(-) diff --git a/app/inventory.py b/app/inventory.py index 71c4477..9cec5c2 100755 --- a/app/inventory.py +++ b/app/inventory.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse from collections import defaultdict +from faros_config import FarosConfig import ipaddress import json import os @@ -11,6 +12,17 @@ IP_RESERVATIONS = '/data/ip_addresses' +class PydanticEncoder(json.JSONEncoder): + def default(self, obj): + obj_has_dict = getattr(obj, "dict", False) + if obj_has_dict and callable(obj_has_dict): + return obj.dict(exclude_none=True) + elif isinstance(obj, ipaddress._IPAddressBase): + return str(obj) + else: + return json.JSONEncoder.default(self, obj) + + class InventoryGroup(object): def __init__(self, parent, name): @@ -33,7 +45,7 @@ class Inventory(object): _data = {"_meta": {"hostvars": defaultdict(dict)}} def __init__(self, mode=0, host=None): - if mode==1: + if mode == 1: # host info requested # current, only list and none are implimented raise NotImplementedError() @@ -41,15 +53,11 @@ def __init__(self, mode=0, host=None): self._mode = mode self._host = host - def __del__(self): - if self._mode == 0: - print(self.to_json()) - def host(self, name): return self._data['_meta']['hostvars'].get(name) def group(self, name): - if name in self_data: + if name in self._data: return InventoryGroup(self, name) else: return None @@ -77,18 +85,17 @@ def add_host(self, name, group=None, hostname=None, **hostvars): self._data['_meta']['hostvars'][name].update(hostvars) def to_json(self): - return json.dumps(self._data, sort_keys=True, - indent=4, separators=(',', ': ')) + return json.dumps(self._data, sort_keys=True, indent=4, + separators=(',', ': '), cls=PydanticEncoder) class IPAddressManager(dict): - def __init__(self, save_file, subnet, subnet_mask): + def __init__(self, save_file, subnet): super().__init__() self._save_file = save_file # parse the subnet definition into a static and dynamic pool - subnet = ipaddress.ip_network(f'{subnet}/{subnet_mask}', strict=False) divided = subnet.subnets() self._static_pool = next(divided) self._dynamic_pool = next(divided) @@ -106,7 +113,7 @@ def __init__(self, save_file, subnet, subnet_mask): # load the last saved state try: restore = pickle.load(open(save_file, 'rb')) - except: + except: # noqa: E722 restore = {} self.update(restore) @@ -157,139 +164,167 @@ def reverse_ptr_zone(self): class Config(object): - _last_key = None - - def __getitem__(self, key): - return self.get(key) - - def get(self, key, default=None): - self._last_key = key - val = os.environ.get(key, default) - try: - return val.replace('\\n', '\n') - except AttributeError: - return val - - @property - def error(self): - return f'\n\033[31mThere was an error parsing the configuration\nPlease check the value for {self._last_key}.\033[0m\n\n' + shim_var_keys = [ + 'WAN_INT', + 'BASTION_IP_ADDR', + 'BASTION_INTERFACES', + 'BASTION_HOST_NAME', + 'BASTION_SSH_USER', + 'CLUSTER_DOMAIN', + 'CLUSTER_NAME', + 'BOOT_DRIVE', + ] + + def __init__(self, yaml_file): + self.shim_vars = {} + for var in self.shim_var_keys: + self.shim_vars[var] = os.getenv(var) + config = FarosConfig.from_yaml(yaml_file) + self.network = config.network + self.bastion = config.bastion + self.cluster = config.cluster + self.proxy = config.proxy def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--list', action = 'store_true') - parser.add_argument('--verify', action = 'store_true') - parser.add_argument('--host', action = 'store') + parser.add_argument('--list', action='store_true') + parser.add_argument('--verify', action='store_true') + parser.add_argument('--host', action='store') args = parser.parse_args() return args + def main(config, ipam, inv): # GATHER INFORMATION FOR EXTRA NODES - extra_nodes = json.loads(config.get('EXTRA_NODES', '[]')) - for idx, item in enumerate(extra_nodes): - addr = ipam.get(item['mac'], item.get('ip')) - extra_nodes[idx].update({'ip': addr}) + for node in config.network.lan.dhcp.extra_reservations: + addr = ipam.get(node.mac, str(node.ip)) + node.ip = ipaddress.IPv4Address(addr) # CREATE INVENTORY - inv.add_group('all', None, + inv.add_group( + 'all', None, ansible_ssh_private_key_file=SSH_PRIVATE_KEY, - cluster_name=config['CLUSTER_NAME'], - cluster_domain=config['CLUSTER_DOMAIN'], - admin_password=config['ADMIN_PASSWORD'], - pull_secret=json.loads(config['PULL_SECRET']), - mgmt_provider=config['MGMT_PROVIDER'], - mgmt_user=config['MGMT_USER'], - mgmt_password=config['MGMT_PASSWORD'], - install_disk=config['BOOT_DRIVE'], + cluster_name=config.shim_vars['CLUSTER_NAME'], + cluster_domain=config.shim_vars['CLUSTER_DOMAIN'], + admin_password=config.bastion.become_pass, + pull_secret=json.loads(config.cluster.pull_secret), + mgmt_provider=config.cluster.management.provider, + mgmt_user=config.cluster.management.user, + mgmt_password=config.cluster.management.password, + install_disk=config.shim_vars['BOOT_DRIVE'], loadbalancer_vip=ipam['loadbalancer'], dynamic_ip_range=ipam.dynamic_pool, reverse_ptr_zone=ipam.reverse_ptr_zone, - subnet=config['SUBNET'], - subnet_mask=config['SUBNET_MASK'], - wan_ip=config['BASTION_IP_ADDR'], - extra_nodes=extra_nodes, - ignored_macs=config['IGNORE_MACS'], - dns_forwarders=[item['server'] for item in json.loads(config.get('DNS_FORWARDERS', '[]'))], - proxy=config['PROXY']=="True", - proxy_http=config.get('PROXY_HTTP', ''), - proxy_https=config.get('PROXY_HTTPS', ''), - proxy_noproxy=[item['dest'] for item in json.loads(config.get('PROXY_NOPROXY', '[]'))], - proxy_ca=config.get('PROXY_CA', '')) + subnet=str(config.network.lan.subnet.network_address), + subnet_mask=config.network.lan.subnet.prefixlen, + wan_ip=config.shim_vars['BASTION_IP_ADDR'], + extra_nodes=config.network.lan.dhcp.extra_reservations, + ignored_macs=config.network.lan.dhcp.ignore_macs, + dns_forwarders=config.network.lan.dns_forward_resolvers, + proxy=config.proxy is not None, + proxy_http=config.proxy.http if config.proxy is not None else '', + proxy_https=config.proxy.https if config.proxy is not None else '', + proxy_noproxy=config.proxy.noproxy if config.proxy is not None else [], + proxy_ca=config.proxy.ca if config.proxy is not None else '' + ) infra = inv.add_group('infra') - router = infra.add_group('router', - wan_interface=config['WAN_INT'], - lan_interfaces=json.loads(config['ROUTER_LAN_INT']), - all_interfaces=config['BASTION_INTERFACES'].split(), - allowed_services=json.loads(config['ALLOWED_SERVICES'])) + router = infra.add_group( + 'router', + wan_interface=config.shim_vars['WAN_INT'], + lan_interfaces=config.network.lan.interfaces, + all_interfaces=config.shim_vars['BASTION_INTERFACES'].split(), + allowed_services=config.network.port_forward + ) # ROUTER INTERFACES - router.add_host('wan', - config['BASTION_IP_ADDR'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) - router.add_host('lan', + router.add_host( + 'wan', config.shim_vars['BASTION_IP_ADDR'], + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) + router.add_host( + 'lan', ipam['bastion'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) # DNS NODE - router.add_host('dns', + router.add_host( + 'dns', ipam['bastion'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) # DHCP NODE - router.add_host('dhcp', + router.add_host( + 'dhcp', ipam['bastion'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) # LOAD BALANCER NODE - router.add_host('loadbalancer', + router.add_host( + 'loadbalancer', ipam['loadbalancer'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) # BASTION NODE bastion = infra.add_group('bastion_hosts') - bastion.add_host(config['BASTION_HOST_NAME'], - ipam['bastion'], - ansible_become_pass=config['ADMIN_PASSWORD'], - ansible_ssh_user=config['BASTION_SSH_USER']) + bastion.add_host( + config.shim_vars['BASTION_HOST_NAME'], + ipam['bastion'], + ansible_become_pass=config.bastion.become_pass, + ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] + ) # CLUSTER NODES cluster = inv.add_group('cluster') # BOOTSTRAP NODE ip = ipam['bootstrap'] - cluster.add_host('bootstrap', ip, - ansible_ssh_user='core', - node_role='bootstrap') + cluster.add_host( + 'bootstrap', ip, + ansible_ssh_user='core', + node_role='bootstrap' + ) # CLUSTER CONTROL PLANE NODES cp = cluster.add_group('control_plane', node_role='master') - node_defs = json.loads(config['CP_NODES']) - for count, node in enumerate(node_defs): - ip = ipam[node['mac']] - mgmt_ip = ipam[node['mgmt_mac']] - cp.add_host(node['name'], ip, - mac_address=node['mac'], - mgmt_mac_address=node['mgmt_mac'], - mgmt_hostname=mgmt_ip, - ansible_ssh_user='core', - cp_node_id=count) - if node.get('install_drive'): - cp.host(node['name'])['install_disk'] = node['install_drive'] + for count, node in enumerate(config.cluster.nodes): + ip = ipam[node.mac] + mgmt_ip = ipam[node.mgmt_mac] + cp.add_host( + node.name, ip, + mac_address=node.mac, + mgmt_mac_address=node.mgmt_mac, + mgmt_hostname=mgmt_ip, + ansible_ssh_user='core', + cp_node_id=count + ) + if node.install_drive is not None: + cp.host(node['name'])['install_disk'] = node.install_drive # VIRTUAL NODES - virt = inv.add_group('virtual', - mgmt_provider='kvm', - mgmt_hostname='bastion', - install_disk='vda') + virt = inv.add_group( + 'virtual', + mgmt_provider='kvm', + mgmt_hostname='bastion', + install_disk='vda' + ) virt.add_host('bootstrap') # MGMT INTERFACES - mgmt = inv.add_group('management', - ansible_ssh_user=config['MGMT_USER'], - ansible_ssh_pass=config['MGMT_PASSWORD']) - for count, node in enumerate(node_defs): - mgmt.add_host(node['name'] + '-mgmt', ipam[node['mgmt_mac']], - mac_address=node['mgmt_mac']) + mgmt = inv.add_group( + 'management', + ansible_ssh_user=config.cluster.management.user, + ansible_ssh_pass=config.cluster.management.password + ) + for node in config.cluster.nodes: + mgmt.add_host( + node.name + '-mgmt', ipam[node.mgmt_mac], + mac_address=node.mgmt_mac + ) if __name__ == "__main__": @@ -302,13 +337,14 @@ def main(config, ipam, inv): else: mode = 1 - # INITIALIZE CONFIG HANDLER - config = Config() + # INTIALIZE CONFIG + config = Config('/data/config.yml') # INTIALIZE IPAM ipam = IPAddressManager( IP_RESERVATIONS, - config['SUBNET'], config['SUBNET_MASK']) + config.network.lan.subnet + ) # INITIALIZE INVENTORY inv = Inventory(mode, args.host) @@ -316,6 +352,9 @@ def main(config, ipam, inv): # CREATE INVENTORY try: main(config, ipam, inv) + if mode == 0: + print(inv.to_json()) + except Exception as e: if mode == 2: sys.stderr.write(config.error) From 991953ba8f851684b69c46025a4467a31c8502bd Mon Sep 17 00:00:00 2001 From: James Harmison Date: Tue, 29 Dec 2020 11:51:14 -0500 Subject: [PATCH 06/10] Added validation of config --- app/cli/validate | 1 + app/playbooks/validate.d/all | 1 + app/playbooks/validate.d/config/main.sh | 5 +++++ app/playbooks/validate.d/config/validate.yml | 6 ++++++ 4 files changed, 13 insertions(+) create mode 120000 app/cli/validate create mode 120000 app/playbooks/validate.d/all create mode 100644 app/playbooks/validate.d/config/main.sh create mode 100644 app/playbooks/validate.d/config/validate.yml diff --git a/app/cli/validate b/app/cli/validate new file mode 120000 index 0000000..1fa21ed --- /dev/null +++ b/app/cli/validate @@ -0,0 +1 @@ +.command \ No newline at end of file diff --git a/app/playbooks/validate.d/all b/app/playbooks/validate.d/all new file mode 120000 index 0000000..30fa1ce --- /dev/null +++ b/app/playbooks/validate.d/all @@ -0,0 +1 @@ +config \ No newline at end of file diff --git a/app/playbooks/validate.d/config/main.sh b/app/playbooks/validate.d/config/main.sh new file mode 100644 index 0000000..f6e3318 --- /dev/null +++ b/app/playbooks/validate.d/config/main.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +ME=$(dirname $0) + +ansible-playbook $ME/validate.yml $@ || exit 1 diff --git a/app/playbooks/validate.d/config/validate.yml b/app/playbooks/validate.d/config/validate.yml new file mode 100644 index 0000000..629f7b4 --- /dev/null +++ b/app/playbooks/validate.d/config/validate.yml @@ -0,0 +1,6 @@ +--- +- name: Validate configuration and inventory + hosts: control_plane,management,infra + tasks: + - ping: + - setup: From 0e0666e4cc9be58ef9d68c1f604c30312d3a4f90 Mon Sep 17 00:00:00 2001 From: James Harmison Date: Mon, 28 Dec 2020 16:04:52 -0500 Subject: [PATCH 07/10] Ignore egg-info for development mode installs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 46b4574..bd406d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .pyo __pycache__ +*.egg-info *.swp data venv From 43d3dbfc51803242ea2592c4c79a967330fc2e87 Mon Sep 17 00:00:00 2001 From: James Harmison Date: Tue, 29 Dec 2020 12:43:58 -0500 Subject: [PATCH 08/10] Bumped major version to demonstrate breaking changes in config --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 6016e8a..0062ac9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.6.0 +5.0.0 From 9724ac4927c818f0b9e66a2c0d514fe8c9b9b4ab Mon Sep 17 00:00:00 2001 From: jharmison Date: Fri, 22 Jan 2021 10:26:10 -0500 Subject: [PATCH 09/10] Removed faros-config from monorepo structure (#1) Faros Config now lives in a separate [repository](https://github.com/project-faros/faros-config) in order to enable it to have its own release cycle and separate, dedicated CI and testing. --- Dockerfile | 3 +- app/inventory.py | 13 +---- data.skel/config.yml | 3 ++ pre/faros_config/README.md | 7 --- pre/faros_config/VERSION.txt | 1 - pre/faros_config/example_config.yml | 65 ----------------------- pre/faros_config/faros_config/__init__.py | 22 -------- pre/faros_config/faros_config/bastion.py | 5 -- pre/faros_config/faros_config/cluster.py | 29 ---------- pre/faros_config/faros_config/common.py | 6 --- pre/faros_config/faros_config/network.py | 43 --------------- pre/faros_config/faros_config/proxy.py | 9 ---- pre/faros_config/setup.cfg | 13 ----- pre/faros_config/setup.py | 4 -- pre/faros_config/test_config.py | 11 ---- requirements.txt | 1 + 16 files changed, 7 insertions(+), 228 deletions(-) delete mode 100644 pre/faros_config/README.md delete mode 100644 pre/faros_config/VERSION.txt delete mode 100644 pre/faros_config/example_config.yml delete mode 100644 pre/faros_config/faros_config/__init__.py delete mode 100644 pre/faros_config/faros_config/bastion.py delete mode 100644 pre/faros_config/faros_config/cluster.py delete mode 100644 pre/faros_config/faros_config/common.py delete mode 100644 pre/faros_config/faros_config/network.py delete mode 100644 pre/faros_config/faros_config/proxy.py delete mode 100644 pre/faros_config/setup.cfg delete mode 100644 pre/faros_config/setup.py delete mode 100644 pre/faros_config/test_config.py diff --git a/Dockerfile b/Dockerfile index 5db21db..81adceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,10 +15,11 @@ COPY requirements.txt /deps/python_requirements.txt COPY requirements.yml /deps/ansible_requirements.yml COPY home /root COPY pre /pre + RUN set -ex; \ microdnf update; \ microdnf install python3 jq openssh-clients tar sshpass findutils telnet less ncurses; \ - pip3 install --user -r /deps/python_requirements.txt /pre/faros_config; \ + pip3 install --user -r /deps/python_requirements.txt; \ ansible-galaxy collection install -r /deps/ansible_requirements.yml; \ microdnf clean all; \ rpm -i /pre/ilorest-3.0.1-7.x86_64.rpm; \ diff --git a/app/inventory.py b/app/inventory.py index 9cec5c2..69b0a91 100755 --- a/app/inventory.py +++ b/app/inventory.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse from collections import defaultdict -from faros_config import FarosConfig +from faros_config import FarosConfig, PydanticEncoder import ipaddress import json import os @@ -12,17 +12,6 @@ IP_RESERVATIONS = '/data/ip_addresses' -class PydanticEncoder(json.JSONEncoder): - def default(self, obj): - obj_has_dict = getattr(obj, "dict", False) - if obj_has_dict and callable(obj_has_dict): - return obj.dict(exclude_none=True) - elif isinstance(obj, ipaddress._IPAddressBase): - return str(obj) - else: - return json.JSONEncoder.default(self, obj) - - class InventoryGroup(object): def __init__(self, parent, name): diff --git a/data.skel/config.yml b/data.skel/config.yml index 75cf923..f1862c3 100644 --- a/data.skel/config.yml +++ b/data.skel/config.yml @@ -4,6 +4,9 @@ network: lan: subnet: 192.168.8.0/24 interfaces: [] + dhcp: + ignore_macs: [] + extra_reservations: [] bastion: become_pass: admin cluster: diff --git a/pre/faros_config/README.md b/pre/faros_config/README.md deleted file mode 100644 index 5d87dcf..0000000 --- a/pre/faros_config/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Faros Config - -This small library is used to validate configuration provided to Project Faros. It contains the necessary parts to load a configuration from the environment, mix it with configuration from a templated YAML file, and provide a single Python object that is easy for the Inventory to work with while generating variables for hosts. - -## NOTE - -This library is not intended to be used outside of Project Faros, or indeed outside of the Project Faros cluster-manager container. diff --git a/pre/faros_config/VERSION.txt b/pre/faros_config/VERSION.txt deleted file mode 100644 index 6e8bf73..0000000 --- a/pre/faros_config/VERSION.txt +++ /dev/null @@ -1 +0,0 @@ -0.1.0 diff --git a/pre/faros_config/example_config.yml b/pre/faros_config/example_config.yml deleted file mode 100644 index bc37659..0000000 --- a/pre/faros_config/example_config.yml +++ /dev/null @@ -1,65 +0,0 @@ -network: - port_forward: - - SSH to Bastion - - HTTPS to Cluster API - - HTTP to Cluster Apps - - HTTPS to Cluster Apps - - HTTPS to Cockpit Panel - lan: - subnet: 192.168.8.0/24 - interfaces: - - eno1 - - eno5 - dns_forward_resolvers: - - 10.1.1.1 - dhcp: - ignore_macs: - - name: node-0-eno1 - mac: da:d5:de:ad:be:ef - - name: node-0-eno2 - mac: da:d5:de:ad:be:ef - - name: node-0-eno3 - mac: da:d5:de:ad:be:ef - extra_reservations: - - name: wifi - mac: da:d5:de:ad:be:ef - ip: 192.168.8.127 - - name: chassis - mac: da:d5:de:ad:be:ef - ip: 192.168.8.50 - - name: bastion-ilo - mac: da:d5:de:ad:be:ef - ip: 192.168.8.51 - - name: client - mac: da:d5:de:ad:be:ef - ip: 192.168.8.52 -bastion: - become_pass: -cluster: - pull_secret: '{"auths":{"cloud.openshift.com":{"auth":"sometoken","email":"some@example.com"}}}' # etc - management: - provider: ilo - user: Administrator - password: - nodes: - - name: node-0 - mac: da:d5:de:ad:be:ef - mgmt_mac: fe:eb:da:ed:5d:ad - - name: node-1 - mac: da:d5:de:ad:be:ef - mgmt_mac: fe:eb:da:ed:5d:ad - - name: node-2 - mac: da:d5:de:ad:be:ef - mgmt_mac: fe:eb:da:ed:5d:ad -proxy: - http: proxy.example.com - https: secure-proxy.example.com - noproxy: - - registry.access.redhat.com - ca: | - # Proxy Server Certificate - -----BEGIN CERTIFICATE----- - MIIGlTCCBH2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjjELMAkGA1UEBhMCVVMx - ... - epmW5U8YK4yf - -----END CERTIFICATE---- diff --git a/pre/faros_config/faros_config/__init__.py b/pre/faros_config/faros_config/__init__.py deleted file mode 100644 index 6bd9245..0000000 --- a/pre/faros_config/faros_config/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -from pydantic import BaseModel -from typing import Optional -import yaml - -from .network import NetworkConfig -from .bastion import BastionConfig -from .cluster import ClusterConfig -from .proxy import ProxyConfig - - -class FarosConfig(BaseModel): - network: NetworkConfig - bastion: BastionConfig - cluster: ClusterConfig - proxy: Optional[ProxyConfig] - - @classmethod - def from_yaml(cls, yaml_file: str) -> 'FarosConfig': - with open(yaml_file) as f: - config = yaml.safe_load(f) - - return cls.parse_obj(config) diff --git a/pre/faros_config/faros_config/bastion.py b/pre/faros_config/faros_config/bastion.py deleted file mode 100644 index b342bc9..0000000 --- a/pre/faros_config/faros_config/bastion.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class BastionConfig(BaseModel): - become_pass: str diff --git a/pre/faros_config/faros_config/cluster.py b/pre/faros_config/faros_config/cluster.py deleted file mode 100644 index 0aa46f6..0000000 --- a/pre/faros_config/faros_config/cluster.py +++ /dev/null @@ -1,29 +0,0 @@ -from pydantic import BaseModel, constr -from typing import List, Optional - -from .common import StrEnum - -MacAddress = constr(regex=r'(([0-9A-Fa-f]{2}[-:]){5}[0-9A-Fa-f]{2})|(([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})') # noqa: E501 - - -class ManagementProviderItem(StrEnum): - ILO = "ilo" - - -class ManagementConfig(BaseModel): - provider: ManagementProviderItem - user: str - password: str - - -class NodeConfig(BaseModel): - name: str - mac: MacAddress - mgmt_mac: MacAddress - install_drive: Optional[str] - - -class ClusterConfig(BaseModel): - pull_secret: str - management: ManagementConfig - nodes: List[NodeConfig] diff --git a/pre/faros_config/faros_config/common.py b/pre/faros_config/faros_config/common.py deleted file mode 100644 index 6ba3d8c..0000000 --- a/pre/faros_config/faros_config/common.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - - -class StrEnum(str, Enum): - def __str__(self): - return self.value diff --git a/pre/faros_config/faros_config/network.py b/pre/faros_config/faros_config/network.py deleted file mode 100644 index 15e5e0e..0000000 --- a/pre/faros_config/faros_config/network.py +++ /dev/null @@ -1,43 +0,0 @@ -from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network -from pydantic import BaseModel, constr -from typing import List, Union - -from .common import StrEnum - -MacAddress = constr(regex=r'(([0-9A-Fa-f]{2}[-:]){5}[0-9A-Fa-f]{2})|(([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})') # noqa: E501 - - -class PortForwardConfigItem(StrEnum): - SSH_TO_BASTION = "SSH to Bastion" - HTTPS_TO_CLUSTER_API = "HTTPS to Cluster API" - HTTP_TO_CLUSTER_APPS = "HTTP to Cluster Apps" - HTTPS_TO_CLUSTER_APPS = "HTTPS to Cluster Apps" - HTTPS_TO_COCKPIT_PANEL = "HTTPS to Cockpit Panel" - - -class NameMacPair(BaseModel): - name: str - mac: MacAddress - - -class NameMacIpSet(BaseModel): - name: str - mac: MacAddress - ip: Union[IPv4Address, IPv6Address] - - -class DhcpConfig(BaseModel): - ignore_macs: List[NameMacPair] - extra_reservations: List[NameMacIpSet] - - -class LanConfig(BaseModel): - subnet: Union[IPv4Network, IPv6Network] - interfaces: List[str] - dns_forward_resolvers: List[Union[IPv4Address, IPv6Address]] - dhcp: DhcpConfig - - -class NetworkConfig(BaseModel): - port_forward: List[PortForwardConfigItem] - lan: LanConfig diff --git a/pre/faros_config/faros_config/proxy.py b/pre/faros_config/faros_config/proxy.py deleted file mode 100644 index 0a11369..0000000 --- a/pre/faros_config/faros_config/proxy.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel -from typing import List - - -class ProxyConfig(BaseModel): - http: str - https: str - noproxy: List[str] - ca: str diff --git a/pre/faros_config/setup.cfg b/pre/faros_config/setup.cfg deleted file mode 100644 index 0236cdf..0000000 --- a/pre/faros_config/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[metadata] -name = faros-config -version = file: VERSION.txt -description = A configuration handler for Project Faros -long_description = file: README.md -long_description_content_type = text/markdown -license = GNU GPL v3 - -[options] -packages = find: -install_requires = - PyYAML==5.3.1 - pydantic==1.7.3 diff --git a/pre/faros_config/setup.py b/pre/faros_config/setup.py deleted file mode 100644 index c823345..0000000 --- a/pre/faros_config/setup.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -setup() diff --git a/pre/faros_config/test_config.py b/pre/faros_config/test_config.py deleted file mode 100644 index a559f2d..0000000 --- a/pre/faros_config/test_config.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 -# This should grow to be a proper library with actual tests at some point - -from faros_config import FarosConfig - -config = FarosConfig.from_yaml('example_config.yml') - -print(config) -print() -for port in config.network.port_forward: - print(port) diff --git a/requirements.txt b/requirements.txt index 24533cc..a760b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ ansible<2.11 +faros-config==0.1.3 # for management/ilo provider python-hpilo==4.4.3 From 70149d816f95cf6ca3850ede8b746916d7df316d Mon Sep 17 00:00:00 2001 From: James Harmison Date: Fri, 29 Jan 2021 09:30:21 -0500 Subject: [PATCH 10/10] Shift inventory into faros-config. --- app/inventory.py | 353 +---------------------------------------------- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 352 deletions(-) mode change 100755 => 100644 app/inventory.py diff --git a/app/inventory.py b/app/inventory.py old mode 100755 new mode 100644 index 69b0a91..be0fef5 --- a/app/inventory.py +++ b/app/inventory.py @@ -1,355 +1,6 @@ #!/usr/bin/env python3 -import argparse -from collections import defaultdict -from faros_config import FarosConfig, PydanticEncoder -import ipaddress -import json -import os import sys -import pickle -SSH_PRIVATE_KEY = '/data/id_rsa' -IP_RESERVATIONS = '/data/ip_addresses' +from faros_config.inventory.cli import main - -class InventoryGroup(object): - - def __init__(self, parent, name): - self._parent = parent - self._name = name - - def add_group(self, name, **groupvars): - return(self._parent.add_group(name, self._name, **groupvars)) - - def add_host(self, name, hostname=None, **hostvars): - return(self._parent.add_host(name, self._name, hostname, **hostvars)) - - def host(self, name): - return self._parent.host(name) - - -class Inventory(object): - - _modes = ['list', 'host', 'verify', 'none'] - _data = {"_meta": {"hostvars": defaultdict(dict)}} - - def __init__(self, mode=0, host=None): - if mode == 1: - # host info requested - # current, only list and none are implimented - raise NotImplementedError() - - self._mode = mode - self._host = host - - def host(self, name): - return self._data['_meta']['hostvars'].get(name) - - def group(self, name): - if name in self._data: - return InventoryGroup(self, name) - else: - return None - - def add_group(self, name, parent=None, **groupvars): - self._data[name] = {'hosts': [], 'vars': groupvars, 'children': []} - - if parent: - if parent not in self._data: - self.add_group(parent) - self._data[parent]['children'].append(name) - - return InventoryGroup(self, name) - - def add_host(self, name, group=None, hostname=None, **hostvars): - if not group: - group = 'all' - if group not in self._data: - self.add_group(group) - - if hostname: - hostvars.update({'ansible_host': hostname}) - - self._data[group]['hosts'].append(name) - self._data['_meta']['hostvars'][name].update(hostvars) - - def to_json(self): - return json.dumps(self._data, sort_keys=True, indent=4, - separators=(',', ': '), cls=PydanticEncoder) - - -class IPAddressManager(dict): - - def __init__(self, save_file, subnet): - super().__init__() - self._save_file = save_file - - # parse the subnet definition into a static and dynamic pool - divided = subnet.subnets() - self._static_pool = next(divided) - self._dynamic_pool = next(divided) - self._generator = self._static_pool.hosts() - - # calculate reverse dns zone - classful_prefix = [32, 24, 16, 8, 0] - classful = subnet - while classful.prefixlen not in classful_prefix: - classful = classful.supernet() - host_octets = classful_prefix.index(classful.prefixlen) - self._reverse_ptr_zone = \ - '.'.join(classful.reverse_pointer.split('.')[host_octets:]) - - # load the last saved state - try: - restore = pickle.load(open(save_file, 'rb')) - except: # noqa: E722 - restore = {} - self.update(restore) - - # reserve the first ip for the bastion - _ = self['bastion'] - - def __getitem__(self, key): - key = key.lower() - try: - return super().__getitem__(key) - except KeyError: - new_ip = self._next_ip() - self[key] = new_ip - return new_ip - - def __setitem__(self, key, value): - return super().__setitem__(key.lower(), value) - - def _next_ip(self): - used_ips = list(self.values()) - loop = True - - while loop: - new_ip = next(self._generator).exploded - loop = new_ip in used_ips - return new_ip - - def get(self, key, value=None): - if value and value not in self.values(): - self[key] = value - return self[key] - - def save(self): - with open(self._save_file, 'wb') as handle: - pickle.dump(dict(self), handle) - - @property - def static_pool(self): - return str(self._static_pool) - - @property - def dynamic_pool(self): - return str(self._dynamic_pool) - - @property - def reverse_ptr_zone(self): - return str(self._reverse_ptr_zone) - - -class Config(object): - shim_var_keys = [ - 'WAN_INT', - 'BASTION_IP_ADDR', - 'BASTION_INTERFACES', - 'BASTION_HOST_NAME', - 'BASTION_SSH_USER', - 'CLUSTER_DOMAIN', - 'CLUSTER_NAME', - 'BOOT_DRIVE', - ] - - def __init__(self, yaml_file): - self.shim_vars = {} - for var in self.shim_var_keys: - self.shim_vars[var] = os.getenv(var) - config = FarosConfig.from_yaml(yaml_file) - self.network = config.network - self.bastion = config.bastion - self.cluster = config.cluster - self.proxy = config.proxy - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--list', action='store_true') - parser.add_argument('--verify', action='store_true') - parser.add_argument('--host', action='store') - args = parser.parse_args() - return args - - -def main(config, ipam, inv): - # GATHER INFORMATION FOR EXTRA NODES - for node in config.network.lan.dhcp.extra_reservations: - addr = ipam.get(node.mac, str(node.ip)) - node.ip = ipaddress.IPv4Address(addr) - - # CREATE INVENTORY - inv.add_group( - 'all', None, - ansible_ssh_private_key_file=SSH_PRIVATE_KEY, - cluster_name=config.shim_vars['CLUSTER_NAME'], - cluster_domain=config.shim_vars['CLUSTER_DOMAIN'], - admin_password=config.bastion.become_pass, - pull_secret=json.loads(config.cluster.pull_secret), - mgmt_provider=config.cluster.management.provider, - mgmt_user=config.cluster.management.user, - mgmt_password=config.cluster.management.password, - install_disk=config.shim_vars['BOOT_DRIVE'], - loadbalancer_vip=ipam['loadbalancer'], - dynamic_ip_range=ipam.dynamic_pool, - reverse_ptr_zone=ipam.reverse_ptr_zone, - subnet=str(config.network.lan.subnet.network_address), - subnet_mask=config.network.lan.subnet.prefixlen, - wan_ip=config.shim_vars['BASTION_IP_ADDR'], - extra_nodes=config.network.lan.dhcp.extra_reservations, - ignored_macs=config.network.lan.dhcp.ignore_macs, - dns_forwarders=config.network.lan.dns_forward_resolvers, - proxy=config.proxy is not None, - proxy_http=config.proxy.http if config.proxy is not None else '', - proxy_https=config.proxy.https if config.proxy is not None else '', - proxy_noproxy=config.proxy.noproxy if config.proxy is not None else [], - proxy_ca=config.proxy.ca if config.proxy is not None else '' - ) - - infra = inv.add_group('infra') - router = infra.add_group( - 'router', - wan_interface=config.shim_vars['WAN_INT'], - lan_interfaces=config.network.lan.interfaces, - all_interfaces=config.shim_vars['BASTION_INTERFACES'].split(), - allowed_services=config.network.port_forward - ) - # ROUTER INTERFACES - router.add_host( - 'wan', config.shim_vars['BASTION_IP_ADDR'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - router.add_host( - 'lan', - ipam['bastion'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - # DNS NODE - router.add_host( - 'dns', - ipam['bastion'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - # DHCP NODE - router.add_host( - 'dhcp', - ipam['bastion'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - # LOAD BALANCER NODE - router.add_host( - 'loadbalancer', - ipam['loadbalancer'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - - # BASTION NODE - bastion = infra.add_group('bastion_hosts') - bastion.add_host( - config.shim_vars['BASTION_HOST_NAME'], - ipam['bastion'], - ansible_become_pass=config.bastion.become_pass, - ansible_ssh_user=config.shim_vars['BASTION_SSH_USER'] - ) - - # CLUSTER NODES - cluster = inv.add_group('cluster') - # BOOTSTRAP NODE - ip = ipam['bootstrap'] - cluster.add_host( - 'bootstrap', ip, - ansible_ssh_user='core', - node_role='bootstrap' - ) - # CLUSTER CONTROL PLANE NODES - cp = cluster.add_group('control_plane', node_role='master') - for count, node in enumerate(config.cluster.nodes): - ip = ipam[node.mac] - mgmt_ip = ipam[node.mgmt_mac] - cp.add_host( - node.name, ip, - mac_address=node.mac, - mgmt_mac_address=node.mgmt_mac, - mgmt_hostname=mgmt_ip, - ansible_ssh_user='core', - cp_node_id=count - ) - if node.install_drive is not None: - cp.host(node['name'])['install_disk'] = node.install_drive - - # VIRTUAL NODES - virt = inv.add_group( - 'virtual', - mgmt_provider='kvm', - mgmt_hostname='bastion', - install_disk='vda' - ) - virt.add_host('bootstrap') - - # MGMT INTERFACES - mgmt = inv.add_group( - 'management', - ansible_ssh_user=config.cluster.management.user, - ansible_ssh_pass=config.cluster.management.password - ) - for node in config.cluster.nodes: - mgmt.add_host( - node.name + '-mgmt', ipam[node.mgmt_mac], - mac_address=node.mgmt_mac - ) - - -if __name__ == "__main__": - # PARSE ARGUMENTS - args = parse_args() - if args.list: - mode = 0 - elif args.verify: - mode = 2 - else: - mode = 1 - - # INTIALIZE CONFIG - config = Config('/data/config.yml') - - # INTIALIZE IPAM - ipam = IPAddressManager( - IP_RESERVATIONS, - config.network.lan.subnet - ) - - # INITIALIZE INVENTORY - inv = Inventory(mode, args.host) - - # CREATE INVENTORY - try: - main(config, ipam, inv) - if mode == 0: - print(inv.to_json()) - - except Exception as e: - if mode == 2: - sys.stderr.write(config.error) - sys.exit(1) - raise(e) - - # DONE - ipam.save() - sys.exit(0) +main(sys.argv[1:]) diff --git a/requirements.txt b/requirements.txt index a760b86..858babe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ansible<2.11 -faros-config==0.1.3 +faros-config==0.2.0 # for management/ilo provider python-hpilo==4.4.3