diff --git a/README.md b/README.md index 1bc3fbc..3b810a7 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,31 @@ ipaperftest --test GroupSizeTest --threads 1500 ipaperftest --test GroupSizeTest --threads 1500 --number-of-subgroups 3 ``` +### CertIssuanceTest + +Find the limit of the IPA API to issue new certificates. + +A set number of clients is enrolled then services for each client are created. + +For each service an ipa-getcert request is issued. There is little effort made +to ensure that these are all run at the same time but in the end this more +closely mirrors a live installation. + +#### Options +Rather than declaring a bunch of new options some are reused. The available options +are: + +- `cert-requests`: number of certificates to request for each client +- `clients`: number of clients to enroll +- `wsgi-processes`: number of WSGI processes to enable (default=4) + +Sample execution: + +``` +ipaperftest --test CertIssuanceTest --amount 70 --cert-requests 5 +ipaperftest --test CertIssuanceTest --amount 70 --cert-requests 5 --wsgi-processes 8 +``` + ## Creating test users For client authentication test we need a lot of users to test against. diff --git a/create-test-data.py b/create-test-data.py index 0159702..3345ff4 100755 --- a/create-test-data.py +++ b/create-test-data.py @@ -17,6 +17,7 @@ def __init__( users=50000, hosts=40000, host_prefix="client", + services=0, number_of_subgroups=0, outfile=None, ): @@ -85,6 +86,8 @@ def __init__( self.hosts = hosts self.hostgroups = hostgroups + self.services = services + self.number_of_subgroups = number_of_subgroups self.hostgroups_per_host = hostgroups_per_host @@ -206,6 +209,20 @@ def __init__( 'ipaUniqueID': ['autogenerate'], } + self.service_defaults = { + 'objectClass': [ + 'ipakrbprincipal', + 'ipaobject', + 'ipaservice', + 'krbprincipal', + 'krbprincipalaux', + 'krbticketpolicyaux', + 'pkiuser', + 'top', + ], + 'ipaUniqueID': ['autogenerate'], + } + self.sudo_defaults = { 'objectClass': [ 'ipasudorule', @@ -330,6 +347,37 @@ def hostgroupname_generator(self, start, stop, step=1): for i in range(start, stop, step): yield 'hostgroup{}'.format(i) + def gen_service(self, servicename, hostname): + service = dict(self.service_defaults) + service['dn'] = 'krbprincipalname={servicename}/{hostname}@{realm},' \ + 'cn=services,cn=accounts,{suffix}'.format( + servicename=servicename, + hostname=hostname, + realm=self.realm, + suffix=self.basedn + ) + service['krbPrincipalName'] = ['{servicename}/{hostname}@{realm}'.format( + servicename=servicename, + hostname=hostname, + realm=self.realm + )] + service['krbCanonicalName'] = service['krbPrincipalName'] + service['ipaKrbPrincipalAlias'] = service['krbPrincipalName'] + service['managedBy'] = [ + 'fqdn={hostname},cn=computers,cn=accounts,{suffix}'.format( + hostname=hostname, + suffix=self.basedn)] + return service + + def generate_services(self): + for i in range(0, self.hosts, 1): + hostname = '{}{:03d}.{}'.format( + self.host_prefix, i, self.domain + ) + for j in range(0, self.services, 1): + service = self.gen_service(f'service{j}', hostname) + self.put_entry(service) + def gen_sudorule( self, name, user_members=(), usergroup_members=(), @@ -671,6 +719,7 @@ def put_entry(self, entry): class IPATestDataLDIF(IPADataLDIF): def do_magic(self): self.gen_users_and_groups() + self.generate_services() def username_generator(self, start, stop, step=1, hostname=None): for i in range(start, stop, step): @@ -729,6 +778,8 @@ def put_entry(self, entry): @click.option("--hosts", default=500, help="Number of hosts to create.", type=int) @click.option("--host-prefix", default="client", help="hostname prefix") +@click.option("--services", default=0, help="Number of services per host to create.", + type=int) @click.option("--outfile", default=None, help="LDIF output file") @click.option("--with-groups", default=False, help="Create user groups.", is_flag=True) @@ -744,6 +795,7 @@ def main( users_per_host, hosts, host_prefix, + services, with_groups, with_hostgroups, with_sudo, @@ -761,6 +813,7 @@ def main( users=users_per_host, hosts=hosts, host_prefix=host_prefix, + services=services, number_of_subgroups=number_of_subgroups, outfile=outfile, ) diff --git a/setup.py b/setup.py index c4ae3aa..71eed4a 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ 'ipaperftest.plugins': [ 'apitest = ipaperftest.plugins.apitest', 'authenticationtest = ipaperftest.plugins.authenticationtest', + 'certissuetest = ipaperftest.plugins.certissuetest', 'enrollmenttest = ipaperftest.plugins.enrollmenttest', 'groupsizetest = ipaperftest.plugins.groupsizetest', ], diff --git a/src/ipaperftest/core/constants.py b/src/ipaperftest/core/constants.py index ab8887b..91214c2 100644 --- a/src/ipaperftest/core/constants.py +++ b/src/ipaperftest/core/constants.py @@ -607,3 +607,45 @@ def getLevelName(level): cmd: "ldapadd -x -D 'cn=Directory Manager' -w password -f userdata.ldif" chdir: /root """ + +ANSIBLE_CERTISSUANCETEST_SERVER_CONFIG_PLAYBOOK = """ +--- +- name: Add services after enrollment + hosts: ipaserver + become: yes + tasks: + - synchronize: + src: "{{{{ item }}}}" + dest: "/root" + mode: push + use_ssh_args: yes + with_items: + - create-test-data.py + - package: + name: python3-pip + - command: + cmd: "pip3 install click" + - command: + cmd: "python3 create-test-data.py --hosts {amount} --outfile userdata.ldif --users-per-host 0 --services {services}" + chdir: /root + - command: + cmd: "ldapadd -x -D 'cn=Directory Manager' -w password -f userdata.ldif" + chdir: /root +""" + +ANSIBLE_CERTISSUANCETEST_SERVER_TUNING_PLAYBOOK = """ +--- +- name: Tune the server WSGI parameters + hosts: ipaserver + become: yes + tasks: + - name: "Tune WSGI" + lineinfile: + path: /etc/httpd/conf.d/ipa.conf + regexp: '^WSGIDaemonProcess' + line: WSGIDaemonProcess ipa processes={wsgi_processes} threads=1 maximum-requests=500 \\ + - name: "Restart httpd" + service: + name: httpd + state: restarted +""" diff --git a/src/ipaperftest/core/main.py b/src/ipaperftest/core/main.py index 9f6fe62..9f7ddb4 100644 --- a/src/ipaperftest/core/main.py +++ b/src/ipaperftest/core/main.py @@ -109,6 +109,7 @@ def run(self, ctx): type=click.Choice(["EnrollmentTest", "APITest", "AuthenticationTest", + "CertIssuanceTest", "GroupSizeTest"])) @click.option( "--client-image", @@ -187,6 +188,8 @@ def run(self, ctx): help="Number of sub groups for Groupsize test", default=0, ) +@click.option("--cert-requests", default=0, help="Number of certificates to request") +@click.option("--wsgi-processes", default=4, help="Number of WSGI processes") @click.pass_context def main( ctx, @@ -211,6 +214,8 @@ def main( auth_spread=0, expected_result_type="no_errors", number_of_subgroups=0, + cert_requests=0, + wsgi_processes=4, ): tests = RunTest(['ipaperftest.registry']) diff --git a/src/ipaperftest/plugins/certissuetest.py b/src/ipaperftest/plugins/certissuetest.py new file mode 100644 index 0000000..c66c16e --- /dev/null +++ b/src/ipaperftest/plugins/certissuetest.py @@ -0,0 +1,262 @@ +# +# Copyright (C) 2024 FreeIPA Contributors see COPYING for license +# + +import os +import random +import resource +import subprocess as sp +import time +from datetime import datetime + +from ipaperftest.core.plugin import Plugin, Result +from ipaperftest.core.constants import ( + SUCCESS, + WARNING, + ERROR, + ANSIBLE_ENROLLMENTTEST_CLIENT_CONFIG_PLAYBOOK, + ANSIBLE_COUNT_IPA_HOSTS_PLAYBOOK, + ANSIBLE_CERTISSUANCETEST_SERVER_TUNING_PLAYBOOK, + ANSIBLE_CERTISSUANCETEST_SERVER_CONFIG_PLAYBOOK) +from ipaperftest.plugins.registry import registry + + +@registry +class CertIssuanceTest(Plugin): + + def __init__(self, registry): + super().__init__(registry) + self.custom_logs = ["getcert.log", ] + + def generate_clients(self, ctx): + + for i in range(ctx.params['amount']): + idx = str(i).zfill(3) + machine_name = "client{}".format(idx) + yield ( + { + "hostname": "%s.%s" % (machine_name, self.domain.lower()), + "type": "client" + } + ) + + def validate_options(self, ctx): + if not ctx.params.get('threads'): + raise RuntimeError('threads number is required') + + def install_server(self, ctx): + if ctx.params["ad_threads"] > 0: + # install AD server + self.run_ansible_playbook_from_template( + ANSIBLE_CERTISSUANCETEST_SERVER_CONFIG_PLAYBOOK, + "authenticationtest_ad_server_setup", {}, ctx + ) + super().install_server(ctx) + + def run(self, ctx): + sp.run(["cp", "create-test-data.py", "runner_metadata/"]) + + # Configure clients before installation + args = { + "server_ip": self.provider.hosts["server"], + "domain": self.domain + } + self.run_ansible_playbook_from_template( + ANSIBLE_ENROLLMENTTEST_CLIENT_CONFIG_PLAYBOOK, + "certissuetest_client_config", args, ctx + ) + + # Clients installation + client_cmds = [ + "sudo ipa-client-install -p admin -w password -U " + "--enable-dns-updates --no-nisdomain -N", + ] + processes = {} + non_client_hosts = 0 + installed = 0 + sleep_time = 20 + for host, ip in self.provider.hosts.items(): + if not host.startswith("client"): + if host.startswith("server") or host.startswith("replica"): + non_client_hosts += 1 + continue + installed += 1 + # spread the client install time to hopefully have all pass + if installed % 30 == 0: + sleep_time += 20 + cmds = ["sleep {}".format(sleep_time + random.randrange(1, 10))] + client_cmds + proc = self.run_ssh_command(" && ".join(cmds), ip, ctx, False) + processes[host] = proc + + print("Waiting for client installs to be completed...") + self.clients_succeeded = 0 + clients_returncodes = "" + for host, proc in processes.items(): + proc.communicate() + returncode = proc.returncode + rc_str = "Host " + host + " returned " + str(returncode) + clients_returncodes += rc_str + "\n" + print(rc_str) + if returncode == 0: + self.clients_succeeded += 1 + print("Clients succeeded: %s" % str(self.clients_succeeded)) + print("Return codes written to sync directory.") + with open("sync/returncodes", "w") as f: + f.write(clients_returncodes) + + # Check all hosts have been registered in server + ansible_ret = self.run_ansible_playbook_from_template( + ANSIBLE_COUNT_IPA_HOSTS_PLAYBOOK, + "enrollmenttest_count_hosts", {}, ctx + ) + server_ip = self.provider.hosts["server"] + host_find_output = ansible_ret.get_fact_cache(server_ip)["host_find_output"] + try: + if ( + int(host_find_output) == (self.clients_succeeded + non_client_hosts) + and len(self.provider.hosts.keys()) == (self.clients_succeeded + + non_client_hosts) + ): + yield Result(self, SUCCESS, msg="All clients enrolled succesfully.") + else: + yield Result(self, ERROR, + error="Client installs succeeded number (%s) " + "does not match host-find output (%s)." % + (self.clients_succeeded, host_find_output)) + except ValueError: + yield Result(self, ERROR, + error="Failed to convert host-find output to int. " + "Value was: %s" % host_find_output) + + args = { + "amount": ctx.params["amount"], + "services": ctx.params["cert_requests"], + "wsgi_processes": ctx.params["wsgi_processes"], + } + self.run_ansible_playbook_from_template( + ANSIBLE_CERTISSUANCETEST_SERVER_CONFIG_PLAYBOOK, + "certissuancetest_server_config", args, ctx + ) + self.run_ansible_playbook_from_template( + ANSIBLE_CERTISSUANCETEST_SERVER_TUNING_PLAYBOOK, + "certissuancetest_server_tuning", args, ctx + ) + + # Client authentications will be triggered at now + 1min per 20 clients + # wait_time = max(int(len(self.provider.hosts.keys()) / 20), 1) * 60 + client_wait_time = int(time.time()) + 30 + + # Now that all the client installs are done, fire off the + # certificate requests (for now whether all installs are ok or not) + resource.setrlimit(resource.RLIMIT_NOFILE, (16384, 16384)) + processes = [] + for host, ip in self.provider.hosts.items(): + if not host.startswith("client"): + continue + for i in range(ctx.params["cert_requests"]): + service_cmd = 'sudo ipa-getcert request -K service{}/{}.{} '\ + '-f /etc/pki/tls/certs/service{}.pem ' \ + '-k /etc/pki/tls/private/service{}.key -v -w >> request.log 2>&1' \ + .format(i, host, self.domain.lower(), i, i) + try: + proc = self.run_ssh_command(service_cmd, ip, ctx, False) + processes.append(proc) + except IOError: + print("Length of procs ", len(processes)) + raise + + print("Waiting for certificate issuance to be completed...") + + start_time = time.time() + for proc in processes: + proc.communicate() + self.execution_time = time.time() - start_time - client_wait_time + + # Get the getcert output + processes = {} + cmds = [] + for host, ip in self.provider.hosts.items(): + if not host.startswith("client"): + continue + for i in range(ctx.params["cert_requests"]): + cmds.append('sudo ipa-getcert list > getcert.log') + proc = self.run_ssh_command(" && ".join(cmds), ip, ctx, False) + processes[host] = proc + + print("Waiting for collection of getcert output to be completed...") + + start_time = time.time() + for host, proc in processes.items(): + proc.communicate() + self.execution_time = time.time() - start_time - client_wait_time + + return + + def post_process_logs(self, ctx): + """ Calculate number of succeeded threads """ + + total_successes = 0 + total_requested = 0 + for host in os.listdir("sync"): + if not host.startswith("client"): + continue + + logpath = "sync/{}/getcert.log".format(host) + try: + logstr = open(logpath).readlines() + except FileNotFoundError: + yield Result(self, WARNING, msg="File %s not found" % logpath) + continue + + n_requested = 0 + n_succeeded = 0 + for line in logstr: + if "status:" not in line: + continue + + n_requested += 1 + if 'MONITORING' in line: + n_succeeded += 1 + + if n_requested > 0: + percentage = round((n_succeeded / n_requested) * 100) + else: + percentage = 0 + if percentage == 100: + yield Result(self, SUCCESS, msg="All threads on %s succeeded." % host) + else: + yield Result(self, ERROR, + error="Not all threads on %s succeded: %s/%s (%s)." + % (host, n_succeeded, n_requested, percentage)) + + total_successes += n_succeeded + total_requested += n_requested + + if total_requested == 0: + yield Result(self, ERROR, + error="None of the requests succeeded.") + total_percentage = round((total_successes / total_requested) * 100) + + yield Result(self, SUCCESS, msg="{} requests out of {} succeeded ({}%)".format( + total_successes, total_requested, total_percentage), successes=total_successes) + + if total_percentage == 100: + msg = "success" + yield Result(self, SUCCESS, msg="All requests succeded.") + else: + msg = "fails" + yield Result(self, ERROR, + error="Not all requests succeeded: %s/%s (%s)." + % (total_successes, total_requested, total_percentage)) + + if total_requested != total_successes: + msg = "fails" + + self.results_archive_name = \ + "CertIssuanceTest-{}-{}-{}clients-{}requests-{}issued-{}".format( + datetime.now().strftime("%FT%H%MZ"), + self.provider.server_image.replace("/", ""), + ctx.params["amount"], + total_requested, + total_successes, + msg,)