volcreate-logs

#!/usr/bin/perl
our $VERSION = '2.6 (2024-07-25)';
#
# volcreate-logs -- Create and grant quota to log volumes in AFS.
#
# Written by Russ Allbery <eagle@eyrie.org>
# Copyright 2002, 2004, 2011, 2012, 2013
#     The Board of Trustees of the Leland Stanford Junior University
#
# Updated by Bill MacAllister <bill@ca-zephyr.org>
# Copyright 2018
#    Bill MacAllister <bill@ca-zephyr.org>
#
# This program is free software; you may redistribute it and/or modify it
# under the same terms as Perl itself.

##############################################################################
# Modules and declarations
##############################################################################

use 5.006;
use strict;
use warnings;

use vars qw($JUSTPRINT $MAIL $MAILOPEN);
use subs qw(system);

use Getopt::Long qw(GetOptions);
use POSIX        qw(strftime);

# AFS::Utils is loaded to get the setpag function if a ticket cache is
# specified on the command line.

##############################################################################
# Site configuration
##############################################################################

# The address to which to send mailed reports.  Required if the -m option was
# given.
our $LOGS_ADDRESS;

# The path to the config file specifying what to do.
our $LOGS_CONFIG = '/etc/afs-admin-tools/log-volumes';

# The default root path for ticket files, used for the -k option if a relative
# path to a ticket file was given.
our $LOGS_TICKETS = '/var/run';

# The path to aklog, used to obtain AFS tokens from a Kerberos ticket.
our $AKLOG = 'aklog';

# The path to volcreate, used to create volumes.
our $VOLCREATE = 'volcreate';

# The full path to fs and vos.  vos may be in an sbin directory, which may
# not be on the user's path by default, so check there first.
our $FS  = 'fs';
our $VOS = grep { -x $_ } qw(/usr/local/sbin/vos /usr/sbin/vos);
$VOS ||= 'vos';

# Maximum quota
our $LOGS_MAX_QUOTA = '50G';

# Cell root used for validation
our $CELL_ROOT = '/afs/@cell';

# Load the configuration file if it exists.
if (-f '/etc/afs-admin-tools/config') {
    require '/etc/afs-admin-tools/config';
}

##############################################################################
# Overrides
##############################################################################

# We override system to honor the global $JUSTPRINT variable.  It otherwise
# works the same way as system normally does.
sub system {
    if ($JUSTPRINT) {
        print "@_\n";
        return 0;
    } else {
        return CORE::system(@_);
    }
}

##############################################################################
# Reporting
##############################################################################

# Start a mail message to $LOGS_ADDRESS on the MAIL file descriptor.
sub start_mail {
    my ($sendmail) = grep { -x $_ } qw(/usr/sbin/sendmail /usr/lib/sendmail);
    $sendmail ||= '/usr/lib/sendmail';
    my $date = strftime("%Y-%m-%d", localtime);
    open(MAIL, "| $sendmail -t -oi -oem")
      or die "Cannot fork $sendmail: $!";
    print MAIL "To: $LOGS_ADDRESS\n";
    print MAIL "Subject: Log volume report ($date)\n";
    print MAIL "\n";
    $MAILOPEN = 1;
}

# Report an action.  Checks $MAIL to see if the report should be via e-mail,
# and if so, checks $MAILOPEN to see if we've already started the mail
# message.
sub report {
    if ($MAIL) {
        if (!$MAILOPEN) {
            start_mail;
        }
        print MAIL @_;
    } else {
        print @_;
    }
}

##############################################################################
# Volume quota checking
##############################################################################

# This subroutine takes a quota specification and return the value in
# kilobytes.  The input is assumed to be megabytes if the units are
# not specified.

sub kbytes {
    my ($quota) = @_;
    my $quota_kbytes;
    if ($quota =~ /^(\d+)$/xms) {
        $quota_kbytes = $quota * 1024;
    } elsif ($quota =~ /^(\d+)([kmgt])/xmsi) {
        my $q = $1;
        my $m = $2;
        if (lc($m) eq 't') {
            $quota_kbytes = $q * 1024 * 1024 * 1024;
        } elsif (lc($m) eq 'g') {
            $quota_kbytes = $q * 1024 * 1024;
        } elsif (lc($m) eq 'm') {
            $quota_kbytes = $q * 1024;
        } elsif (lc($m) eq 'k') {
            $quota_kbytes = $q;
        } else {
            die "ERROR: invalid unit of measure ($m)";
        }
    }
    if (!$quota_kbytes) {
        die "ERROR: invalid quota value ($quota)";
    }
    return $quota_kbytes;
}

# This subroutine takes a quota specification in kbytes and returns
# a human friendly form.

sub display_kbytes {
    my ($quota_kbytes) = @_;
    my @sizes          = ('T', 'G', 'M');
    my $quota          = "${quota_kbytes}K";
    my $i              = $quota_kbytes;
    my $u              = 'K';
    while ($i > 1024) {
        if (@sizes) {
            $i     = $i / 1024;
            $u     = pop(@sizes);
            $quota = "${i}$u";
        } else {
            last;
        }
    }
    return $quota;
}

# Takes the volume name, the full path to a mount point for a volume, and the
# minimum quota that that volume should have.  Increases the quota of the
# volume to at least the minimum and by 10% if the volume is within 90% of its
# quota.
sub check_quota {
    my ($volume, $mountpoint, $quota_min) = @_;
    my $minimum   = kbytes($quota_min);
    my @listquota = `$FS listquota -path $mountpoint 2>&1`;

    # The first line is either an error or headers.  Check to see if it is
    # an error, and if not, ignore it.  The second line is our quota
    # information, in the form volume name, quota, used amount, used
    # percentage, and partition percentage.
    if ($listquota[0] =~ /^fs: /) {
        warn $listquota[0];
        warn "Unable to get quota information for $volume\n";
        return;
    }
    shift @listquota;
    my ($volname, $quota, $used, $percent) = split(' ', $listquota[0]);
    if (!($percent =~ s/%(<\<)?$//)) {
        warn "Unable to parse: $listquota[0]";
        warn "Unable to get quota information for $volume\n";
        return;
    }

    # Figure out if we need to do anything.
    my $newquota = $quota;
    if ($percent > 90) {
        $newquota = int($quota * 1.1 + 0.9);
    }
    $newquota = $minimum if ($newquota < $minimum);
    if ($newquota != $quota) {
        my $s = display_kbytes($newquota);
        report "Setting quota to $s MB for $volume\n";
        if (
            system($FS, 'setquota', '-path', $mountpoint, '-max', $newquota)
            != 0)
        {
            warn "$FS setquota $mountpoint $newquota exited with status ",
              ($? >> 8), "\n";
            warn "Unable to set quota for $volume\n";
        }
    }
}

##############################################################################
# Finding volumes
##############################################################################

# Given a path, run fs lsmount on it and return the volume name, or undef if
# this path isn't a mount point.
sub lsmount {
    my ($path) = @_;
    if ($path =~ /[\\\';]/) {
        die "$0: invalid character in $path";
    }
    my $volume = `$FS lsmount '$path' 2>&1`;
    if ($volume =~ /is a mount point for volume \'\#([^\']+)\'/) {
        return $1;
    } else {
        return undef;
    }
}

# Given the base path and the current month and year, find the full path to
# the volume associated with that month and year.  Returns the full path and
# the volume name in a list.
sub find_volume_month {
    my ($base, $month, $year) = @_;
    $month = sprintf("%02d", $month);
    my ($volume, $path);
    if (-d "$base/$year/$month") {
        $volume = lsmount("$base/$year/$month");
        $path   = "$base/$year/$month" if defined $volume;
    }
    return ($volume, $path);
}

# Given the base path and the current year, find the full path to the volume
# associated with that year for logs where we only create one volume per year.
# Returns the full path and the volume name in a list.
sub find_volume_year {
    my ($base, $year) = @_;
    my ($volume, $path);
    if (-d "$base/$year") {
        $volume = lsmount("$base/$year");
        if (defined $volume) {
            $path = "$base/$year";
        }
    }
    return ($volume, $path);
}

##############################################################################
# Creating volumes
##############################################################################

# Given a volume name, mount point, quota, and flag saying whether to be
# quiet, create that volume.  Performs some basic consistency checks to make
# sure that the quota isn't too large and that the volume or mount point don't
# already exist.
sub create_volume {
    my ($volume, $mountpoint, $quota, $quiet) = @_;
    my $quota_kbytes = kbytes($quota);
    my $max          = kbytes($LOGS_MAX_QUOTA);
    if ($quota_kbytes > $max) {
        my $s = display_kbytes($quota_kbytes);
        my $m = display_kbytes($max);
        warn "$volume not created: ${s} exceeds maximum of $m\n";
        return;
    }
    my $cell_root_rw = reverse($CELL_ROOT);
    $cell_root_rw =~ s%(.*?)/%$1./%xms;
    $cell_root_rw = reverse($cell_root_rw);
    $mountpoint =~ s%$CELL_ROOT%$cell_root_rw%;
    if (-d "$mountpoint") {
        warn "$volume not created: $mountpoint already exists\n";
        return;
    }
    if (!$JUSTPRINT
        && system("$VOS examine -id $volume >/dev/null 2>&1") == 0)
    {
        warn "$volume not created: volume already exists\n";
        return;
    }

    # Create the volume.
    my @command = ('-t', 'logs', $volume, $quota_kbytes, $mountpoint);
    if ($quiet) {
        unshift(@command, '-q');
    }
    if (system($VOLCREATE, @command) != 0) {
        warn "$VOLCREATE -t logs $volume $quota_kbytes $mountpoint exited ",
          "with status ", ($? >> 8), "\n";
        warn "$volume not fully created, please check\n";
    } else {
        my $s = display_kbytes($quota_kbytes);
        report "Created $volume with quota ${s}\n";
    }
    return;
}

# For a yearly log volume, create all of the month subdirectories.  Takes the
# base directory of the new log volume.
sub create_month_dirs {
    my ($base) = @_;
    for (1 .. 12) {
        my $month = sprintf("%02d", $_);
        if ($JUSTPRINT) {
            print "mkdir $base/$month\n";
        } else {
            mkdir("$base/$month", 0755)
              or warn "Could not create $base/$month: $!\n";
        }
    }
}

# Given a year, month, configuration hash for a log volume, and flag saying
# whether to be quiet, make sure that a volume for that year and month already
# exists.  If it doesn't, create it.
sub create_log_volume {
    my ($year, $month, $config, $quiet) = @_;
    my ($path, $volume);
    if ($$config{type} eq 'monthly') {
        $path   = sprintf("$$config{path}/%04d/%02d", $year, $month);
        $volume = sprintf("%s.%04d%02d", $$config{name}, $year, $month);
        if (!-d "$$config{path}/$year") {
            if ($JUSTPRINT) {
                print "mkdir $$config{path}/$year\n";
            } elsif (!mkdir("$$config{path}/$year", 0755)) {
                warn "Could not create $$config{path}/$year: $!\n";
                warn "$volume not created\n";
                return;
            }
        }
    } elsif ($$config{type} eq 'yearly') {
        $path   = sprintf("$$config{path}/%04d", $year);
        $volume = sprintf("%s.%04d", $$config{name}, $year);
    }
    if (!-d $path) {
        create_volume($volume, $path, $$config{quota}, $quiet);
        create_month_dirs($path) if $$config{type} eq 'yearly';
    }
}

##############################################################################
# Configuration parsing
##############################################################################

# Parse a log section, returning an anonymous hash representing the log
# configuration.
sub parse_log {
    my ($config) = @_;
    my %log;
    while (<$config>) {
        next if /^\s*\#/;
        next if /^\s*$/;
        if (/^\s*\}/) {
            my $okay = 1;
            for (qw/name path type quota/) {
                if (!exists $log{$_}) {
                    $okay = 0;
                }
            }
            if ($okay) {
                return \%log;
            } else {
                warn "Missing attributes in log group ending on line $.\n";
                return undef;
            }
        }
        my ($key, $value) = split(/:\s+/, $_, 2);
        $key   =~ s/^\s+//;
        $value =~ s/\s+$//;
        if (!$value) {
            warn "Parse error in log group on line $.\n";
            next;
        }
        if (!($key =~ /^(name|path|type|quota)$/)) {
            warn "Unknown attribute on line $. ($key)\n";
            next;
        }
        if ($key eq 'quota') {
            if ($value =~ /^\d+$/xms) {
                $value = kbytes("${value}M");
            }
        }
        $log{$key} = $value;
    }
    warn "Unterminated log group on line $.\n";
    return undef;
}

# Parse the configuration file.  Returns a list of anonymous hashes containing
# configuration information for each log.
sub parse_config {
    my @config;
    open(CONFIG, $LOGS_CONFIG) or die "Cannot open $LOGS_CONFIG: $!";
    while (<CONFIG>) {
        if (/^\s*\#/) {
            next;
        }
        if (/^\s*$/) {
            next;
        }
        if (/^\s*log\s+\{\s*$/) {
            my $config = parse_log(\*CONFIG);
            if ($config) {
                push(@config, $config);
            }
        } else {
            warn "Parse error in configuration file on line $.\n";
            next;
        }
    }
    return @config;
}

##############################################################################
# Main routine
##############################################################################

# Parse command line options.
my ($help, $quiet, $ticket, $version);
Getopt::Long::config('bundling', 'no_ignore_case');
GetOptions(
    'n|dry-run|just-print' => \$JUSTPRINT,
    'h|help'               => \$help,
    'k|ticket-cache=s'     => \$ticket,
    'm|mail'               => \$MAIL,
    'q|quiet'              => \$quiet,
    'v|version'            => \$version
) or exit 1;
if ($help) {
    exec('perldoc', '-t', $0) or die "Cannot fork: $!";
    exit 1;
} elsif ($version) {
    print "volcreate-logs $VERSION\n";
    exit 1;
}
if ($MAIL and not $LOGS_ADDRESS) {
    die "\$LOGS_ADDRESS must be set in /etc/afs-admin-tools/config for -m";
}

# If a ticket cache was specified on the command line, obtain Kerberos tickets
# from that cache and run aklog to get a token.
if ($ticket) {
    require AFS::Utils;
    if (!($ticket =~ m%^/%)) {
        $ticket = $LOGS_TICKETS . '/' . $ticket;
    }
    $ENV{KRB5CCNAME} = $ticket;
    AFS::Utils::setpag()      or die "$0: unable to setpag: $!";
    CORE::system($AKLOG) == 0 or die "$0: unable to obtain tokens";
}

# Parse the configuration and get the current time.
my @config = parse_config;
my ($day, $month, $year) = (localtime)[3 .. 5];
$month++;
$year += 1900;

# If a particular name or set of names were given on the command line, limit
# our actions to just those names.
if (@ARGV) {
    my %names = map { $_ => 1 } @ARGV;
    @config = grep { $names{ $$_{name} } } @config;
    if (!(@ARGV == @config)) {
        my %found = map { $$_{name} => 1 } @config;
        for (sort keys %names) {
            if (!$found{$_}) {
                warn "No configuration found for $_";
            }
        }
        exit 1;
    }
}

# For each log configuration, update the quota of the current volume if
# necessary and then create the new volume if it's past the 20th of the month.
for my $config (@config) {
    my ($path, $volume);
    if ($$config{type} eq 'monthly') {
        ($volume, $path) = find_volume_month($$config{path}, $month, $year);
    } elsif ($$config{type} eq 'yearly') {
        ($volume, $path) = find_volume_year($$config{path}, $year);
    } else {
        warn "Unknown type $$config{type} for $$config{name}\n";
        next;
    }
    if ($path) {
        check_quota($volume, $path, $$config{quota});
    } else {
        create_log_volume($year, $month, $config, $quiet);
    }
    if ($day >= 20) {
        my $month = $month;
        my $year  = $year;
        $month++;
        if ($month > 12) {
            $month = 1;
            $year++;
        }
        create_log_volume($year, $month, $config, $quiet);
    }
}

__END__

##############################################################################
# Documentation
##############################################################################

=for stopwords
ACL AFS PAG .YYYY .YYYMM afs-admin-tools aklog fs -hmnqv newsyslog
subdirectories volcreate volcreate's volcreate-logs vos logs.service.YYYYMM
YYYYMM

=head1 NAME

volcreate-logs - Create and grant quota to log volumes in AFS

=head1 SYNOPSIS

B<volcreate-logs> [B<-hmnqv>] [B<-k> I<ticket-cache>]

=head1 DESCRIPTION

This program maintains the size and existence of log volumes in AFS,
creating new ones for upcoming months or years as necessary and increasing
their quota as necessary if they're getting too full.  It reports all of
its actions, by default to standard out but optionally through mail with
the B<-m> option.

The rules for each set of log volumes are defined in the program
configuration file, in F</etc/afs-admin-tools/log-volumes> by default.  It
uses B<volcreate> to create all new log volumes, and therefore initial
ACLs for new log volumes can be set using B<volcreate>'s ACL handling
capabilities.  See L<volcreate(1)> for more information.

In order to use B<volcreate-logs> to manage a set of volumes, the volumes
must use standard directory structure and naming conventions.  This means
that under some root log directory, volumes must be organized like:

    ROOT/<year>/<month>

where <year> is the four-digit year and <month> is the two-digit month
(with zero-padding where necessary).  If there is a separate volume for
each month, those volumes are mounted at locations that look like the
above; if there is only a single volume for each year's worth of logs,
that volume is mounted at ROOT/<year> and will have subdirectories for
each month created automatically.  The log volumes are named
C<logs.service.YYYYMM> where C<service> is some string representing the
service being logged and C<YYYYMM> is the four-digit year and two-digit
month.  If the volume only holds logs for one year, the volume will end
with C<YYYY> instead.

New volumes will be created for the following month on any date after the
twentieth of the month (log volumes will only be created if the
appropriate directory does not already exist).  Existing log volumes will
have their quota and usage checked.  If the quota is under the configured
size, it will be increased to match.  If the usage is over 90%, the volume
quota will be increased by 10%, rounded up to the nearest 10MB.

=head1 OPTIONS

=over 4

=item B<-h>, B<--help>

Print out this documentation (which is done simply by feeding the script
to C<perldoc -t>).

=item B<-k> I<path>, B<--ticket-cache>=I<path>

Indicates that the Kerberos v4 ticket cache found at I<path> should be
used to obtain an AFS token.  If this option is used, the AFS::Utils Perl
module must be available.  A new PAG will be created for the script, the
K4 ticket cache set to I<path>, and B<aklog> will be run to get a token.

If I<path> is relative, it is assumed to be relative to F</var/run> (or
whatever path is set in the $LOGS_TICKETS configuration variable).

=item B<-m>, B<--mail>

Report via e-mail rather than to standard out.  Sends an e-mail report to
the address set in the $LOGS_ADDRESS configuration variable, which must be
set.  See L<CONFIGURATION> below.

=item B<-n>, B<--dry-run>, B<--just-print>

Don't take any action other than inspection.  Instead, just print to
standard out each command that would be executed.

=item B<-q>, B<--quiet>

Suppress any additional output other than the one line per action taken
(done mostly by adding the B<-q> option to B<volcreate>).

=item B<-v>, B<--version>

Print out the version of B<volcreate-logs> and exit.

=back

=head1 CONFIGURATION

=head2 General Settings

B<volcreate-logs> loads configuration settings from
F</etc/afs-admin-tools/config> if that file exists.  If it exists, it must
be Perl code suitable for loading with C<require>.  This means that each
line of the configuration file should be of the form:

    our $VARIABLE = VALUE;

where C<$VARIABLE> is the configuration variable being set and C<VALUE> is
the value to set it to (which should be enclosed in quotes if it's not a
number).  The file should end with:

    1;

so that Perl knows the file was loaded correctly.

The supported configuration variables are:

=over 4

=item $LOGS_ADDRESS

The address to which to send mailed reports.  If the B<-m> option is
given, this configuration variable must be set.

=item $LOGS_CONFIG

The configuration file specifying what log volumes to manage.  See L<Log
Volumes> below for the syntax.  The default path is
F</etc/afs-admin-tools/log-volumes>.

=item $LOGS_TICKETS

The default root path for ticket files, used for the B<-k> option if a
relative path to a ticket file was given.  The default value is
F</var/run>.

=item $CELL_ROOT

The root of the AFS cell.  It not specified then /afs/@cell is used.

=item $LOGS_MAX_QUOTA

The maximum quota value that will be set.  If not specified then
the default value is 50G, i.e. 50 gigabytes.

=item $AKLOG

The full path to B<aklog>, used to obtain AFS tokens from a Kerberos
ticket if the B<-k> option was given.  If this variable is not set,
B<volcreate-logs> defaults to looking for B<aklog> on the user's PATH.

=item $FS

The full path to the AFS B<fs> utility.  If this variable is not set,
B<volcreate-logs> defaults to looking for B<fs> on the user's PATH.

=item $VOLCREATE

The full path to the B<volcreate> utility.  If this variable is not set,
B<volcreate-logs> defaults to looking for B<volcreate> on the user's PATH.

=item $VOS

The full path to the AFS B<vos> utility.  If this variable is not set,
B<volcreate-logs> defaults to F</usr/local/sbin/vos> or F</usr/sbin/vos>
if they exist, and otherwise looks for B<vos> on the user's PATH.

=back

=head2 Log Volumes

The file set by the $LOGS_CONFIG configuration variable, defaulting to
F</etc/afs-admin-tools/log-volumes>, specifies the collections of log
volumes managed by this program.  The configuration will be used to
increase quota or create new log volumes as needed.  The syntax is:

    log {
        name: <volume-name>
        path: <base-path>
        type: monthly | yearly
        quota: <quota>[k|m|g|t]
    }

where <volume-name> is the base name of the volume (C<.YYYY> or C<.YYYYMM>
will be appended), <base-path> is the root of the log structure (its
immediate subdirectories must be the year directories), type is either
monthly to create a new volume for each month or yearly to create one
volume for each year of logs, and <quota> is the starting quota for each
volume in MB if the units are not specified.

Blank lines and lines beginning with C<#> are ignored.

=head1 EXAMPLES

Process all of the current rules, reporting actions to standard out.

    volcreate-logs

Do the same, but instead send the report via e-mail.

    volcreate-logs -m

Do the same, but obtain AFS credentials from F</var/run/lsdb.k5.tgt>:

    volcreate-logs -m -k lsdb.k5.tgt

Check to see what needs to be done, but don't actually do it.  Instead,
just print the commands that would be executed to standard out:

    volcreate-logs -n

Only check and reset quota or create new log volumes for the config file
entries with a name value of C<logs.cgi> or C<logs.www>:

    volcreate-logs logs.cgi logs.www

=head1 FILES

=over 4

=item F</etc/afs-admin-tools/log-volumes>

The default configuration file specifying the collections of log volumes
managed by this program.  The path to this file may be overridden by
setting the $LOGS_CONFIG configuration variable.

=back

=head1 AUTHOR

Original script by Russ Allbery <eagle@eyrie.org>. Updated with
Auristor support by Bill MacAllister <bill@ca-zephyr.org>.

=head1 COPYRIGHT AND LICENSE

Copyright 2002, 2004, 2011, 2012 The Board of Trustees of the Leland Stanford
Junior University.

Copyright 2018 Bill MacAllister <bill@ca-zephyr.org>

This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

=head1 SEE ALSO

L<newsyslog(1)>, L<volcreate(1)>

The current version of newsyslog is available from its web site at
L<http://www.eyrie.org/~eagle/software/newsyslog/>.

This script is part of the afs-admin-tools package.  The most recent
version is available from the afs-admin-tools web page at
L<http://www.eyrie.org/~eagle/software/afs-admin-tools/>.

=cut