Skip to content
This repository has been archived by the owner on Feb 15, 2023. It is now read-only.

Commit

Permalink
chore(release): merge release-21.10.next in 21.10.x (#269)
Browse files Browse the repository at this point in the history 
* fix(proxy): close zmq internal connections (#163)

* fix(proxy): uninitialized value in modulus log (#164)

MON-11593

* security(core): rsa keys no more world readable (#165)

* fix(core): signal race condition during stop (#166)

* fix(core): signal race condition during stop - should be ok now (#167)

* fix(autodisco): handle advanced option for service discovery - use Safe for security (#170)

* add(package): centreon audit module configuration (#171)

* add(package): centreon audit script (#172)

* chore(install): update to version to 21.10.1 (#175)

* add(module): new httpserver (#173)

* enh(httpserverng): manage not installed basic_auth_plus (#177)

* fix(core): wrong module management (#178)

* fix(proxy): pull mode communication crash (#179)

* fix(secu): add SQ pipeline timeout (#182)

* fix(chore): use github action env usage (#183)

* fix(httpserver): timeout and parse token (#186)

* fix(contrib): gorgone_audit.pl unitialized value (#187)

* enh(audit): better system detection (#188)

* enh(audit): better system detection 2 (#189)

* fix(build): disable centos8 packaging (#193)

* fix(build): disable centos8 packaging

* Update Jenkinsfile

* fix(chore): dependabot github automation (#200)

* chore(install): update version to 21.10.2 (#201)

* enable el8 build and packaging 21.10.x (#210)

* enable el8 build and packaging 21.10.x

* Update Jenkinsfile

* MON-12908: add centreon-mbi etl optimized (#220)

* MON-12908: add centreon-mbi deps Try::Tiny (#222)

* MON-12908: centreon-mbi etl - fix daily option (#224)

* backport mbi-etl (#225)

* MON-12908: centreon-mbi etl - fix monthly perfdata (#226)

* fix(ad): use string for ml config label values (#230)

* fix(mbi): manage mariadb 10.3.x (#242)

* enh(mbi): add etl kill action (#246)

* enh(mbi): add etl status action (#247)

* enh(legacy): cbd reload after remote-server import (#232)

* fix(mbi): stuck if there is an issue during the planning (#250)

* (action) add tar_insecure_extra_mode option (#254)

* chore(install): update version to 21.10.3

Co-authored-by: qgarnier <[email protected]>
Co-authored-by: Charles Gautier <[email protected]>
Co-authored-by: sc979 <[email protected]>
Co-authored-by: Thomas Arnaud <[email protected]>
  • Loading branch information
@tuntoja @garnier-quentin
@chgautier @sc979 @Nohzoh
5 people authored Sep 23, 2022
1 parent 2741ccc commit 8eb227e
Show file tree
Hide file tree
Showing 63 changed files with 11,046 additions and 18 deletions.
382 changes: 382 additions & 0 deletions contrib/mbi/centreonBIETL
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,382 @@
#!/usr/bin/perl

use warnings;
use strict;
use FindBin;
use lib "$FindBin::Bin";
# to be launched from contrib directory
use lib "$FindBin::Bin/../";

gorgone::script::centreonBIETL->new()->run();

package gorgone::script::centreonBIETL;

use strict;
use warnings;
use Data::Dumper;
use gorgone::modules::centreon::mbi::libs::Utils;
use gorgone::standard::misc;
use gorgone::class::http::http;
use JSON::XS;

use base qw(gorgone::class::script);

sub new {
my $class = shift;
my $self = $class->SUPER::new(
'centreonBIETL',
centreon_db_conn => 0,
centstorage_db_conn => 0,
noconfig => 0
);

bless $self, $class;

$self->{moptions}->{rebuild} = 0;
$self->{moptions}->{daily} = 0;
$self->{moptions}->{import} = 0;
$self->{moptions}->{dimensions} = 0;
$self->{moptions}->{event} = 0;
$self->{moptions}->{perfdata} = 0;
$self->{moptions}->{start} = '';
$self->{moptions}->{end} = '';
$self->{moptions}->{create_tables} = 0;
$self->{moptions}->{ignore_databin} = 0;
$self->{moptions}->{centreon_only} = 0;
$self->{moptions}->{nopurge} = 0;

$self->add_options(
'url:s' => \$self->{url},
'status' => \$self->{status},
'r' => \$self->{moptions}->{rebuild},
'd' => \$self->{moptions}->{daily},
'I' => \$self->{moptions}->{import},
'D' => \$self->{moptions}->{dimensions},
'E' => \$self->{moptions}->{event},
'P' => \$self->{moptions}->{perfdata},
's:s' => \$self->{moptions}->{start},
'e:s' => \$self->{moptions}->{end},
'c' => \$self->{moptions}->{create_tables},
'i' => \$self->{moptions}->{ignore_databin},
'C' => \$self->{moptions}->{centreon_only},
'p' => \$self->{moptions}->{nopurge}
);
return $self;
}

sub init {
my $self = shift;
$self->SUPER::init();

$self->{url} = 'http://127.0.0.1:8085' if (!defined($self->{url}) || $self->{url} eq '');
$self->{http} = gorgone::class::http::http->new(logger => $self->{logger});

return if (defined($self->{status}));

my $utils = gorgone::modules::centreon::mbi::libs::Utils->new($self->{logger});
if ($utils->checkBasicOptions($self->{moptions}) == 1) {
exit(1);
}

if ($self->{moptions}->{create_tables} == 0 &&
$self->{moptions}->{import} == 0 &&
$self->{moptions}->{dimensions} == 0 &&
$self->{moptions}->{event} == 0 &&
$self->{moptions}->{perfdata} == 0) {
$self->{moptions}->{import} = 1;
$self->{moptions}->{dimensions} = 1;
$self->{moptions}->{event} = 1;
$self->{moptions}->{perfdata} = 1;
}
}

sub json_decode {
my ($self, %options) = @_;

my $decoded;
eval {
$decoded = JSON::XS->new->utf8->decode($options{content});
};
if ($@) {
$self->{logger}->writeLogError("cannot decode json response: $@");
exit(1);
}

return $decoded;
}

sub run_etl {
my ($self) = @_;

my ($code, $content) = $self->{http}->request(
http_backend => 'curl',
method => 'POST',
hostname => '',
full_url => $self->{url} . '/api/centreon/mbietl/run',
query_form_post => JSON::XS->new->utf8->encode($self->{moptions}),
header => [
'Accept-Type: application/json; charset=utf-8',
'Content-Type: application/json; charset=utf-8',
],
curl_opt => ['CURLOPT_SSL_VERIFYPEER => 0', 'CURLOPT_POSTREDIR => CURL_REDIR_POST_ALL'],
warning_status => '',
unknown_status => '',
critical_status => ''
);

if ($self->{http}->get_code() < 200 || $self->{http}->get_code() >= 300) {
$self->{logger}->writeLogError("Login error [code: '" . $self->{http}->get_code() . "'] [message: '" . $self->{http}->get_message() . "']");
exit(1);
}

my $decoded = $self->json_decode(content => $content);
if (!defined($decoded->{token})) {
$self->{logger}->writeLogError('cannot get token');
exit(1);
}

$self->{token} = $decoded->{token};
}

sub display_messages {
my ($self, %options) = @_;

if (defined($options{data}->{messages})) {
foreach (@{$options{data}->{messages}}) {
if ($_->[0] eq 'D') {
$self->{logger}->writeLogDebug($_->[1])
} elsif ($_->[0] eq 'I') {
$self->{logger}->writeLogInfo($_->[1]);
} elsif ($_->[0] eq 'E') {
$self->{logger}->writeLogError($_->[1]);
}
}
}
}

sub get_etl_log {
my ($self) = @_;

my $log_id;
while (1) {
my $get_param = [];
if (defined($log_id)) {
$get_param = ['id=' . $log_id];
}

my ($code, $content) = $self->{http}->request(
http_backend => 'curl',
method => 'GET',
hostname => '',
full_url => $self->{url} . '/api/log/' . $self->{token},
get_param => $get_param,
header => [
'Accept-Type: application/json; charset=utf-8'
],
curl_opt => ['CURLOPT_SSL_VERIFYPEER => 0', 'CURLOPT_POSTREDIR => CURL_REDIR_POST_ALL'],
warning_status => '',
unknown_status => '',
critical_status => ''
);

if ($self->{http}->get_code() < 200 || $self->{http}->get_code() >= 300) {
$self->{logger}->writeLogError("Login error [code: '" . $self->{http}->get_code() . "'] [message: '" . $self->{http}->get_message() . "']");
exit(1);
}

my $decoded = $self->json_decode(content => $content);
if (!defined($decoded->{data})) {
$self->{logger}->writeLogError("Cannot get log information");
exit(1);
}

my $stop = 0;
foreach (@{$decoded->{data}}) {
my $data = $self->json_decode(content => $_->{data});
next if (defined($log_id) && $log_id >= $_->{id});
$log_id = $_->{id};

if ($_->{code} == 600) {
$self->display_messages(data => $data);
} elsif ($_->{code} == 1) {
$self->display_messages(data => $data);
$stop = 1;
} elsif ($_->{code} == 2) {
$self->display_messages(data => $data);
$stop = 1;
}
}

last if ($stop == 1);
sleep(2);
}
}

sub get_etl_status {
my ($self) = @_;

my ($code, $content) = $self->{http}->request(
http_backend => 'curl',
method => 'GET',
hostname => '',
full_url => $self->{url} . '/api/centreon/mbietl/status',
header => [
'Accept-Type: application/json; charset=utf-8',
'Content-Type: application/json; charset=utf-8',
],
curl_opt => ['CURLOPT_SSL_VERIFYPEER => 0', 'CURLOPT_POSTREDIR => CURL_REDIR_POST_ALL'],
warning_status => '',
unknown_status => '',
critical_status => ''
);

if ($self->{http}->get_code() < 200 || $self->{http}->get_code() >= 300) {
$self->{logger}->writeLogError("Login error [code: '" . $self->{http}->get_code() . "'] [message: '" . $self->{http}->get_message() . "']");
exit(1);
}

my $decoded = $self->json_decode(content => $content);
if (!defined($decoded->{token})) {
$self->{logger}->writeLogError('cannot get token');
exit(1);
}

my $token = $decoded->{token};
my $log_id;
my $result;

while (1) {
my $get_param = [];
if (defined($log_id)) {
$get_param = ['id=' . $log_id];
}

my ($code, $content) = $self->{http}->request(
http_backend => 'curl',
method => 'GET',
hostname => '',
full_url => $self->{url} . '/api/log/' . $token,
get_param => $get_param,
header => [
'Accept-Type: application/json; charset=utf-8'
],
curl_opt => ['CURLOPT_SSL_VERIFYPEER => 0', 'CURLOPT_POSTREDIR => CURL_REDIR_POST_ALL'],
warning_status => '',
unknown_status => '',
critical_status => ''
);

if ($self->{http}->get_code() < 200 || $self->{http}->get_code() >= 300) {
$self->{logger}->writeLogError("Login error [code: '" . $self->{http}->get_code() . "'] [message: '" . $self->{http}->get_message() . "']");
exit(1);
}

my $decoded = $self->json_decode(content => $content);
if (!defined($decoded->{data})) {
$self->{logger}->writeLogError("Cannot get log information");
exit(1);
}

my $stop = 0;
foreach (@{$decoded->{data}}) {
my $data = $self->json_decode(content => $_->{data});
next if (defined($log_id) && $log_id >= $_->{id});
$log_id = $_->{id};

if ($_->{code} == 1) {
$self->{logger}->writeLogError('cannot get etl status');
exit(1);
} elsif ($_->{code} == 2) {
$result = $data;
$stop = 1;
}
}

last if ($stop == 1);
sleep(2);
}

print "ETL status: $result->{statusStr}\n";
if ($result->{statusStr} ne 'ready') {
print "planning: $result->{planningStr}\n";
foreach ('import', 'dimensions', 'event', 'perfdata') {
next if (!defined($result->{sections}->{$_}));

print " $_ status: $result->{sections}->{$_}->{statusStr}";
if (defined($result->{sections}->{$_}->{steps_total})) {
print " ($result->{sections}->{$_}->{steps_executed}/$result->{sections}->{$_}->{steps_total})";
}
print "\n";
}
}
}

sub run {
my $self = shift;

$self->SUPER::run();

if (defined($self->{status})) {
$self->get_etl_status();
} else {
$self->run_etl();
$self->get_etl_log();
}
}

__END__
=head1 NAME
centreonBIETL - script to execute mbi etl
=head1 SYNOPSIS
centreonBIETL [options]
=head1 OPTIONS
=over 8
=item B<--url>
Specify the api url (default: 'http://127.0.0.1:8085').
=item B<--severity>
Set the script log severity (default: 'info').
=item B<--help>
Print a brief help message and exits.
Execution modes
-c Create the reporting database model
-d Daily execution to calculate statistics on yesterday
-r Rebuild mode to calculate statitics on a historical period. Can be used with:
Extra arguments for options -d and -r (if none of the following is specified, these one are selected by default: -IDEP):
-I Extract data from the monitoring server
Extra arguments for option -I:
-C Extract only Centreon configuration database only. Works with option -I.
-i Ignore perfdata extraction from monitoring server
-o Extract only perfdata from monitoring server
-D Calculate dimensions
-E Calculate event and availability statistics
-P Calculate perfdata statistics
Common options for -rIDEP:
-s Start date in format YYYY-MM-DD.
By default, the program uses the data retention period from Centreon BI configuration
-e End date in format YYYY-MM-DD.
By default, the program uses the data retention period from Centreon BI configuration
-p Do not empty statistic tables, delete only entries for the processed period.
Does not work on raw data tables, only on Centreon BI statistics tables.
=back
=head1 DESCRIPTION
B<centreonBIETL>
=cut
Loading

0 comments on commit 8eb227e

Please sign in to comment.