From 1a86882e58cda9338549e0ef6020ed29452250c1 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 7 Sep 2021 14:46:28 +0200 Subject: [PATCH 01/15] ARGO-3287 Fetch service URL from GOCDB --- src/modules/NCG/SiteInfo/GOCDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/NCG/SiteInfo/GOCDB.pm b/src/modules/NCG/SiteInfo/GOCDB.pm index 698c956..f308b31 100644 --- a/src/modules/NCG/SiteInfo/GOCDB.pm +++ b/src/modules/NCG/SiteInfo/GOCDB.pm @@ -212,6 +212,8 @@ sub getData { } $self->{SITEDB}->hostAttribute($hostname, 'URL', $value); $self->{SITEDB}->hostAttribute($hostname, $serviceType."_URL", $value); + $self->{SITEDB}->hostAttribute($hostname, 'GOCDB_SERVICE_URL', $value); + $self->{SITEDB}->hostAttribute($hostname, $serviceType."_GOCDB_SERVICE_URL", $value); } } } From 27dd3a292a94392b2d563cef6b65212c00f15baf Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 9 Sep 2021 11:32:32 +0200 Subject: [PATCH 02/15] ARGO-3287 Fetch service URL from GOCDB --- src/modules/NCG/SiteInfo/GOCDB.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/NCG/SiteInfo/GOCDB.pm b/src/modules/NCG/SiteInfo/GOCDB.pm index f308b31..829ec0f 100644 --- a/src/modules/NCG/SiteInfo/GOCDB.pm +++ b/src/modules/NCG/SiteInfo/GOCDB.pm @@ -197,6 +197,7 @@ sub getData { } } foreach $elem ($site->getElementsByTagName("URL")) { + next unless ($elem->getParentNode->getTagName() eq "SERVICE_ENDPOINT"); my $child = $elem->getFirstChild; if ($child) { my $value = $child->getNodeValue(); @@ -219,7 +220,7 @@ sub getData { } foreach my $endpoint ($site->getElementsByTagName("ENDPOINT")) { my $monitored = ""; - foreach $elem ($site->getElementsByTagName("ENDPOINT_MONITORED")) { + foreach $elem ($endpoint->getElementsByTagName("ENDPOINT_MONITORED")) { my $value = $elem->getFirstChild->getNodeValue(); if ($value) { $monitored = $value; From f8c59cc10d0d3acb98bacd786499214b63613956 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 21 Sep 2021 17:03:24 +0200 Subject: [PATCH 03/15] ARGO-3328 lots of Missing in EOSC-PORTAL --- src/modules/NCG/SiteInfo/EOSC.pm | 1 + src/modules/NCG/SiteSet/EOSC.pm | 1 + 2 files changed, 2 insertions(+) diff --git a/src/modules/NCG/SiteInfo/EOSC.pm b/src/modules/NCG/SiteInfo/EOSC.pm index 53cb6d7..8ac067b 100644 --- a/src/modules/NCG/SiteInfo/EOSC.pm +++ b/src/modules/NCG/SiteInfo/EOSC.pm @@ -109,6 +109,7 @@ sub getData my $url; my $hostname; my $serviceSiteName = $service->{'SITENAME-SERVICEGROUP'} || next; + $serviceSiteName =~ s/\,//g; next if ( $serviceSiteName ne $sitename ); if ( !$service->{'Service Unique ID'} || !$service->{SERVICE_TYPE} ) { $self->error("Entry for $serviceSiteName is missing 'Service Unique ID' and/or 'SERVICE_TYPE'"); diff --git a/src/modules/NCG/SiteSet/EOSC.pm b/src/modules/NCG/SiteSet/EOSC.pm index a420eac..308e6db 100644 --- a/src/modules/NCG/SiteSet/EOSC.pm +++ b/src/modules/NCG/SiteSet/EOSC.pm @@ -102,6 +102,7 @@ sub getData foreach my $site (@{$jsonRef}) { my $sitename = $site->{'SITENAME-SERVICEGROUP'}; + $sitename =~ s/\,//g; my $country = $site->{'COUNTRY_NAME'}; $self->verbose ("Found site: $sitename"); if (!exists $self->{SITES}->{$sitename}) { From 8bf16c3dbf2201d1f18b7d6323dfbec1f7e3179c Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 21 Sep 2021 17:20:43 +0200 Subject: [PATCH 04/15] ARGO-3328 lots of Missing in EOSC-PORTAL --- argo-ncg.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/argo-ncg.spec b/argo-ncg.spec index fc9c0db..c1d226a 100644 --- a/argo-ncg.spec +++ b/argo-ncg.spec @@ -15,6 +15,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root BuildArch: noarch Requires: perl-libwww-perl > 5.833-2 Requires: psmisc +Requires: perl(JSON::XS) %if 0%{?el7:1} Requires: perl(LWP::Protocol::https) %endif From cb2dd016c6359e837ba94f039d498f191321364c Mon Sep 17 00:00:00 2001 From: eimamagi Date: Fri, 22 Oct 2021 03:18:08 +0200 Subject: [PATCH 05/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/LocalMetrics/WEBAPI.pm | 32 ++-- src/modules/NCG/SiteSet/WEBAPI.pm | 199 +++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 src/modules/NCG/SiteSet/WEBAPI.pm diff --git a/src/modules/NCG/LocalMetrics/WEBAPI.pm b/src/modules/NCG/LocalMetrics/WEBAPI.pm index f76bbf3..0345508 100644 --- a/src/modules/NCG/LocalMetrics/WEBAPI.pm +++ b/src/modules/NCG/LocalMetrics/WEBAPI.pm @@ -55,6 +55,9 @@ sub new if (! $self->{VO}) { $self->{VO} = 'ops'; } + if (! exists $self->{TIMEOUT}) { + $self->{TIMEOUT} = $self->{DEFAULT_HTTP_TIMEOUT}; + } $self; } @@ -167,18 +170,18 @@ sub getData { =head1 NAME -NCG::LocalMetrics::POEM +NCG::LocalMetrics::WEBAPI =head1 DESCRIPTION -The NCG::LocalMetrics::POEM module extends NCG::LocalMetrics module. -Module extracts metric information from hard-coded POEM. +The NCG::LocalMetrics::WEBAPI module extends NCG::LocalMetrics module. +Module extracts metric information from ARGO WEBAPI. =head1 SYNOPSIS - use NCG::LocalMetrics::POEM; + use NCG::LocalMetrics::WEBAPI; - my $lms = NCG::LocalMetrics::POEM->new( { SITEDB=> $sitedb} ); + my $lms = NCG::LocalMetrics::WEBAPI->new( { SITEDB=> $sitedb} ); $lms->getData(); @@ -190,19 +193,24 @@ Module extracts metric information from hard-coded POEM. =item C - $siteInfo = NCG::LocalMetrics::POEM->new( $options ); + $siteInfo = NCG::LocalMetrics::WEBAPI->new( $options ); -Creates new NCG::LocalMetrics::POEM instance. Argument $options is hash +Creates new NCG::LocalMetrics::WEBAPI instance. Argument $options is hash reference that can contain following elements: - POEM_FILE - file containing JSON definition - (default: ) - - POEM_ROOT_URL - POEM JSON API root URL - (default: http://localhost/poem_sync) + WEBAPI_ROOT_URL - WEBAPI JSON API root URL + (default: https://api.argo.grnet.gr/) + PROFILES - list of profiles for which metrics will be fetched + + VO - VO to be associated with profiles by default (EGI-specific) + (default: ops) + METRIC_CONFIG - metric configuration structure fetched from NCG::MetricConfig module + TIMEOUT - HTTP timeout + - default: DEFAULT_HTTP_TIMEOUT inherited from NCG + TOKEN - token used for POEM API authentication (default: ) diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm new file mode 100644 index 0000000..5c16380 --- /dev/null +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -0,0 +1,199 @@ +#!/usr/bin/perl -w +# +# Nagios configuration generator (WLCG probe based) +# Copyright (c) 2007 Emir Imamagic +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package NCG::SiteSet::WEBAPI; + +use strict; +use warnings; +use NCG::SiteSet; +use vars qw(@ISA); +use JSON; +use LWP::UserAgent; + +@ISA=("NCG::SiteSet"); + +my $DEFAULT_WEBAPI_ROOT_URL = "https://api.argo.grnet.gr/"; +my $DEFAULT_WEBAPI_ROOT_URL_SUFFIX = "/api/v2/topology/groups"; + +sub new +{ + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = $class->SUPER::new(@_); + # set default values + if (! $self->{WEBAPI_ROOT_URL}) { + $self->{WEBAPI_ROOT_URL} = $DEFAULT_WEBAPI_ROOT_URL; + } + + if (! exists $self->{SITE_MONITORED}) { + $self->{SITE_MONITORED} = 'Y'; + } + + if (! exists $self->{PROD_STATUS}) { + $self->{PROD_STATUS} = 'Production'; + } + + if (! exists $self->{CERT_STATUS}) { + $self->{CERT_STATUS} = 'Certified'; + } + + if (! $self->{TOKEN}) { + $self->error("Authentication token must be defined."); + return; + } + if (! exists $self->{TIMEOUT}) { + $self->{TIMEOUT} = $self->{DEFAULT_HTTP_TIMEOUT}; + } + + $self; +} + +sub getData { + my $self = shift; + my $sitename = shift || $self->{SITENAME}; + my $poemService = {}; + my $url; + + my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); + $ua->agent("NCG::SiteSet::WEBAPI"); + $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; + $url .= '?type=NGI'; + my @tags; + if ($self->{CERT_STATUS}) { + push @tags, 'certification:' . $self->{CERT_STATUS}; + } + if ($self->{PROD_STATUS}) { + push @tags, 'infrastructure:' . $self->{PROD_STATUS}; + } + if ($self->{SCOPE}) { + push @tags, 'scope:' . $self->{SCOPE}; + } + if (@tags) { + $url .= '&tags='.join(',',@tags); + } + my $req = HTTP::Request->new(GET => $url); + $req->header('x-api-key' => $self->{TOKEN}); + $req->header('Accept' => 'application/json'); + $req->header('Content-Type' => 'application/json'); + my $res = $self->safeHTTPSCall($ua,$req); + if (!$res->is_success) { + $self->error("Could not get results from WEBAPI: ".$res->status_line); + return; + } + + my $jsonRef; + eval { + $jsonRef = from_json($res->content); + }; + if ($@) { + $self->error("Error parsing JSON response: ".$@); + return; + } + if ( !$jsonRef ) { + $self->error("JSON response is empty"); + return; + } + if ( ref $jsonRef ne "HASH" ) { + $self->error("JSON response is not a hash:".$res->content); + return; + } + if ( ! exists $jsonRef->{status} ) { + $self->error("JSON response doesn't contain status hash:".$res->content); + return; + } + if ( ! exists $jsonRef->{status}->{code} ) { + $self->error("JSON response doesn't contain status code:".$res->content); + return; + } + if ( $jsonRef->{status}->{code} ne '200' ) { + $self->error("Status code is not equal 200:".$res->content); + return; + } + + foreach my $site (@{$jsonRef->{data}}) { + my $sitename = $site->{subgroup}; + my $roc = $site->{group}; + + $self->verbose ("Found site: $sitename."); + + if (!exists $self->{SITES}->{$sitename}) { + $self->{SITES}->{$sitename} = NCG::SiteDB->new ({SITENAME=>$sitename, ROC=>$roc}); + } else { + $self->{SITES}->{$sitename}->siteROC($roc) if ($roc); + } + } + + 1; +} + + +=head1 NAME + +NCG::SiteSet::WEBAPI + +=head1 DESCRIPTION + +The NCG::SiteSet::WEBAPI module extends NCG::SiteSet module. +Module extracts list of sites from ARGO WEBAPI component. + +=head1 SYNOPSIS + + use NCG::SiteSet::WEBAPI; + + my $lms = NCG::SiteSet::WEBAPI->new( { SITEDB=> $sitedb} ); + + $lms->getData(); + +=cut + +=head1 METHODS + +=over + +=item C + + $siteInfo = NCG::SiteSet::WEBAPI->new( $options ); + +Creates new NCG::SiteSet::WEBAPI instance. Argument $options is hash +reference that can contain following elements: + WEBAPI_ROOT_URL - WEBAPI JSON API root URL + (default: https://api.argo.grnet.gr) + + PROD_STATUS - production status of site + (default: Production) + + CERT_STATUS - certification status of site + (default: Certified) + + SCOPE - scope of sites + (default: ) + + TIMEOUT - HTTP timeout + (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) + + TOKEN - token used for WEBAPI API authentication + (default: ) + +=back + +=head1 SEE ALSO + +NCG::SiteSet + +=cut + +1; From 7537fc080db32b91c6db2585cf73ea092534a4df Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 2 Nov 2021 13:24:49 +0100 Subject: [PATCH 06/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 215 +++++++++++++++++++++++++++++ src/modules/NCG/SiteSet/WEBAPI.pm | 2 - 2 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 src/modules/NCG/SiteInfo/WEBAPI.pm diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm new file mode 100644 index 0000000..6d91ea5 --- /dev/null +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -0,0 +1,215 @@ +#!/usr/bin/perl -w +# +# Nagios configuration generator (WLCG probe based) +# Copyright (c) 2007 Emir Imamagic +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package NCG::SiteInfo::WEBAPI; + +use strict; +use warnings; +use NCG::SiteInfo; +use vars qw(@ISA); +use JSON; +use LWP::UserAgent; + +@ISA=("NCG::SiteInfo"); + +my $DEFAULT_WEBAPI_ROOT_URL = "https://api.argo.grnet.gr/"; +my $DEFAULT_WEBAPI_ROOT_URL_SUFFIX = "/api/v2/topology/endpoints"; + +sub new +{ + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = $class->SUPER::new(@_); + # set default values + if (! $self->{WEBAPI_ROOT_URL}) { + $self->{WEBAPI_ROOT_URL} = $DEFAULT_WEBAPI_ROOT_URL; + } + + if (! exists $self->{NODE_MONITORED}) { + $self->{NODE_MONITORED} = '1'; + } + + if (! $self->{TOKEN}) { + $self->error("Authentication token must be defined."); + return; + } + if (! exists $self->{TIMEOUT}) { + $self->{TIMEOUT} = $self->{DEFAULT_HTTP_TIMEOUT}; + } + + if (! exists $self->{VO}) { + $self->{VO} = 'ops'; + } + + $self; +} + +sub getData { + my $self = shift; + my $sitename = shift || $self->{SITENAME}; + my $poemService = {}; + my $url; + + my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); + $ua->agent("NCG::SiteInfo::WEBAPI"); + $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; + $url .= '?type=SITES&group='.$sitename; + my @tags; + if ($self->{NODE_MONITORED}) { + push @tags, 'monitored:' . $self->{NODE_MONITORED}; + } + if ($self->{SCOPE}) { + push @tags, 'scope:' . $self->{SCOPE}; + } + if (@tags) { + $url .= '&tags='.join(',',@tags); + } + my $req = HTTP::Request->new(GET => $url); + $req->header('x-api-key' => $self->{TOKEN}); + $req->header('Accept' => 'application/json'); + $req->header('Content-Type' => 'application/json'); + my $res = $self->safeHTTPSCall($ua,$req); + if (!$res->is_success) { + $self->error("Could not get results from WEBAPI: ".$res->status_line); + return; + } + + my $jsonRef; + eval { + $jsonRef = from_json($res->content); + }; + if ($@) { + $self->error("Error parsing JSON response: ".$@); + return; + } + if ( !$jsonRef ) { + $self->error("JSON response is empty"); + return; + } + if ( ref $jsonRef ne "HASH" ) { + $self->error("JSON response is not a hash:".$res->content); + return; + } + if ( ! exists $jsonRef->{status} ) { + $self->error("JSON response doesn't contain status hash:".$res->content); + return; + } + if ( ! exists $jsonRef->{status}->{code} ) { + $self->error("JSON response doesn't contain status code:".$res->content); + return; + } + if ( $jsonRef->{status}->{code} ne '200' ) { + $self->error("Status code is not equal 200:".$res->content); + return; + } + + foreach my $service (@{$jsonRef->{data}}) { + my $sitename = $service->{group}; + my $hostname = $service->{hostname}; + my $serviceType = $service->{service}; + + $self->{SITEDB}->addHost($hostname); + $self->{SITEDB}->addService($hostname, $serviceType); + $self->{SITEDB}->addVO($hostname, $serviceType, $self->{VO}); + $self->{SITEDB}->siteLDAP($hostname) if ($serviceType eq 'Site-BDII'); + + foreach my $tag (keys %{$service->{tags}}) { + if ( $tag =~ /^info_ext_(\S+)$/i ) { + $self->{SITEDB}->hostAttribute($hostname, $1, $service->{tags}->{$tag}); + } elsif ( $tag eq 'info_URL' ) { + my $url; + my $value = $service->{tags}->{$tag}; + eval { + $url = url($value); + }; + unless ($@) { + eval { $self->{SITEDB}->hostAttribute($hostname, 'PORT', $url->port) if ($url->port); }; + eval { $self->{SITEDB}->hostAttribute($hostname, 'PATH', $url->path) if ($url->path); }; + eval { $self->{SITEDB}->hostAttribute($hostname, 'SSL', 0) if ($url->scheme && $url->scheme eq 'https'); }; + } + $self->{SITEDB}->hostAttribute($hostname, 'URL', $value); + $self->{SITEDB}->hostAttribute($hostname, $serviceType."_URL", $value); + $self->{SITEDB}->hostAttribute($hostname, 'GOCDB_SERVICE_URL', $value); + $self->{SITEDB}->hostAttribute($hostname, $serviceType."_GOCDB_SERVICE_URL", $value); + } elsif ( $tag eq 'info_service_endpoint_URL' ) { + foreach my $url ( split (/, /, $service->{tags}->{$tag}) ) { + $self->{SITEDB}->hostAttribute($hostname, $serviceType."_URL", $url); + } + } + } + } + + 1; +} + + +=head1 NAME + +NCG::SiteInfo::WEBAPI + +=head1 DESCRIPTION + +The NCG::SiteInfo::WEBAPI module extends NCG::SiteInfo module. +Module extracts list of sites from ARGO WEBAPI component. + +=head1 SYNOPSIS + + use NCG::SiteInfo::WEBAPI; + + my $lms = NCG::SiteInfo::WEBAPI->new( { SITEDB=> $sitedb} ); + + $lms->getData(); + +=cut + +=head1 METHODS + +=over + +=item C + + $siteInfo = NCG::SiteInfo::WEBAPI->new( $options ); + +Creates new NCG::SiteInfo::WEBAPI instance. Argument $options is hash +reference that can contain following elements: + WEBAPI_ROOT_URL - WEBAPI JSON API root URL + (default: https://api.argo.grnet.gr) + + PROD_STATUS - production status of site + (default: Production) + + CERT_STATUS - certification status of site + (default: Certified) + + SCOPE - scope of sites + (default: ) + + TIMEOUT - HTTP timeout + (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) + + TOKEN - token used for WEBAPI API authentication + (default: ) + +=back + +=head1 SEE ALSO + +NCG::SiteInfo + +=cut + +1; diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index 5c16380..322236b 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -64,8 +64,6 @@ sub new sub getData { my $self = shift; - my $sitename = shift || $self->{SITENAME}; - my $poemService = {}; my $url; my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); From 9cad7225344ca31d002dda7fb402e487bfe5283b Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 9 Nov 2021 09:03:47 +0100 Subject: [PATCH 07/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 2 +- src/modules/NCG/SiteSet/WEBAPI.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 6d91ea5..1cef41e 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -73,7 +73,7 @@ sub getData { push @tags, 'monitored:' . $self->{NODE_MONITORED}; } if ($self->{SCOPE}) { - push @tags, 'scope:' . $self->{SCOPE}; + push @tags, 'scope:*'.$self->{SCOPE}.'*'; } if (@tags) { $url .= '&tags='.join(',',@tags); diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index 322236b..98110a6 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -78,7 +78,7 @@ sub getData { push @tags, 'infrastructure:' . $self->{PROD_STATUS}; } if ($self->{SCOPE}) { - push @tags, 'scope:' . $self->{SCOPE}; + push @tags, 'scope:*'.$self->{SCOPE}.'*'; } if (@tags) { $url .= '&tags='.join(',',@tags); From 7858a239f5a281a42a328cc5ef28a9e6197016ff Mon Sep 17 00:00:00 2001 From: eimamagi Date: Fri, 12 Nov 2021 02:00:18 +0100 Subject: [PATCH 08/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 1cef41e..2ed72b9 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -23,6 +23,7 @@ use NCG::SiteInfo; use vars qw(@ISA); use JSON; use LWP::UserAgent; +use URI::URL; @ISA=("NCG::SiteInfo"); From 9f6e162a426dd386ec847cf1888d5b0dc7f82bdf Mon Sep 17 00:00:00 2001 From: eimamagi Date: Fri, 26 Nov 2021 06:01:50 +0100 Subject: [PATCH 09/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 11 +++++++++-- src/modules/NCG/SiteSet/WEBAPI.pm | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 2ed72b9..6cccba7 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -51,6 +51,9 @@ sub new if (! exists $self->{TIMEOUT}) { $self->{TIMEOUT} = $self->{DEFAULT_HTTP_TIMEOUT}; } + if (! exists $self->{TYPE}) { + $self->{TYPE} = 'SITES'; + } if (! exists $self->{VO}) { $self->{VO} = 'ops'; @@ -68,7 +71,7 @@ sub getData { my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); $ua->agent("NCG::SiteInfo::WEBAPI"); $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; - $url .= '?type=SITES&group='.$sitename; + $url .= '?type=' . $self->{TYPE} . '&group=' . $sitename; my @tags; if ($self->{NODE_MONITORED}) { push @tags, 'monitored:' . $self->{NODE_MONITORED}; @@ -131,7 +134,7 @@ sub getData { foreach my $tag (keys %{$service->{tags}}) { if ( $tag =~ /^info_ext_(\S+)$/i ) { $self->{SITEDB}->hostAttribute($hostname, $1, $service->{tags}->{$tag}); - } elsif ( $tag eq 'info_URL' ) { + } elsif ( $tag eq 'info_URL' || $tag eq 'info.URL' ) { my $url; my $value = $service->{tags}->{$tag}; eval { @@ -199,6 +202,10 @@ reference that can contain following elements: SCOPE - scope of sites (default: ) + TYPE - type of groups fetched from WEBAPI, EGI uses SITES, + most other tenants SERVICEGROUPS + (default: SITES) + TIMEOUT - HTTP timeout (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index 98110a6..890f584 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -51,6 +51,10 @@ sub new $self->{CERT_STATUS} = 'Certified'; } + if (! exists $self->{TYPE}) { + $self->{TYPE} = 'NGI'; + } + if (! $self->{TOKEN}) { $self->error("Authentication token must be defined."); return; @@ -69,7 +73,7 @@ sub getData { my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); $ua->agent("NCG::SiteSet::WEBAPI"); $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; - $url .= '?type=NGI'; + $url .= '?type' . $self->{TYPE}; my @tags; if ($self->{CERT_STATUS}) { push @tags, 'certification:' . $self->{CERT_STATUS}; @@ -180,6 +184,10 @@ reference that can contain following elements: SCOPE - scope of sites (default: ) + TYPE - type of groups fetched from WEBAPI, EGI uses NGI, + most other tenants PROJECT + (default: NGI) + TIMEOUT - HTTP timeout (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) From d8abea5bddd22fdf4ca1fedce4d951ba08c77699 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 9 Dec 2021 07:20:03 +0100 Subject: [PATCH 10/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 6cccba7..02aaedb 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -59,6 +59,10 @@ sub new $self->{VO} = 'ops'; } + if (! exists $self->{USE_IDS}) { + $self->{USE_IDS} = 0; + } + $self; } @@ -123,10 +127,20 @@ sub getData { foreach my $service (@{$jsonRef->{data}}) { my $sitename = $service->{group}; - my $hostname = $service->{hostname}; + my $hostname = $service->{hostname}; my $serviceType = $service->{service}; - $self->{SITEDB}->addHost($hostname); + if (exists $service->{tags}->{hostname}) { + $hostname = $service->{tags}->{hostname}; + } + + if ($self->{USE_IDS} && exists $service->{tags}->{info_id} && $service->{tags}->{info_id}) { + $self->{SITEDB}->addHost($hostname, $service->{tags}->{info_id}); + $hostname .= '_' . $service->{tags}->{info_id}; + } else { + $self->{SITEDB}->addHost($hostname); + } + $self->{SITEDB}->addService($hostname, $serviceType); $self->{SITEDB}->addVO($hostname, $serviceType, $self->{VO}); $self->{SITEDB}->siteLDAP($hostname) if ($serviceType eq 'Site-BDII'); @@ -153,6 +167,8 @@ sub getData { foreach my $url ( split (/, /, $service->{tags}->{$tag}) ) { $self->{SITEDB}->hostAttribute($hostname, $serviceType."_URL", $url); } + } elsif ( $tag =~ /^vo_(\S+?)_attr_(\S+)$/ ) { + $self->{SITEDB}->hostAttributeVO($hostname, $2, $1, $service->{tags}->{$tag}); } } } @@ -202,6 +218,9 @@ reference that can contain following elements: SCOPE - scope of sites (default: ) + USE_IDS - use IDs for Nagios hostnames + (default: false) + TYPE - type of groups fetched from WEBAPI, EGI uses SITES, most other tenants SERVICEGROUPS (default: SITES) From 3d4b78820363976f2816fe41e23587db7062b883 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 9 Dec 2021 07:26:08 +0100 Subject: [PATCH 11/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 18 ++++++++---------- src/modules/NCG/SiteSet/WEBAPI.pm | 25 +++++++------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 02aaedb..7f013a1 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -206,21 +206,13 @@ Module extracts list of sites from ARGO WEBAPI component. Creates new NCG::SiteInfo::WEBAPI instance. Argument $options is hash reference that can contain following elements: - WEBAPI_ROOT_URL - WEBAPI JSON API root URL - (default: https://api.argo.grnet.gr) - PROD_STATUS - production status of site - (default: Production) - - CERT_STATUS - certification status of site - (default: Certified) + NODE_MONITORED - filter only endpoints with tag monitored + (default: ) SCOPE - scope of sites (default: ) - USE_IDS - use IDs for Nagios hostnames - (default: false) - TYPE - type of groups fetched from WEBAPI, EGI uses SITES, most other tenants SERVICEGROUPS (default: SITES) @@ -231,6 +223,12 @@ reference that can contain following elements: TOKEN - token used for WEBAPI API authentication (default: ) + USE_IDS - use IDs for Nagios hostnames + (default: false) + + WEBAPI_ROOT_URL - WEBAPI JSON API root URL + (default: https://api.argo.grnet.gr) + =back =head1 SEE ALSO diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index 890f584..be82a91 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -39,18 +39,6 @@ sub new $self->{WEBAPI_ROOT_URL} = $DEFAULT_WEBAPI_ROOT_URL; } - if (! exists $self->{SITE_MONITORED}) { - $self->{SITE_MONITORED} = 'Y'; - } - - if (! exists $self->{PROD_STATUS}) { - $self->{PROD_STATUS} = 'Production'; - } - - if (! exists $self->{CERT_STATUS}) { - $self->{CERT_STATUS} = 'Certified'; - } - if (! exists $self->{TYPE}) { $self->{TYPE} = 'NGI'; } @@ -172,14 +160,12 @@ Module extracts list of sites from ARGO WEBAPI component. Creates new NCG::SiteSet::WEBAPI instance. Argument $options is hash reference that can contain following elements: - WEBAPI_ROOT_URL - WEBAPI JSON API root URL - (default: https://api.argo.grnet.gr) - - PROD_STATUS - production status of site - (default: Production) CERT_STATUS - certification status of site - (default: Certified) + (default: ) + + PROD_STATUS - production status of site + (default: ) SCOPE - scope of sites (default: ) @@ -194,6 +180,9 @@ reference that can contain following elements: TOKEN - token used for WEBAPI API authentication (default: ) + WEBAPI_ROOT_URL - WEBAPI JSON API root URL + (default: https://api.argo.grnet.gr) + =back =head1 SEE ALSO From bf61c858a46a66677995197c5e6e843582bebad7 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Tue, 28 Dec 2021 11:32:58 +0100 Subject: [PATCH 12/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 7f013a1..4dc3c58 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -134,9 +134,9 @@ sub getData { $hostname = $service->{tags}->{hostname}; } - if ($self->{USE_IDS} && exists $service->{tags}->{info_id} && $service->{tags}->{info_id}) { - $self->{SITEDB}->addHost($hostname, $service->{tags}->{info_id}); - $hostname .= '_' . $service->{tags}->{info_id}; + if ($self->{USE_IDS} && exists $service->{tags}->{info_ID} && $service->{tags}->{info_ID}) { + $self->{SITEDB}->addHost($hostname, $service->{tags}->{info_ID}); + $hostname .= '_' . $service->{tags}->{info_ID}; } else { $self->{SITEDB}->addHost($hostname); } @@ -148,6 +148,8 @@ sub getData { foreach my $tag (keys %{$service->{tags}}) { if ( $tag =~ /^info_ext_(\S+)$/i ) { $self->{SITEDB}->hostAttribute($hostname, $1, $service->{tags}->{$tag}); + } elsif ( $tag =~ /^info_bdii_(\S+)$/i ) { + $self->{SITEDB}->hostAttribute($hostname, $1, $service->{tags}->{$tag}); } elsif ( $tag eq 'info_URL' || $tag eq 'info.URL' ) { my $url; my $value = $service->{tags}->{$tag}; @@ -167,6 +169,8 @@ sub getData { foreach my $url ( split (/, /, $service->{tags}->{$tag}) ) { $self->{SITEDB}->hostAttribute($hostname, $serviceType."_URL", $url); } + } elsif ( $tag =~ /^info_(\S+)$/i ) { + $self->{SITEDB}->hostAttribute($hostname, $1, $service->{tags}->{$tag}); } elsif ( $tag =~ /^vo_(\S+?)_attr_(\S+)$/ ) { $self->{SITEDB}->hostAttributeVO($hostname, $2, $1, $service->{tags}->{$tag}); } From 00db2e2f3454e6becc848dc2ca45938f1bb78a1a Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 20 Jan 2022 16:10:19 +0100 Subject: [PATCH 13/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteInfo/WEBAPI.pm | 32 +++++++---------------------- src/modules/NCG/SiteSet/WEBAPI.pm | 33 +++++------------------------- 2 files changed, 12 insertions(+), 53 deletions(-) diff --git a/src/modules/NCG/SiteInfo/WEBAPI.pm b/src/modules/NCG/SiteInfo/WEBAPI.pm index 4dc3c58..3e2f6a5 100644 --- a/src/modules/NCG/SiteInfo/WEBAPI.pm +++ b/src/modules/NCG/SiteInfo/WEBAPI.pm @@ -40,10 +40,6 @@ sub new $self->{WEBAPI_ROOT_URL} = $DEFAULT_WEBAPI_ROOT_URL; } - if (! exists $self->{NODE_MONITORED}) { - $self->{NODE_MONITORED} = '1'; - } - if (! $self->{TOKEN}) { $self->error("Authentication token must be defined."); return; @@ -51,9 +47,6 @@ sub new if (! exists $self->{TIMEOUT}) { $self->{TIMEOUT} = $self->{DEFAULT_HTTP_TIMEOUT}; } - if (! exists $self->{TYPE}) { - $self->{TYPE} = 'SITES'; - } if (! exists $self->{VO}) { $self->{VO} = 'ops'; @@ -75,17 +68,13 @@ sub getData { my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); $ua->agent("NCG::SiteInfo::WEBAPI"); $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; - $url .= '?type=' . $self->{TYPE} . '&group=' . $sitename; - my @tags; - if ($self->{NODE_MONITORED}) { - push @tags, 'monitored:' . $self->{NODE_MONITORED}; - } - if ($self->{SCOPE}) { - push @tags, 'scope:*'.$self->{SCOPE}.'*'; - } - if (@tags) { - $url .= '&tags='.join(',',@tags); + + if ($self->{FILTER}) { + $url .= '?' . $self->{FILTER} . '&group=' . $sitename; + } else { + $url .= '?group=' . $sitename; } + my $req = HTTP::Request->new(GET => $url); $req->header('x-api-key' => $self->{TOKEN}); $req->header('Accept' => 'application/json'); @@ -211,16 +200,9 @@ Module extracts list of sites from ARGO WEBAPI component. Creates new NCG::SiteInfo::WEBAPI instance. Argument $options is hash reference that can contain following elements: - NODE_MONITORED - filter only endpoints with tag monitored + FILTER - filter query that will be forwarded (default: ) - SCOPE - scope of sites - (default: ) - - TYPE - type of groups fetched from WEBAPI, EGI uses SITES, - most other tenants SERVICEGROUPS - (default: SITES) - TIMEOUT - HTTP timeout (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index be82a91..5d36ecd 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -39,10 +39,6 @@ sub new $self->{WEBAPI_ROOT_URL} = $DEFAULT_WEBAPI_ROOT_URL; } - if (! exists $self->{TYPE}) { - $self->{TYPE} = 'NGI'; - } - if (! $self->{TOKEN}) { $self->error("Authentication token must be defined."); return; @@ -61,20 +57,11 @@ sub getData { my $ua = LWP::UserAgent->new( timeout=>$self->{TIMEOUT}, env_proxy=>1 ); $ua->agent("NCG::SiteSet::WEBAPI"); $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; - $url .= '?type' . $self->{TYPE}; - my @tags; - if ($self->{CERT_STATUS}) { - push @tags, 'certification:' . $self->{CERT_STATUS}; - } - if ($self->{PROD_STATUS}) { - push @tags, 'infrastructure:' . $self->{PROD_STATUS}; - } - if ($self->{SCOPE}) { - push @tags, 'scope:*'.$self->{SCOPE}.'*'; - } - if (@tags) { - $url .= '&tags='.join(',',@tags); + + if ($self->{FILTER}) { + $url .= '?' . $self->{FILTER} . '&group=' . $sitename; } + my $req = HTTP::Request->new(GET => $url); $req->header('x-api-key' => $self->{TOKEN}); $req->header('Accept' => 'application/json'); @@ -161,19 +148,9 @@ Module extracts list of sites from ARGO WEBAPI component. Creates new NCG::SiteSet::WEBAPI instance. Argument $options is hash reference that can contain following elements: - CERT_STATUS - certification status of site - (default: ) - - PROD_STATUS - production status of site + FILTER - filter query that will be forwarded (default: ) - SCOPE - scope of sites - (default: ) - - TYPE - type of groups fetched from WEBAPI, EGI uses NGI, - most other tenants PROJECT - (default: NGI) - TIMEOUT - HTTP timeout (default: DEFAULT_HTTP_TIMEOUT inherited from NCG) From 6cd53009a3c2924cdac4af449d0962952d4a330a Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 20 Jan 2022 16:39:44 +0100 Subject: [PATCH 14/15] ARGO-2280 Integrate WEBAPI as topology source in argo-ncg --- src/modules/NCG/SiteSet/WEBAPI.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/NCG/SiteSet/WEBAPI.pm b/src/modules/NCG/SiteSet/WEBAPI.pm index 5d36ecd..b1d4630 100644 --- a/src/modules/NCG/SiteSet/WEBAPI.pm +++ b/src/modules/NCG/SiteSet/WEBAPI.pm @@ -59,7 +59,7 @@ sub getData { $url = $self->{WEBAPI_ROOT_URL} . $DEFAULT_WEBAPI_ROOT_URL_SUFFIX; if ($self->{FILTER}) { - $url .= '?' . $self->{FILTER} . '&group=' . $sitename; + $url .= '?' . $self->{FILTER}; } my $req = HTTP::Request->new(GET => $url); From fc5b3d722bb6f9b75ad5da6845180831448b2a30 Mon Sep 17 00:00:00 2001 From: eimamagi Date: Thu, 20 Jan 2022 18:35:51 +0100 Subject: [PATCH 15/15] Version bump. --- argo-ncg.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/argo-ncg.spec b/argo-ncg.spec index c1d226a..e65961a 100644 --- a/argo-ncg.spec +++ b/argo-ncg.spec @@ -5,7 +5,7 @@ Summary: ARGO Nagios config generator Name: argo-ncg -Version: 0.4.12 +Version: 0.4.13 Release: 1%{?dist} License: ASL 2.0 Group: Network/Monitoring @@ -96,6 +96,8 @@ if [ -f /etc/init.d/ncg ] ; then fi %changelog +* Thu Jan 20 2022 Emir Imamagic