Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PHRAS-4085_data-volumes-api #4529

Merged
merged 12 commits into from
Jul 17, 2024
Merged

PHRAS-4085_data-volumes-api #4529

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
bcb456e
new route `/api/v3/monitor/data/?oauth_token=xxx&blocksize=16ko`
jygaulier Jun 26, 2024
8744a23
add `unit` parameter for size restults ('', 'o', ''ko', 'mo', 'go'), …
jygaulier Jun 27, 2024
65a5bc3
new units "octet", "octets"
jygaulier Jun 27, 2024
0ab38c6
fix round error (don't sum rounded values: round the real sum)
jygaulier Jun 27, 2024
db021b6
add infos about downloads;
jygaulier Jun 27, 2024
82ef3d0
add `...&details=1` url parameter to get values by collection and sub…
jygaulier Jun 27, 2024
5757e00
add column size in LazaretFiles table
aynsix Jul 4, 2024
d27a62d
add api monitor data by databox
aynsix Jul 4, 2024
c3d7813
Merge branch 'master' into PHRAS-4085_data-volumes-api
nmaillat Jul 5, 2024
c93bd18
fix size
aynsix Jul 10, 2024
1f68fee
Merge branch 'master' into PHRAS-4085_data-volumes-api
nmaillat Jul 17, 2024
7924e36
Merge branch 'master' into PHRAS-4085_data-volumes-api
nmaillat Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
312 changes: 223 additions & 89 deletions lib/Alchemy/Phrasea/Controller/Api/V3/V3MonitorDataController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,87 +39,16 @@ private function unitToMultiplier(string $unit)
*/
public function indexAction(Request $request)
{
$getDetails = $request->get('details', '0') === '1';

$stopwatch = new Stopwatch("controller");
$matches = [];
if(preg_match("/^(\\d+)\\s*([a-z]*)$/i", $request->get('blocksize', '1'), $matches) !== 1) {
throw new Exception("bad 'blocksize' parameter");
}
$matches[] = ''; // if no unit, force
if(($mutiplier = $this->unitToMultiplier($matches[2])) === false) {
throw new Exception("bad 'blocksize' unit");
}
$blocksize = (int)($matches[1]) * $mutiplier;

if( ($divider = $this->unitToMultiplier($unit = $request->get('unit', '')) ) === false) {
throw new Exception("bad 'unit' parameter");
}
$sqlDivider = $divider === 1 ? '' : (' / ' . $divider);
list($getDetails, $blocksize, $divider, $unit, $sqlByColl, $sqlByName, $sqlByDb) = $this->getParamsFromRequest($request);

$ret = [
'unit' => $divider === 1 ? $unit : ucfirst($unit), // octet => octet ; mo => Mo
'databoxes' => []
];

if($getDetails) {

$sqlByColl = "SELECT COALESCE(r.`coll_id`, '?') AS `coll_id`,
COALESCE(c.`asciiname`, CONCAT('_',r.`coll_id`), '?') AS `asciiname`, s.`name`,
SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s LEFT JOIN `record` AS r ON r.`record_id`=s.`record_id`
LEFT JOIN `coll` AS c ON r.`coll_id`=c.`coll_id`
GROUP BY r.`coll_id`, s.`name`;";

$sqlByName = "SELECT s.`name`,
SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s
GROUP BY s.`name`;";
}
$sqlByDb = "SELECT SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s";

foreach($this->app->getDataboxes() as $databox) {

if($getDetails) {

// get volumes grouped by collection and subdef

$collections = [];
$stmt = $databox->get_connection()->prepare($sqlByColl);
$stmt->execute();
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
if (!array_key_exists($row['coll_id'], $collections)) {
$collections[$row['coll_id']] = [
'coll_id' => $row['coll_id'],
'name' => $row['asciiname'],
'subdefs' => []
];
}
$collections[$row['coll_id']]['subdefs'][$row['name']] = [
'count' => (int)$row['n'],
'size' => round($row['size'], 2),
'disksize' => round($row['disksize'], 2)
];
}
$stmt->closeCursor();

// get volumes by subdef

$subdefs = [];
$stmt = $databox->get_connection()->prepare($sqlByName);
$stmt->execute();
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
$subdefs[$row['name']]['count'] = (int)$row['n'];
$subdefs[$row['name']]['size'] = round($row['size'], 2);
$subdefs[$row['name']]['disksize'] = round($row['disksize'], 2);
}
$stmt->closeCursor();
}

foreach ($this->app->getDataboxes() as $databox) {
// get volumes by db

$stmt = $databox->get_connection()->prepare($sqlByDb);
Expand All @@ -128,16 +57,18 @@ public function indexAction(Request $request)
$stmt->closeCursor();

$ret['databoxes'][$databox->get_sbas_id()] = [
'sbas_id' => $databox->get_sbas_id(),
'viewname' => $databox->get_viewname(),
'count' => (int)$row['n'],
'size' => round($row['size'], 2),
'disksize' => round($row['disksize'], 2)
'sbas_id' => $databox->get_sbas_id(),
'viewname' => $databox->get_viewname(),
'count' => (int)$row['n'],
'size' => round($row['size'], 2),
'disksize' => round($row['disksize'], 2)
];

if($getDetails) {
$ret['databoxes'][$databox->get_sbas_id()]['collections'] = $collections;
$ret['databoxes'][$databox->get_sbas_id()]['subdefs'] = $subdefs;
if ($getDetails) {
list($collections, $subdefs) = $this->getVolumeDetails($databox, $sqlByColl, $sqlByName);

$ret['databoxes'][$databox->get_sbas_id()]['collections'] = $collections;
$ret['databoxes'][$databox->get_sbas_id()]['subdefs'] = $subdefs;
}
}

Expand All @@ -151,9 +82,9 @@ public function indexAction(Request $request)
$n = 0;
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
try {
$data = unserialize($row['data']);
$size += $data['size'];
$disksize += ceil($data['size'] / $blocksize) * $blocksize;
$data = unserialize($row['data']);
$size += $data['size'];
$disksize += ceil($data['size'] / $blocksize) * $blocksize;
$n++;
}
catch (\Exception $e) {
Expand All @@ -169,15 +100,218 @@ public function indexAction(Request $request)
$stmt->closeCursor();

$ret['downloads'] = [
'count' => $n,
'days_oldest' => (int)$row['oldest'],
'expired' => (int)$row['expired'],
'size' => round($size / $divider, 2),
'disksize' => round($disksize / $divider, 2)
'count' => $n,
'days_oldest' => (int)$row['oldest'],
'expired' => (int)$row['expired'],
'size' => round($size / $divider, 2),
'disksize' => round($disksize / $divider, 2)
];

$sql = "SELECT count(*) AS n , SUM(`size`) AS size FROM `LazaretFiles` WHERE size IS NOT NULL";
$stmt = $this->getApplicationBox()->get_connection()->prepare($sql);
$stmt->execute();

$row = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();

$disksize = ceil($row['size'] / $blocksize) * $blocksize;;

$ret['lazaret'] = [
'count' => $row['n'],
'size' => round($row['size'] / $divider, 2),
'disksize' => round($disksize / $divider, 2)
];

return Result::create($request, $ret)->createResponse([$stopwatch]);
}

/**
* monitor info for app by databox
* @param Request $request
*/
public function perDataboxAction(Request $request)
{
$stopwatch = new Stopwatch("controller");
$databoxId = $request->get('databox_id');

list($getDetails, $blocksize, $divider, $unit, $sqlByColl, $sqlByName, $sqlByDb) = $this->getParamsFromRequest($request);

$ret = [
'unit' => $divider === 1 ? $unit : ucfirst($unit), // octet => octet ; mo => Mo
'databox' => []
];

$databox = $this->findDataboxById($databoxId);

// get volumes by db

$stmt = $databox->get_connection()->prepare($sqlByDb);
$stmt->execute();
$row = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();

$ret['databox'] = [
'sbas_id' => $databox->get_sbas_id(),
'viewname' => $databox->get_viewname(),
'count' => (int)$row['n'],
'size' => round($row['size'], 2),
'disksize' => round($row['disksize'], 2)
];

if ($getDetails) {
list($collections, $subdefs) = $this->getVolumeDetails($databox, $sqlByColl, $sqlByName);

$ret['databox']['collections'] = $collections;
$ret['databox']['subdefs'] = $subdefs;
}

// get volumes of downloads

$sql = "SELECT `data` FROM `Tokens` WHERE `type`='download'";
$stmt = $this->getApplicationBox()->get_connection()->prepare($sql);
$stmt->execute();
$size = 0;
$disksize = 0;
$n = 0;
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
try {
$found = false;
$data = unserialize($row['data']);
foreach ($data['files'] as $file) {
// get only for the needed databoxId
if ($file['databox_id'] == $databoxId) {
$found = true;
foreach ($file['subdefs'] as $subdef) {
$size += $subdef['size'];
$disksize += ceil($subdef['size'] / $blocksize) * $blocksize;
}
}
}

if ($found) {
$n++;
}
}
catch (\Exception $e) {
// ignore
}
}

$stmt->closeCursor();

$ret['downloads'] = [
'sbas_id' => $databoxId,
'count' => $n,
'size' => round($size / $divider, 2),
'disksize' => round($disksize / $divider, 2)
];

// get lazaret volume for the databox

$sql = "SELECT count(*) AS n , SUM(`size`) AS size ".
" FROM `LazaretFiles` AS L ".
" LEFT JOIN `bas` AS b ON L.`base_id`=b.`base_id`".
" WHERE L.`size` IS NOT NULL AND b.`sbas_id`=". $databoxId;


$stmt = $this->getApplicationBox()->get_connection()->prepare($sql);
$stmt->execute();

$row = $stmt->fetch(PDO::FETCH_ASSOC);
$stmt->closeCursor();

$disksize = ceil($row['size'] / $blocksize) * $blocksize;;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NO...
disksize is per file, eg. if blocksize=16ko, the disksize of every file is a multiple of 16ko :

  • 10 small files (1 ko) = 10 * 16 ko ==> 160 ko (good)

here we get 10 * 1 ko = 10 ko ==> 16 ko (wrong)

Do blocksize, divider etc in sql, like in other requests.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok


$ret['lazaret'] = [
'sbas_id' => $databoxId,
'count' => $row['n'],
'size' => round($row['size'] / $divider, 2),
'disksize' => round($disksize / $divider, 2)
];

return Result::create($request, $ret)->createResponse([$stopwatch]);
}

private function getParamsFromRequest(Request $request)
{
$getDetails = $request->get('details', '0') === '1';

$matches = [];
if(preg_match("/^(\\d+)\\s*([a-z]*)$/i", $request->get('blocksize', '1'), $matches) !== 1) {
throw new Exception("bad 'blocksize' parameter");
}
$matches[] = ''; // if no unit, force
if(($mutiplier = $this->unitToMultiplier($matches[2])) === false) {
throw new Exception("bad 'blocksize' unit");
}
$blocksize = (int)($matches[1]) * $mutiplier;

if( ($divider = $this->unitToMultiplier($unit = $request->get('unit', '')) ) === false) {
throw new Exception("bad 'unit' parameter");
}
$sqlDivider = $divider === 1 ? '' : (' / ' . $divider);

$sqlByColl = "";
$sqlByName = "";

if ($getDetails) {

$sqlByColl = "SELECT COALESCE(r.`coll_id`, '?') AS `coll_id`,
COALESCE(c.`asciiname`, CONCAT('_',r.`coll_id`), '?') AS `asciiname`, s.`name`,
SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s LEFT JOIN `record` AS r ON r.`record_id`=s.`record_id`
LEFT JOIN `coll` AS c ON r.`coll_id`=c.`coll_id`
GROUP BY r.`coll_id`, s.`name`;";

$sqlByName = "SELECT s.`name`,
SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s
GROUP BY s.`name`;";
}
$sqlByDb = "SELECT SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`,
SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize`
FROM `subdef` AS s";

return [$getDetails, $blocksize, $divider, $unit, $sqlByColl, $sqlByName, $sqlByDb];
}

private function getVolumeDetails(\databox $databox, $sqlByColl, $sqlByName)
{
// get volumes grouped by collection and subdef

$collections = [];
$stmt = $databox->get_connection()->prepare($sqlByColl);
$stmt->execute();
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
if (!array_key_exists($row['coll_id'], $collections)) {
$collections[$row['coll_id']] = [
'coll_id' => $row['coll_id'],
'name' => $row['asciiname'],
'subdefs' => []
];
}
$collections[$row['coll_id']]['subdefs'][$row['name']] = [
'count' => (int)$row['n'],
'size' => round($row['size'], 2),
'disksize' => round($row['disksize'], 2)
];
}
$stmt->closeCursor();

// get volumes by subdef

$subdefs = [];
$stmt = $databox->get_connection()->prepare($sqlByName);
$stmt->execute();
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
$subdefs[$row['name']]['count'] = (int)$row['n'];
$subdefs[$row['name']]['size'] = round($row['size'], 2);
$subdefs[$row['name']]['disksize'] = round($row['disksize'], 2);
}
$stmt->closeCursor();

return [$collections, $subdefs];
}
}
8 changes: 8 additions & 0 deletions lib/Alchemy/Phrasea/ControllerProvider/Api/V3.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ public function connect(Application $app)
->before('controller.api.v1:ensureAdmin')
;

/**
* @uses V3MonitorDataController::perDataboxAction()
*/
$controllers->get('databoxes/{databox_id}/monitor/data/', 'controller.api.v3.monitorData:perDataboxAction')
->before('controller.api.v1:ensureAdmin')
->assert('databox_id', '\d+')
;

/**
* @uses V3SearchController::helloAction()
*/
Expand Down
Loading