Skip to content

Commit

Permalink
more efficient pagination system - remove unused getTotalPackageEstimate
Browse files Browse the repository at this point in the history
  • Loading branch information
@Digitalone1
Digitalone1 committed Dec 23, 2022
1 parent f1d45fe commit 0473c0e
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 111 deletions.
97 changes: 31 additions & 66 deletions src/database.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ async function getPackageCollectionByName(packArray) {

return command.count !== 0
? { ok: true, content: command }
: { ok: false, content: `No packages found.`, short: "Not Found" };
: { ok: false, content: "No packages found.", short: "Not Found" };
} catch (err) {
return { ok: false, content: err, short: "Server Error" };
}
Expand All @@ -578,7 +578,7 @@ async function getPackageCollectionByID(packArray) {

return command.count !== 0
? { ok: true, content: command }
: { ok: false, content: `No packages found.`, short: "Not Found" };
: { ok: false, content: "No packages found.", short: "Not Found" };
} catch (err) {
return { ok: false, content: err, short: "Server Error" };
}
Expand Down Expand Up @@ -976,47 +976,6 @@ async function getFeaturedThemes() {
return await getPackageCollectionByName(featuredThemeArray.content);
}

/**
* @async
* @function getTotalPackageEstimate
* @desc Returns an estimate of how many rows are included in the packages SQL table.
* Used to aid in trunication and page generation of Link headers for large requests.
* @returns {object} A server status object returning the number of the packages
* along with the number of the pages calculated on the paginated_amount.
*/
async function getTotalPackageEstimate() {
try {
sqlStorage ??= setupSQL();

const command = await sqlStorage`
SELECT reltuples AS estimate FROM pg_class WHERE relname = 'packages';
`;

if (command.count === 0) {
return {
ok: false,
content: `Unable to query total row count estimate.`,
short: "Server Error",
};
}

const total = command[0].estimate;
const quotient = Math.trunc(total / paginated_amount);
const remainder = total % paginated_amount;
const pages = quotient + (remainder > 0 ? 1 : 0);

return {
ok: true,
content: {
total: total,
pages: pages
}
};
} catch (err) {
return { ok: false, content: err, short: "Server Error" };
}
}

/**
* @async
* @function getUserByName
Expand Down Expand Up @@ -1327,7 +1286,7 @@ async function getStarringUsersByPointer(pointer) {
* @function simpleSearch
* @description The current Fuzzy-Finder implementation of search. Ideally eventually
* will use a more advanced search method.
* @returns {object} A server status object containing the search status object.
* @returns {object} A server status object containing the results and the pagination object.
*/
async function simpleSearch(term, page, dir, sort, themes = false) {
try {
Expand All @@ -1337,8 +1296,12 @@ async function simpleSearch(term, page, dir, sort, themes = false) {
// lowercase format (see atom-backend issue #86).
const lcterm = term.toLowerCase();

const limit = paginated_amount;
const offset = page > 1 ? (page - 1) * limit : 0;

const command = await sqlStorage`
SELECT p.data, p.downloads, (p.stargazers_count + p.original_stargazers) AS stargazers_count, v.semver
SELECT p.data, p.downloads, (p.stargazers_count + p.original_stargazers) AS stargazers_count,
v.semver, COUNT(*) OVER() AS query_result_count
FROM packages AS p INNER JOIN versions AS v ON (p.pointer = v.package) AND (v.status = 'latest')
WHERE pointer IN (
SELECT pointer
Expand All @@ -1349,27 +1312,27 @@ async function simpleSearch(term, page, dir, sort, themes = false) {
ORDER BY ${
sort === "relevance" ? sqlStorage`downloads` : sqlStorage`${sort}`
}
${dir === "desc" ? sqlStorage`DESC` : sqlStorage`ASC`};
${dir === "desc" ? sqlStorage`DESC` : sqlStorage`ASC`}
LIMIT ${limit}
OFFSET ${offset};
`;

const resultCount = command.count;
if (resultCount === 0) {
return { ok: false, content: `No packages found.`, short: "Not Found" };
if (command.count === 0) {
return { ok: false, content: "No packages found.", short: "Not Found" };
}

const resultCount = command[0].query_result_count;
const quotient = Math.trunc(resultCount / paginated_amount);
const remainder = resultCount % paginated_amount;
const totalPages = quotient + (remainder > 0 ? 1 : 0);

// search status object to return
// { page: Number, total: Number, result: Array };
return {
ok: true,
content: {
content: command,
pagination: {
count: resultCount,
page: (page < totalPages) ? page : totalPages,
total: totalPages,
result: command.slice((page - 1) * paginated_amount, page * paginated_amount)
total: totalPages
}
};
} catch (err) {
Expand Down Expand Up @@ -1420,14 +1383,17 @@ async function getUserCollectionById(ids) {
* @param {string} dir - String flag for asc/desc order.
* @param {string} method - The column name the results have to be sorted by.
* @param {boolean} [themes=false] - Optional Parameter to specify if this should only return themes.
* @returns {object} A server status object containing the search status object.
* @returns {object} A server status object containing the results and the pagination object.
*/
async function getSortedPackages(page, dir, method, themes = false) {
// Here will be a monolithic function for returning sortable packages arrays.
// We must keep in mind that all the endpoint handler knows is the
// page, sort method, and direction. We must figure out the rest here.
// only knowing we have a valid sort method provided.

const limit = paginated_amount;
const offset = page > 1 ? (page - 1) * limit : 0;

try {
sqlStorage ??= setupSQL();

Expand Down Expand Up @@ -1456,33 +1422,33 @@ async function getSortedPackages(page, dir, method, themes = false) {
}

const command = await sqlStorage`
SELECT p.data, p.downloads, (p.stargazers_count + p.original_stargazers) AS stargazers_count, v.semver
SELECT p.data, p.downloads, (p.stargazers_count + p.original_stargazers) AS stargazers_count,
v.semver, COUNT(*) OVER() AS query_result_count
FROM packages AS p INNER JOIN versions AS v ON (p.pointer = v.package) AND (v.status = 'latest')
${
themes === true
? sqlStorage`WHERE package_type = 'theme'`
: sqlStorage``
}
ORDER BY ${orderType} ${
dir === "desc" ? sqlStorage`DESC` : sqlStorage`ASC`
};
dir === "desc" ? sqlStorage`DESC` : sqlStorage`ASC`
}
LIMIT ${limit}
OFFSET ${offset};
`;

const resultCount = command.count;

const resultCount = command[0]?.query_result_count ?? 0;
const quotient = Math.trunc(resultCount / paginated_amount);
const remainder = resultCount % paginated_amount;
const totalPages = quotient + (remainder > 0 ? 1 : 0);

// search status object to return
// { page: Number, total: Number, result: Array };
return {
ok: true,
content: {
content: command,
pagination: {
count: resultCount,
page: (page < totalPages) ? page : totalPages,
total: totalPages,
result: command.slice((page - 1) * paginated_amount, page * paginated_amount)
total: totalPages
}
};
} catch (err) {
Expand All @@ -1499,7 +1465,6 @@ module.exports = {
removePackageByName,
removePackageVersion,
getFeaturedPackages,
getTotalPackageEstimate,
getSortedPackages,
getUserByName,
getUserByNodeID,
Expand Down
37 changes: 20 additions & 17 deletions src/handlers/package_handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,25 +38,25 @@ async function getPackages(req, res) {
direction: query.dir(req),
};

const searchStatus = await database.getSortedPackages(
const packages = await database.getSortedPackages(
params.page,
params.direction,
params.sort
);

if (!searchStatus.ok) {
if (!packages.ok) {
logger.generic(
3,
`getPackages-getSortedPackages Not OK: ${searchStatus.content}`
`getPackages-getSortedPackages Not OK: ${packages.content}`
);
await common.handleError(req, res, searchStatus, 1001);
await common.handleError(req, res, packages, 1001);
return;
}

const page = searchStatus.content.page;
const totPage = searchStatus.content.total;
const page = packages.pagination.page;
const totPage = packages.pagination.total;
const packObjShort = await utils.constructPackageObjectShort(
searchStatus.content.result
packages.content
);

// The endpoint using this function needs an array.
Expand All @@ -75,7 +75,7 @@ async function getPackages(req, res) {
}

res.append("Link", link);
res.append("Query-Total", searchStatus.content.count);
res.append("Query-Total", packages.pagination.count);

res.status(200).json(packArray);
logger.httpLog(req, res);
Expand Down Expand Up @@ -283,15 +283,15 @@ async function getPackagesSearch(req, res) {
// side. This is only an effort to get this working quickly and should be changed later.
// This also means for now, the default sorting method will be downloads, not relevance.

const searchStatus = await database.simpleSearch(
const packs = await database.simpleSearch(
params.query,
params.page,
params.direction,
params.sort
);

if (!searchStatus.ok) {
if (searchStatus.short == "Not Found") {
if (!packs.ok) {
if (packs.short == "Not Found") {
logger.generic(
4,
"getPackagesSearch-simpleSearch Responding with Empty Array for Not Found Status"
Expand All @@ -305,15 +305,18 @@ async function getPackagesSearch(req, res) {
}
logger.generic(
3,
`getPackagesSearch-simpleSearch Not OK: ${searchStatus.content}`
`getPackagesSearch-simpleSearch Not OK: ${packs.content}`
);
await common.handleError(req, res, searchStatus, 1007);
await common.handleError(req, res, packs, 1007);
return;
}

const page = searchStatus.content.page;
const totPage = searchStatus.content.total;
const newPacks = await utils.constructPackageObjectShort(searchStatus.content.result);
const page = packs.pagination.page;
const totPage = packs.pagination.total;
const newPacks = await utils.constructPackageObjectShort(
packs.content
);

let packArray = null;

if (Array.isArray(newPacks)) {
Expand Down Expand Up @@ -348,7 +351,7 @@ async function getPackagesSearch(req, res) {
}

res.append("Link", link);
res.append("Query-Total", searchStatus.content.count);
res.append("Query-Total", packs.pagination.count);

res.status(200).json(packArray);
logger.httpLog(req, res);
Expand Down
37 changes: 20 additions & 17 deletions src/handlers/theme_handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,26 +61,26 @@ async function getThemes(req, res) {
direction: query.dir(req),
};

const searchStatus = await database.getSortedPackages(
const packages = await database.getSortedPackages(
params.page,
params.direction,
params.sort,
true
);

if (!searchStatus.ok) {
if (!packages.ok) {
logger.generic(
3,
`getThemes-getSortedPackages Not OK: ${searchStatus.content}`
`getThemes-getSortedPackages Not OK: ${packages.content}`
);
await common.handleError(req, res, searchStatus);
await common.handleError(req, res, packages);
return;
}

const page = searchStatus.content.page;
const totPage = searchStatus.content.total;
const page = packages.pagination.page;
const totPage = packages.pagination.total;
const packObjShort = await utils.constructPackageObjectShort(
searchStatus.content.result
packages.content
);

const packArray = Array.isArray(packObjShort) ? packObjShort : [packObjShort];
Expand All @@ -98,7 +98,7 @@ async function getThemes(req, res) {
}

res.append("Link", link);
res.append("Query-Total", searchStatus.content.count);
res.append("Query-Total", packages.pagination.count);

res.status(200).json(packArray);
logger.httpLog(req, res);
Expand All @@ -121,16 +121,16 @@ async function getThemesSearch(req, res) {
query: query.query(req),
};

const searchStatus = await database.simpleSearch(
const packs = await database.simpleSearch(
params.query,
params.page,
params.direction,
params.sort,
true
);

if (!searchStatus.ok) {
if (searchStatus.short == "Not Found") {
if (!packs.ok) {
if (packs.short == "Not Found") {
logger.generic(
4,
"getThemesSearch-simpleSearch Responding with Empty Array for Not Found Status"
Expand All @@ -139,14 +139,17 @@ async function getThemesSearch(req, res) {
logger.httpLog(req, res);
return;
}
logger.generic(3, `getThemesSearch-simpleSearch Not OK: ${searchStatus.content}`);
await common.handleError(req, res, searchStatus);
logger.generic(3, `getThemesSearch-simpleSearch Not OK: ${packs.content}`);
await common.handleError(req, res, packs);
return;
}

const page = searchStatus.content.page;
const totPage = searchStatus.content.total;
const newPacks = await utils.constructPackageObjectShort(searchStatus.content.result);
const page = packs.pagination.page;
const totPage = packs.pagination.total;
const newPacks = await utils.constructPackageObjectShort(
packs.content
);

let packArray = null;

if (Array.isArray(newPacks)) {
Expand Down Expand Up @@ -178,7 +181,7 @@ async function getThemesSearch(req, res) {
}

res.append("Link", link);
res.append("Query-Total", searchStatus.content.count);
res.append("Query-Total", packs.pagination.count);

res.status(200).json(packArray);
logger.httpLog(req, res);
Expand Down
Loading

0 comments on commit 0473c0e

Please sign in to comment.