-
-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Updated items endpoint to use official list
- Loading branch information
1 parent
b34d32b
commit c89fd08
Showing
6 changed files
with
526 additions
and
153 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,62 +1,7 @@ | ||
| const got = require('got'); | ||
| const { Profile } = require('skyblock-parser'); | ||
| const config = require('../config'); | ||
| const { logger, removeFormatting } = require('../util/utility'); | ||
|
|
||
| // Submit items to the items service | ||
| async function checkItems(members = {}) { | ||
| // We can control sample size to prevent overload | ||
| if (Math.random() >= Number(config.ITEMS_PERCENT)) { | ||
| return; | ||
| } | ||
| let inventories = []; | ||
| // Merge all inventories | ||
| Object.values(members).forEach((member) => { | ||
| inventories = inventories.concat(member.inventory || []); | ||
| }); | ||
| // Remove empty inventory slots | ||
| inventories = inventories.filter((i) => 'name' in i); | ||
| // Get unique items | ||
| try { | ||
| const items = [...new Set(inventories.map((i) => i.attributes.id))] | ||
| .flatMap((id) => { | ||
| const item = inventories.find((i) => i.attributes.id === id); | ||
| // Filter unfit items | ||
| if (![null, undefined].includes(item.attributes.modifier) | ||
| || id.startsWith('MAP:') | ||
| || item.name === '§fnull' | ||
| || item.name.includes('⚚') | ||
| || !/[!-~]/.test(item.name) || !/[!-~]/.test(item.type) | ||
| || !item.name.match(/[a-z]/i) | ||
| || item.attributes.wood_singularity_count | ||
| || item.attributes.rarity_upgrades) return []; | ||
| return [{ | ||
| id, | ||
| name: removeFormatting(item.name).replace(/✦|✪/g, '').trim(), | ||
| tier: item.rarity, | ||
| category: item.type || 'misc', | ||
| damage: item.damage || null, | ||
| item_id: item.item_id, | ||
| texture: item.attributes.texture || null, | ||
| }]; | ||
| }); | ||
| try { | ||
| await got.post(config.ITEMS_HOST, { | ||
| json: items, | ||
| responseType: 'json', | ||
| }); | ||
| } catch (error) { | ||
| logger.warn(`Failed to insert item updates, is the items service running? ${error}`); | ||
| } | ||
| } catch { | ||
| // we don't mind | ||
| } | ||
| } | ||
|
|
||
| async function processSkyBlock(profile) { | ||
| const data = await new Profile(profile); | ||
| checkItems(data.members); | ||
| return data; | ||
| return new Profile(profile); | ||
| } | ||
|
|
||
| module.exports = processSkyBlock; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,111 +1,72 @@ | ||
| /* | ||
| * Worker to generate SkyBlock item schema from inventories processed by web | ||
| * Worker to generate SkyBlock item schema | ||
| */ | ||
| const express = require('express'); | ||
| const bodyParser = require('body-parser'); | ||
| const compression = require('compression'); | ||
| const config = require('../config'); | ||
| const bukkitToId = require('../util/bukkitToId.json'); | ||
| const redis = require('../store/redis'); | ||
| const { | ||
| logger, invokeInterval, | ||
| logger, invokeInterval, generateJob, getData, | ||
| } = require('../util/utility'); | ||
|
|
||
| const app = express(); | ||
| const port = config.PORT || config.ITEMS_PORT; | ||
|
|
||
| let discoveredItems; | ||
| let bazaarProducts = []; | ||
| let itemList = {}; | ||
| const updateQueue = new Set(); | ||
|
|
||
| (async function init() { | ||
| // TODO - expose this in sb-parser | ||
| function getSkinHash(base64) { | ||
| let texture = null; | ||
| try { | ||
| itemList = JSON.parse(await redis.get('skyblock_items')) || {}; | ||
| const items = Object.keys(itemList); | ||
| logger.info(`Caching existing ${items.length} item IDs`); | ||
| discoveredItems = new Set(items); | ||
| } catch (error) { | ||
| logger.error(`Failed retrieving skyblock_items from redis: ${error}`); | ||
| texture = JSON.parse(Buffer.from(base64, 'base64').toString()).textures.SKIN.url.split('/').pop(); | ||
| } catch (e) { | ||
| // do nothing | ||
| } | ||
| }()); | ||
| return texture; | ||
| } | ||
|
|
||
| /* | ||
| * Retrieve items resource from Hypixel API and do some processing | ||
| */ | ||
| async function updateItemList() { | ||
| const itemsObject = {}; | ||
| const { url } = generateJob('skyblock_items'); | ||
| const { items } = await getData(redis, url); | ||
| items.forEach((item) => { | ||
| const { | ||
| id, name, tier, category, skin, durability, ...rest | ||
| } = item; | ||
| const obj = { | ||
| id, | ||
| name, | ||
| item_id: bukkitToId[item?.material] || 0, | ||
| ...rest, | ||
| }; | ||
| if (tier) { | ||
| obj.tier = tier.toLowerCase(); | ||
| } | ||
| if (category) { | ||
| obj.category = category.toLowerCase(); | ||
| } | ||
| if (durability) { | ||
| obj.damage = durability; | ||
| } | ||
| if (skin) { | ||
| obj.texture = getSkinHash(skin); | ||
| } | ||
| itemsObject[id] = obj; | ||
| }); | ||
| // Get bazaar status | ||
| try { | ||
| logger.info('Updating item list to redis...'); | ||
| await redis.set('skyblock_items', JSON.stringify(itemList)); | ||
| const data = await redis.get('skyblock_bazaar'); | ||
| const bazaarProducts = Object.keys(JSON.parse(data)); | ||
| bazaarProducts.forEach((id) => { | ||
| if (Object.hasOwn(itemsObject, id)) { | ||
| itemsObject[id].bazaar = true; | ||
| } | ||
| }); | ||
| } catch (error) { | ||
| logger.error(`Failed updating skyblock_items: ${error}`); | ||
| logger.error(`Failed getting bazaar data: ${error}`); | ||
| } | ||
| } | ||
|
|
||
| async function updateBazaar() { | ||
| try { | ||
| const data = await redis.get('skyblock_bazaar'); | ||
| bazaarProducts = Object.keys(JSON.parse(data)); | ||
| try { | ||
| logger.info('[Bazaar] Updated item IDs'); | ||
| let changes = 0; | ||
| discoveredItems.forEach((id) => { | ||
| if (bazaarProducts.includes(id)) { | ||
| itemList[id].bazaar = true; | ||
| changes += 1; | ||
| } | ||
| }); | ||
| if (changes > 0) { | ||
| logger.info(`Discovered ${changes} new bazaar items!`); | ||
| updateItemList(); | ||
| } | ||
| } catch (error) { | ||
| logger.error(error.message); | ||
| } | ||
| logger.info('Updating item list to redis...'); | ||
| await redis.set('skyblock_items', JSON.stringify(itemsObject)); | ||
| } catch (error) { | ||
| logger.error(`Failed updating bazaar products: ${error}`); | ||
| logger.error(`Failed updating skyblock_items: ${error}`); | ||
| } | ||
| } | ||
|
|
||
| invokeInterval(updateBazaar, 60 * 60 * 1000); | ||
|
|
||
| app.use(compression()); | ||
| app.use(bodyParser.json()); | ||
| app.post('/', (request, response, _callback) => { | ||
| const items = request.body; | ||
| let updates = false; | ||
| items.forEach((item) => { | ||
| const { id } = item; | ||
| if (!discoveredItems.has(id) || updateQueue.has(id)) { | ||
| updates = true; | ||
| logger.info(`Found new item ID ${id}`); | ||
| if (item.texture === null) { | ||
| delete item.texture; | ||
| } | ||
| if (item.damage === null || item.texture) { | ||
| delete item.damage; | ||
| } | ||
| delete item.id; | ||
| itemList[id] = item; | ||
| discoveredItems.add(id); | ||
| if (updateQueue.has(id)) { | ||
| updateQueue.delete(id); | ||
| } | ||
| } | ||
| }); | ||
| if (updates) { | ||
| updateItemList(); | ||
| } | ||
| response.json({ status: 'ok' }); | ||
| }); | ||
| app.delete('/:id', (request, response, _callback) => { | ||
| const { id } = request.params; | ||
| if (id in itemList) { | ||
| logger.info(`Adding item ${id} to update queue`); | ||
| updateQueue.add(id); | ||
| } | ||
| response.json({ status: 'ok' }); | ||
| }); | ||
| app.use((error, _, response) => response.status(500).json({ | ||
| error, | ||
| })); | ||
| const server = app.listen(port, () => { | ||
| const host = server.address().address; | ||
| logger.info(`[ITEMS] listening at http://${host}:${port}`); | ||
| }); | ||
| invokeInterval(updateItemList, 15 * 60 * 1000); |
Oops, something went wrong.