From 2e1c7bb4d0c4ad793d1a432d2d3073258a8a1965 Mon Sep 17 00:00:00 2001 From: David Neilsen Date: Fri, 4 Sep 2020 00:13:39 +1200 Subject: [PATCH] Add ranking tracker --- debug/rankings.php | 122 +++++++++++++++++++++++++++++++++ package-lock.json | 157 +++++++++++++++++++++++++++++++++++++++++++ package.json | 4 ++ scrapes/.gitignore | 2 + src/scrape/scrape.ts | 41 +++++++++++ 5 files changed, 326 insertions(+) create mode 100644 debug/rankings.php create mode 100644 scrapes/.gitignore create mode 100644 src/scrape/scrape.ts diff --git a/debug/rankings.php b/debug/rankings.php new file mode 100644 index 00000000..99446f03 --- /dev/null +++ b/debug/rankings.php @@ -0,0 +1,122 @@ + $players) { + $labels[] = $time; + foreach ($players as $player => $rank) { + if (count($datasets) > 15 && !in_array($player, $mySnakes)) { + continue; + } + $datasets[$player] = [ + 'label' => $player, + 'fill' => false, + 'data' => [], + ]; + } + } + foreach ($labels as $dataIndex => $time) { + foreach ($datasets as &$dataset) { + $dataset['data'][$dataIndex] = null; + foreach ($rankings[$time] as $player => $rank) { + if ($player == $dataset['label']) { + $dataset['data'][$dataIndex] = (int) $rank; + break; + } + } + } + } + $datasets = array_values($datasets); + return [$labels, $datasets, str_replace('.json', '', basename($file))]; +} +?> + + + + + + + + + + +
+
+ + +
+ +
+ + +
+
+ + + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index aac4134e..bb881d01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1175,6 +1175,15 @@ "integrity": "sha512-UBEgZTFg+tAvGZrJpTfTrRbBHXdcA3GErBs+QfIM1K8Gnzl7WghYc7gsbWRQ7rXvpnxmbffAFLPkKl32XEx8IQ==", "dev": true }, + "@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "dev": true, + "requires": { + "axios": "*" + } + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -1185,6 +1194,15 @@ "@types/node": "*" } }, + "@types/cheerio": { + "version": "0.22.21", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.21.tgz", + "integrity": "sha512-aGI3DfswwqgKPiEOTaiHV2ZPC9KEhprpgEbJnv0fZl3SGX0cGgEva1126dGrMC6AJM6v/aihlUgJn9M5DbDZ/Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/color": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@types/color/-/color-3.0.1.tgz", @@ -1681,6 +1699,15 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, + "axios": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.20.0.tgz", + "integrity": "sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==", + "dev": true, + "requires": { + "follow-redirects": "^1.10.0" + } + }, "babel-plugin-dynamic-import-node": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", @@ -1827,6 +1854,12 @@ "type-is": "~1.6.17" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", + "dev": true + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -2298,6 +2331,20 @@ "supports-color": "^5.3.0" } }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "dev": true, + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", @@ -2815,6 +2862,24 @@ } } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "dev": true, + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "dev": true + }, "cycle": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", @@ -3080,12 +3145,47 @@ "integrity": "sha1-v9N0EHX9f7GlsuE2WMX0vsd3NvM=", "dev": true }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dev": true, + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", "dev": true }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dev": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dot-prop": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz", @@ -3186,6 +3286,12 @@ "once": "^1.4.0" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -3747,6 +3853,12 @@ "readable-stream": "^2.3.6" } }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==", + "dev": true + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -5147,6 +5259,33 @@ "integrity": "sha1-OgPtwiFLyjtmQko+eVk0lQnLA1E=", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dev": true, + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -6365,6 +6504,15 @@ "lazy": "~1.0.11" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "requires": { + "boolbase": "~1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -6715,6 +6863,15 @@ "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", "dev": true }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", diff --git a/package.json b/package.json index 09e21e48..80499065 100644 --- a/package.json +++ b/package.json @@ -16,13 +16,17 @@ "@babel/core": "^7.11.5", "@babel/preset-env": "^7.11.5", "@types/angular": "^1.7.2", + "@types/axios": "^0.14.0", + "@types/cheerio": "^0.22.21", "@types/color": "^3.0.1", "@types/express": "^4.17.8", "@types/jquery": "^3.5.1", "@types/node": "^11.15.21", "@types/websocket": "^1.0.1", + "axios": "^0.20.0", "babelify": "^10.0.0", "browserify": "^16.5.2", + "cheerio": "^1.0.0-rc.3", "color": "^3.1.2", "concurrently": "^5.3.0", "forever": "^0.15.3", diff --git a/scrapes/.gitignore b/scrapes/.gitignore new file mode 100644 index 00000000..c96a04f0 --- /dev/null +++ b/scrapes/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/src/scrape/scrape.ts b/src/scrape/scrape.ts new file mode 100644 index 00000000..3d4567fd --- /dev/null +++ b/src/scrape/scrape.ts @@ -0,0 +1,41 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; +import * as fs from 'fs'; + +const time = new Date().getTime(); + +async function fetchData(url) { + const result = await axios.get(url); + return cheerio.load(result.data); +} + +async function scrape(name, url) { + const scrapeFile = `${__dirname}/../../scrapes/${name}.json`; + let dates: any; + try { + dates = fs.readFileSync(scrapeFile); + dates = JSON.parse(dates); + } catch (error) { + console.error(error); + dates = {}; + } + + const $ = await fetchData(url); + const ranks = {}; + $('#tab-leaderboard table tbody tr').each((index, element) => { + const rank = $(element).find('.rank').text().trim(); + const nameColumn = $(element).find('td').eq(2); + const name = $(nameColumn).find('a').first().text().trim(); + ranks[name] = rank; + }); + console.log(name, ranks); + dates[time] = ranks; + + fs.writeFileSync(scrapeFile, JSON.stringify(dates, null, 4)); +} + +Promise.all([ + scrape('summer-challenge-squads', 'https://play.battlesnake.com/arena/summer-challenge-squads/'), + scrape('summer-veteran', 'https://play.battlesnake.com/arena/summer-veteran/'), + scrape('global', 'https://play.battlesnake.com/arena/global/'), +]);