Feature/improve tokenlist workflow (#179)

* feat: Improve version handling

* feat: Validate valid logoURI

* feat: Verbose error messages for failed validation

* feat: Add CI check

* chore: Update README

.github/workflows/tokenlist.yml

@@ -0,0 +1,27 @@
+name: Verify token lists
+
+on:
+  pull_request:
+    paths:
+      - "packages/token-lists/src/tokens/**"
+      - "packages/token-lists/lists/**"
+
+jobs:
+  verifyTokenLists:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version: 14.x
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Check if tokenlists were updated correctly
+        working-directory: ./packages/token-lists
+        run: yarn ci-check

packages/token-lists/README.md

@@ -10,16 +10,18 @@ URLs to external lists are stored in `token-lists.json`, if you want your list t
 
 - Add an array of tokens under `src/tokens`
 - Add `checksum:newlistname`, `generate:newlistname`, `makelist:newlistname` command to `package.json` analogous to PancakeSwap default and extended list scripts.
-- Modify `checksum.ts`, `buildList.ts` and `default.test.ts` to handle new list
+- Modify `checksum.ts`, `buildList.ts`, `ci-check.ts`, and `default.test.ts` to handle new list
 
 ## How to add new tokens to PancakeSwap (extended) token list
 
 Note - this is not something we expect pull requests for.  
 Unless you've been specifically asked by someone from PCS team please do no submit PRs to be listed on default PCS list. You can still trade your tokens on PCS exchange by pasting your address into the token field.
 
-- Update version in `package.json`
 - Add new tokens to `src/tokens/pancakeswap-extended.json` file
 - Run `yarn makelist:pcs-extended`
+  - By default new list will have patch version number bumped by 1 (e.g. `2.0.1` -> `2.0.2`).
+  - If you want to bump minor version add `minor` after makelist command `yarn makelist:pcs-extended minor`
+  - If you want to bump major version add `major` after makelist command `yarn makelist:pcs-extended major`
 - If tests pass - new token list will be created under `lists` directory
 
 For list to be considered valid it need to satisfy the following criteria:
@@ -30,7 +32,7 @@ For list to be considered valid it need to satisfy the following criteria:
 
 ## How to update Top100 Token list
 
-Note - this is not something we expect pull requests for. 
+Note - this is not something we expect pull requests for.
 
 ```shell script
 # Fetch the Top100 Tokens on PancakeSwap v2, and update list.

packages/token-lists/package.json

@@ -20,7 +20,8 @@
     "generate:pcs-top-15": "yarn test && yarn build && node ./dist generate pancakeswap-top-15",
     "makelist:pcs-top-15": "yarn checksum:pcs-top-15 && yarn generate:pcs-top-15",
     "fetch:pcs-top-100": "yarn build && node ./dist fetch",
-    "test": "jest"
+    "test": "jest",
+    "ci-check": "yarn build && node ./dist ci-check"
   },
   "dependencies": {
     "@ethersproject/address": "^5.1.0",

packages/token-lists/src/buildList.ts

@@ -1,12 +1,27 @@
 import fs from "fs";
 import path from "path";
 import { TokenList } from "@uniswap/token-lists";
-import { version } from "../package.json";
+import { version as pancakeswapDefaultVersion } from "../lists/pancakeswap-default.json";
+import { version as pancakeswapExtendedVersion } from "../lists/pancakeswap-extended.json";
+import { version as pancakeswapTop15Version } from "../lists/pancakeswap-top-15.json";
+import { version as pancakeswapTop100Version } from "../lists/pancakeswap-top-100.json";
 import pancakeswapDefault from "./tokens/pancakeswap-default.json";
 import pancakeswapExtended from "./tokens/pancakeswap-extended.json";
 import pancakeswapTop100 from "./tokens/pancakeswap-top-100.json";
 import pancakeswapTop15 from "./tokens/pancakeswap-top-15.json";
 
+export enum VersionBump {
+  "major" = "major",
+  "minor" = "minor",
+  "patch" = "patch",
+}
+
+type Version = {
+  major: number;
+  minor: number;
+  patch: number;
+};
+
 const lists = {
   "pancakeswap-default": {
     list: pancakeswapDefault,
@@ -15,6 +30,7 @@ const lists = {
     logoURI:
       "https://assets.trustwalletapp.com/blockchains/smartchain/assets/0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82/logo.png",
     sort: false,
+    currentVersion: pancakeswapDefaultVersion,
   },
   "pancakeswap-extended": {
     list: pancakeswapExtended,
@@ -23,6 +39,7 @@ const lists = {
     logoURI:
       "https://assets.trustwalletapp.com/blockchains/smartchain/assets/0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82/logo.png",
     sort: true,
+    currentVersion: pancakeswapExtendedVersion,
   },
   "pancakeswap-top-100": {
     list: pancakeswapTop100,
@@ -31,6 +48,7 @@ const lists = {
     logoURI:
       "https://assets.trustwalletapp.com/blockchains/smartchain/assets/0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82/logo.png",
     sort: true,
+    currentVersion: pancakeswapTop100Version,
   },
   "pancakeswap-top-15": {
     list: pancakeswapTop15,
@@ -39,20 +57,30 @@ const lists = {
     logoURI:
       "https://assets.trustwalletapp.com/blockchains/smartchain/assets/0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82/logo.png",
     sort: true,
+    currentVersion: pancakeswapTop15Version,
   },
 };
 
-export const buildList = (listName: string): TokenList => {
-  const [major, minor, patch] = version.split(".").map((versionNumber) => parseInt(versionNumber, 10));
-  const { list, name, keywords, logoURI, sort } = lists[listName];
+const getNextVersion = (currentVersion: Version, versionBump?: VersionBump) => {
+  const { major, minor, patch } = currentVersion;
+  switch (versionBump) {
+    case VersionBump.major:
+      return { major: major + 1, minor, patch };
+    case VersionBump.minor:
+      return { major, minor: minor + 1, patch };
+    case VersionBump.patch:
+    default:
+      return { major, minor, patch: patch + 1 };
+  }
+};
+
+export const buildList = (listName: string, versionBump?: VersionBump): TokenList => {
+  const { list, name, keywords, logoURI, sort, currentVersion } = lists[listName];
+  const version = getNextVersion(currentVersion, versionBump);
   return {
     name,
     timestamp: new Date().toISOString(),
-    version: {
-      major,
-      minor,
-      patch,
-    },
+    version,
     logoURI,
     keywords,
     // sort them by symbol for easy readability (not applied to default list)

packages/token-lists/src/ci-check.ts

@@ -0,0 +1,62 @@
+import srcDefault from "./tokens/pancakeswap-default.json";
+import srcExtended from "./tokens/pancakeswap-extended.json";
+import srcTop100 from "./tokens/pancakeswap-top-100.json";
+import srcTop15 from "./tokens/pancakeswap-top-15.json";
+import defaultList from "../lists/pancakeswap-default.json";
+import extendedtList from "../lists/pancakeswap-extended.json";
+import top15List from "../lists/pancakeswap-top-15.json";
+import top100tList from "../lists/pancakeswap-top-100.json";
+
+const lists = [
+  {
+    name: "pancakeswap-default",
+    src: srcDefault,
+    actual: defaultList,
+  },
+  {
+    name: "pancakeswap-extended",
+    src: srcExtended,
+    actual: extendedtList,
+  },
+  {
+    name: "pancakeswap-top-15",
+    src: srcTop15,
+    actual: top15List,
+  },
+  {
+    name: "pancakeswap-top-100",
+    src: srcTop100,
+    actual: top100tList,
+  },
+];
+
+const compareLists = (listPair) => {
+  const { name, src, actual } = listPair;
+  if (src.length !== actual.tokens.length) {
+    throw Error(
+      `List ${name} seems to be not properly regenerated. Soure file has ${src.length} tokens but actual list has ${actual.tokens.length}. Did you forget to run yarn makelist?`
+    );
+  }
+  src.sort((t1, t2) => (t1.address < t2.address ? -1 : 1));
+  actual.tokens.sort((t1, t2) => (t1.address < t2.address ? -1 : 1));
+  src.forEach((srcToken, index) => {
+    if (JSON.stringify(srcToken) !== JSON.stringify(actual.tokens[index])) {
+      throw Error(
+        `List ${name} seems to be not properly regenerated. Tokens from src/tokens directory don't match up with the final list. Did you forget to run yarn makelist?`
+      );
+    }
+  });
+};
+
+/**
+ * Check in CI that author properly updated token list
+ * i.e. not just changed token list in src/tokens but also regenerated lists with yarn makelist command.
+ * Github Action runs only on change in src/tokens directory.
+ */
+const ciCheck = (): void => {
+  lists.forEach((listPair) => {
+    compareLists(listPair);
+  });
+};
+
+export default ciCheck;

packages/token-lists/src/index.ts

@@ -1,20 +1,25 @@
-import { buildList, saveList } from "./buildList";
+import { buildList, saveList, VersionBump } from "./buildList";
 import checksumAddresses from "./checksum";
+import ciCheck from "./ci-check";
 import topTokens from "./top-100";
 
 const command = process.argv[2];
 const listName = process.argv[3];
+const versionBump = process.argv[4];
 
 switch (command) {
   case "checksum":
     checksumAddresses(listName);
     break;
   case "generate":
-    saveList(buildList(listName), listName);
+    saveList(buildList(listName, versionBump as VersionBump), listName);
     break;
   case "fetch":
     topTokens();
     break;
+  case "ci-check":
+    ciCheck();
+    break;
   default:
     console.info("Unknown command");
     break;