diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 317572a3..36c54da6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,6 @@ jobs: pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --exclude=venv,__pycache__ --count --ignore=E1,E2,E3,E501,W1,W2,W3,W5 --show-source --statistics --max-complexity=10 --max-line-length=100 - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - #flake8 . --count --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | #pip install pytest diff --git a/README.md b/README.md index 7d582315..d1a63476 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # DAO-Analyzer -It is a tool to visualize DAO metrics. Currently, it shows DAO from [DAOstack](https://daostack.io/), [DAOhaus](https://daohaus.club/). +It is a tool to visualize DAO metrics. Currently, it shows DAO from [DAOstack](https://daostack.io/), [DAOhaus](https://daohaus.club/), and [Aragon](https://aragon.org/). ## Available metrics * DAOstack: + * DAO: + * Months which the DAO has registered activity * Reputation Holders: * New reputation holders * Active reputation holders @@ -20,6 +22,8 @@ It is a tool to visualize DAO metrics. Currently, it shows DAO from [DAOstack](h * Success rate of the stakes by type * DAOhaus: + * DAO: + * Months which the DAO has registered activity * Members: * New members * Active members @@ -33,8 +37,24 @@ It is a tool to visualize DAO metrics. Currently, it shows DAO from [DAOstack](h * Proposals outcome * Proposals type +* Aragon: + * DAO: + * Months which the DAO has registered activity + * Token Holders: + * Active token holders + * Votes: + * New votes + * Votes's outcome + * Casted votes: + * Casted votes by support + * Active voters + * Transactions: + * New transactions + * Aragon apps: + * Installed apps + ## Architecture -There is available a class diagram of the [DAOstack app](https://github.com/Grasia/dao-analyzer/blob/master/src/apps/daostack/class_diagram.png), and the [DAOhaus app](https://github.com/Grasia/dao-analyzer/blob/master/src/apps/daohaus/class_diagram.png). +There is available a class diagram of the [DAOstack app](https://github.com/Grasia/dao-analyzer/blob/master/src/apps/daostack/class_diagram.png), the [DAOhaus app](https://github.com/Grasia/dao-analyzer/blob/master/src/apps/daohaus/class_diagram.png), and the [Aragon app](https://github.com/Grasia/dao-analyzer/blob/master/src/apps/aragon/class_diagram.png). ## Download Enter in your terminal (git must be installed) and write down: diff --git a/cache_scripts/api_requester.py b/cache_scripts/api_requester.py index 8dc22444..ff698d71 100644 --- a/cache_scripts/api_requester.py +++ b/cache_scripts/api_requester.py @@ -15,10 +15,20 @@ class ApiRequester: ELEMS_PER_CHUNK: int = 1000 + DAOSTACK: int = 0 DAOHAUS: int = 1 - __DAOSTACK_URL: str = 'https://api.thegraph.com/subgraphs/name/daostack/master' - __DAOHAUS_URL: str = 'https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus' + ARAGON_MAINNET: int = 2 + ARAGON_TOKENS: int = 3 + ARAGON_VOTING: int = 4 + ARAGON_FINANCE: int = 5 + + __URL_DAOSTACK: str = 'https://api.thegraph.com/subgraphs/name/daostack/master' + __URL_DAOHAUS: str = 'https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus' + __URL_ARAGON_MAINNET: str = 'https://api.thegraph.com/subgraphs/name/aragon/aragon-mainnet' + __URL_ARAGON_TOKENS: str = 'https://api.thegraph.com/subgraphs/name/aragon/aragon-tokens-mainnet' + __URL_ARAGON_VOTING: str = 'https://api.thegraph.com/subgraphs/name/aragon/aragon-voting-mainnet' + __URL_ARAGON_FINANCE: str = 'https://api.thegraph.com/subgraphs/name/aragon/aragon-finance-mainnet' def __init__(self, endpoint: int) -> None: @@ -30,9 +40,17 @@ def __get_endpoint(self, endpoint: int) -> str: url: str = '' if endpoint is self.DAOSTACK: - url = self.__DAOSTACK_URL + url = self.__URL_DAOSTACK elif endpoint is self.DAOHAUS: - url = self.__DAOHAUS_URL + url = self.__URL_DAOHAUS + elif endpoint is self.ARAGON_MAINNET: + url = self.__URL_ARAGON_MAINNET + elif endpoint is self.ARAGON_TOKENS: + url = self.__URL_ARAGON_TOKENS + elif endpoint is self.ARAGON_VOTING: + url = self.__URL_ARAGON_VOTING + elif endpoint is self.ARAGON_FINANCE: + url = self.__URL_ARAGON_FINANCE return url diff --git a/cache_scripts/aragon/__init__.py b/cache_scripts/aragon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cache_scripts/aragon/collectors/__init__.py b/cache_scripts/aragon/collectors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cache_scripts/aragon/collectors/apps.py b/cache_scripts/aragon/collectors/apps.py new file mode 100644 index 00000000..2bb0e45e --- /dev/null +++ b/cache_scripts/aragon/collectors/apps.py @@ -0,0 +1,65 @@ +""" + Descp: Script to fetch app's data and store it. + + Created on: 15-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +APP_QUERY: str = '{{apps(first: {0}, skip: {1}\ +){{id isForwarder isUpgradeable repoName repoAddress organization{{id}} }}}}' + +META_KEY: str = 'apps' + + +def _request_apps(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_MAINNET) + print("Requesting App\'s data ...") + start: datetime = datetime.now() + + apps: List[Dict] = requester.n_requests(query=APP_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'App\'s data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return apps + + +def _transform_to_df(apps: List[Dict]) -> pd.DataFrame: + for app in apps: + org: str = app['organization']['id'] + del app['organization'] + app['organizationId'] = org + + return pd.DataFrame(apps) + + +def update_apps(meta_data: Dict) -> None: + apps: List[Dict] = _request_apps(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(apps=apps) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(apps) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_apps(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/cast.py b/cache_scripts/aragon/collectors/cast.py new file mode 100644 index 00000000..5808d017 --- /dev/null +++ b/cache_scripts/aragon/collectors/cast.py @@ -0,0 +1,69 @@ +""" + Descp: Script to fetch Cast data and store it. Cast means vote. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +CAST_QUERY: str = '{{casts(first: {0}, skip: {1}\ +){{id voteId voter supports voterStake createdAt vote{{orgAddress appAddress}} }}}}' + +META_KEY: str = 'casts' + + +def _request_casts(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_VOTING) + print("Requesting Cast data ...") + start: datetime = datetime.now() + + casts: List[Dict] = requester.n_requests(query=CAST_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Cast data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return casts + + +def _transform_to_df(casts: List[Dict]) -> pd.DataFrame: + for cast in casts: + org: str = cast['vote']['orgAddress'] + app: str = cast['vote']['appAddress'] + + del cast['vote'] + + cast['orgAddress'] = org + cast['appAddress'] = app + + return pd.DataFrame(casts) + + +def update_casts(meta_data: Dict) -> None: + casts: List[Dict] = _request_casts(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(casts=casts) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(casts) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_casts(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/mini_me_token.py b/cache_scripts/aragon/collectors/mini_me_token.py new file mode 100644 index 00000000..6b127c70 --- /dev/null +++ b/cache_scripts/aragon/collectors/mini_me_token.py @@ -0,0 +1,60 @@ +""" + Descp: Script to fetch MiniMeToken's data and store it. + + Created on: 15-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +MINI_ME_TOKEN_QUERY: str = '{{miniMeTokens(first: {0}, skip: {1}\ +){{id address totalSupply transferable name symbol orgAddress appAddress}}}}' + +META_KEY: str = 'miniMeTokens' + + +def _request_mini_me_tokens(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_TOKENS) + print("Requesting Mini me token\'s data ...") + start: datetime = datetime.now() + + tokens: List[Dict] = requester.n_requests(query=MINI_ME_TOKEN_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Mini me token\'s data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return tokens + + +def _transform_to_df(tokens: List[Dict]) -> pd.DataFrame: + return pd.DataFrame(tokens) + + +def update_tokens(meta_data: Dict) -> None: + tokens: List[Dict] = _request_mini_me_tokens(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(tokens=tokens) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(tokens) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_tokens(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/organizations.py b/cache_scripts/aragon/collectors/organizations.py new file mode 100644 index 00000000..37f90e60 --- /dev/null +++ b/cache_scripts/aragon/collectors/organizations.py @@ -0,0 +1,68 @@ +""" + Descp: Script to fetch organization's data and store it. + + Created on: 15-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +ORGANIZATION_QUERY: str = '{{organizations(first: {0}, skip: {1}\ +){{id createdAt recoveryVault}}}}' + +META_KEY: str = 'organizations' + + +def _request_organizations(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_MAINNET) + print("Requesting Organization\'s data ...") + start: datetime = datetime.now() + + orgs: List[Dict] = requester.n_requests(query=ORGANIZATION_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Organization\'s data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return orgs + + +def _transform_to_df(orgs: List[Dict]) -> pd.DataFrame: + if not orgs: + return pd.DataFrame() + + df: pd.DataFrame = pd.DataFrame(orgs) + + #TODO: temporal solution to non-attribute name + df['name'] = df['id'].tolist() + + return df + + +def update_organizations(meta_data: Dict) -> None: + orgs: List[Dict] = _request_organizations(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(orgs=orgs) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(orgs) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_organizations(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/repo.py b/cache_scripts/aragon/collectors/repo.py new file mode 100644 index 00000000..fb929bcd --- /dev/null +++ b/cache_scripts/aragon/collectors/repo.py @@ -0,0 +1,58 @@ +""" + Descp: Script to fetch repo data and store it, app schema is an instances of repo. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +REPO_QUERY: str = '{{repos(first: {0}, skip: {1}){{id address name node appCount}}}}' +META_KEY: str = 'repos' + + +def _request_repos(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_MAINNET) + print("Requesting Repo data ...") + start: datetime = datetime.now() + + repos: List[Dict] = requester.n_requests(query=REPO_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Repo data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return repos + + +def _transform_to_df(repos: List[Dict]) -> pd.DataFrame: + return pd.DataFrame(repos) + + +def update_repos(meta_data: Dict) -> None: + repos: List[Dict] = _request_repos(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(repos=repos) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(repos) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_repos(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/token_holders.py b/cache_scripts/aragon/collectors/token_holders.py new file mode 100644 index 00000000..0a4b4586 --- /dev/null +++ b/cache_scripts/aragon/collectors/token_holders.py @@ -0,0 +1,82 @@ +""" + Descp: Script to fetch TokenHolder's data and store it. + + Created on: 15-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester +from aragon.collectors.mini_me_token import META_KEY as TOKEN_KEY + + +TOKEN_HOLDER_QUERY: str = '{{tokenHolders(first: {0}, skip: {1}\ +){{id address tokenAddress balance}}}}' + +TOKEN_QUERY: str = '{{miniMeTokens(first: {0}, skip: {1}\ +){{address orgAddress}}}}' + +META_KEY: str = 'tokenHolders' + + +def _request_token_holders(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_TOKENS) + print("Requesting Token Holders\'s data ...") + start: datetime = datetime.now() + + holders: List[Dict] = requester.n_requests(query=TOKEN_HOLDER_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Token Holders\'s data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return holders + + +def _transform_to_df(holders: List[Dict]) -> pd.DataFrame: + if not holders: + return pd.DataFrame() + + df: pd.DataFrame = pd.DataFrame(holders) + + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_TOKENS) + tokens: List[Dict] = requester.n_requests( + query=TOKEN_QUERY, + skip_n=0, + result_key=TOKEN_KEY) + + # List[Dict[str, str]] -> Dict[str, str] + tokens: Dict[str, str] = {token['address']: token['orgAddress'] for token in tokens} + + tokenAddreses: List[str] = df['tokenAddress'].tolist() + orgAddrs: List[str] = [tokens[x] for x in tokenAddreses] + + df['organizationAddress'] = orgAddrs + return df + + +def update_holders(meta_data: Dict) -> None: + holders: List[Dict] = _request_token_holders(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(holders=holders) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(holders) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_holders(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/transaction.py b/cache_scripts/aragon/collectors/transaction.py new file mode 100644 index 00000000..992de177 --- /dev/null +++ b/cache_scripts/aragon/collectors/transaction.py @@ -0,0 +1,60 @@ +""" + Descp: Script to fetch Transaction's data and store it. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +TRANSACTION_QUERY: str = '{{transactions(first: {0}, skip: {1}\ +){{id orgAddress appAddress token entity isIncoming amount date reference }}}}' + +META_KEY: str = 'transactions' + + +def _request_transactions(current_rows: int) -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_FINANCE) + print("Requesting Transaction's data ...") + start: datetime = datetime.now() + + transactions: List[Dict] = requester.n_requests(query=TRANSACTION_QUERY, skip_n=current_rows, + result_key=META_KEY) + + print(f'Transaction\'s data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return transactions + + +def _transform_to_df(transactions: List[Dict]) -> pd.DataFrame: + return pd.DataFrame(transactions) + + +def update_transactions(meta_data: Dict) -> None: + transactions: List[Dict] = _request_transactions(current_rows=meta_data[META_KEY]['rows']) + df: pd.DataFrame = _transform_to_df(transactions=transactions) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + if os.path.isfile(filename): + df.to_csv(filename, mode='a', header=False, index=False) + else: + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(transactions) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_transactions(meta_data=meta) diff --git a/cache_scripts/aragon/collectors/vote.py b/cache_scripts/aragon/collectors/vote.py new file mode 100644 index 00000000..d63ffc96 --- /dev/null +++ b/cache_scripts/aragon/collectors/vote.py @@ -0,0 +1,60 @@ +""" + Descp: Script to fetch vote data and store it. + In Aragon context, Vote means proposal, and cast means vote. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import pandas as pd +from typing import Dict, List +from datetime import datetime, date + +from api_requester import ApiRequester + + +VOTE_QUERY: str = '{{votes(first: {0}, skip: {1})\ +{{id orgAddress appAddress creator metadata executed executedAt startDate \ +supportRequiredPct minAcceptQuorum yea nay voteNum votingPower}}}}' + +META_KEY: str = 'votes' + + +def _request_votes() -> List[Dict]: + requester: ApiRequester = ApiRequester(endpoint=ApiRequester.ARAGON_VOTING) + print("Requesting Vote data ...") + start: datetime = datetime.now() + + votes: List[Dict] = requester.n_requests(query=VOTE_QUERY, skip_n=0, + result_key=META_KEY) + + print(f'Vote data requested in {round((datetime.now() - start).total_seconds(), 2)}s') + return votes + + +def _transform_to_df(votes: List[Dict]) -> pd.DataFrame: + return pd.DataFrame(votes) + + +def update_votes(meta_data: Dict) -> None: + votes: List[Dict] = _request_votes() + df: pd.DataFrame = _transform_to_df(votes=votes) + + filename: str = os.path.join('datawarehouse', 'aragon', f'{META_KEY}.csv') + + # Always rewrite the whole file cause it is more efficient to do it than request all the open proposals. + df.to_csv(filename, index=False) + + print(f'Data stored in {filename}.\n') + + # update meta + meta_data[META_KEY]['rows'] = len(votes) + meta_data[META_KEY]['lastUpdate'] = str(date.today()) + + +if __name__ == '__main__': + meta: dict = {META_KEY: {'rows': 0}} + update_votes(meta_data=meta) diff --git a/cache_scripts/aragon/main.py b/cache_scripts/aragon/main.py new file mode 100644 index 00000000..88d1162f --- /dev/null +++ b/cache_scripts/aragon/main.py @@ -0,0 +1,91 @@ +""" + Descp: Main script to create Aragon datawarehouse, it call all the collectors. + + Created on: 15-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import os +import json +from typing import Dict, List, Callable + +import aragon.collectors.organizations as organizations +import aragon.collectors.apps as apps +import aragon.collectors.mini_me_token as token +import aragon.collectors.token_holders as holders +import aragon.collectors.repo as repos +import aragon.collectors.vote as votes +import aragon.collectors.cast as casts +import aragon.collectors.transaction as transactions + +DIRS: str = os.path.join('datawarehouse', 'aragon') +META_PATH: str = os.path.join(DIRS, 'meta.json') +KEYS: List[str] = [ + organizations.META_KEY, + apps.META_KEY, + token.META_KEY, + holders.META_KEY, + repos.META_KEY, + votes.META_KEY, + casts.META_KEY, + transactions.META_KEY, +] # add here new keys +COLLECTORS: List[Callable] = [ + organizations.update_organizations, + apps.update_apps, + token.update_tokens, + holders.update_holders, + repos.update_repos, + votes.update_votes, + casts.update_casts, + transactions.update_transactions, +] # add new collectors + + +def _fill_empty_keys(meta_data: Dict) -> Dict: + meta_fill: Dict = meta_data + + for k in KEYS: + if k not in meta_data: + meta_fill[k] = {'rows': 0} + + return meta_fill + + +def _get_meta_data() -> Dict: + meta_data: Dict + + if os.path.isfile(META_PATH): + with open(META_PATH) as json_file: + meta_data = json.load(json_file) + else: + meta_data = dict() # there are not previous executions + + return _fill_empty_keys(meta_data=meta_data) + + +def _write_meta_data(meta: Dict) -> None: + with open(META_PATH, 'w+') as outfile: + json.dump(meta, outfile) + + print(f'Updated meta-data in {META_PATH}') + + +def run() -> None: + print('------------- Updating Aragon\' datawarehouse -------------\n') + if not os.path.isdir(DIRS): + os.makedirs(DIRS) + + meta_data: Dict = _get_meta_data() + + for c in COLLECTORS: + c(meta_data) + + _write_meta_data(meta=meta_data) + print('------------- Aragon\' datawarehouse updated -------------\n') + + +if __name__ == '__main__': + run() diff --git a/cache_scripts/daohaus/collectors/proposal_collector.py b/cache_scripts/daohaus/collectors/proposal_collector.py index e59279e0..8ed07cd8 100644 --- a/cache_scripts/daohaus/collectors/proposal_collector.py +++ b/cache_scripts/daohaus/collectors/proposal_collector.py @@ -21,8 +21,9 @@ sponsored sponsoredAt processed didPass yesShares noShares}}}}' O_PROPOSAL_QUERY: str = '{{proposal(id: \"{0}\")\ -{{id yesVotes noVotes sponsor sponsored sponsoredAt processed didPass yesShares \ -noShares}}}}' +{{id createdAt proposalId molochAddress memberAddress proposer sponsor \ +sharesRequested lootRequested tributeOffered paymentRequested yesVotes noVotes \ +sponsored sponsoredAt processed didPass yesShares noShares}}}}' META_KEY: str = 'proposals' @@ -73,14 +74,13 @@ def join_data(df: pd.DataFrame, df2: pd.DataFrame, df3: pd.DataFrame) -> pd.Data """ dff: pd.DataFrame = df - dff.set_index('id', inplace=True) - if len(df2) > 0: - df2.set_index('id', inplace=True) - dff.update(df2) + ids: List[str] = df2['id'].tolist() + index = dff[dff['id'].isin(ids)].index + dff.drop(index, inplace=True) + dff = dff.append(df2) if len(df3) > 0: - df3.set_index('id', inplace=True) dff = dff.append(df3) return dff @@ -90,8 +90,8 @@ def update_proposals(meta_data: Dict) -> None: df: pd.DataFrame data: List[Dict] = _request_proposals(current_rows=meta_data[META_KEY]['rows']) - df3: pd.DataFrame = _transform_to_df(data=data) + size: int = len(df3) filename: str = os.path.join('datawarehouse', 'daohaus', 'proposals.csv') @@ -103,7 +103,8 @@ def update_proposals(meta_data: Dict) -> None: df2: pd.DataFrame = pd.DataFrame(open_prop) df = join_data(df=df, df2=df2, df3=df3) - df.to_csv(filename) + df.to_csv(filename, index=False) + size = len(df) # save all proposals else: @@ -112,7 +113,7 @@ def update_proposals(meta_data: Dict) -> None: print(f'Data stored in {filename}.\n') # update meta - meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(data) + meta_data[META_KEY]['rows'] = size meta_data[META_KEY]['lastUpdate'] = str(date.today()) diff --git a/cache_scripts/daohaus/main.py b/cache_scripts/daohaus/main.py index 93f65393..a0c095f6 100644 --- a/cache_scripts/daohaus/main.py +++ b/cache_scripts/daohaus/main.py @@ -18,19 +18,26 @@ DIRS: str = os.path.join('datawarehouse', 'daohaus') META_PATH: str = os.path.join(DIRS, 'meta.json') +KEYS: List[str] = [ + moloch.META_KEY, + member.META_KEY, + vote.META_KEY, + rage_quit.META_KEY, + proposal.META_KEY, +] # add here new keys +COLLECTORS: List = [ + moloch.update_moloches, + member.update_members, + vote.update_votes, + rage_quit.update_rage_quits, + proposal.update_proposals, +] # add new collectors def _fill_empty_keys(meta_data: Dict) -> Dict: meta_fill: Dict = meta_data - keys: List[str] = [ - moloch.META_KEY, - member.META_KEY, - vote.META_KEY, - rage_quit.META_KEY, - proposal.META_KEY, - ] # add here new keys - - for k in keys: + + for k in KEYS: if k not in meta_data: meta_fill[k] = {'rows': 0} @@ -44,7 +51,7 @@ def _get_meta_data() -> Dict: with open(META_PATH) as json_file: meta_data = json.load(json_file) else: - meta_data = dict() # there is not previous executions + meta_data = dict() # there are not previous executions return _fill_empty_keys(meta_data=meta_data) @@ -63,16 +70,7 @@ def run() -> None: meta_data: Dict = _get_meta_data() - # add new collectors - collectors: List = [ - moloch.update_moloches, - member.update_members, - vote.update_votes, - rage_quit.update_rage_quits, - proposal.update_proposals, - ] - - for c in collectors: + for c in COLLECTORS: c(meta_data) _write_meta_data(meta=meta_data) diff --git a/cache_scripts/daostack/collectors/proposal_collector.py b/cache_scripts/daostack/collectors/proposal_collector.py index ff751e2a..3dfaf0fe 100644 --- a/cache_scripts/daostack/collectors/proposal_collector.py +++ b/cache_scripts/daostack/collectors/proposal_collector.py @@ -22,8 +22,10 @@ genesisProtocolParams{{queuedVoteRequiredPercentage}} dao{{id}} }}}}' O_PROPOSAL_QUERY: str = '{{proposal(id: \"{0}\")\ -{{id stage preBoostedAt boostedAt executedAt totalRepWhenExecuted executionState \ -expiresInQueueAt votesFor votesAgainst winningOutcome stakesFor stakesAgainst}}}}' +{{id proposer stage createdAt preBoostedAt boostedAt closingAt executedAt \ +totalRepWhenExecuted totalRepWhenCreated executionState \ +expiresInQueueAt votesFor votesAgainst winningOutcome stakesFor stakesAgainst \ +genesisProtocolParams{{queuedVoteRequiredPercentage}} dao{{id}} }}}}' META_KEY: str = 'proposals' OUT_FILE: str = os.path.join('datawarehouse', 'daostack', 'proposals.csv') @@ -86,14 +88,13 @@ def join_data(df: pd.DataFrame, df2: pd.DataFrame, df3: pd.DataFrame) -> pd.Data """ dff: pd.DataFrame = df - dff.set_index('id', inplace=True) - if len(df2) > 0: - df2.set_index('id', inplace=True) - dff.update(df2) + ids: List[str] = df2['id'].tolist() + index = dff[dff['id'].isin(ids)].index + dff.drop(index, inplace=True) + dff = dff.append(df2) if len(df3) > 0: - df3.set_index('id', inplace=True) dff = dff.append(df3) return dff @@ -105,16 +106,18 @@ def update_proposals(meta_data: Dict) -> None: proposals: List[Dict] = _request_proposals(current_rows= meta_data[META_KEY]['rows']) df3: pd.DataFrame = _transform_to_df(proposals=proposals) + size: int = len(df3) # fetch new proposals and update opened proposals if os.path.isfile(OUT_FILE): df = pd.read_csv(OUT_FILE, header=0) open_prop: List[Dict] = _request_open_proposals(ids=_get_opened_proposals(df)) - df2: pd.DataFrame = pd.DataFrame(open_prop) + df2: pd.DataFrame = _transform_to_df(proposals=open_prop) df = join_data(df=df, df2=df2, df3=df3) - df.to_csv(OUT_FILE) + df.to_csv(OUT_FILE, index=False) + size = len(df) # save all proposals else: @@ -123,7 +126,7 @@ def update_proposals(meta_data: Dict) -> None: print(f'Data stored in {OUT_FILE}.\n') # update meta - meta_data[META_KEY]['rows'] = meta_data[META_KEY]['rows'] + len(proposals) + meta_data[META_KEY]['rows'] = size meta_data[META_KEY]['lastUpdate'] = str(date.today()) diff --git a/cache_scripts/daostack/main.py b/cache_scripts/daostack/main.py index 93a68986..a1d95d9b 100644 --- a/cache_scripts/daostack/main.py +++ b/cache_scripts/daostack/main.py @@ -18,19 +18,26 @@ DIRS: str = os.path.join('datawarehouse', 'daostack') META_PATH: str = os.path.join(DIRS, 'meta.json') +KEYS: List[str] = [ + dao.META_KEY, + rep_h.META_KEY, + vote.META_KEY, + stake.META_KEY, + proposal.META_KEY +] # add here new keys +COLLECTORS: List = [ + dao.update_daos, + rep_h.update_rep_holders, + vote.update_votes, + stake.update_stakes, + proposal.update_proposals +]# add new collectors def _fill_empty_keys(meta_data: Dict) -> Dict: meta_fill: Dict = meta_data - keys: List[str] = [ - dao.META_KEY, - rep_h.META_KEY, - vote.META_KEY, - stake.META_KEY, - proposal.META_KEY - ] # add here new keys - - for k in keys: + + for k in KEYS: if k not in meta_data: meta_fill[k] = {'rows': 0} @@ -44,7 +51,7 @@ def _get_meta_data() -> Dict: with open(META_PATH) as json_file: meta_data = json.load(json_file) else: - meta_data = dict() # there is not previous executions + meta_data = dict() # there are not previous executions return _fill_empty_keys(meta_data=meta_data) @@ -63,16 +70,7 @@ def run() -> None: meta_data: Dict = _get_meta_data() - # add new collectors - collectors: List = [ - dao.update_daos, - rep_h.update_rep_holders, - vote.update_votes, - stake.update_stakes, - proposal.update_proposals - ] - - for c in collectors: + for c in COLLECTORS: c(meta_data) _write_meta_data(meta=meta_data) diff --git a/cache_scripts/main.py b/cache_scripts/main.py index f84de6ef..fa4101d9 100644 --- a/cache_scripts/main.py +++ b/cache_scripts/main.py @@ -1,7 +1,9 @@ import daostack.main as daostack import daohaus.main as daohaus +import aragon.main as aragon if __name__ == '__main__': daostack.run() daohaus.run() + aragon.run() diff --git a/src/apps/aragon/__init__.py b/src/apps/aragon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/business/__init__.py b/src/apps/aragon/business/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/business/app_service.py b/src/apps/aragon/business/app_service.py new file mode 100644 index 00000000..902d9209 --- /dev/null +++ b/src/apps/aragon/business/app_service.py @@ -0,0 +1,307 @@ +""" + Descp: Manage the application logic, and it's used to interconect the + data_access and presentation layers. + + Created on: 2-mar-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +from typing import Dict, List, Callable +import dash_html_components as html + +from src.app import app +import src.apps.common.presentation.dashboard_view.dashboard_view as view +import src.apps.common.presentation.dashboard_view.controller as view_cont +from src.apps.common.data_access.daos.organization_dao\ + import OrganizationListDao +import src.apps.aragon.data_access.requesters.cache_requester as cache +from src.apps.common.business.transfers.organization import OrganizationList +from src.apps.common.presentation.charts.chart_controller import ChartController +from src.apps.common.presentation.charts.layout.chart_pane_layout \ + import ChartPaneLayout +from src.apps.common.presentation.charts.layout.figure.figure import Figure +from src.apps.common.presentation.charts.layout.figure.bar_figure import BarFigure +from src.apps.common.presentation.charts.layout.figure.multi_bar_figure import MultiBarFigure +import src.apps.aragon.data_access.daos.metric.metric_dao_factory as s_factory +from src.apps.common.business.i_metric_adapter import IMetricAdapter +from src.apps.aragon.business.metric_adapter.basic_adapter import BasicAdapter +from src.apps.aragon.business.metric_adapter.installed_apps import InstalledApps +from src.apps.aragon.business.metric_adapter.cast_type import CastType +from src.apps.aragon.business.metric_adapter.vote_outcome import VoteOutcome + +from src.apps.aragon.resources.strings import TEXT +from src.apps.common.resources.strings import TEXT as COMMON_TEXT +import src.apps.common.resources.colors as COLOR + + +_aragon_service = None + +def get_service(): + """ + Singelton object. + """ + global _aragon_service + + if not _aragon_service: + _aragon_service = AragonService() + + return _aragon_service + + +class AragonService(): + _TOKEN_HOLDER: int = 0 + _VOTE: int = 1 + _CAST: int = 2 + _TRANSACTION: int = 3 + _APP: int = 4 + _ORGANIZATION: int = 5 + + def __init__(self): + # app state + self.__orgs: OrganizationList = None + self.__controllers: Dict[int, List[ChartController]] = { + self._TOKEN_HOLDER: list(), + self._VOTE: list(), + self._CAST: list(), + self._TRANSACTION: list(), + self._APP: list(), + self._ORGANIZATION: list(), + } + + + @property + def organizations(self) -> OrganizationList: + if not self.__orgs: + orgs: OrganizationList = OrganizationListDao(cache.CacheRequester( + srcs=[cache.ORGANIZATIONS])).get_organizations() + if not orgs.is_empty(): + self.__orgs = orgs + + return self.__orgs + + + @property + def are_panes(self) -> bool: + """ + Checks if panes and their controllers are already created. + """ + is_empty: bool = False + + for _, v in self.__controllers.items(): + is_empty = is_empty or (len(v) != 0) + + return is_empty + + + def get_layout(self) -> html.Div: + """ + Returns the app's layout. + """ + orgs: OrganizationList = self.organizations + + if not self.are_panes: + view_cont.bind_callbacks( + app=app, + section_id=TEXT['css_id_organization']) + + return view.generate_layout( + labels=orgs.get_dict_representation(), + sections=self.__get_sections(), + color_app=COMMON_TEXT['css_color_aragon'] + ) + + + def __get_sections(self) -> Dict[str, List[Callable]]: + """ + Returns a dict with each section filled with a callable function which + returns the chart layout + """ + l_token_holders: List[Callable] = list() + l_vote: List[Callable] = list() + l_cast: List[Callable] = list() + l_transaction: List[Callable] = list() + l_app: List[Callable] = list() + l_organization: List[Callable] = list() + + # Panes are already created. + if self.are_panes: + l_token_holders = [c.layout.get_layout for c in self.__controllers[self._TOKEN_HOLDER]] + l_vote = [c.layout.get_layout for c in self.__controllers[self._VOTE]] + l_cast = [c.layout.get_layout for c in self.__controllers[self._CAST]] + l_transaction = [c.layout.get_layout for c in self.__controllers[self._TRANSACTION]] + l_app = [c.layout.get_layout for c in self.__controllers[self._APP]] + l_organization = [c.layout.get_layout for c in self.__controllers[self._ORGANIZATION]] + else: + l_token_holders = self.__get_token_holder_charts() + l_vote = self.__get_vote_charts() + l_cast = self.__get_cast_charts() + l_transaction = self.__get_transaction_charts() + l_app = self.__get_app_charts() + l_organization = self.__get_organization_charts() + + return { + COMMON_TEXT['no_data_selected']: { + 'callables': l_organization, + 'css_id': TEXT['css_id_organization'], + }, + TEXT['title_section_token_holders']: { + 'callables': l_token_holders, + 'css_id': TEXT['css_id_token_holder'], + }, + TEXT['title_section_vote']: { + 'callables': l_vote, + 'css_id': TEXT['css_id_vote'], + }, + TEXT['title_section_cast']: { + 'callables': l_cast, + 'css_id': TEXT['css_id_cast'] + }, + TEXT['title_section_transaction']: { + 'callables': l_transaction, + 'css_id': TEXT['css_id_transactions'], + }, + TEXT['title_section_app']: { + 'callables': l_app, + 'css_id': TEXT['css_id_app'], + }, + } + + + def __get_organization_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # active organization + charts.append(self.__create_chart( + title=TEXT['title_active_organization'], + adapter=BasicAdapter( + metric_id=s_factory.ACTIVE_ORGANIZATION, + organizations=call), + figure=BarFigure(), + cont_key=self._ORGANIZATION + )) + + return charts + + + def __get_token_holder_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # active token holders + charts.append(self.__create_chart( + title=TEXT['title_active_token_holders'], + adapter=BasicAdapter( + metric_id=s_factory.ACTIVE_TOKEN_HOLDERS, + organizations=call), + figure=BarFigure(), + cont_key=self._TOKEN_HOLDER + )) + + return charts + + + def __get_vote_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # new votes + charts.append(self.__create_chart( + title=TEXT['title_new_votes'], + adapter=BasicAdapter( + metric_id=s_factory.NEW_VOTES, + organizations=call), + figure=BarFigure(), + cont_key=self._VOTE + )) + + # vote's outcome + charts.append(self.__create_chart( + title=TEXT['title_vote_outcome'], + adapter=VoteOutcome(organizations=call), + figure=MultiBarFigure(MultiBarFigure.STACK), + cont_key=self._VOTE + )) + + return charts + + + def __get_cast_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # cast type + charts.append(self.__create_chart( + title=TEXT['title_cast_type'], + adapter=CastType(organizations=call), + figure=MultiBarFigure(bar_type=MultiBarFigure.STACK), + cont_key=self._CAST + )) + + # active voters + charts.append(self.__create_chart( + title=TEXT['title_active_voters'], + adapter=BasicAdapter( + metric_id=s_factory.ACTIVE_VOTERS, + organizations=call), + figure=BarFigure(), + cont_key=self._CAST + )) + return charts + + + def __get_transaction_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # new transactions + charts.append(self.__create_chart( + title=TEXT['title_new_transactions'], + adapter=BasicAdapter( + metric_id=s_factory.NEW_TRANSACTIONS, + organizations=call), + figure=BarFigure(), + cont_key=self._TRANSACTION + )) + return charts + + + def __get_app_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # installed apps + charts.append(self.__create_chart( + title=TEXT['title_installed_apps'], + adapter=InstalledApps(organizations=call), + figure=BarFigure(), + cont_key=self._APP + )) + self.__controllers[self._APP][-1].layout.configuration.disable_subtitles() + + return charts + + def __create_chart(self, title: str, adapter: IMetricAdapter, figure: Figure + , cont_key: int) -> Callable: + """ + Creates the chart layout and its controller, and returns a callable + to get the html representation. + """ + css_id: str = f"{TEXT['pane_css_prefix']}{ChartPaneLayout.pane_id()}" + layout: ChartPaneLayout = ChartPaneLayout( + title=title, + css_id=css_id, + figure=figure + ) + layout.configuration.set_color(color=COLOR.DARK_BLUE) + layout.configuration.set_css_border(css_border=TEXT['css_pane_border']) + + controller: ChartController = ChartController( + css_id=css_id, + layout=layout, + adapter=adapter) + + self.__controllers[cont_key].append(controller) + return layout.get_layout diff --git a/src/apps/aragon/business/metric_adapter/__init__.py b/src/apps/aragon/business/metric_adapter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/daohaus/business/metric_adapter/active_voters.py b/src/apps/aragon/business/metric_adapter/basic_adapter.py similarity index 81% rename from src/apps/daohaus/business/metric_adapter/active_voters.py rename to src/apps/aragon/business/metric_adapter/basic_adapter.py index 57e38109..4c4d5ce4 100644 --- a/src/apps/daohaus/business/metric_adapter/active_voters.py +++ b/src/apps/aragon/business/metric_adapter/basic_adapter.py @@ -1,8 +1,8 @@ """ Descp: Class to adapt StakedSeries in a chart. This class is used to adapt - the 'active voters' metric + basic metrics. - Created on: 5-oct-2020 + Created on: 19-oct-2020 Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui @@ -14,15 +14,16 @@ from src.apps.common.business.transfers.stacked_serie import StackedSerie from src.apps.common.business.transfers.organization import OrganizationList from src.apps.common.business.i_metric_adapter import IMetricAdapter -import src.apps.daohaus.data_access.daos.metric.\ +import src.apps.aragon.data_access.daos.metric.\ metric_dao_factory as s_factory -class ActiveVoters(IMetricAdapter): +class BasicAdapter(IMetricAdapter): DATE_FORMAT: str = '%b, %Y' - def __init__(self, organizations: OrganizationList) -> None: + def __init__(self, metric_id: int, organizations: OrganizationList) -> None: self.__organizations = organizations + self.__metric = metric_id @property @@ -36,7 +37,7 @@ def get_plot_data(self, o_id: str) -> Dict: """ dao = s_factory.get_dao( ids=self.__organizations.get_ids_from_id(o_id), - metric=s_factory.ACTIVE_VOTERS + metric=self.__metric ) metric: StackedSerie = dao.get_metric() @@ -52,7 +53,7 @@ def get_plot_data(self, o_id: str) -> Dict: 'name': '', 'color': color, 'type': 'date', - 'x_format': ActiveVoters.DATE_FORMAT, + 'x_format': self.DATE_FORMAT, 'last_serie_elem': metric.get_last_serie_elem(), 'last_value': metric.get_last_value(0), 'diff': metric.get_diff_last_values(), diff --git a/src/apps/aragon/business/metric_adapter/cast_type.py b/src/apps/aragon/business/metric_adapter/cast_type.py new file mode 100644 index 00000000..3151538d --- /dev/null +++ b/src/apps/aragon/business/metric_adapter/cast_type.py @@ -0,0 +1,67 @@ +""" + Descp: Class to adapt StakedSeries in a chart. This class is used to adapt + the 'votes by type' metric + + Created on: 21-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +from typing import Dict + +import src.apps.common.resources.colors as Color +from src.apps.aragon.resources.strings import TEXT +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.organization import OrganizationList +from src.apps.common.business.i_metric_adapter import IMetricAdapter +import src.apps.aragon.data_access.daos.metric.\ + metric_dao_factory as s_factory + +class CastType(IMetricAdapter): + + DATE_FORMAT: str = '%b, %Y' + + def __init__(self, organizations: OrganizationList) -> None: + self.__organizations = organizations + + + @property + def organizations(self) -> OrganizationList: + return self.__organizations + + + def get_plot_data(self, o_id: str) -> Dict: + """ + Returns the metric data in a Dict using o_id param. + """ + dao = s_factory.get_dao( + ids=self.organizations.get_ids_from_id(o_id), + metric=s_factory.CAST_TYPE + ) + metric: StackedSerie = dao.get_metric() + + last_value: int = metric.get_last_value(0) + metric.get_last_value(1) + diff: float = metric.get_diff_last_values(add_stacks=True) + + return { + 'type1': { + 'y': metric.get_i_stack(0), + 'color': Color.LIGHT_RED, + 'name': TEXT['votes_against'], + }, + 'type2': { + 'y': metric.get_i_stack(1), + 'color': Color.LIGHT_GREEN, + 'name': TEXT['votes_for'], + }, + 'common': { + 'x': metric.get_serie(), + 'type': 'date', + 'x_format': self.DATE_FORMAT, + 'ordered_keys': ['type1', 'type2'], + }, + 'last_serie_elem': metric.get_last_serie_elem(), + 'last_value': last_value, + 'diff': diff, + } diff --git a/src/apps/aragon/business/metric_adapter/installed_apps.py b/src/apps/aragon/business/metric_adapter/installed_apps.py new file mode 100644 index 00000000..e28081b8 --- /dev/null +++ b/src/apps/aragon/business/metric_adapter/installed_apps.py @@ -0,0 +1,53 @@ +""" + Descp: This class is used to adapt installed apps metrics. + + Created on: 20-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +from typing import Dict, List + +import src.apps.common.resources.colors as Color +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.organization import OrganizationList +from src.apps.common.business.i_metric_adapter import IMetricAdapter +import src.apps.aragon.data_access.daos.metric.\ + metric_dao_factory as s_factory + +class InstalledApps(IMetricAdapter): + + def __init__(self, organizations: OrganizationList) -> None: + self.__organizations = organizations + + + @property + def organizations(self) -> OrganizationList: + return self.__organizations + + + def get_plot_data(self, o_id: str) -> Dict: + """ + Returns the metric data in a Dict using o_id param. + """ + dao = s_factory.get_dao( + ids=self.__organizations.get_ids_from_id(o_id), + metric=s_factory.INSTALLED_APPS + ) + metric: StackedSerie = dao.get_metric() + + y: List[float] = metric.get_i_stack(0) + color = [Color.LIGHT_BLUE] * len(y) + + return { + 'x': metric.get_serie(), + 'y': y, + 'name': '', + 'color': color, + 'type': 'category', + 'x_format': '', + 'last_serie_elem': '', + 'last_value': '', + 'diff': '', + } diff --git a/src/apps/aragon/business/metric_adapter/vote_outcome.py b/src/apps/aragon/business/metric_adapter/vote_outcome.py new file mode 100644 index 00000000..46953752 --- /dev/null +++ b/src/apps/aragon/business/metric_adapter/vote_outcome.py @@ -0,0 +1,67 @@ +""" + Descp: Class to adapt StakedSeries in a chart. This class is used to adapt + the 'proposal outcome' metric + + Created on: 9-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +from typing import Dict + +import src.apps.common.resources.colors as Color +from src.apps.aragon.resources.strings import TEXT +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.organization import OrganizationList +from src.apps.common.business.i_metric_adapter import IMetricAdapter +import src.apps.aragon.data_access.daos.metric.\ + metric_dao_factory as s_factory + +class VoteOutcome(IMetricAdapter): + + DATE_FORMAT: str = '%b, %Y' + + def __init__(self, organizations: OrganizationList) -> None: + self.__organizations = organizations + + + @property + def organizations(self) -> OrganizationList: + return self.__organizations + + + def get_plot_data(self, o_id: str) -> Dict: + """ + Returns the metric data in a Dict using o_id param. + """ + dao = s_factory.get_dao( + ids=self.organizations.get_ids_from_id(o_id), + metric=s_factory.VOTE_OUTCOME + ) + metric: StackedSerie = dao.get_metric() + + last_value: int = metric.get_last_value(0) + metric.get_last_value(1) + diff: float = metric.get_diff_last_values(add_stacks=True) + + return { + 'type1': { + 'y': metric.get_i_stack(0), + 'color': Color.LIGHT_RED, + 'name': TEXT['rejected_votes'], + }, + 'type2': { + 'y': metric.get_i_stack(1), + 'color': Color.LIGHT_GREEN, + 'name': TEXT['approved_votes'], + }, + 'common': { + 'x': metric.get_serie(), + 'type': 'date', + 'x_format': self.DATE_FORMAT, + 'ordered_keys': ['type1', 'type2'], + }, + 'last_serie_elem': metric.get_last_serie_elem(), + 'last_value': last_value, + 'diff': diff, + } diff --git a/src/apps/aragon/class_diagram.png b/src/apps/aragon/class_diagram.png new file mode 100644 index 00000000..d4200e08 Binary files /dev/null and b/src/apps/aragon/class_diagram.png differ diff --git a/src/apps/aragon/data_access/__init__.py b/src/apps/aragon/data_access/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/data_access/daos/__init__.py b/src/apps/aragon/data_access/daos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/data_access/daos/metric/__init__.py b/src/apps/aragon/data_access/daos/metric/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/data_access/daos/metric/metric_dao_factory.py b/src/apps/aragon/data_access/daos/metric/metric_dao_factory.py new file mode 100644 index 00000000..2b682ac5 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/metric_dao_factory.py @@ -0,0 +1,77 @@ +""" + Descp: This file is used as factory to create a DAO metric + + Created on: 19-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +from typing import List + +from src.apps.common.data_access.daos.metric.metric_dao \ + import MetricDao +import src.apps.aragon.data_access.requesters.cache_requester as cache +from src.apps.aragon.data_access.daos.metric.strategy.st_new_additions import StNewAdditions +from src.apps.aragon.data_access.daos.metric.strategy.st_installed_apps import StInstalledApps +from src.apps.aragon.data_access.daos.metric.strategy.st_cast_type import StCastType +from src.apps.aragon.data_access.daos.metric.strategy.st_vote_outcome import StVoteOutcome +from src.apps.aragon.data_access.daos.metric.strategy.st_active_voters import StActiveVoters +from src.apps.aragon.data_access.daos.metric.strategy.st_active_token_holders import StActiveTokenHolders +from src.apps.aragon.data_access.daos.metric.strategy.st_active_organization import StActiveOrganization + + +NEW_VOTES = 0 +NEW_TRANSACTIONS = 1 +INSTALLED_APPS = 2 +CAST_TYPE = 3 +VOTE_OUTCOME = 4 +ACTIVE_VOTERS = 5 +ACTIVE_TOKEN_HOLDERS = 6 +ACTIVE_ORGANIZATION = 7 + + +def get_dao(ids: List[str], metric: int) -> MetricDao: + address_key: str = '' + requester: cache.CacheRequester = None + stg = None + + if metric == NEW_VOTES: + stg = StNewAdditions(typ=StNewAdditions.VOTE) + requester = cache.CacheRequester(srcs=[cache.VOTES]) + address_key = 'orgAddress' + elif metric == NEW_TRANSACTIONS: + stg = StNewAdditions(typ=StNewAdditions.TRANSACTION) + requester = cache.CacheRequester(srcs=[cache.TRANSACTIONS]) + address_key = 'orgAddress' + elif metric == INSTALLED_APPS: + stg = StInstalledApps() + requester = cache.CacheRequester(srcs=[cache.APPS]) + address_key = 'organizationId' + elif metric == CAST_TYPE: + stg = StCastType() + requester = cache.CacheRequester(srcs=[cache.CASTS]) + address_key = 'orgAddress' + elif metric == VOTE_OUTCOME: + stg = StVoteOutcome() + requester = cache.CacheRequester(srcs=[cache.VOTES]) + address_key = 'orgAddress' + elif metric == ACTIVE_VOTERS: + stg = StActiveVoters() + requester = cache.CacheRequester(srcs=[cache.CASTS]) + address_key = 'orgAddress' + elif metric == ACTIVE_TOKEN_HOLDERS: + stg = StActiveTokenHolders() + requester = cache.CacheRequester(srcs=[ + cache.CASTS, + cache.VOTES, + cache.TRANSACTIONS]) + address_key = 'orgAddress' + elif metric == ACTIVE_ORGANIZATION: + stg = StActiveOrganization() + requester = cache.CacheRequester(srcs=[ + cache.CASTS, + cache.VOTES, + cache.TRANSACTIONS]) + address_key = 'orgAddress' + + return MetricDao(ids=ids, strategy=stg, requester=requester, address_key=address_key) diff --git a/src/apps/aragon/data_access/daos/metric/strategy/__init__.py b/src/apps/aragon/data_access/daos/metric/strategy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_active_organization.py b/src/apps/aragon/data_access/daos/metric/strategy/st_active_organization.py new file mode 100644 index 00000000..f95bddb7 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_active_organization.py @@ -0,0 +1,112 @@ +""" + Descp: Strategy pattern to create active DAO(meaning organizations) metric. + It'll show the number of active DAOs each month, + whether just one DAO is selected it will show the month when + it was active. + + Created on: 22-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StActiveOrganization(IMetricStrategy): + __DF_CAST_DATE = 'createdAt' + __DF_VOTE_DATE = 'startDate' + __DF_TRANSACTION_DATE = 'date' + __DF_ID = 'orgAddress' + __DF_DATE = 'date' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_ID] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + used_keys = set([self.__DF_ID, + self.__DF_CAST_DATE, + self.__DF_VOTE_DATE, + self.__DF_TRANSACTION_DATE]) + used_keys = list(used_keys.intersection(set(df.columns))) + + dff.loc[:, used_keys] = dff[used_keys] + return dff + + + def __get_action(self, df: pd.DataFrame, date_col: str) -> pd.DataFrame: + if date_col not in df.columns: + return pd.DataFrame() + + columns: List[str] = [self.__DF_ID, date_col] + dff = df[columns] + dff = dff.dropna(subset=[date_col]) + return dff.rename(columns={date_col: self.__DF_DATE}) + + + def __prepare_df(self, df: pd.DataFrame) -> pd.DataFrame: + """ + It combines the different actions in a common action. + """ + # votes casted + dff: pd.DataFrame = self.__get_action( + df=df, + date_col=self.__DF_CAST_DATE) + + # vote creations + dff = dff.append(self.__get_action( + df=df, + date_col=self.__DF_VOTE_DATE) + , ignore_index=True) + + # transactions + dff = dff.append(self.__get_action( + df=df, + date_col=self.__DF_TRANSACTION_DATE) + , ignore_index=True) + + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + df = self.__prepare_df(df=df) + + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + # join dates-ids + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + # different voters by month + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_active_token_holders.py b/src/apps/aragon/data_access/daos/metric/strategy/st_active_token_holders.py new file mode 100644 index 00000000..d9677d7f --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_active_token_holders.py @@ -0,0 +1,119 @@ +""" + Descp: Strategy pattern to create active token holders metric. + + Created on: 22-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StActiveTokenHolders(IMetricStrategy): + __DF_CAST_DATE = 'createdAt' + __DF_VOTE_DATE = 'startDate' + __DF_TRANSACTION_DATE = 'date' + __DF_CASTER = 'voter' + __DF_PROPOSER = 'creator' + __DF_TRANSACTIONER = 'entity' + __DF_MEMEBER = 'member' + __DF_DATE = 'date' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_MEMEBER] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + used_keys = set([self.__DF_CASTER, + self.__DF_PROPOSER, + self.__DF_TRANSACTIONER, + self.__DF_CAST_DATE, + self.__DF_VOTE_DATE, + self.__DF_TRANSACTION_DATE]) + used_keys = list(used_keys.intersection(set(df.columns))) + + dff.loc[:, used_keys] = dff[used_keys] + return dff + + + def __get_action(self, df: pd.DataFrame, actioner: str, date_col: str) -> pd.DataFrame: + if actioner not in df.columns or date_col not in df.columns: + return pd.DataFrame() + + columns: List[str] = [date_col, actioner] + dff = df[columns] + dff = dff.dropna(subset=[actioner, date_col]) + return dff.rename(columns={ + actioner: self.__DF_MEMEBER, + date_col: self.__DF_DATE}) + + + def __prepare_df(self, df: pd.DataFrame) -> pd.DataFrame: + """ + It combines the different actions in a common action. + """ + # votes casted + dff: pd.DataFrame = self.__get_action( + df=df, + actioner=self.__DF_CASTER, + date_col=self.__DF_CAST_DATE) + + # vote creation + dff = dff.append(self.__get_action( + df=df, + actioner=self.__DF_PROPOSER, + date_col=self.__DF_VOTE_DATE) + , ignore_index=True) + + # transaction + dff = dff.append(self.__get_action( + df=df, + actioner=self.__DF_TRANSACTIONER, + date_col=self.__DF_TRANSACTION_DATE) + , ignore_index=True) + + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + df = self.__prepare_df(df=df) + + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + # join dates-ids + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + # different voters by month + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_active_voters.py b/src/apps/aragon/data_access/daos/metric/strategy/st_active_voters.py new file mode 100644 index 00000000..8d4bb9b5 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_active_voters.py @@ -0,0 +1,60 @@ +""" + Descp: Strategy pattern to create active voters metric. + + Created on: 8-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StActiveVoters(IMetricStrategy): + __DF_DATE = 'createdAt' + __DF_VOTER = 'voter' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_VOTER] + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + # join dates-ids + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + # different voters by month + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_cast_type.py b/src/apps/aragon/data_access/daos/metric/strategy/st_cast_type.py new file mode 100644 index 00000000..0b210593 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_cast_type.py @@ -0,0 +1,74 @@ +""" + Descp: Strategy pattern to create a 'votes by type' metric. + + Created on: 21-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StCastType(IMetricStrategy): + __DF_DATE = 'createdAt' + __DF_COUNT = 'count' + __DF_SUPPORT = 'supports' + __DF_COLS = [__DF_DATE, __DF_SUPPORT] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + serie: Serie = Serie(x=dff[self.__DF_DATE].tolist()) + positives: List = self.__get_outcome(True, df, dff) + negatives: List = self.__get_outcome(False, df, dff) + + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [negatives, positives]) + + return metric + + + def __get_outcome(self, positive: bool, df: pd.DataFrame, + dff: pd.DataFrame) -> List[int]: + + d3f: pd.DataFrame = pd_utl.filter_by_col_value( + df=df, + col=self.__DF_SUPPORT, + value=positive, + filters=[pd_utl.EQ]) + + d3f = d3f.drop(columns=[self.__DF_SUPPORT]) + d3f = d3f.append(dff, ignore_index=True) + d3f.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + d3f.sort_values(self.__DF_DATE, inplace=True) + + return d3f[self.__DF_COUNT].tolist() diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_installed_apps.py b/src/apps/aragon/data_access/daos/metric/strategy/st_installed_apps.py new file mode 100644 index 00000000..e09c6609 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_installed_apps.py @@ -0,0 +1,78 @@ +""" + Descp: Strategy pattern to create a metric of installed apps. + + Created on: 20-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StInstalledApps(IMetricStrategy): + __MAX_APPS: int = 15 + __OTHERS: str = 'Others' + __DF_ID: str = 'id' + __DF_NAME: str = 'repoName' + __DF_COUNT: str = 'count' + __DF_COLS: List[str] = [__DF_ID, __DF_NAME] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # count pair id-name + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + + #remove apps which has not name + df.dropna(subset=[self.__DF_NAME], inplace=True) + + # count name repetitions (should be repoAddress, however not all of them have an entry) + df = pd_utl.count_cols_repetitions(df, [self.__DF_NAME], self.__DF_COUNT) + df.sort_values(self.__DF_COUNT, inplace=True, ascending=False) + + df = self.__adjust_values(df=df) + + serie: Serie = Serie(x=df[self.__DF_NAME].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric + + + def __adjust_values(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + + if len(dff) <= self.__MAX_APPS: + return dff + + names: List[str] = dff[self.__DF_NAME].tolist()[:self.__MAX_APPS] + installs: List[str] = dff[self.__DF_COUNT].tolist()[:self.__MAX_APPS] + + # add the sum of the others + names.append(self.__OTHERS) + other_installs: List[str] = dff[self.__DF_COUNT].tolist()[self.__MAX_APPS:] + installs.append(sum(other_installs)) + + return pd.DataFrame( + { + self.__DF_NAME: names, + self.__DF_COUNT: installs + }) diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_new_additions.py b/src/apps/aragon/data_access/daos/metric/strategy/st_new_additions.py new file mode 100644 index 00000000..14ebc0d3 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_new_additions.py @@ -0,0 +1,82 @@ +""" + Descp: Strategy pattern to create a metric of new additions. + + Created on: 19-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StNewAdditions(IMetricStrategy): + VOTE = 0 + TRANSACTION = 1 + __TYPES: List[int] = [VOTE, TRANSACTION] + + __DF_VOTE_DATE = 'startDate' + __DF_TRA_DATE = 'date' + __DF_DATE = 'createdAt' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_COUNT] + + + def __init__(self, typ: int) -> None: + self.__typ: int = self.__get_type(typ) + + + def __get_type(self, typ: int) -> int: + """ + Checks if typ exists, if not return by default the first type + """ + return typ if typ in self.__TYPES else self.__TYPES[0] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + + if self.__typ is self.VOTE: + dff.rename(columns={self.__DF_VOTE_DATE: self.__DF_DATE}, inplace=True) + elif self.__typ is self.TRANSACTION: + dff.rename(columns={self.__DF_TRA_DATE: self.__DF_DATE}, inplace=True) + + dff.loc[:, [self.__DF_DATE]] = dff[[self.__DF_DATE]] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + + dff = pd_utl.get_df_from_lists([idx, 0], self.__DF_COLS) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/aragon/data_access/daos/metric/strategy/st_vote_outcome.py b/src/apps/aragon/data_access/daos/metric/strategy/st_vote_outcome.py new file mode 100644 index 00000000..b9447bf9 --- /dev/null +++ b/src/apps/aragon/data_access/daos/metric/strategy/st_vote_outcome.py @@ -0,0 +1,136 @@ +""" + Descp: Strategy pattern to create a votation outcome metric. + + Created on: 21-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd +from typing import List + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StVoteOutcome(IMetricStrategy): + __DF_DATE = 'startDate' + __DF_EXECUTED = 'executed' + __DF_YEA = 'yea' + __DF_NAY = 'nay' + __DF_SUPPORT = 'supportRequiredPct' + __DF_VOTING = 'votingPower' + __DF_QUORUM = 'minAcceptQuorum' + __DF_PASS = 'didPass' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_YEA, __DF_NAY, __DF_SUPPORT, __DF_VOTING, __DF_QUORUM] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + # just take closed proposals + # dff = pd_utl.filter_by_col_value( + # df=dff, + # col=self.__DF_EXECUTED, + # value=True, + # filters=[pd_utl.EQ]) + + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + df = self.__calculate_outcome(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE, self.__DF_PASS], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + serie: Serie = Serie(x=dff[self.__DF_DATE].tolist()) + passes: List = self.__get_outcome(True, df, dff) + rejections: List = self.__get_outcome(False, df, dff) + + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [rejections, passes]) + + return metric + + + def __get_outcome(self, has_pass: bool, df: pd.DataFrame, + dff: pd.DataFrame) -> List[int]: + + d3f: pd.DataFrame = pd_utl.filter_by_col_value( + df=df, + col=self.__DF_PASS, + value=has_pass, + filters=[pd_utl.EQ]) + + d3f = d3f.drop(columns=[self.__DF_PASS]) + d3f = d3f.append(dff, ignore_index=True) + d3f.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + d3f.sort_values(self.__DF_DATE, inplace=True) + + return d3f[self.__DF_COUNT].tolist() + + + def __calculate_outcome(self, df: pd.DataFrame) -> pd.DataFrame: + """ + outcome = (((positive voted tokens) - (negative voted tokens)) / total voted tokens * 100) >= support required + and + ((positive voted tokens) / total tokens * 100) >= minimum accepted quorum + """ + dff: pd.DataFrame = df + + has_pass: List[bool] = [] + yea: List[int] = [int(x) for x in dff[self.__DF_YEA].tolist()] + nay: List[int] = [int(x) for x in dff[self.__DF_NAY].tolist()] + total: List[int] = [int(x) for x in dff[self.__DF_VOTING].tolist()] + l_support: List[int] = [int(x) for x in dff[self.__DF_SUPPORT].tolist()] + l_quorum: List[int] = [int(x) for x in dff[self.__DF_QUORUM].tolist()] + + for i in range(len(dff)): + support: int = self.__percentage(l_support[i]) + quorum: int = self.__percentage(l_quorum[i]) + + positive_support: int = self.__ratio((yea[i] - nay[i]), (yea[i] + nay[i])) * 100 + total_support: int = self.__ratio(yea[i], total[i]) * 100 + + has_pass.append( (positive_support >= support) and (total_support >= quorum) ) + + dff.loc[:, self.__DF_PASS] = has_pass + dff.loc[:, [self.__DF_DATE, self.__DF_PASS]] = \ + dff[[self.__DF_DATE, self.__DF_PASS]] + + return dff + + + def __percentage(self, value: int) -> int: + """ + transform big int Aragon percentage into a normal percentage + """ + if value <= 100: + return value + + return value % 101 + + + def __ratio(self, num: int, divider: int) -> int: + if divider == 0: + return 0 + + return num / divider diff --git a/src/apps/aragon/data_access/requesters/__init__.py b/src/apps/aragon/data_access/requesters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/data_access/requesters/cache_requester.py b/src/apps/aragon/data_access/requesters/cache_requester.py new file mode 100644 index 00000000..0b5fde9c --- /dev/null +++ b/src/apps/aragon/data_access/requesters/cache_requester.py @@ -0,0 +1,70 @@ +""" + Descp: This class is used to load data from the datawarehouse. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import pandas as pd +import os +from typing import List + +from src.apps.common.data_access.requesters.irequester import IRequester + +CACHE_PATH: str = os.path.join('datawarehouse', 'aragon') +APPS: str = os.path.join(CACHE_PATH, 'apps.csv') +CASTS: str = os.path.join(CACHE_PATH, 'casts.csv') +MINI_ME_TOKENS: str = os.path.join(CACHE_PATH, 'miniMeTokens.csv') +ORGANIZATIONS: str = os.path.join(CACHE_PATH, 'organizations.csv') +REPOS: str = os.path.join(CACHE_PATH, 'repos.csv') +TOKEN_HOLDERS: str = os.path.join(CACHE_PATH, 'tokenHolders.csv') +TRANSACTIONS: str = os.path.join(CACHE_PATH, 'transactions.csv') +VOTES: str = os.path.join(CACHE_PATH, 'votes.csv') +ALL_FILES: List[str] = [ + APPS, + CASTS, + MINI_ME_TOKENS, + ORGANIZATIONS, + REPOS, + TOKEN_HOLDERS, + TRANSACTIONS, + VOTES +] + + +class CacheRequester(IRequester): + def __init__(self, srcs: List[str]): + self.__srcs = srcs + + + def request(self, *args) -> pd.DataFrame: + """ + Gets data from datawarehouse. + Arguments: + * args: Its not used + Return: + a pandas dataframe with all the data loaded. If the src does not + exist, it will return an empty dataframe. + """ + df: pd.DataFrame = pd.DataFrame() + for src in self.__srcs: + if os.path.isfile(src): + df = pd.concat([df, pd.read_csv(src, header=0)], axis=0, ignore_index=True) + + return df + + + @classmethod + def is_cache_available(cls) -> bool: + """ + Checks whether the cache is available. + Return: + True if it is, False if it is not. + """ + available: bool = True + for filename in CACHE_PATH: + available &= os.path.isfile(filename) + + return available diff --git a/src/apps/aragon/presentation/__init__.py b/src/apps/aragon/presentation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/resources/__init__.py b/src/apps/aragon/resources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/aragon/resources/strings.py b/src/apps/aragon/resources/strings.py new file mode 100644 index 00000000..7f2e7aea --- /dev/null +++ b/src/apps/aragon/resources/strings.py @@ -0,0 +1,36 @@ +""" + Descp: This file is used to store Aragon app's text. + + Created on: 16-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +TEXT: dict = { + 'approved_votes': 'Approved votes', + 'css_pane_border': 'aragon-border', + 'css_id_app': 'aragon-apps', + 'css_id_cast': 'aragon-casts', + 'css_id_organization': 'aragon-organization', + 'css_id_token_holder': 'aragon-token-holders', + 'css_id_transactions': 'aragon-transactions', + 'css_id_vote': 'aragon-votes', + 'pane_css_prefix': 'aragon-pane', + 'rejected_votes': 'Rejected votes', + 'title_active_organization': 'Months which the DAO has registered activity', + 'title_active_token_holders': 'Active token holders', + 'title_active_voters': 'Active voters', + 'title_cast_type': 'Casted votes by support', + 'title_installed_apps': 'Installed apps', + 'title_new_transactions': 'New transactions', + 'title_new_votes': 'New votes', + 'title_section_token_holders': 'Token Holders', + 'title_section_vote': 'Votes', + 'title_section_cast': 'Casted votes', + 'title_section_transaction': 'Transactions', + 'title_section_app': 'Aragon apps', + 'title_vote_outcome': 'Vote\'s outcome', + 'votes_against': 'Casted votes against', + 'votes_for': 'Casted votes for', +} diff --git a/src/apps/common/presentation/dashboard_view/__init__.py b/src/apps/common/presentation/dashboard_view/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/common/presentation/dashboard_view/controller.py b/src/apps/common/presentation/dashboard_view/controller.py new file mode 100644 index 00000000..a0ec5663 --- /dev/null +++ b/src/apps/common/presentation/dashboard_view/controller.py @@ -0,0 +1,26 @@ +""" + Descp: Dashboard view controller to manage the callbacks. + + Created on: 22-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +from dash.dependencies import Input, Output, State + +from src.apps.common.resources.strings import TEXT + +def bind_callbacks(app, section_id: str) -> None: + + @app.callback( + Output(section_id, 'children'), + [Input('org-dropdown', 'value')], + [State('org-dropdown', 'options')] + ) + def organization_section_name(value: str, options: dict) -> str: + if not value: + return TEXT['no_data_selected'] + + result = list(filter(lambda x: x['value'] == value, options)) + + return result[0]['label'] diff --git a/src/apps/common/presentation/dashboard_view.py b/src/apps/common/presentation/dashboard_view/dashboard_view.py similarity index 89% rename from src/apps/common/presentation/dashboard_view.py rename to src/apps/common/presentation/dashboard_view/dashboard_view.py index 6faca092..07e9fd22 100644 --- a/src/apps/common/presentation/dashboard_view.py +++ b/src/apps/common/presentation/dashboard_view/dashboard_view.py @@ -43,15 +43,15 @@ def __generate_selector(labels: List[Dict[str, str]]) -> html.Div: def __generate_sections(sections: Dict[str, List[Callable]], color_app: str) -> html.Div: children: List = list() - for name, callables in sections.items(): + for name, data in sections.items(): charts = list() - for chart_pane in callables: + for chart_pane in data['callables']: charts.append(chart_pane()) sec = html.Div( className='section', children=[ - html.Div(name, className=f'title-section {color_app}'), + html.Div(name, id=data['css_id'], className=f'title-section {color_app}'), html.Div(children=charts, className='graph-section') ], ) diff --git a/src/apps/common/presentation/main_view/main_view.py b/src/apps/common/presentation/main_view/main_view.py index c1aca869..c3853ad4 100644 --- a/src/apps/common/presentation/main_view/main_view.py +++ b/src/apps/common/presentation/main_view/main_view.py @@ -50,6 +50,11 @@ def __generate_body() -> html.Div: title=TEXT['daostack'], bt_id='daostack-bt', css_class='daostack'), + __generate_ecosystem_pane( + img=f'{REL_PATH}aragon.png', + title=TEXT['aragon'], + bt_id='aragon-bt', + css_class='aragon'), __generate_ecosystem_pane( img=f'{REL_PATH}daohaus.png', title=TEXT['daohaus'], diff --git a/src/apps/common/presentation/main_view/main_view_controller.py b/src/apps/common/presentation/main_view/main_view_controller.py index 73f4f0b7..3858b18d 100644 --- a/src/apps/common/presentation/main_view/main_view_controller.py +++ b/src/apps/common/presentation/main_view/main_view_controller.py @@ -13,13 +13,15 @@ from src.apps.common.presentation.main_view.main_view import generate_layout import src.apps.daostack.business.app_service as daostack import src.apps.daohaus.business.app_service as daohaus +import src.apps.aragon.business.app_service as aragon from src.apps.common.resources.strings import TEXT -def bind_callbacks(app) -> None: +def bind_callbacks(app) -> None: # noqa: C901 # Callbacks need to be loaded twice. daostack.get_service().get_layout() daohaus.get_service().get_layout() + aragon.get_service().get_layout() @app.callback( @@ -27,7 +29,6 @@ def bind_callbacks(app) -> None: [Input('url', 'pathname')] ) def display_page(pathname): - #print(pathname) if pathname == TEXT['url_main']: return generate_layout( header_title=TEXT['app_title'], @@ -42,6 +43,11 @@ def display_page(pathname): header_title=TEXT['app_title_daohaus'], app_color=TEXT['css_color_daohaus'], body=daohaus.get_service().get_layout()) + elif pathname == TEXT['url_aragon']: + return generate_layout( + header_title=TEXT['app_title_aragon'], + app_color=TEXT['css_color_aragon'], + body=aragon.get_service().get_layout()) else: return TEXT['not_found'] @@ -49,12 +55,13 @@ def display_page(pathname): @app.callback( Output('url', 'pathname'), [Input('daostack-bt', 'n_clicks'), - Input('daohaus-bt', 'n_clicks')] + Input('daohaus-bt', 'n_clicks'), + Input('aragon-bt', 'n_clicks')] ) - def load_ecosystem(bt_daostack: int, bt_daohaus: int) -> str: + def load_ecosystem(bt_daostack: int, bt_daohaus: int, bt_aragon: int) -> str: ctx = dash.callback_context - if not bt_daostack and not bt_daohaus: + if not bt_daostack and not bt_daohaus and not bt_aragon: raise PreventUpdate trigger = ctx.triggered[0]['prop_id'].split('.')[0] @@ -64,5 +71,7 @@ def load_ecosystem(bt_daostack: int, bt_daohaus: int) -> str: pathname = TEXT['url_daostack'] elif trigger == 'daohaus-bt': pathname = TEXT['url_daohaus'] + elif trigger == 'aragon-bt': + pathname = TEXT['url_aragon'] return pathname diff --git a/src/apps/common/resources/strings.py b/src/apps/common/resources/strings.py index 0f4a14a9..d91ee148 100644 --- a/src/apps/common/resources/strings.py +++ b/src/apps/common/resources/strings.py @@ -12,13 +12,16 @@ 'active_users_title': 'Active reputation holders', 'all_orgs': 'All DAOs', 'app_title': 'DAO-Analyzer', + 'app_title_aragon': 'Aragon-Analyzer', 'app_title_daohaus': 'DAOhaus-Analyzer', 'app_title_daostack': 'DAOstack-Analyzer', + 'aragon': 'Aragon', 'bt_analyze': 'Analyze', 'cc_image_url': 'https://i.creativecommons.org/l/by/4.0/88x31.png', 'cc_license_text': 'This site is licensed under a Creative Commons Attribution 4.0 International License', 'cc_url': 'https://creativecommons.org/licenses/by/4.0/', 'css_color_app': 'app-color', + 'css_color_aragon': 'aragon-color', 'css_color_daohaus': 'daohaus-color', 'css_color_daostack': 'daostack-color', 'dao_selector_title': 'Select your DAO: ', @@ -42,6 +45,7 @@ 'p2p_models_url': 'https://p2pmodels.eu/', 'spanish_ministry_image_url': 'https://encrypted-tbn0.gstatic.com/images?q=tbn%3AANd9GcRfq3CJzLUtY8NUahJOUBdn35aAMHMhqcihyA&usqp=CAU', 'spanish_ministry_url': 'https://www.ciencia.gob.es/', + 'url_aragon': '/aragon', 'url_main': '/', 'url_daostack': '/daostack', 'url_daohaus': '/daohaus', diff --git a/src/apps/daohaus/business/app_service.py b/src/apps/daohaus/business/app_service.py index f019911f..6d954038 100644 --- a/src/apps/daohaus/business/app_service.py +++ b/src/apps/daohaus/business/app_service.py @@ -11,7 +11,9 @@ from typing import Dict, List, Callable import dash_html_components as html -import src.apps.common.presentation.dashboard_view as view +from src.app import app +import src.apps.common.presentation.dashboard_view.dashboard_view as view +import src.apps.common.presentation.dashboard_view.controller as view_cont from src.apps.common.data_access.daos.organization_dao\ import OrganizationListDao import src.apps.daohaus.data_access.requesters.cache_requester as cache @@ -25,7 +27,6 @@ from src.apps.common.business.i_metric_adapter import IMetricAdapter from src.apps.daohaus.business.metric_adapter.basic_adapter import BasicAdapter from src.apps.daohaus.business.metric_adapter.votes_type import VotesType -from src.apps.daohaus.business.metric_adapter.active_voters import ActiveVoters from src.apps.daohaus.business.metric_adapter.proposal_outcome import ProposalOutcome from src.apps.daohaus.business.metric_adapter.proposal_type import ProposalType import src.apps.daohaus.data_access.daos.metric.metric_dao_factory as s_factory @@ -53,6 +54,7 @@ class DaohausService(): _VOTE: int = 1 _RAGE_QUIT: int = 2 _PROPOSAL: int = 3 + _ORGANIZATION: int = 4 def __init__(self): # app state @@ -62,6 +64,7 @@ def __init__(self): self._VOTE: list(), self._RAGE_QUIT: list(), self._PROPOSAL: list(), + self._ORGANIZATION: list(), } @@ -94,6 +97,12 @@ def get_layout(self) -> html.Div: Returns the app's layout. """ orgs: OrganizationList = self.organizations + + if not self.are_panes: + view_cont.bind_callbacks( + app=app, + section_id=TEXT['css_id_organization']) + return view.generate_layout( labels=orgs.get_dict_representation(), sections=self.__get_sections(), @@ -110,6 +119,7 @@ def __get_sections(self) -> Dict[str, List[Callable]]: l_vote: List[Callable] = list() l_rage_q: List[Callable] = list() l_proposal: List[Callable] = list() + l_organization: List[Callable] = list() # Panes are already created. if self.are_panes: @@ -117,20 +127,55 @@ def __get_sections(self) -> Dict[str, List[Callable]]: l_vote = [c.layout.get_layout for c in self.__controllers[self._VOTE]] l_rage_q = [c.layout.get_layout for c in self.__controllers[self._RAGE_QUIT]] l_proposal = [c.layout.get_layout for c in self.__controllers[self._PROPOSAL]] + l_organization = [c.layout.get_layout for c in self.__controllers[self._ORGANIZATION]] else: l_member = self.__get_member_charts() l_vote = self.__get_vote_charts() l_rage_q = self.__get_rage_quits_charts() l_proposal = self.__get_proposal_charts() + l_organization = self.__get_organization_charts() return { - TEXT['title_member']: l_member, - TEXT['title_rage_quits']: l_rage_q, - TEXT['title_vote']: l_vote, - TEXT['title_proposal']: l_proposal, + COMMON_TEXT['no_data_selected']: { + 'callables': l_organization, + 'css_id': TEXT['css_id_organization'], + }, + TEXT['title_member']: { + 'callables': l_member, + 'css_id': TEXT['css_id_member'], + }, + TEXT['title_rage_quits']: { + 'callables': l_rage_q, + 'css_id': TEXT['css_id_rage_quit'], + }, + TEXT['title_vote']: { + 'callables': l_vote, + 'css_id': TEXT['css_id_vote'], + }, + TEXT['title_proposal']: { + 'callables': l_proposal, + 'css_id': TEXT['css_id_proposal'], + }, } + def __get_organization_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # active organizations + charts.append(self.__create_chart( + title=TEXT['title_active_organization'], + adapter=BasicAdapter( + metric_id=s_factory.ACTIVE_ORGANIZATION, + organizations=call), + figure=BarFigure(), + cont_key=self._ORGANIZATION + )) + return charts + + + def __get_member_charts(self) -> List[Callable[[], html.Div]]: charts: List[Callable] = list() call: Callable = self.organizations @@ -172,7 +217,9 @@ def __get_vote_charts(self) -> List[Callable[[], html.Div]]: # active voters charts.append(self.__create_chart( title=TEXT['title_active_voters'], - adapter=ActiveVoters(call), + adapter=BasicAdapter( + metric_id=s_factory.ACTIVE_VOTERS, + organizations=call), figure=BarFigure(), cont_key=self._VOTE )) diff --git a/src/apps/daohaus/class_diagram.png b/src/apps/daohaus/class_diagram.png index 9a080239..fca1a4f3 100644 Binary files a/src/apps/daohaus/class_diagram.png and b/src/apps/daohaus/class_diagram.png differ diff --git a/src/apps/daohaus/data_access/daos/metric/metric_dao_factory.py b/src/apps/daohaus/data_access/daos/metric/metric_dao_factory.py index 248eac4d..6bf2cc21 100644 --- a/src/apps/daohaus/data_access/daos/metric/metric_dao_factory.py +++ b/src/apps/daohaus/data_access/daos/metric/metric_dao_factory.py @@ -17,6 +17,7 @@ from src.apps.daohaus.data_access.daos.metric.strategy.st_proposal_outcome import StProposalOutcome from src.apps.daohaus.data_access.daos.metric.strategy.st_active_members import StActiveMembers from src.apps.daohaus.data_access.daos.metric.strategy.st_proposal_type import StProposalType +from src.apps.daohaus.data_access.daos.metric.strategy.st_active_organization import StActiveOrganization NEW_MEMBERS = 0 VOTES_TYPE = 1 @@ -26,6 +27,7 @@ PROPOSALS_OUTCOME = 5 ACTIVE_MEMBERS = 6 PROPOSAL_TYPE = 7 +ACTIVE_ORGANIZATION = 8 def get_dao(ids: List[str], metric: int) -> MetricDao: @@ -59,5 +61,11 @@ def get_dao(ids: List[str], metric: int) -> MetricDao: elif metric == PROPOSAL_TYPE: stg = StProposalType() requester = cache.CacheRequester(srcs=[cache.PROPOSALS]) + elif metric == ACTIVE_ORGANIZATION: + stg = StActiveOrganization() + requester = cache.CacheRequester(srcs=[ + cache.PROPOSALS, + cache.RAGE_QUITS, + cache.VOTES]) return MetricDao(ids=ids, strategy=stg, requester=requester, address_key='molochAddress') diff --git a/src/apps/daohaus/data_access/daos/metric/strategy/st_active_organization.py b/src/apps/daohaus/data_access/daos/metric/strategy/st_active_organization.py new file mode 100644 index 00000000..69838a1f --- /dev/null +++ b/src/apps/daohaus/data_access/daos/metric/strategy/st_active_organization.py @@ -0,0 +1,64 @@ +""" + Descp: Strategy pattern to create active DAO(meaning organizations) metric. + It'll show the number of active DAOs each month, + whether just one DAO is selected it will show the month when + it was active. + + Created on: 23-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" +import pandas as pd + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StActiveOrganization(IMetricStrategy): + __DF_DATE = 'createdAt' + __DF_ID = 'molochAddress' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_ID] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + # join dates-ids + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + # different voters by month + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/daohaus/resources/strings.py b/src/apps/daohaus/resources/strings.py index 2f98c20d..7e329580 100644 --- a/src/apps/daohaus/resources/strings.py +++ b/src/apps/daohaus/resources/strings.py @@ -10,6 +10,11 @@ TEXT: dict = { 'approved_proposals': 'Approved proposals', 'css_pane_border': 'daohaus-border', + 'css_id_member': 'daohaus-members', + 'css_id_organization': 'daohaus-organization', + 'css_id_rage_quit': 'daohaus-rage-quit', + 'css_id_vote': 'daohaus-vote', + 'css_id_proposal': 'daohaus-proposal', 'pane_css_prefix': 'daohaus-pane', 'proposal_donation': 'Donation proposals', 'proposal_grant': 'Grant proposals', @@ -17,6 +22,7 @@ 'proposal_other': 'Others', 'rejected_proposals': 'Rejected proposals', 'title_active_members': 'Active members', + 'title_active_organization': 'Months which the DAO has registered activity', 'title_active_voters': 'Active voters', 'title_member': 'Members', 'title_new_members': 'New Members', diff --git a/src/apps/daostack/business/app_service.py b/src/apps/daostack/business/app_service.py index 71ab105b..a0070816 100644 --- a/src/apps/daostack/business/app_service.py +++ b/src/apps/daostack/business/app_service.py @@ -11,7 +11,9 @@ from typing import Dict, List, Callable import dash_html_components as html -import src.apps.common.presentation.dashboard_view as view +from src.app import app +import src.apps.common.presentation.dashboard_view.dashboard_view as view +import src.apps.common.presentation.dashboard_view.controller as view_cont from src.apps.common.data_access.daos.organization_dao\ import OrganizationListDao import src.apps.daostack.data_access.daos.metric.\ @@ -61,6 +63,7 @@ class DaostackService(): _VOTE: int = 1 _STAKE: int = 2 _PROPOSAL: int = 3 + _ORGANIZATION: int = 4 def __init__(self): # app state @@ -70,6 +73,7 @@ def __init__(self): self._VOTE: list(), self._STAKE: list(), self._PROPOSAL: list(), + self._ORGANIZATION: list(), } @@ -102,6 +106,12 @@ def get_layout(self) -> html.Div: Returns the app's layout. """ orgs: OrganizationList = self.organizations + + if not self.are_panes: + view_cont.bind_callbacks( + app=app, + section_id=TEXT['css_id_organization']) + return view.generate_layout( labels=orgs.get_dict_representation(), sections=self.__get_sections(), @@ -118,6 +128,7 @@ def __get_sections(self) -> Dict[str, List[Callable]]: l_vote: List[Callable] = list() l_stake: List[Callable] = list() l_proposal: List[Callable] = list() + l_organization: List[Callable] = list() # Panes are already created. if self.are_panes: @@ -125,20 +136,53 @@ def __get_sections(self) -> Dict[str, List[Callable]]: l_vote = [c.layout.get_layout for c in self.__controllers[self._VOTE]] l_stake = [c.layout.get_layout for c in self.__controllers[self._STAKE]] l_proposal = [c.layout.get_layout for c in self.__controllers[self._PROPOSAL]] + l_organization = [c.layout.get_layout for c in self.__controllers[self._ORGANIZATION]] else: l_rep_h = self.__get_rep_holder_charts() l_vote = self.__get_vote_charts() l_stake = self.__get_stake_charts() l_proposal = self.__get_proposal_charts() + l_organization = self.__get_organization_charts() return { - TEXT['rep_holder_title']: l_rep_h, - TEXT['vote_title']: l_vote, - TEXT['stake_title']: l_stake, - TEXT['proposal_title']: l_proposal, + COMMON_TEXT['no_data_selected']: { + 'callables': l_organization, + 'css_id': TEXT['css_id_organization'], + }, + TEXT['rep_holder_title']: { + 'callables': l_rep_h, + 'css_id': TEXT['css_id_reputation_holders'], + }, + TEXT['vote_title']: { + 'callables': l_vote, + 'css_id': TEXT['css_id_votes'], + }, + TEXT['stake_title']: { + 'callables': l_stake, + 'css_id': TEXT['css_id_stake'], + }, + TEXT['proposal_title']: { + 'callables': l_proposal, + 'css_id': TEXT['css_id_proposal'], + }, } + def __get_organization_charts(self) -> List[Callable[[], html.Div]]: + charts: List[Callable] = list() + call: Callable = self.organizations + + # active organizations + charts.append(self.__create_chart( + title=TEXT['title_active_organization'], + adapter=MetricAdapter(s_factory.ACTIVE_ORGANIZATION, call), + figure=BarFigure(), + cont_key=self._ORGANIZATION + )) + + return charts + + def __get_rep_holder_charts(self) -> List[Callable[[], html.Div]]: """ Creates charts of reputation holder section, this includes diff --git a/src/apps/daostack/class_diagram.png b/src/apps/daostack/class_diagram.png index 092e5c66..d082d354 100644 Binary files a/src/apps/daostack/class_diagram.png and b/src/apps/daostack/class_diagram.png differ diff --git a/src/apps/daostack/data_access/daos/metric/metric_dao_factory.py b/src/apps/daostack/data_access/daos/metric/metric_dao_factory.py index 10524265..c07a20cf 100644 --- a/src/apps/daostack/data_access/daos/metric/metric_dao_factory.py +++ b/src/apps/daostack/data_access/daos/metric/metric_dao_factory.py @@ -23,6 +23,8 @@ st_total_votes_stakes_option as st_tvso from src.apps.daostack.data_access.daos.metric.strategy.st_active_users\ import StActiveUsers +from src.apps.daostack.data_access.daos.metric.strategy.st_active_organization\ + import StActiveOrganization NEW_USERS = 0 NEW_PROPOSALS = 1 @@ -37,6 +39,7 @@ TOTAL_VOTES_OPTION = 10 TOTAL_STAKES_OPTION = 11 ACTIVE_USERS = 12 +ACTIVE_ORGANIZATION = 13 def get_dao(ids: List[str], metric: int) -> MetricDao: # noqa: C901 @@ -85,5 +88,11 @@ def get_dao(ids: List[str], metric: int) -> MetricDao: # noqa: C901 cache.PROPOSALS, cache.VOTES, cache.STAKES]) + elif metric == ACTIVE_ORGANIZATION: + stg = StActiveOrganization() + requester = cache.CacheRequester(srcs=[ + cache.PROPOSALS, + cache.VOTES, + cache.STAKES]) return MetricDao(ids=ids, strategy=stg, requester=requester, address_key='dao') diff --git a/src/apps/daostack/data_access/daos/metric/strategy/st_active_organization.py b/src/apps/daostack/data_access/daos/metric/strategy/st_active_organization.py new file mode 100644 index 00000000..18329050 --- /dev/null +++ b/src/apps/daostack/data_access/daos/metric/strategy/st_active_organization.py @@ -0,0 +1,66 @@ +""" + Descp: Strategy pattern to create active DAO(meaning organizations) metric. + It'll show the number of active DAOs each month, + whether just one DAO is selected it will show the month when + it was active. + + Created on: 23-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import pandas as pd + +from src.apps.common.data_access.daos.metric.imetric_strategy \ + import IMetricStrategy + +from src.apps.common.business.transfers.stacked_serie import StackedSerie +from src.apps.common.business.transfers.serie import Serie +import src.apps.common.data_access.pandas_utils as pd_utl + + +class StActiveOrganization(IMetricStrategy): + __DF_DATE = 'createdAt' + __DF_ID = 'dao' + __DF_COUNT = 'count' + __DF_COLS = [__DF_DATE, __DF_ID] + + + def clean_df(self, df: pd.DataFrame) -> pd.DataFrame: + dff: pd.DataFrame = df + dff.loc[:, self.__DF_COLS] = dff[self.__DF_COLS] + return dff + + + def process_data(self, df: pd.DataFrame) -> StackedSerie: + if pd_utl.is_an_empty_df(df): + return StackedSerie() + + df = self.clean_df(df=df) + + # takes just the month + df = pd_utl.unix_to_date(df, self.__DF_DATE) + df = pd_utl.transform_to_monthly_date(df, self.__DF_DATE) + + # join dates-ids + df = pd_utl.count_cols_repetitions(df, self.__DF_COLS, self.__DF_COUNT) + # different users by month + df = pd_utl.count_cols_repetitions(df, [self.__DF_DATE], self.__DF_COUNT) + + # generates a time series + idx = pd_utl.get_monthly_serie_from_df(df, self.__DF_DATE) + dff = pd_utl.get_df_from_lists([idx, 0], [self.__DF_DATE, self.__DF_COUNT]) + dff = pd_utl.datetime_to_date(dff, self.__DF_DATE) + + # joinning all the data in a unique dataframe + df = df.append(dff, ignore_index=True) + df.drop_duplicates(subset=self.__DF_DATE, keep="first", inplace=True) + df.sort_values(self.__DF_DATE, inplace=True) + + serie: Serie = Serie(x=df[self.__DF_DATE].tolist()) + metric: StackedSerie = StackedSerie( + serie = serie, + y_stack = [df[self.__DF_COUNT].tolist()]) + + return metric diff --git a/src/apps/daostack/resources/strings.py b/src/apps/daostack/resources/strings.py index 2c3e7d9d..105f2d03 100644 --- a/src/apps/daostack/resources/strings.py +++ b/src/apps/daostack/resources/strings.py @@ -14,6 +14,11 @@ 'boost': 'Rate of boosted proposals that were approved', 'boost_fail': 'Boosted rejections', 'boost_pass': 'Boosted approvals', + 'css_id_organization': 'daostack-organization', + 'css_id_proposal': 'daostack-proposal', + 'css_id_reputation_holders': 'daostack-reputation-holders', + 'css_id_stake': 'daostack-stake', + 'css_id_votes': 'daostack-votes', 'css_pane_border': 'daostack-border', 'different_stakers_title': 'Different stakers', 'different_voters_title': 'Different voters', @@ -33,6 +38,7 @@ 'rep_holder_title': 'Reputation holders', 'rel_pass': 'Relative majority approval', 'stake_title': 'Stakes', + 'title_active_organization': 'Months which the DAO has registered activity', 'total_stakes_option_title': 'Total stakes option', 'total_stakes_title': 'Total stakes', 'total_votes_option_title': 'Total votes option', diff --git a/src/assets/aragon.png b/src/assets/aragon.png new file mode 100644 index 00000000..ea6f1740 Binary files /dev/null and b/src/assets/aragon.png differ diff --git a/src/assets/stylesheet.css b/src/assets/stylesheet.css index e0d849f3..ffd4525e 100644 --- a/src/assets/stylesheet.css +++ b/src/assets/stylesheet.css @@ -187,6 +187,10 @@ border: 3px #FFCC80 solid; } +.aragon:hover { + border: 3px #90CAF9 solid; +} + .app-color { background-color: #CFD8DC; } @@ -199,6 +203,10 @@ background-color: #FFCC80; } +.aragon-color { + background-color:#90CAF9; +} + .daostack-border { border: 1px #A5D6A7 solid; border-top: 0; @@ -207,4 +215,9 @@ .daohaus-border { border: 1px#FFCC80 solid; border-top: 0; +} + +.aragon-border { + border: 1px#90CAF9 solid; + border-top: 0; } \ No newline at end of file diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_aragon_st_new_additions.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_aragon_st_new_additions.py new file mode 100644 index 00000000..910ccf14 --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_aragon_st_new_additions.py @@ -0,0 +1,62 @@ +""" + Descp: New additions test. + + Created on: 19-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_new_additions import StNewAdditions + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StNewMembersTest(unittest.TestCase): + + def test_vote_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=1, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'trash': 'trash', 'startDate': bl.unix()},#today_year-(today_month-3)-01T00:00:00+00:00 + {'trash': 'trash', 'startDate': bl.add(month=1).change(day=25).unix()},#today_year-(today_month-2)-25T00:00:00+00:00 + {'trash': 'trash', 'startDate': bl.unix()},#today_year-(today_month-2)-25T00:00:00+00:00 + {'trash': 'trash', 'startDate': bl.add(month=1).change(day=1, hour=23).unix()},#today_year-(today_month-1)-01T23:00:00+00:00 + {'trash': 'trash', 'startDate': bl.sub(month=1).change(minute=59, second=59).unix()},#today_year-(today_month-2)-01T23:59:59+00:00 + {'trash': 'trash', 'startDate': bl.unix()},#today_year-(today_month-2)-01T23:59:59+00:00 + {'trash': 'trash', 'startDate': bl.add(month=2, second=1).change(day=21).unix()},#today_year-today_month-21T00:00:00+00:00 + ]) + strategy: StNewAdditions = StNewAdditions(typ=StNewAdditions.VOTE) + result: StackedSerie = strategy.process_data(df=in_df) + out: List[int] = [1, 4, 1, 1] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_transactons_members_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=4, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'date': bl.unix()},#today_year-(today_month-3)-04T00:00:00+00:00 + {'date': bl.add(month=1).change(day=16).unix()},#today_year-(today_month-2)-16T00:00:00+00:00 + {'date': bl.unix()},#today_year-(today_month-2)-16T00:00:00+00:00 + {'date': bl.change(day=9, hour=23).unix()},#today_year-(today_month-2)-09T23:00:00+00:00 + {'date': bl.sub(month=1).change(minute=59, second=59).unix()},#today_year-(today_month-3)-01T23:59:59+00:00 + ]) + strategy: StNewAdditions = StNewAdditions(typ=StNewAdditions.TRANSACTION) + result: StackedSerie = strategy.process_data(df=in_df) + out: List[int] = [2, 3, 0, 0] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_organization_aragon.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_organization_aragon.py new file mode 100644 index 00000000..2f298ef2 --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_organization_aragon.py @@ -0,0 +1,49 @@ +""" + Descp: Active organization test. + + Created on: 23-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_active_organization import StActiveOrganization + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StActiveOrganizationTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[int]) -> None: + strategy: StActiveOrganization = StActiveOrganization() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=27, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'orgAddress': 1, 'trash': 'trash', 'createdAt': bl.unix()},#today_year-(today_month-3)-27T00:00:00+00:00 + {'orgAddress': 1, 'trash': 'trash', 'createdAt': bl.change(hour=5).unix()},#today_year-(today_month-3)-27T05:00:00+00:00 + {'orgAddress': 1, 'trash': 'trash', 'date': bl.add(month=2).unix()},#today_year-(today_month-1)-27T05:00:00+00:00 + {'orgAddress': 2, 'trash': 'trash', 'date': bl.change(day=3, hour=23).unix()},#today_year-(today_month-1)-03T23:00:00+00:00 + {'orgAddress': 2, 'trash': 'trash', 'startDate': bl.add(month=1).change(minute=59, second=59).unix()},#today_year-today_month-03T23:59:59+00:00 + {'orgAddress': 3, 'trash': 'trash', 'date': bl.unix()},#today_year-today_month-03T23:59:59+00:00 + {'orgAddress': 2, 'trash': 'trash', 'startDate': bl.add(second=1).change(day=7, hour=17).unix()},#today_year-today_month-07T17:00:00+00:00 + ]) + out: List[int] = [1, 0, 2, 2] + + self.__check_lists(df=in_df, out=out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_token_holders.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_token_holders.py new file mode 100644 index 00000000..cf691252 --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_active_token_holders.py @@ -0,0 +1,49 @@ +""" + Descp: Active token holders test. + + Created on: 22-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_active_token_holders import StActiveTokenHolders + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StActiveTokenHoldersTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[int]) -> None: + strategy: StActiveTokenHolders = StActiveTokenHolders() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=27, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'voter': 1, 'trash': 'trash', 'createdAt': bl.unix()},#today_year-(today_month-3)-27T00:00:00+00:00 + {'voter': 1, 'trash': 'trash', 'createdAt': bl.change(hour=5).unix()},#today_year-(today_month-3)-27T05:00:00+00:00 + {'entity': 1, 'trash': 'trash', 'date': bl.add(month=2).unix()},#today_year-(today_month-1)-27T05:00:00+00:00 + {'entity': 2, 'trash': 'trash', 'date': bl.change(day=3, hour=23).unix()},#today_year-(today_month-1)-03T23:00:00+00:00 + {'creator': 2, 'trash': 'trash', 'startDate': bl.add(month=1).change(minute=59, second=59).unix()},#today_year-today_month-03T23:59:59+00:00 + {'entity': 3, 'trash': 'trash', 'date': bl.unix()},#today_year-today_month-03T23:59:59+00:00 + {'creator': 2, 'trash': 'trash', 'startDate': bl.add(second=1).change(day=7, hour=17).unix()},#today_year-today_month-07T17:00:00+00:00 + ]) + out: List[int] = [1, 0, 2, 2] + + self.__check_lists(df=in_df, out=out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_cast_type.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_cast_type.py new file mode 100644 index 00000000..d84cb568 --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_cast_type.py @@ -0,0 +1,75 @@ +""" + Descp: Casted votes by supports metric strategy test. + + Created on: 21-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_cast_type import StCastType + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StCastTypeTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[List[int]]) -> None: + strategy: StCastType = StCastType() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out[0], result.get_i_stack(i_stack=0)) # test negatives + self.assertListEqual(out[1], result.get_i_stack(i_stack=1)) # test positives + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=1, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'supports': True, 'createdAt': bl.unix()},#today_year-(today_month-3)-01T00:00:00+00:00 + {'supports': True, 'createdAt': bl.add(month=1).change(day=25).unix()},#today_year-(today_month-2)-25T00:00:00+00:00 + {'supports': True, 'createdAt': bl.unix()},#today_year-(today_month-2)-25T00:00:00+00:00 + {'supports': False, 'createdAt': bl.add(month=1).change(day=1, hour=23).unix()},#today_year-(today_month-1)-01T23:00:00+00:00 + {'supports': False, 'createdAt': bl.change(minute=59, second=59).unix()},#today_year-(today_month-1)-01T23:59:59+00:00 + {'supports': True, 'createdAt': bl.add(month=1).unix()},#today_year-today_month-01T23:59:59+00:00 + {'supports': False, 'createdAt': bl.add(second=1).change(day=21).unix()},#today_year-today_month-21T00:00:00+00:00 + ]) + + out: List[List[int]] = [ + [0, 0, 2, 1], # number of votes against + [1, 2, 0, 1], # number of votes for + ] + + self.__check_lists(df=in_df, out=out) + + + def test_cast_against(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=1, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'supports': False, 'createdAt': bl.unix()},#today_year-(today_month-3)-01T00:00:00+00:00 + {'supports': False, 'createdAt': bl.add(month=1).change(day=11).unix()},#today_year-(today_month-2)-11T00:00:00+00:00 + {'supports': False, 'createdAt': bl.unix()},#today_year-(today_month-2)-11T00:00:00+00:00 + {'supports': False, 'createdAt': bl.sub(month=1).change(day=3, hour=5).unix()},#today_year-(today_month-3)-03T5:00:00+00:00 + {'supports': False, 'createdAt': bl.add(month=1).change(day=1, minute=59, second=59).unix()},#today_year-(today_month-2)-01T23:59:59+00:00 + {'supports': False, 'createdAt': bl.unix()},#today_year-(today_month-2)-01T23:59:59+00:00 + {'supports': False, 'createdAt': bl.add(month=1).change(day=21, hour=0, minute=0, second=0).unix()},#today_year-(today_month-1)-21T00:00:00+00:00 + ]) + + out: List[List[int]] = [ + [2, 4, 1, 0], # number of votes against + [0, 0, 0, 0]# number of votes for + ] + + self.__check_lists(df=in_df, out=out) + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_installed_apps.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_installed_apps.py new file mode 100644 index 00000000..58392aba --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_installed_apps.py @@ -0,0 +1,86 @@ +""" + Descp: Installed apps test. + + Created on: 20-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List, Dict +import pandas as pd + +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_installed_apps import StInstalledApps + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StInstalledAppsTest(unittest.TestCase): + + def test_repeated_names(self): + + in_df: pd.DataFrame = pd.DataFrame([ + {'trash': 'trash', 'id': '1', 'repoName': '1'}, + {'trash': 'trash', 'id': '2', 'repoName': '2'}, + {'trash': 'trash', 'id': '3', 'repoName': '3'}, + {'trash': 'trash', 'id': '4', 'repoName': '1'}, + ]) + strategy: StInstalledApps = StInstalledApps() + result: StackedSerie = strategy.process_data(df=in_df) + out: List[int] = [2, 1, 1] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_repeated_non_names(self): + + in_df: pd.DataFrame = pd.DataFrame([ + {'trash': 'trash', 'id': '1', 'repoName': '1'}, + {'trash': 'trash', 'id': '2', 'repoName': '2'}, + {'trash': 'trash', 'id': '3', 'repoName': None}, + {'trash': 'trash', 'id': '4', 'repoName': None}, + ]) + strategy: StInstalledApps = StInstalledApps() + result: StackedSerie = strategy.process_data(df=in_df) + out: List[int] = [1, 1] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_repetitions(self): + + in_df: pd.DataFrame = pd.DataFrame([ + {'trash': 'trash', 'id': '1', 'repoName': '1'}, + {'trash': 'trash', 'id': '1', 'repoName': '1'}, + {'trash': 'trash', 'id': '1', 'repoName': '1'}, + ]) + strategy: StInstalledApps = StInstalledApps() + result: StackedSerie = strategy.process_data(df=in_df) + out: List[int] = [1] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_others(self): + max_elems: int = 15 + + in_data: List[Dict[str, str]] = [ + {'id': '1', 'repoName': '1'}, + {'id': '2', 'repoName': '1'}, + {'id': '3', 'repoName': '1'}, + ] + \ + [{'id': f'{i}', 'repoName': f'{i}'} for i in range(4, max_elems+6)] + + in_df: pd.DataFrame = pd.DataFrame(in_data) + strategy: StInstalledApps = StInstalledApps() + result: StackedSerie = strategy.process_data(df=in_df) + + out: List[int] = [3] + ([1] * (max_elems-1)) + [3] + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_vote_outcome.py b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_vote_outcome.py new file mode 100644 index 00000000..68d23aac --- /dev/null +++ b/test/unit/apps/aragon/data_access/daos/metric/strategy/test_st_vote_outcome.py @@ -0,0 +1,52 @@ +""" + Descp: Vote outcome metric strategy test. + + Created on: 21-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.aragon.data_access.daos.metric.strategy.\ + st_vote_outcome import StVoteOutcome + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StVoteOutcomeTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[List[int]]) -> None: + strategy: StVoteOutcome = StVoteOutcome() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out[0], result.get_i_stack(i_stack=0)) # test rejections + self.assertListEqual(out[1], result.get_i_stack(i_stack=1)) # test passes + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=16, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'yea': 0, 'nay': 1, 'supportRequiredPct': 15, 'minAcceptQuorum': 100000000000000000, 'votingPower': 5,'startDate': bl.unix()},#today_year-(today_month-3)-16T00:00:00+00:00 + {'yea': 6, 'nay': 2, 'supportRequiredPct': 50, 'minAcceptQuorum': 50, 'votingPower': 9,'startDate': bl.add(month=1).change(day=5).unix()},#today_year-(today_month-2)-05T00:00:00+00:00 + {'yea': 1, 'nay': 0, 'supportRequiredPct': 100000000000000000, 'minAcceptQuorum': 33, 'votingPower': 3,'startDate': bl.unix()},#today_year-(today_month-2)-05T00:00:00+00:00 + {'yea': 5, 'nay': 4, 'supportRequiredPct': 33, 'minAcceptQuorum': 15, 'votingPower': 9,'startDate': bl.add(month=2).change(day=1, hour=23, minute=59, second=59).unix()},#today_year-today_month-01T23:59:59+00:00 + {'yea': 0, 'nay': 0, 'supportRequiredPct': 0, 'minAcceptQuorum': 0, 'votingPower': 00000000000000000,'startDate': bl.unix()},#today_year-today_month-01T23:59:59+00:00 + {'yea': 4, 'nay': 0, 'supportRequiredPct': 1000000000000000000, 'minAcceptQuorum': 500000000000000000, 'votingPower': 10,'startDate': bl.add(second=1).change(day=27).unix()},#today_year-today_month-27T00:00:00+00:00 + ]) + out: List[List[int]] = [ + [1, 0, 0, 2], # number of rejected proposals + [0, 2, 0, 1] # number of approved proposals + ] + + self.__check_lists(df=in_df, out=out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_active_organization_daohaus.py b/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_active_organization_daohaus.py new file mode 100644 index 00000000..9104b1da --- /dev/null +++ b/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_active_organization_daohaus.py @@ -0,0 +1,49 @@ +""" + Descp: Active organization test. + + Created on: 23-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.daohaus.data_access.daos.metric.strategy.\ + st_active_organization import StActiveOrganization + +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StActiveOrganizationTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[int]) -> None: + strategy: StActiveOrganization = StActiveOrganization() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=3).change(day=27, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'molochAddress': 1, 'trash': 'trash', 'createdAt': bl.unix()},#today_year-(today_month-3)-27T00:00:00+00:00 + {'molochAddress': 1, 'trash': 'trash', 'createdAt': bl.change(hour=5).unix()},#today_year-(today_month-3)-27T05:00:00+00:00 + {'molochAddress': 1, 'trash': 'trash', 'createdAt': bl.add(month=2).unix()},#today_year-(today_month-1)-27T05:00:00+00:00 + {'molochAddress': 2, 'trash': 'trash', 'createdAt': bl.change(day=3, hour=23).unix()},#today_year-(today_month-1)-03T23:00:00+00:00 + {'molochAddress': 2, 'trash': 'trash', 'createdAt': bl.add(month=1).change(minute=59, second=59).unix()},#today_year-today_month-03T23:59:59+00:00 + {'molochAddress': 3, 'trash': 'trash', 'createdAt': bl.unix()},#today_year-today_month-03T23:59:59+00:00 + {'molochAddress': 2, 'trash': 'trash', 'createdAt': bl.add(second=1).change(day=7, hour=17).unix()},#today_year-today_month-07T17:00:00+00:00 + ]) + out: List[int] = [1, 0, 2, 2] + + self.__check_lists(df=in_df, out=out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_new_additions.py b/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_new_additions.py index a6e65078..442cc202 100644 --- a/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_new_additions.py +++ b/test/unit/apps/daohaus/data_access/daos/metric/strategy/test_st_new_additions.py @@ -1,5 +1,5 @@ """ - Descp: New members test. + Descp: New additions test. Created on: 7-oct-2020 diff --git a/test/unit/apps/daostack/data_access/daos/metric/strategy/test_st_active_organization_daostack.py b/test/unit/apps/daostack/data_access/daos/metric/strategy/test_st_active_organization_daostack.py new file mode 100644 index 00000000..4eaabbfb --- /dev/null +++ b/test/unit/apps/daostack/data_access/daos/metric/strategy/test_st_active_organization_daostack.py @@ -0,0 +1,49 @@ +""" + Descp: Tester for active organization strategy. + + Created on: 23-oct-2020 + + Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui + +""" + +import unittest +from typing import List +import pandas as pd + +from test.mocks.unix_date_builder import UnixDateBuilder +from src.apps.daostack.data_access.daos.metric.strategy.\ + st_active_organization import StActiveOrganization +from src.apps.common.business.transfers.stacked_serie import StackedSerie + + +class StActiveOrganizationTest(unittest.TestCase): + + def __check_lists(self, df: pd.DataFrame, out: List[int]) -> None: + strategy: StActiveOrganization = StActiveOrganization() + result: StackedSerie = strategy.process_data(df=df) + + self.assertListEqual(out, result.get_i_stack(i_stack=0)) + + + def test_process_data(self): + bl: UnixDateBuilder = UnixDateBuilder() + bl.sub(month=2).change(day=25, hour=0, minute=0, second=0) + + in_df: pd.DataFrame = pd.DataFrame([ + {'dao': '0', 'trash': 'trash', 'createdAt': bl.unix()}, #today_year-(today_month-2)-25T00:00:00+00:00 + {'dao': '0', 'trash': 'trash', 'createdAt': bl.unix()}, #today_year-(today_month-2)-25T00:00:00+00:00 + {'dao': '0', 'trash': 'trash', 'createdAt': bl.add(month=1).change(day=1, hour=23).unix()}, #today_year-(today_month-1)-01T23:00:00+00:00 + {'dao': '1', 'trash': 'trash', 'createdAt': bl.sub(month=1).change(minute=59, second=59).unix()}, #today_year-(today_month-2)-01T23:59:59+00:00 + {'dao': '1', 'trash': 'trash', 'createdAt': bl.unix()}, #today_year-(today_month-2)-01T23:59:59+00:00 + {'dao': '3', 'trash': 'trash', 'createdAt': bl.add(month=2).change(day=15, hour=0).unix()}, #today_year-today_month-15T00:00:00+00:00 + {'dao': '2', 'trash': 'trash', 'createdAt': bl.change(day=21).unix()}, #today_year-today_month-21T00:00:00+00:00 + {'dao': '3', 'trash': 'trash', 'createdAt': bl.unix()}, #today_year-today_month-21T00:00:00+00:00 + ]) + out: List[int] = [2, 1, 2] + + self.__check_lists(df=in_df, out=out) + + +if __name__ == "__main__": + unittest.main()