diff --git a/README.md b/README.md index b0c922e..61f8814 100644 --- a/README.md +++ b/README.md @@ -15,50 +15,56 @@ The database contains the following fields: 5. Meta (a JSON encoded field of whatever fields each source provides) ``` +## Installation +```bash +$ pip install unscrapulous +``` + +## Usage +```bash +$ unscrapulous --config=config.toml --output=output.csv +``` + ## Development ```bash $ git clone git@github.com:themousepotato/unscrapulous.git $ cd unscrapulous $ curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - $ poetry install -$ poetry shell -``` - -## Usage -```bash -$ ./unscrapulous.py --config=config.toml --output=output.csv +$ poetry build +$ pip install dist/unscrapulous-*.whl ``` ## Config The `config.toml` file has the following format: ```toml [scrapers] -arbitration-awards-bse = false -arbitration-awards-nse = false -bse-defaulter-and-expelled-members = false -icex-defaulter-members = false -icex-expelled-members = false -income-tax-defaulters = false -irda-blacklisted = false -mca-company-defaulter-list = false -mca-director-defaulter-list = false -mca-director-disqualified-list = false -mca-proclaimed-offenders-ind = false -mcx-action-ap = false -mcx-defaulter-members = false -mcx-secretaries-defaulter-list = false -mse-arbitral-awards = false -ncdex-suspended-defaulted-expelled-debarred-members = false -nse-defaulted-members = false -nse-expelled-members = false -nse-regulatory-defaulting-clients = false -sebi-debarred-bse = false -sebi-debarred-nse = false -sfio-convicted = true -sfio-proclaimed-offenders = false -unsc-1988 = false -unsc-consolidated-list = false -wildlife-crime-convicts = true +arbitration_awards_bse = false +arbitration_awards_nse = false +bse_defaulter_and_expelled_members = false +icex_defaulter_members = false +icex_expelled_members = false +income_tax_defaulters = false +irda_blacklisted = false +mca_company_defaulter_list = false +mca_director_defaulter_list = false +mca_director_disqualified_list = false +mca_proclaimed_offenders_ind = false +mcx_action_ap = false +mcx_defaulter_members = false +mcx_secretaries_defaulter_list = false +mse_arbitral_awards = false +ncdex_suspended_defaulted_expelled_debarred_members = false +nse_defaulted_members = false +nse_expelled_members = false +nse_regulatory_defaulting_clients = false +sebi_debarred_bse = true +sebi_debarred_nse = true +sfio_convicted = false +sfio_proclaimed_offenders = false +unsc_1988 = false +unsc_consolidated_list = false +wildlife_crime_convicts = false ``` ## Roadmap diff --git a/pyproject.toml b/pyproject.toml index cbb3dbf..46c1477 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "unscrapulous" -version = "0.1.0" +version = "0.1.1" description = "A utility that scrapes lists of unscrupulous entities (barred from doing financial business) published by various legal institutions" authors = ["Navaneeth Suresh "] license = "MIT License" @@ -21,6 +21,9 @@ toml = "^0.10.2" [tool.poetry.dev-dependencies] +[tool.poetry.scripts] +unscrapulous = 'unscrapulous.unscrapulous:main' + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/unscrapulous/arbitration-awards-bse.py b/unscrapulous/arbitration_awards_bse.py similarity index 95% rename from unscrapulous/arbitration-awards-bse.py rename to unscrapulous/arbitration_awards_bse.py index da3484b..b756d98 100644 --- a/unscrapulous/arbitration-awards-bse.py +++ b/unscrapulous/arbitration_awards_bse.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.bseindia.com/investors/ArbitAwards.aspx' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/arbitration-awards-nse.py b/unscrapulous/arbitration_awards_nse.py similarity index 96% rename from unscrapulous/arbitration-awards-nse.py rename to unscrapulous/arbitration_awards_nse.py index fc1e85d..5a61a10 100644 --- a/unscrapulous/arbitration-awards-nse.py +++ b/unscrapulous/arbitration_awards_nse.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www1.nseindia.com/invest/dynaContent/arbitration_award.jsp?requestPage=main&qryFlag=yes' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/bse-defaulter-and-expelled-members.py b/unscrapulous/bse_defaulter_and_expelled_members.py similarity index 95% rename from unscrapulous/bse-defaulter-and-expelled-members.py rename to unscrapulous/bse_defaulter_and_expelled_members.py index c63c6a2..e21c698 100644 --- a/unscrapulous/bse-defaulter-and-expelled-members.py +++ b/unscrapulous/bse_defaulter_and_expelled_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.bseindia.com/static/members/List_defaulters_Expelled_members.aspx' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/config.toml b/unscrapulous/config.toml index fff6ee0..c1f1b62 100644 --- a/unscrapulous/config.toml +++ b/unscrapulous/config.toml @@ -1,27 +1,27 @@ [scrapers] -arbitration-awards-bse = false -arbitration-awards-nse = false -bse-defaulter-and-expelled-members = false -icex-defaulter-members = false -icex-expelled-members = false -income-tax-defaulters = false -irda-blacklisted = false -mca-company-defaulter-list = false -mca-director-defaulter-list = false -mca-director-disqualified-list = false -mca-proclaimed-offenders-ind = false -mcx-action-ap = false -mcx-defaulter-members = false -mcx-secretaries-defaulter-list = false -mse-arbitral-awards = false -ncdex-suspended-defaulted-expelled-debarred-members = false -nse-defaulted-members = false -nse-expelled-members = false -nse-regulatory-defaulting-clients = false -sebi-debarred-bse = false -sebi-debarred-nse = false -sfio-convicted = true -sfio-proclaimed-offenders = false -unsc-1988 = false -unsc-consolidated-list = false -wildlife-crime-convicts = true +arbitration_awards_bse = false +arbitration_awards_nse = false +bse_defaulter_and_expelled_members = false +icex_defaulter_members = false +icex_expelled_members = false +income_tax_defaulters = false +irda_blacklisted = false +mca_company_defaulter_list = false +mca_director_defaulter_list = false +mca_director_disqualified_list = false +mca_proclaimed_offenders_ind = false +mcx_action_ap = false +mcx_defaulter_members = false +mcx_secretaries_defaulter_list = false +mse_arbitral_awards = false +ncdex_suspended_defaulted_expelled_debarred_members = false +nse_defaulted_members = false +nse_expelled_members = false +nse_regulatory_defaulting_clients = false +sebi_debarred_bse = true +sebi_debarred_nse = true +sfio_convicted = false +sfio_proclaimed_offenders = false +unsc_1988 = false +unsc_consolidated_list = false +wildlife_crime_convicts = false diff --git a/unscrapulous/icex-defaulter-members.py b/unscrapulous/icex_defaulter_members.py similarity index 97% rename from unscrapulous/icex-defaulter-members.py rename to unscrapulous/icex_defaulter_members.py index 8063806..748cc92 100644 --- a/unscrapulous/icex-defaulter-members.py +++ b/unscrapulous/icex_defaulter_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.icexindia.com/membership/expelled-defaulter-surrendered-members' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/icex-expelled-members.py b/unscrapulous/icex_expelled_members.py similarity index 97% rename from unscrapulous/icex-expelled-members.py rename to unscrapulous/icex_expelled_members.py index ed89470..ffaf592 100644 --- a/unscrapulous/icex-expelled-members.py +++ b/unscrapulous/icex_expelled_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.icexindia.com/membership/expelled-defaulter-surrendered-members' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/income-tax-defaulters.py b/unscrapulous/income_tax_defaulters.py similarity index 98% rename from unscrapulous/income-tax-defaulters.py rename to unscrapulous/income_tax_defaulters.py index bbfa9ec..8803441 100644 --- a/unscrapulous/income-tax-defaulters.py +++ b/unscrapulous/income_tax_defaulters.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'http://office.incometaxindia.gov.in/administration/_layouts/15/inplview.aspx?List={5A26177B-D7A0-4251-843D-5E6C0B3C3DF2}&View={D8DD9754-8FD1-4D72-9908-727646E99CA0}&ViewCount=450&IsXslView=TRUE&IsCSR=TRUE&Paged=TRUE&p_ID=' FILE_URL = 'http://office.incometaxindia.gov.in/administration/Lists/Tax%20Defaulters/AllItems.aspx' diff --git a/unscrapulous/irda-blacklisted.py b/unscrapulous/irda_blacklisted.py similarity index 97% rename from unscrapulous/irda-blacklisted.py rename to unscrapulous/irda_blacklisted.py index dc59b6d..f22d3be 100644 --- a/unscrapulous/irda-blacklisted.py +++ b/unscrapulous/irda_blacklisted.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * import datetime SOURCE = 'https://agencyportal.irdai.gov.in/PublicAccess/BlackListedAgent.aspx' diff --git a/unscrapulous/mca-company-defaulter-list.py b/unscrapulous/mca_company_defaulter_list.py similarity index 96% rename from unscrapulous/mca-company-defaulter-list.py rename to unscrapulous/mca_company_defaulter_list.py index d7aab93..eade8d8 100644 --- a/unscrapulous/mca-company-defaulter-list.py +++ b/unscrapulous/mca_company_defaulter_list.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'http://www.mca.gov.in' SOURCE = 'http://www.mca.gov.in/MinistryV2/defaultercompanieslist.html' diff --git a/unscrapulous/mca-director-defaulter-list.py b/unscrapulous/mca_director_defaulter_list.py similarity index 96% rename from unscrapulous/mca-director-defaulter-list.py rename to unscrapulous/mca_director_defaulter_list.py index 88a1b41..f49357c 100644 --- a/unscrapulous/mca-director-defaulter-list.py +++ b/unscrapulous/mca_director_defaulter_list.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'http://www.mca.gov.in' SOURCE = 'http://www.mca.gov.in/MinistryV2/defaulterdirectorslist.html' diff --git a/unscrapulous/mca-director-disqualified-list.py b/unscrapulous/mca_director_disqualified_list.py similarity index 96% rename from unscrapulous/mca-director-disqualified-list.py rename to unscrapulous/mca_director_disqualified_list.py index 17ee52d..d19a738 100644 --- a/unscrapulous/mca-director-disqualified-list.py +++ b/unscrapulous/mca_director_disqualified_list.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'http://www.mca.gov.in' SOURCE = 'http://www.mca.gov.in/MinistryV2/disqualifieddirectorslist.html' diff --git a/unscrapulous/mca-proclaimed-offenders-ind.py b/unscrapulous/mca_proclaimed_offenders_ind.py similarity index 96% rename from unscrapulous/mca-proclaimed-offenders-ind.py rename to unscrapulous/mca_proclaimed_offenders_ind.py index f944dee..5fff938 100644 --- a/unscrapulous/mca-proclaimed-offenders-ind.py +++ b/unscrapulous/mca_proclaimed_offenders_ind.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'http://www.mca.gov.in' SOURCE = 'http://www.mca.gov.in/MinistryV2/proclaimedoffenders.html' diff --git a/unscrapulous/mcx-action-ap.py b/unscrapulous/mcx_action_ap.py similarity index 98% rename from unscrapulous/mcx-action-ap.py rename to unscrapulous/mcx_action_ap.py index bd552d8..c11c36e 100644 --- a/unscrapulous/mcx-action-ap.py +++ b/unscrapulous/mcx_action_ap.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'https://www.mcxindia.com' SOURCE = 'https://www.mcxindia.com/membership/notice-board/notice-board-disciplinary-action' diff --git a/unscrapulous/mcx-defaulter-members.py b/unscrapulous/mcx_defaulter_members.py similarity index 98% rename from unscrapulous/mcx-defaulter-members.py rename to unscrapulous/mcx_defaulter_members.py index b2b6bae..bd2af28 100644 --- a/unscrapulous/mcx-defaulter-members.py +++ b/unscrapulous/mcx_defaulter_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCE = 'https://www.mcxindia.com' SOURCE = 'https://www.mcxindia.com/membership/notice-board/notice-board-disciplinary-action' diff --git a/unscrapulous/mcx-secretaries-defaulter-list.py b/unscrapulous/mcx_secretaries_defaulter_list.py similarity index 95% rename from unscrapulous/mcx-secretaries-defaulter-list.py rename to unscrapulous/mcx_secretaries_defaulter_list.py index 247cfe9..2ffc39a 100644 --- a/unscrapulous/mcx-secretaries-defaulter-list.py +++ b/unscrapulous/mcx_secretaries_defaulter_list.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.mcxindia.com/Investor-Services/defaulters/defaulters-list' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/mse-arbitral-awards.py b/unscrapulous/mse_arbitral_awards.py similarity index 96% rename from unscrapulous/mse-arbitral-awards.py rename to unscrapulous/mse_arbitral_awards.py index f2728a2..212bdc1 100644 --- a/unscrapulous/mse-arbitral-awards.py +++ b/unscrapulous/mse_arbitral_awards.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.msei.in/investors/list-of-arbitrators' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/ncdex-suspended-defaulted-expelled-debarred-members.py b/unscrapulous/ncdex_suspended_defaulted_expelled_debarred_members.py similarity index 98% rename from unscrapulous/ncdex-suspended-defaulted-expelled-debarred-members.py rename to unscrapulous/ncdex_suspended_defaulted_expelled_debarred_members.py index a6dbc58..2897a36 100644 --- a/unscrapulous/ncdex-suspended-defaulted-expelled-debarred-members.py +++ b/unscrapulous/ncdex_suspended_defaulted_expelled_debarred_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://ncdex.com/suspended_member/latest_info' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/nse-defaulted-members.py b/unscrapulous/nse_defaulted_members.py similarity index 96% rename from unscrapulous/nse-defaulted-members.py rename to unscrapulous/nse_defaulted_members.py index 82bc7cd..64aae04 100644 --- a/unscrapulous/nse-defaulted-members.py +++ b/unscrapulous/nse_defaulted_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www1.nseindia.com/invest/json/def_members.json' FILE_PARENT_URL = 'https://www1.nseindia.com/invest/resources/download/' diff --git a/unscrapulous/nse-expelled-members.py b/unscrapulous/nse_expelled_members.py similarity index 96% rename from unscrapulous/nse-expelled-members.py rename to unscrapulous/nse_expelled_members.py index eda57ae..187ebba 100644 --- a/unscrapulous/nse-expelled-members.py +++ b/unscrapulous/nse_expelled_members.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www1.nseindia.com/invest/json/exp_members.json' FILE_PARENT_URL = 'https://www1.nseindia.com/invest/resources/download/' diff --git a/unscrapulous/nse-regulatory-defaulting-clients.py b/unscrapulous/nse_regulatory_defaulting_clients.py similarity index 96% rename from unscrapulous/nse-regulatory-defaulting-clients.py rename to unscrapulous/nse_regulatory_defaulting_clients.py index b993183..8a86a28 100644 --- a/unscrapulous/nse-regulatory-defaulting-clients.py +++ b/unscrapulous/nse_regulatory_defaulting_clients.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.nseindia.com/regulations/exchange-defaulting-clients' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/sebi-debarred-bse.py b/unscrapulous/sebi_debarred_bse.py similarity index 97% rename from unscrapulous/sebi-debarred-bse.py rename to unscrapulous/sebi_debarred_bse.py index 0e5ac5e..e6ffb97 100644 --- a/unscrapulous/sebi-debarred-bse.py +++ b/unscrapulous/sebi_debarred_bse.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * PARENT_SOURCES = ['https://www.bseindia.com', 'https://www.bseindia.com/investors/'] SOURCE = 'https://www.bseindia.com/investors/debent.aspx' diff --git a/unscrapulous/sebi-debarred-nse.py b/unscrapulous/sebi_debarred_nse.py similarity index 96% rename from unscrapulous/sebi-debarred-nse.py rename to unscrapulous/sebi_debarred_nse.py index 6edc578..accffb9 100644 --- a/unscrapulous/sebi-debarred-nse.py +++ b/unscrapulous/sebi_debarred_nse.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.nseindia.com/regulations/member-sebi-debarred-entities' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/sfio-convicted.py b/unscrapulous/sfio_convicted.py similarity index 96% rename from unscrapulous/sfio-convicted.py rename to unscrapulous/sfio_convicted.py index 71c5540..4168cb7 100644 --- a/unscrapulous/sfio-convicted.py +++ b/unscrapulous/sfio_convicted.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://sfio.nic.in/' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/sfio-proclaimed-offenders.py b/unscrapulous/sfio_proclaimed_offenders.py similarity index 96% rename from unscrapulous/sfio-proclaimed-offenders.py rename to unscrapulous/sfio_proclaimed_offenders.py index 0393a13..45972e4 100644 --- a/unscrapulous/sfio-proclaimed-offenders.py +++ b/unscrapulous/sfio_proclaimed_offenders.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://sfio.nic.in/' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/unsc-1988.py b/unscrapulous/unsc_1988.py similarity index 97% rename from unscrapulous/unsc-1988.py rename to unscrapulous/unsc_1988.py index 00c6e24..f30f2ff 100644 --- a/unscrapulous/unsc-1988.py +++ b/unscrapulous/unsc_1988.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://scsanctions.un.org/taliban/' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/unsc-consolidated-list.py b/unscrapulous/unsc_consolidated_list.py similarity index 97% rename from unscrapulous/unsc-consolidated-list.py rename to unscrapulous/unsc_consolidated_list.py index 9a4602d..521c5af 100644 --- a/unscrapulous/unsc-consolidated-list.py +++ b/unscrapulous/unsc_consolidated_list.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'https://www.un.org/securitycouncil/content/un-sc-consolidated-list' OUTPUT_DIR = '/tmp/unscrapulous/files' diff --git a/unscrapulous/unscrapulous.py b/unscrapulous/unscrapulous.py index a8d8395..2641ad7 100644 --- a/unscrapulous/unscrapulous.py +++ b/unscrapulous/unscrapulous.py @@ -4,7 +4,8 @@ # Usage: ./unscrapulous.py --config=config.toml --output=results.csv from subprocess import call -from utils import * +from importlib import import_module +from unscrapulous.utils import * import argparse import toml @@ -22,9 +23,10 @@ def main(): config = toml.load(config_path) - filenames = [f for f in config['scrapers'] if config['scrapers'][f]] - for filename in filenames: - call(['python', filename + '.py']) + submodules = [f for f in config['scrapers'] if config['scrapers'][f]] + for submodule in submodules: + mod = import_module(f'unscrapulous.{submodule}') + mod.main() csv_files = [os.path.join(OUTPUT_DIR, f) for f in os.listdir(OUTPUT_DIR) if f.endswith('.csv')] merge_csvs(filenames=csv_files, output_filename=output_filename, delete=True) diff --git a/unscrapulous/wildlife-crime-convicts.py b/unscrapulous/wildlife_crime_convicts.py similarity index 94% rename from unscrapulous/wildlife-crime-convicts.py rename to unscrapulous/wildlife_crime_convicts.py index 956ab7e..93b59d3 100644 --- a/unscrapulous/wildlife-crime-convicts.py +++ b/unscrapulous/wildlife_crime_convicts.py @@ -1,7 +1,7 @@ #!/usr/bin/python #-*- coding: utf-8 -*- -from utils import * +from unscrapulous.utils import * SOURCE = 'http://wccb.gov.in/Content/Convicts.aspx' OUTPUT_DIR = '/tmp/unscrapulous/files'