Updated the logic for the plugins page.

Now it has a simple heuristic to sort the plugins by activity.

tools/blacklisted.txt

@@ -1,4 +1,5 @@
 errbotio/errbot
+YaroslavMolchan/lctv
 kongluoxing/TomBot
 TheArchives/Inter
 Offbeatmammal/jsErrLog
@@ -556,3 +557,115 @@ LauJangit/Simulation-Of-Quantum-Communication-Error
 FrederikWR/course-01405_algebraic_error_correcting_codes-2017
 darrenyaoyao/OhmyNN_for_ASR_error_on_dialog_system
 Alan-g-s/Cell-Microsatelite-Data-Processing-and-Analysis-for-Imputation-Errors
+aleju/gan-error-avoidance
+PhyrionX/ErReSuLoNaDorH
+andersx/errorlearn
+chrisjbryant/errant
+suetAndTie/ClarkeErrorGrid
+mhsiddiqui/django-error-report
+blakegall/ErraticBot
+seanpianka/Fatal-Error
+qedsoftware/django-force-error
+sovamakarosh/Prtg_Err
+errezeta/errez
+commit-live-admin/errors
+srishti88/errors
+salcedo/errorpages
+AAAwesomer/ErrorRecognition
+amaiellu/ErrorChecker
+mwerevu/ErrorComp
+dingusagar/Error404
+AndriiMazur/ErrorDecoder
+tbec/ErrorControlCoding
+pandahuang/ErrorTypeRecognition
+Nadiah16/ErrorCountAP
+ErrorFeed/ErrorFeed-Python
+priyabagaria/error-analysis
+Foolings/Trail-Error
+ForumOrganisation/errache-bot
+antspy/cntkError
+sabrinadowla14/flaskError
+MorozYaroslavKN-B/Test-error
+ErrolLin/errol.pa
+luismiguelnarvaez/demoError
+weijlander/ScaleErrors
+twhunt/backwards_error
+manexagirrezabal/errima-bertsolaritzan
+hzhang-wx/light_errata
+FreakyBytes/errbot-docker
+johnjosephmorgan/error-analysis
+Nelestya/BeautifullError
+rasika-a/errand_mapper
+rasika-a/errand_mapper
+eduardogpg/tinyintError
+briantical/Personal-Errands
+klawal/Hour-Error
+JayHyeonwoo/error_work
+zencore-dobetter/zencore-errors
+ErrolX3/ErrolX3.github.io
+Bharathkumar-nb/Poco-Error-Detector
+jpulec/errbot-heroku-deploy
+im-shyam/django_error_assist
+A-Setiabudi/HoldError-ANN
+anuragrana/Error_Logging_Django
+ncrmro/polymorphic_django_grapene_error
+RedHatHackFest/spark-error-selector
+bbbbb34016/get_the_error_counts
+angelusmx/error_parser_QSA
+zeran4/mnist_trial_and_error
+drkutuzov/arrays-with-errorbars
+Axeleik/False_merge_errosion
+alepers/error-responsibility-routing-thesis
+gpanther/blobstore-error-poc
+vmdowney/oclc-error-report-tools
+keisks/error-repair-parsing
+imalic3/python-word-error-rate
+hoionazun/HTTPSecondErrorTest
+jwilk/python-syntax-errors
+kylehovey/ERRNO_Picture_Daemon
+finnd/korymbus_sample_case_error
+OkBuilds/BuckwWatchmanErrorReproduce
+krnkl/appengine-route-svc-error
+derekhendrickx/find-my-flac-errors
+zetayue/DiffusionErrorAnalyzer
+lucianoRM/error_logger_server
+JeffreyEnglish/DissolveWithError
+comjoueur/snake_con_error
+hcoura/forbes_scrapy_error
+piruty/square_error_app
+OscarJHernandez/linear_fit_w_errors
+wooyek/django-error-views
+comjoueur/snake_con_error
+devoidheiligenschein/million-song-errythang
+willgeorgejr/system-32-error
+claudiusalp/SpellErrorCorrection
+jwfrizzell/udacity_causes_of_error
+zarechnev/stepik_errors_and_exceptions
+petebrowne/django-adminactions-forbidden-error
+krisys/django-error-email-throttle
+martin056/HandleErrorsDemo
+MrSimonC/Galaxy-Error-Log-Content
+MrSimonC/Galaxy-Error-Log-Interface
+igor-pea/Time-Error-Correction
+aary/buck-shared-library-error-report
+evandrocoan/SublimeSelectAllSpellingErrors
+lowlevel86/multiple-variable-trial-and-error
+mzelinka/klein2013-cloud-error-metrics
+NinjaWolf064/Python-3.4.3-Errors
+robodair/yapsy_multiprocess_pool_import_error
+hainan89/EffectOfLocalizationErrorsOnDecisionMaking
+Hem-Bhatt/Bit-error-propagation-in-block-cipher-modes
+Bala9626/Hamming-De-hamming-and-error-detection
+ODYTRON/Challenge-Modules-Classes-Error-Handling-And-List-Comprehension
+deanishe/alfred-errnum
+failtale/error-parser
+frozenfoxx/rpi-errbot
+cobhuni/errors_fixer
+Dhvani35729/error_deep
+gokceuludogan/SpellErrorCorrector
+marcosvsmorais/nagios-interface-check-errors
+mathemage/bars-n-errors
+jemdiggity/test_clinkage_errors
+cafecco/k-error-linear-complexity
+andrewpgit/spacewalk-api-errata
+realraum/r3bot-errbot

tools/extras.txt

@@ -1,9 +1,73 @@
+gamkedo-la/robokedo
+openstack/oslo.tools
+tormich/yobabot
+bigmlab/bigmbot
+elopio/snappy-m-o
+mPharma/bot-demo
+RQstudio/errbottest-tensorflow
+sde-melo/hipchat-errbot-japanese
+labs-holic/FutBot
+EdoPut/errbot-ebay
+arykalin/errbot-getinfo
+obihann/SantaBot
+PythonSanSebastian/pyperr
+jerrykan/recordwho
+brandverity/tcs_bot
+J4LP/pingbot
+unitycoders/uc_pyircbot
+ilkka/jutibot
+prsnnami/chatops
+zsoobhan/nibbler
+kaytwo/lockbot
+debojitkakoti/atraey
+lagenorhynque/muse-bot
+fookatchu/GoogleCloudVisionParser
+Team488/meeting-bot
+chiel1980/chatbot-plugins
+staircaseJapes/slack
+rdo-infra/rdobot
+xi/lunchbot
+nelsonam/meryl
+fookatchu/LinkParser
+lekum/ansiblebot
+jlu368/slackcitizenshiptester
+kgutwin/lyncbot
+rvm-xx/weather-bot
+kkc/gogobot
+MaxwellBo/SlackVCS
+lanthos/chillbot
+ZaneRL9/IpreoChatOps
+youske/errbot-backend-chatwork
+teran-mckinney/potatoclicker
+lazam/FunBot
+srayneau/eva
+ministry-of-love/contractor
+davvi/bender
+roman8422/waiter_bot
+Axylos/errfite
+Beit-Hatfutsot/jewbot
+tormich/errkeygen
+ValhallaGamePlays/odin
+danie1cohen/errbot-kodi
+0ahab/vpn_hook
+robojase/errbot-chat-points
+t-kenji/errbot-moogle
+jpulec/errbot-deploy
+tormich/errmesos
+markusj/titlebot-ng
+RQstudio/botplugins
+Appleman1234/arisu
+qlixed/nerdobot
+pirxthepilot/ninjam-slack-bot
+matanby/st0ckbot
 aherok/errbot_plugins
 attakei/errbot-teratail-fetch
 austinhappel/sabot
+coala/corobo
 avengerpenguin/alfred
 Axylos/pyfite
 Bloodskate/chatops
+checalov/AliBot
 brandverity/tcs_bot
 chiel1980/chatbot-plugins
 chiel1980/docker-containers

tools/gen_home.py

@@ -17,12 +17,12 @@
     for repo, plugins in repos.items():
         for name, plugin in plugins.items():
             if plugin['path'].startswith('errbot/builtins'):
-                pass
+                continue
             if plugin['repo'][PREFIX_LEN:] in blacklisted:
-                pass
+                continue
             filtered_plugins.append(plugin)
 
-    sorted_plugins = sorted(filtered_plugins, key=lambda plugin: plugin['name'])
+    sorted_plugins = sorted(filtered_plugins, key=lambda plugin: -plugin['score'])
 
     with open('Home.md', 'w') as out:
         out.write(template.render(plugins=sorted_plugins))

tools/plugin-gen.py

@@ -4,6 +4,7 @@
 import requests
 import sys
 from requests.auth import HTTPBasicAuth
+from datetime import datetime
 import logging
 import time
 import configparser
@@ -50,7 +51,6 @@ def save_plugins():
     with open('repos.json', 'w') as f:
         json.dump(plugins,
                   f,
-                  sort_keys=True,
                   indent=2,
                   separators=(',', ': '))
 
@@ -100,26 +100,35 @@ def rate_limit(resp):
     time.sleep(delay)
 
 
+def parse_date(gh_date: str)-> datetime:
+    return datetime.strptime(gh_date, "%Y-%m-%dT%H:%M:%SZ")
+
+
 def check_repo(repo):
-    log.debug('Checking %s...', repo)
-    code_resp = requests.get('https://api.github.com/search/code?q=extension:plug+repo:%s' % repo, auth=AUTH)
+    repo_name = repo['full_name']
+    log.debug('Checking %s...', repo_name)
+    code_resp = requests.get('https://api.github.com/search/code?q=extension:plug+repo:%s' % repo_name, auth=AUTH)
     if code_resp.status_code != 200:
-        log.error('Error getting https://api.github.com/search/code?q=extension:plug+repo:%s', repo)
+        log.error('Error getting https://api.github.com/search/code?q=extension:plug+repo:%s', repo_name)
         log.error('code %d', code_resp.status_code)
         log.error('content %s', code_resp.text)
 
         return
     plug_items = code_resp.json()['items']
     if not plug_items:
-        log.debug('No plugin found in %s, blacklisting it.', repo)
-        add_blacklisted(repo)
+        log.debug('No plugin found in %s, blacklisting it.', repo_name)
+        add_blacklisted(repo_name)
         return
-    avatar_url = get_avatar_url(repo)
+    owner = repo['owner']
+    avatar_url = owner['avatar_url'] if 'avatar_url' in owner else DEFAULT_AVATAR
+
+    days_old = (datetime.now() - parse_date(repo['updated_at'])).days
+    score = repo['stargazers_count'] + repo['watchers_count'] * 2 + repo['forks_count'] - days_old / 25
 
     for plug in plug_items:
-        plugfile_resp = requests.get('https://raw.githubusercontent.com/%s/master/%s' % (repo, plug['path']))
+        plugfile_resp = requests.get('https://raw.githubusercontent.com/%s/master/%s' % (repo_name, plug['path']))
         log.debug('Found a plugin:')
-        log.debug('Repo:  %s', repo)
+        log.debug('Repo:  %s', repo_name)
         log.debug('File:  %s', plug['path'])
         parser = configparser.ConfigParser()
         try:
@@ -142,16 +151,17 @@ def check_repo(repo):
 
             plugin = {
                 'path': plug['path'],
-                'repo': 'https://github.com/{0}'.format(repo),
+                'repo': repo['html_url'],
                 'documentation': doc,
                 'name': name,
                 'python': python,
                 'avatar_url': avatar_url,
+                'score': score,
             }
 
-            repo_entry = plugins.get(repo, {})
+            repo_entry = plugins.get(repo_name, {})
             repo_entry[name] = plugin
-            plugins[repo] = repo_entry
+            plugins[repo_name] = repo_entry
             log.debug('Catalog added plugin %s.', plugin['name'])
         except:
             log.error('Invalid syntax in %s, skipping... ' % plug['path'])
@@ -163,8 +173,8 @@ def check_repo(repo):
     rate_limit(code_resp)
 
 
-def find_plugins():
-    url = 'https://api.github.com/search/repositories?q=err+in:name+language:python&sort=stars&order=desc'
+def find_plugins(query):
+    url = 'https://api.github.com/search/repositories?q=%s+in:name+language:python&sort=stars&order=desc' % query
     while True:
         repo_resp = requests.get(url, auth=AUTH)
         repo_json = repo_resp.json()
@@ -180,9 +190,8 @@ def find_plugins():
             continue
         items = repo_json['items']
 
-        for i, item in enumerate(items):
-            repo = item['full_name']
-            if repo in BLACKLISTED:
+        for i, repo in enumerate(items):
+            if repo['full_name'] in BLACKLISTED:
                 log.debug('Skipping %s.', repo)
                 continue
             check_repo(repo)
@@ -194,12 +203,27 @@ def find_plugins():
 
 
 def main():
-    find_plugins()
+    find_plugins('err')
     # Those are found by global search only available on github UI:
     # https://github.com/search?l=&q=Documentation+extension%3Aplug&ref=advsearch&type=Code&utf8=%E2%9C%93
+    url = 'https://api.github.com/repos/%s'
     with open('extras.txt', 'r') as extras:
-        for repo in extras:
-            check_repo(repo.strip())
+        for repo_name in extras:
+            repo_name = repo_name.strip()
+            repo_resp = requests.get(url % repo_name, auth=AUTH)
+            repo = repo_resp.json()
+            if repo.get('message', None) == 'Bad credentials':
+                log.error('Invalid credentials, check your token file, see README.')
+                sys.exit(-1)
+            if 'message' in repo and repo['message'].startswith('API rate limit exceeded for'):
+                log.error('API rate limit hit anyway ... wait for 30s')
+                time.sleep(30)
+                continue
+            if 'message' in repo and repo['message'].startswith('Not Found'):
+                log.error('%s not found.', repo_name)
+            else:
+                check_repo(repo)
+            rate_limit(repo_resp)
 
 if __name__ == "__main__":
     main()

tools/plugins.md

@@ -9,6 +9,7 @@ If you are looking for errbot documentation, it is there: [errbot.io](http://err
 
 - Python {{plugin.python}}
 - Install: `!repos install {{plugin.repo}}`
+- Activity: {{plugin.score}}
 
 ---
 {% endfor %}