diff --git a/liberapay/elsewhere/_base.py b/liberapay/elsewhere/_base.py index 24faedf44f..1d9d10eb6e 100644 --- a/liberapay/elsewhere/_base.py +++ b/liberapay/elsewhere/_base.py @@ -272,9 +272,6 @@ def extract_user_info(self, info, source): r.avatar_url = 'https://seccdn.libravatar.org/avatar/'+gravatar_id r.is_team = self.x_is_team(r, info, False) r.description = self.x_description(r, info, None) - r.extra_info = info - if hasattr(self, 'x_extra_info_drop'): - self.x_extra_info_drop(r.extra_info) return r def get_team_members(self, account, page_url=None): @@ -344,8 +341,6 @@ def get_follows_for(self, account, page_url=None, sess=None): r = self.api_get(account.domain, page_url, sess=sess) follows, count, pages_urls = self.api_paginator(r, self.api_parser(r)) follows = [self.extract_user_info(f, account.domain) for f in follows] - if count == -1 and hasattr(self, 'x_follows_count'): - count = self.x_follows_count(None, account.extra_info, -1) return follows, count, pages_urls def extract_repo_info(self, info): @@ -363,9 +358,6 @@ def extract_repo_info(self, info): r.last_update = parse_date(r.last_update) r.is_fork = self.x_repo_is_fork(r, info, None) r.stars_count = self.x_repo_stars_count(r, info, None) - r.extra_info = info - if hasattr(self, 'x_repo_extra_info_drop'): - self.x_repo_extra_info_drop(r.extra_info) return r def get_repos(self, account, page_url=None, sess=None, refresh=True): @@ -394,8 +386,6 @@ def get_repos(self, account, page_url=None, sess=None, refresh=True): raise TokenExpiredError() # Note: we can't pass the page_url below, because it contains the old user_name return self.get_repos(account, page_url=None, sess=sess, refresh=False) - if count == -1 and hasattr(self, 'x_repos_count'): - count = self.x_repos_count(None, account.extra_info, -1) return repos, count, pages_urls def get_starred_repos(self, account, sess, page_url=None): diff --git a/liberapay/elsewhere/github.py b/liberapay/elsewhere/github.py index 397a2c370a..51cd147952 100644 --- a/liberapay/elsewhere/github.py +++ b/liberapay/elsewhere/github.py @@ -1,6 +1,6 @@ from liberapay.elsewhere._base import PlatformOAuth2 from liberapay.elsewhere._exceptions import CantReadMembership -from liberapay.elsewhere._extractors import key, drop_keys +from liberapay.elsewhere._extractors import key from liberapay.elsewhere._paginators import header_links_paginator @@ -43,7 +43,6 @@ class GitHub(PlatformOAuth2): x_avatar_url = key('avatar_url') x_is_team = key('type', clean=lambda t: t.lower() == 'organization') x_description = key('bio') - x_extra_info_drop = drop_keys(lambda k: k.endswith('_url')) # Repo info extractors x_repo_id = key('id') @@ -54,7 +53,6 @@ class GitHub(PlatformOAuth2): x_repo_is_fork = key('fork') x_repo_stars_count = key('stargazers_count') x_repo_owner_id = key('owner', clean=lambda d: d['id']) - x_repo_extra_info_drop = drop_keys(lambda k: k.endswith('_url')) def get_CantReadMembership_url(self, account): return 'https://github.com/orgs/%s/people' % account.user_name diff --git a/liberapay/elsewhere/twitter.py b/liberapay/elsewhere/twitter.py index 60c362c27c..045575294e 100644 --- a/liberapay/elsewhere/twitter.py +++ b/liberapay/elsewhere/twitter.py @@ -1,5 +1,5 @@ from liberapay.elsewhere._base import PlatformOAuth1 -from liberapay.elsewhere._extractors import drop_keys, key, not_available +from liberapay.elsewhere._extractors import key, not_available from liberapay.elsewhere._paginators import query_param_paginator @@ -36,4 +36,3 @@ class Twitter(PlatformOAuth1): clean=lambda v: v.replace('_normal.', '.')) x_follows_count = key('friends_count') x_description = key('description') - x_extra_info_drop = drop_keys('id') diff --git a/liberapay/models/account_elsewhere.py b/liberapay/models/account_elsewhere.py index 4d6d381ad4..14a818361b 100644 --- a/liberapay/models/account_elsewhere.py +++ b/liberapay/models/account_elsewhere.py @@ -2,14 +2,12 @@ import json from urllib.parse import urlsplit, urlunsplit import uuid -import xml.etree.ElementTree as ET from markupsafe import Markup from oauthlib.oauth2 import InvalidGrantError, TokenExpiredError from pando.utils import utcnow from postgres.orm import Model from psycopg2 import IntegrityError -import xmltodict from ..constants import AVATAR_QUERY, DOMAIN_RE, RATE_LIMITS, SUMMARY_MAX_SIZE from ..cron import logger @@ -122,11 +120,6 @@ def upsert(cls, i): query = AVATAR_QUERY i.avatar_url = urlunsplit((scheme, netloc, path, query, fragment)) - # Serialize extra_info - if isinstance(i.extra_info, ET.Element): - i.extra_info = xmltodict.parse(ET.tostring(i.extra_info)) - i.extra_info = json.dumps(i.extra_info) - d = dict(i.__dict__) d.pop('email', None) cols, vals = zip(*d.items()) diff --git a/liberapay/models/repository.py b/liberapay/models/repository.py index 0cbe264348..a43c0c5807 100644 --- a/liberapay/models/repository.py +++ b/liberapay/models/repository.py @@ -1,4 +1,3 @@ -import json from time import sleep from oauthlib.oauth2 import InvalidGrantError, TokenExpiredError @@ -48,7 +47,6 @@ def upsert_repos(cursor, repos, participant, info_fetched_at): if not repo.owner_id or not repo.last_update: continue repo.participant = participant.id - repo.extra_info = json.dumps(repo.extra_info) repo.info_fetched_at = info_fetched_at cols, vals = zip(*repo.__dict__.items()) on_conflict_set = ','.join('{0}=excluded.{0}'.format(col) for col in cols) diff --git a/liberapay/utils/fake_data.py b/liberapay/utils/fake_data.py index c79ff95a3d..d3d93ea7ef 100644 --- a/liberapay/utils/fake_data.py +++ b/liberapay/utils/fake_data.py @@ -128,7 +128,6 @@ def fake_elsewhere(db, participant, platform): user_id=fake_text_id(), user_name=participant.id, participant=participant.id, - extra_info=None, domain='', ) diff --git a/sql/branch.sql b/sql/branch.sql new file mode 100644 index 0000000000..74da0d338d --- /dev/null +++ b/sql/branch.sql @@ -0,0 +1,4 @@ +SELECT 'after deployment'; + +ALTER TABLE elsewhere DROP COLUMN extra_info; +ALTER TABLE repositories DROP COLUMN extra_info; diff --git a/templates/macros/pagination.html b/templates/macros/pagination.html index cc42256b08..fa21522801 100644 --- a/templates/macros/pagination.html +++ b/templates/macros/pagination.html @@ -23,15 +23,15 @@ % endmacro -% macro simple_pager(current_page, last_page) - % if last_page != 1 +% macro simple_pager(current_page, has_more) + % if current_page > 1 or has_more
{{ _("The top {0} individuals on Liberapay are:", len(individuals)) }}
% else{{ _( - "List of individuals on Liberapay, page {number} of {total}:" - , number=current_page, total=last_page + "List of individuals on Liberapay, page {number}:", number=current_page ) }}
% endif{{ _("Nothing to show.") }}
diff --git a/www/explore/organizations.spt b/www/explore/organizations.spt index ace6d94686..86ec526bc8 100644 --- a/www/explore/organizations.spt +++ b/www/explore/organizations.spt @@ -1,40 +1,28 @@ -from math import ceil - [---] per_page = 18 current_page = request.qs.get_int('page', default=1, minimum=1) -with website.db.get_cursor() as cursor: - norganizations = cursor.one(""" - CREATE TEMP TABLE organizations ON COMMIT DROP AS - SELECT p - , ( SELECT s.content - FROM statements s - WHERE s.participant = p.id - AND s.type = 'summary' - ORDER BY s.lang = %s DESC, s.id - LIMIT 1 - ) AS summary - FROM participants p - WHERE p.kind = 'organization' - AND p.status = 'active' - AND (p.goal > 0 OR p.goal IS NULL) - AND p.hide_receiving IS NOT TRUE - AND p.hide_from_lists = 0 - AND p.receiving > 0; - - SELECT count(1) FROM organizations; - """, (locale.language,)) - last_page = max(int(ceil(norganizations / per_page)), 1) - if current_page > last_page: - response.redirect("?page=%i" % last_page) - organizations = cursor.all(""" - SELECT * - FROM organizations o - ORDER BY convert((o.p).receiving, 'EUR') DESC, (o.p).join_time DESC - LIMIT %s - OFFSET %s - """, (per_page, (current_page - 1) * per_page)) +organizations = website.db_qc5.all(""" + SELECT p + , ( SELECT s.content + FROM statements s + WHERE s.participant = p.id + AND s.type = 'summary' + ORDER BY s.lang = %s DESC, s.id + LIMIT 1 + ) AS summary + FROM participants p + WHERE p.kind = 'organization' + AND p.status = 'active' + AND (p.goal > 0 OR p.goal IS NULL) + AND p.hide_receiving IS NOT TRUE + AND p.hide_from_lists = 0 + AND p.receiving > 0 + ORDER BY convert(p.receiving, 'EUR') DESC, p.join_time DESC + LIMIT %s + OFFSET %s +""", (locale.language, per_page + 1, (current_page - 1) * per_page)) +has_more = len(organizations) > per_page title = _("Explore") subhead = _("Organizations") @@ -52,16 +40,16 @@ subhead = _("Organizations"){{ _("The top {0} organizations on Liberapay are:", len(organizations)) }}
% else{{ _( - "List of organizations on Liberapay, page {number} of {total}:" - , number=current_page, total=last_page + "List of organizations on Liberapay, page {number}:", + number=current_page ) }}
% endif{{ _("Nothing to show.") }}
diff --git a/www/explore/pledges.spt b/www/explore/pledges.spt index 47eb96d313..ea7f278700 100644 --- a/www/explore/pledges.spt +++ b/www/explore/pledges.spt @@ -1,31 +1,19 @@ -from math import ceil - [---] per_page = 18 current_page = request.qs.get_int('page', default=1, minimum=1) -with website.db.get_cursor() as cursor: - npledgees = cursor.one(""" - CREATE TEMP TABLE pledgees ON COMMIT DROP AS - SELECT e, p - FROM participants p - JOIN elsewhere e ON e.participant = p.id - WHERE p.status = 'stub' - AND p.receiving > 0 - AND p.hide_from_lists = 0; - - SELECT count(1) FROM pledgees; - """, (locale.language,)) - last_page = max(int(ceil(npledgees / per_page)), 1) - if current_page > last_page: - response.redirect("?page=%i" % last_page) - pledgees = cursor.all(""" - SELECT (e, p)::elsewhere_with_participant - FROM pledgees - ORDER BY (p).npatrons DESC, convert((p).receiving, 'EUR') DESC, (e).id DESC - LIMIT %s - OFFSET %s - """, (per_page, (current_page - 1) * per_page)) +pledgees = website.db_qc5.all(""" + SELECT (e, p)::elsewhere_with_participant + FROM participants p + JOIN elsewhere e ON e.participant = p.id + WHERE p.status = 'stub' + AND p.receiving > 0 + AND p.hide_from_lists = 0 + ORDER BY p.npatrons DESC, convert(p.receiving, 'EUR') DESC, e.id DESC + LIMIT %s + OFFSET %s +""", (per_page + 1, (current_page - 1) * per_page)) +has_more = len(pledgees) > per_page title = _("Explore") subhead = _("Unclaimed Donations") @@ -52,11 +40,11 @@ subhead = _("Unclaimed Donations") "inviting them to join Liberapay or asking them why they haven't." ) }}{{ _("There are no unclaimed donations right now.") }}
% endif diff --git a/www/explore/repositories.spt b/www/explore/repositories.spt index 9f0d75dcab..9fead8da96 100644 --- a/www/explore/repositories.spt +++ b/www/explore/repositories.spt @@ -1,32 +1,20 @@ -from math import ceil - [---] per_page = 20 current_page = request.qs.get_int('page', default=1, minimum=1) -with website.db.get_cursor() as cursor: - nrepos = cursor.one(""" - CREATE TEMP TABLE repos ON COMMIT DROP AS - SELECT r, p - FROM repositories r - JOIN elsewhere e ON e.platform = r.platform AND e.domain = '' AND e.user_id = r.owner_id - JOIN participants p ON p.id = e.participant - WHERE r.stars_count > 1 - AND r.show_on_profile - AND p.status = 'active'; - - SELECT count(1) FROM repos; - """, (locale.language,)) - last_page = max(int(ceil(nrepos / per_page)), 1) - if current_page > last_page: - response.redirect("?page=%i" % last_page) - repos = cursor.all(""" - SELECT * - FROM repos - ORDER BY (r).stars_count DESC, (r).id DESC - LIMIT %s - OFFSET %s - """, (per_page, (current_page - 1) * per_page)) +repos = website.db_qc5.all(""" + SELECT r, p + FROM repositories r + JOIN elsewhere e ON e.platform = r.platform AND e.domain = '' AND e.user_id = r.owner_id + JOIN participants p ON p.id = e.participant + WHERE r.stars_count > 1 + AND r.show_on_profile + AND p.status = 'active' + ORDER BY r.stars_count DESC, r.id DESC + LIMIT %s + OFFSET %s +""", (per_page + 1, (current_page - 1) * per_page)) +has_more = len(repos) > per_page title = _("Explore") subhead = _("Repositories") @@ -46,13 +34,13 @@ subhead = _("Repositories") ) }} % else{{ _( - "List of repositories currently linked to a Liberapay account, page {number} of {total}:" - , number=current_page, total=last_page + "List of repositories currently linked to a Liberapay account, page {number}:", + number=current_page ) }}
% endif{{ _( - "List of teams on Liberapay, page {number} of {total}:" - , number=current_page, total=last_page -) }}
+{{ _("List of teams on Liberapay, page {number}:", number=current_page) }}
% endif