Skip to content

Commit

Permalink
Created Top-20 List Queries For API
Browse files Browse the repository at this point in the history
Three top-20 lists: page count, incoming links, outgoing links. Also updated __init__ to import the lists.
Secondary edit: Updated .gitignore to ignore custom backend.cfg configuration file.
  • Loading branch information
Unknown committed Feb 26, 2018
1 parent 3423407 commit 84b153a
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 4 deletions.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,7 @@ ENV/
.mypy_cache/

#Pycharm
.idea
.idea

# Backend Configuration File
backend.cfg
4 changes: 1 addition & 3 deletions app/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,4 @@ def default(self, obj):
app.json_encoder = MyJSONEncoder

from app import api, models
from app.views import next_url


from app.views import next_url, lists
130 changes: 130 additions & 0 deletions app/views/lists.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
from app.models import Onions, Links, Pages
from flask.login import login_required
from sqlalchemy import desc
from flask import jsonify
from flask import request
from app import db

"""
Page Count:
SELECT DISTINCT onions.domain, count(pages.url)
FROM pages INNER JOIN onions ON onions.domain = pages.domain
WHERE pages.fault IS NOT NULL AND pages.domain IN (
SELECT domain FROM onions
WHERE online IS TRUE
AND last_online IS NOT NULL)
GROUP BY onions.domain
ORDER BY COUNT(pages.url) DESC LIMIT 20;
"""

@app.route("/api/top20/pages", methods=["GET"])
# @login_required
def top_twenty_page_count():
# Retrieve the top twenty onions in order of page count.
query = db.session.query(
Onions.domain,
db.func.count(Pages.url)
).join(Pages).filter(
Pages.fault == 'none',
Pages.domain.in_(
db.session.query(Onions.domain).filter(
Onions.online == True,
Onions.last_online != None
).all()
)
).order_by(
desc(db.func.count(Pages.url))
).group_by(Onions).limit(20)
results = query.all()
if len(results) is 0:
return jsonify({'object': {}})
return jsonify({'objects': results})


"""
Outgoing Links:
SELECT DISTINCT onions.domain, count(links.domain_to)
FROM links INNER JOIN onions ON onions.domain = links.domain_from
WHERE links.domain_to IN (
SELECT domain FROM onions
WHERE online IS TRUE
AND last_online IS NOT NULL)
AND links.domain_from IN (
SELECT domain FROM onions
WHERE online IS TRUE
AND last_online IS NOT NULL)
GROUP BY onions.domain ORDER BY COUNT(links.domain_to) DESC
LIMIT 20;
"""

@app.route("/api/top20/outlinks", methods=["GET"])
# @login_required
def top_twenty_outlinks():
# Retrieve the top twenty onions in order of outgoing links.
query = db.session.query(
Onions.domain,
db.func.count(Links.domain_to)
).join(Links, Onions.domain == Links.domain_from).filter(
Links.domain_to.in_(
db.session.query(Onions.domain).filter(
Onions.online == True,
Onions.last_online != None
)
),
Links.domain_from.in_(
db.session.query(Onions.domain).filter(
Onions.online == True,
Onions.last_online != None
)
)
).order_by(
desc(db.func.count(Links.domain_to))
).group_by(Onions).limit(20)
results = query.all()
if len(results) is 0:
return jsonify({'object': {}})
return jsonify({'objects': results})

"""
Incoming Links:
SELECT DISTINCT onions.domain, count(links.domain_from)
FROM links INNER JOIN onions ON onions.domain = links.domain_to
WHERE links.domain_to IN (
SELECT domain FROM onions
WHERE online IS TRUE
AND last_online IS NOT NULL)
AND links.domain_from IN (
SELECT domain FROM onions
WHERE online IS TRUE
AND last_online IS NOT NULL)
GROUP BY onions.domain ORDER BY COUNT(links.domain_from) DESC
LIMIT 20;
"""

@app.route("/api/top20/inlinks", methods=["GET"])
# @login_required
def top_twenty_inlinks():
# Retrieve the top twenty onions in order of incoming links.
query = db.session.query(
Onions.domain,
db.func.count(Links.domain_from)
).join(Links, Onions.domain == Links.domain_to).filter(
Links.domain_to.in_(
db.session.query(Onions.domain).filter(
Onions.online == True,
Onions.last_online != None
)
),
Links.domain_from.in_(
db.session.query(Onions.domain).filter(
Onions.online == True,
Onions.last_online != None
)
)
).order_by(
desc(db.func.count(Links.domain_from))
).group_by(Onions).limit(20)
results = query.all()
if len(results) is 0:
return jsonify({'object': {}})
return jsonify({'objects': results})

0 comments on commit 84b153a

Please sign in to comment.