-
Notifications
You must be signed in to change notification settings - Fork 58
/
Copy pathadmin.py
147 lines (119 loc) · 5.12 KB
/
admin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""Renders admin pages for ops and other management tasks.
Currently just ``/admin/responses``, which shows active responses with tasks that
haven't completed yet.
"""
import datetime
import itertools
import logging
from flask import render_template, request
from google.cloud import ndb
from google.cloud.ndb.stats import KindStat, KindPropertyNamePropertyTypeStat
from oauth_dropins.webutil import logs
from oauth_dropins.webutil.util import json_dumps, json_loads
from flask_app import app
import models
import util
# Import source class files so their metaclasses are initialized.
from models import BlogPost, Response, Source
import blogger, bluesky, flickr, github, instagram, mastodon, medium, reddit, tumblr, twitter, wordpress_rest
logger = logging.getLogger(__name__)
NUM_ENTITIES = 10
# Result of this query in BigQuery:
# SELECT count(*) FROM `brid-gy.datastore.Response` WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_RESPONSES = 25313804
# Result of this query in BigQuery:
# SELECT SUM(ARRAY_LENGTH(sent) + ARRAY_LENGTH(unsent) + ARRAY_LENGTH(error) + ARRAY_LENGTH(failed) + ARRAY_LENGTH(skipped))
# FROM `brid-gy.datastore.Response`
# WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_LINKS = 4355678
# Result of this query in BigQuery:
# SELECT SUM(ARRAY_LENGTH(sent))
# FROM `brid-gy.datastore.Response`
# WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_SENT_LINKS = 2060197
@app.route('/admin/responses')
def responses():
"""Find the most recently attempted responses and blog posts with error URLs."""
entities = []
for cls in (Response,): # BlogPost
for e in cls.query().order(-cls.updated):
if (len(entities) >= NUM_ENTITIES or
e.updated < util.now() - datetime.timedelta(hours=1)):
break
elif (not e.error and not e.unsent) or e.status == 'complete':
continue
e.links = [util.pretty_link(u, new_tab=True) for u in e.error + e.failed]
if e.key.kind() == 'Response':
e.response = json_loads(e.response_json)
e.activities = [json_loads(a) for a in e.activities_json]
else:
e.response = {'content': '[BlogPost]'}
e.activities = [{'url': e.key.id()}]
entities.append(e)
return render_template('admin_responses.html', responses=entities, logs=logs)
@app.route('/admin/sources')
def sources():
"""Find sources whose last poll errored out."""
CLASSES = (flickr.Flickr, github.GitHub, mastodon.Mastodon, reddit.Reddit,
bluesky.Bluesky)
queries = [cls.query(Source.status == 'enabled',
Source.poll_status == 'error',
Source.rate_limited.IN((False, None)),
Source.features == 'listen',
).fetch_async(NUM_ENTITIES)
for cls in CLASSES]
return render_template(
'admin_sources.html',
sources=itertools.chain(*[q.get_result() for q in queries]),
logs=logs,
)
@app.route('/admin/mark_complete', methods=['POST'])
def mark_complete():
entities = ndb.get_multi(ndb.Key(urlsafe=u)
for u in request.values.getlist('key'))
for e in entities:
e.status = 'complete'
ndb.put_multi(entities)
return util.redirect('/admin/responses')
@app.route('/admin/disable', methods=['POST'])
def disable():
source = util.load_source()
logger.info(f'Disabling {source.label()}')
source.status = 'disabled'
source.put()
return util.redirect(source.bridgy_path())
@app.route('/admin/stats')
def stats():
"""Collect and report misc lifetime stats.
https://developers.google.com/appengine/docs/python/ndb/admin#Statistics_queries
Used to be on the front page, dropped them during the Flask port in August 2021.
"""
def count(query):
stat = query.get() # no datastore stats when running locally
return stat.count if stat else 0
def kind_count(kind):
return count(KindStat.query(KindStat.kind_name == kind))
num_users = sum(kind_count(cls.__name__) for cls in models.sources.values())
response_count = kind_count('Response')
link_counts = {
property: sum(count(KindPropertyNamePropertyTypeStat.query(
KindPropertyNamePropertyTypeStat.kind_name == kind,
KindPropertyNamePropertyTypeStat.property_name == property,
# specify string because there are also >2M Response entities with null
# values for some of these properties, as opposed to missing altogether,
# which we don't want to include.
KindPropertyNamePropertyTypeStat.property_type == 'String'))
for kind in ('BlogPost', 'Response'))
for property in ('sent', 'unsent', 'error', 'failed', 'skipped')}
return render_template('admin_stats.html', **{
# add comma separator between thousands
k: f'{v:,}' for k, v in {
'users': num_users,
'responses': response_count + ARCHIVED_RESPONSES,
'responses_stored': response_count,
'links': sum(link_counts.values()) + ARCHIVED_LINKS,
'webmentions': link_counts['sent'] + kind_count('BlogPost') + ARCHIVED_SENT_LINKS,
'publishes': kind_count('Publish'),
'blogposts': kind_count('BlogPost'),
'webmentions_received': kind_count('BlogWebmention'),
}.items()})