lub
/
snowstorm-matrix
1
1
Fork 0
Code Issues 1 Pull Requests Packages Releases 4 Wiki Activity

add url cache

Browse Source
master
lub 5 years ago
parent fd27695db1
commit 57e62ea871
1 changed files with 49 additions and 50 deletions
Show Stats Download Patch File Download Diff File

49
scrape.py
Unescape Escape View File

@ -116,22 +116,6 @@ async def main():
} }
} }
} }
next_batch_state = await matrix[mxid].room_get_state_event(room_id=admin_room,
event_type=event_type_prefix+'next_batch',
state_key=mxid)
if 'token' in next_batch_state.content:
try:
sync = await matrix[mxid].sync(timeout=3000,
sync_filter=sync_filter,
since=next_batch_state.content['token'])
next_batch[mxid] = sync.next_batch
continue
except:
pass
# when there is no next_batch token or first sync threw an error,
# then do a first sync without next_batch
print('doing first sync for '+mxid)
sync = await matrix[mxid].sync(timeout=3000, sync = await matrix[mxid].sync(timeout=3000,
sync_filter=sync_filter) sync_filter=sync_filter)
next_batch[mxid] = sync.next_batch next_batch[mxid] = sync.next_batch
@ -139,10 +123,23 @@ async def main():
next_update = datetime.now() next_update = datetime.now()
while True: while True:
if next_update < datetime.now(): if next_update < datetime.now():
# refresh url cache
cache_state = await matrix[next(iter(matrix))].room_get_state_event(room_id=admin_room,
event_type=event_type_prefix+'cache',
state_key='')
if hasattr(cache_state, 'content') and 'url_list' in cache_state.content:
cache = cache_state.content['url_list']
else:
cache = []
# scape all blog posts and process them
blog = get_blog() blog = get_blog()
for post in blog: for post in blog:
if post['url'] not in cache:
# post url not found in cache
mxid = '@'+mxid_prefix+post['game']+':'+homeserver_name mxid = '@'+mxid_prefix+post['game']+':'+homeserver_name
if mxid in matrix: if mxid in matrix:
# announce new post to matrix rooms
content = { content = {
'msgtype': 'm.notice', 'msgtype': 'm.notice',
'body': get_body(post), 'body': get_body(post),
@ -154,6 +151,7 @@ async def main():
message_type='m.room.message', message_type='m.room.message',
content=content) content=content)
else: else:
# no accesstoken for the calculated mxid
content = { content = {
'msgtype': 'm.notice', 'msgtype': 'm.notice',
'body': 'No accesstoken for '+mxid+' available.', 'body': 'No accesstoken for '+mxid+' available.',
@ -167,23 +165,24 @@ async def main():
message_type='m.room.message', message_type='m.room.message',
content=content) content=content)
# add url to cache
cache += [post['url']]
while len(cache) > len(blog):
cache.remove(cache[0])
set_state = await matrix[next(iter(matrix))].room_put_state(room_id=admin_room,
event_type=event_type_prefix+'cache',
content={'url_list': cache})
else:
# no new posts found
pass
next_update = datetime.now() + timedelta(minutes=30) next_update = datetime.now() + timedelta(minutes=30)
next_batch_state = await matrix[mxid].room_get_state_event(room_id=admin_room,
event_type=event_type_prefix+'next_batch',
state_key=mxid)
for mxid in next_batch: for mxid in next_batch:
sync = await matrix[mxid].sync(timeout=10000, sync = await matrix[mxid].sync(timeout=10000,
sync_filter=sync_filter, sync_filter=sync_filter,
since=next_batch[mxid]) since=next_batch[mxid])
next_batch[mxid] = sync.next_batch next_batch[mxid] = sync.next_batch
await matrix[mxid].room_put_state(room_id=admin_room,
event_type=event_type_prefix+'next_batch',
state_key=mxid,
content={'token': next_batch[mxid]})
homeserver_name = environ['HOMESERVER_NAME'] homeserver_name = environ['HOMESERVER_NAME']
homeserver_url = environ['HOMESERVER_URL'] homeserver_url = environ['HOMESERVER_URL']

Write Preview
Loading…
Cancel
Save