diff --git a/scrape.py b/scrape.py
index 8df1215..80e277e 100644
--- a/scrape.py
+++ b/scrape.py
@@ -3,6 +3,7 @@ from os import environ
import requests
import re
from bs4 import BeautifulSoup
+from datetime import datetime, timedelta
import asyncio
from nio import ClientConfig, AsyncClient, LoginResponse, InviteEvent
@@ -18,7 +19,9 @@ async def on_event(room, event):
if hasattr(event, 'membership'):
if event.membership == 'invite':
# automatically join invites
- await matrix[event.source['state_key']].join(room.room_id)
+ print('joining '+room.room_id)
+ join = await matrix[event.source['state_key']].join(room.room_id)
+ print(join)
def get_blog():
url = 'https://news.blizzard.com/en-us/'
html = requests.get(url).text
@@ -38,7 +41,7 @@ def get_blog():
blog.append({
'image': image_url,
- 'game': text_list[0].contents[0],
+ 'game': text_list[0].contents[0].replace(' ', '').lower(),
'title': text_list[1].contents[0],
'description': '',
'url': base_url+feature_html.attrs['href'],
@@ -54,7 +57,7 @@ def get_blog():
blog.append({
'image': image_url,
- 'game': content_html.find(class_='ArticleListItem-subtitle').find(class_='ArticleListItem-labelInner').contents[0],
+ 'game': content_html.find(class_='ArticleListItem-subtitle').find(class_='ArticleListItem-labelInner').contents[0].replace(' ', '').lower(),
'title': content_html.find(class_='ArticleListItem-title').contents[0],
'description': content_html.find(class_='ArticleListItem-description').find(class_='h6').contents[0],
'url': base_url+article_html.find(class_='ArticleLink').attrs['href'],
@@ -82,6 +85,8 @@ def get_formatted_body(post):
return formatted_body
async def main():
+ event_type_prefix = 'de.lubiland.snowstorm-matrix.'
+
next_batch = {}
for game in device:
# initialize new client
@@ -113,12 +118,12 @@ async def main():
}
}
}
- next_batch_state = await matrix[mxid].room_get_state_event(admin_room,
- event_type,
- mxid)
+ next_batch_state = await matrix[mxid].room_get_state_event(room_id=admin_room,
+ event_type=event_type_prefix+'next_batch',
+ state_key=mxid)
if 'token' in next_batch_state.content:
try:
- sync = await matrix[mxid].sync(timeout=30000,
+ sync = await matrix[mxid].sync(timeout=3000,
sync_filter=sync_filter,
since=next_batch_state.content['token'])
next_batch[mxid] = sync.next_batch
@@ -128,19 +133,55 @@ async def main():
pass
# when there is no next_batch token or first sync threw an error,
# then do a first sync without next_batch
- sync = await matrix[mxid].sync(timeout=30000,
+ print('doing first sync for '+mxid)
+ sync = await matrix[mxid].sync(timeout=3000,
sync_filter=sync_filter)
next_batch[mxid] = sync.next_batch
+ next_update = datetime.now()
while True:
+ if next_update < datetime.now():
+ blog = get_blog()
+ for post in blog:
+ mxid = '@'+mxid_prefix+post['game']+':'+homeserver_name
+ if mxid in matrix:
+ content = {
+ 'msgtype': 'm.notice',
+ 'body': get_body(post),
+ 'format': 'org.matrix.custom.html',
+ 'formatted_body': get_formatted_body(post)
+ }
+ for room_id in matrix[mxid].rooms:
+ await matrix[mxid].room_send(room_id=room_id,
+ message_type='m.room.message',
+ content=content)
+ else:
+ content = {
+ 'msgtype': 'm.notice',
+ 'body': 'No accesstoken for '+mxid+' available.',
+ 'formatted_body': ('No accesstoken
for '+
+ ''+mxid+'
available.')
+ }
+ # send the message with the first available matrix client,
+ # because we will always have at least one accesstoken
+ await matrix[next(iter(matrix))].room_send(room_id=admin_room,
+ message_type='m.room.message',
+ content=content)
+
+ next_update = datetime.now() + timedelta(minutes=30)
+
+ next_batch_state = await matrix[mxid].room_get_state_event(room_id=admin_room,
+ event_type=event_type_prefix+'next_batch',
+ state_key=mxid)
+
for mxid in next_batch:
- sync = await matrix[mxid].sync(timeout=30000,
+ sync = await matrix[mxid].sync(timeout=10000,
sync_filter=sync_filter,
since=next_batch[mxid])
next_batch[mxid] = sync.next_batch
await matrix[mxid].room_put_state(room_id=admin_room,
- event_type=event_type,
+ event_type=event_type_prefix+'next_batch',
state_key=mxid,
content={'token': next_batch[mxid]})
@@ -158,15 +199,8 @@ for var in environ:
if (game := re.match('^ACCESSTOKEN_([A-Z]*)_FILE$', var)) is not None:
device[game[1].lower()]['accesstoken'] = get_accesstoken_from_file(environ[var])
-event_type = 'de.lubiland.snowstorm-matrix.next_batch'
matrix = {}
asyncio.run(main())
-
-
-blog = get_blog()
-for post in blog:
- print(get_body(post))
- print(get_formatted_body(post))
\ No newline at end of file