Commit d6a06183 authored by Koen Martens's avatar Koen Martens

unfinished gridtalk, ima and tangle fetchers

parent 8be1d0d0
import requests
from lib.event import Event
from dateutil import parser
from lxml import html
import re
import sys
import pytz
import datetime
from lib.category import Category
from helper.helper import Helper
class GridTalkEvent(Event):
tz_berlin = pytz.timezone('Europe/Berlin')
hgexp = re.compile('secondlife://hypergrid.org:8002:([^/]+)/')
# 02.01.2015, 20:00 - 31.12.2015, 23:59
fromtoRE = re.compile("([0-9]{2})\.([0-9]{2})\.([0-9]{4}), ([0-9]{2}:[0-9]{2}) - ([0-9]{2})\.([0-9]{2})\.([0-9]{4}), ([0-9]{2}:[0-9]{2})")
def __init__(self,url,webcache=None):
super(GridTalkEvent,self).__init__()
self.url=url
self.categories = self.categories + [ Category("grid-gridtalk") ]
self.webcache=webcache
def parseTime(self, s):
m = self.fromtoRE.search(s)
self.start = None
self.end = None
if m!=None:
d1 = m.group(1)
m1 = m.group(2)
y1 = m.group(3)
t1 = m.group(4)
#d2 = m.group(5)
#m2 = m.group(6)
#y2 = m.group(7)
t2 = m.group(8)
date1 = "%s-%s-%s" % (y1,m1,d1)
self.start = GridTalkEvent.tz_berlin.localize(parser.parse(date1+" "+t1))
self.end = GridTalkEvent.tz_berlin.localize(parser.parse(date1+" "+t2))
if t1==t2:
self.end += datetime.timedelta(hours=2)
def fetch(self):
if self.url == None:
raise Exception("fetching GridTalkEvent without url set")
if self.webcache!=None:
r = self.webcache.fetch(self.url)
else:
r = requests.get(self.url)
if r.status_code == 200:
tree = html.fromstring(r.text)
trs = tree.xpath('//div[@id="content"]/div[@class="wrapper"]/table//tr')
if len(trs)!=4:
raise HtmlParsingError("GridTalkEvent can't find table rows")
self.title = trs[0].text_content().strip()
rawtime = trs[1].text_content().strip()
self.parseTime(rawtime)
self.description = trs[3].text_content().strip()
self.categories += [ Category("grid-gridtalk") ]
def __str__(self):
rv = super(GridTalkEvent,self).__str__()
return rv
class GridTalkFetcher:
def __init__(self, eventlist, webcache=None):
self.eventlist=eventlist
self.webcache=webcache
self.helper=Helper()
def fetch(self, limit=0):
url = "http://www.gridtalk.de/calendar.php"
numMonths = 2
while numMonths>0:
r = self.webcache.fetch(url, 1000, 1800)
if r.status_code==200:
tree = html.fromstring(r.text)
events = tree.xpath('//div[contains(@class,"public_event")]/a')
for eventNode in events:
href = eventNode.get('href')
title = eventNode.get('title')
event = GridTalkEvent("http://www.gridtalk.de/" + href, self.webcache)
event.title = title
event.fetch()
event = self.helper.customizeEvent(event)
if event != None:
self.eventlist.add(event)
nextUrlNode = tree.xpath('//div[@class="float_right"]/a[2]/@href')
if len(nextUrlNode)!=1:
raise HtmlParsingError("GridTalkFetcher: unable to find next month link")
url = "http://www.gridtalk.de/" + nextUrlNode[0]
numMonths -= 1
if __name__=='__main__':
from lib.webcache import WebCache
from lib.eventlist import EventList
eventlist = EventList()
cache = WebCache("data/test_gridtalk.cache")
f = GridTalkFetcher(eventlist, cache)
f.fetch(12)
cache.flush()
for event in eventlist:
print str(event)
import requests
from lib.event import Event
from fetcher.icalfetcher import IcalFetcher
import icalendar
from lib.category import Category
from helper.helper import Helper
class InfiniteMetaverseAllianceFetcher(IcalFetcher):
def __init__(self, eventlist, webcache=None):
url="https://calendar.google.com/calendar/ical/2e73vfmkgbi7op4o7j9aplt4rk%40import.calendar.google.com/public/basic.ics"
super(InfiniteMetaverseAllianceFetcher,self).__init__(
url,
[Category("grid-ima")],
eventlist,
webcache,
Helper()
)
self.webcache = webcache
self.minexpiry = 1000
self.maxexpirty = 1800
if __name__=='__main__':
from lib.webcache import WebCache
from lib.eventlist import EventList
eventlist = EventList()
cache = WebCache("data/test_ima.cache")
f = InfiniteMetaverseAllianceFetcher(eventlist, cache)
f.fetch()
cache.flush()
for ev in eventlist:
print str(ev)
import requests
from lib.event import Event
from fetcher.icalfetcher import IcalFetcher
import icalendar
from lib.category import Category
from helper.tangle import TangleHelper
class TangleFetcher(IcalFetcher):
def __init__(self, eventlist, webcache=None):
super(TangleFetcher,self).__init__(
"https://calendar.google.com/calendar/ical/lesliespiritweaver%40gmail.com/public/basic.ics",
[Category("grid-tangle")],
eventlist,
webcache,
TangleHelper()
)
self.webcache = webcache
self.minexpiry = 1000
self.maxexpirty = 1800
if __name__=='__main__':
from lib.webcache import WebCache
from lib.eventlist import EventList
eventlist = EventList()
cache = WebCache("data/test_tangle.cache")
f = TangleFetcher(eventlist, cache)
f.fetch()
cache.flush()
for ev in eventlist:
print str(ev)
import re
import pickle
from datetime import timedelta
from helper import Helper
class TangleHelper(Helper):
hgre = re.compile("EXPO", re.I)
def customizeEvent(self, event):
if TangleHelper.hgre.search(event.title)==None:
return None
event = super(TangleHelper, self).customizeEvent(event)
if event.hgurl=='' or event.hgurl=='-':
if TangleHelper.hgre.search(event.title)!=None:
event.hgurl='tanglegrid.net:8032:EXPO Isle'
else:
event.hgrul='tanglegrid.net:8032:Welcome Island'
return event
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment