import logging import re import requests import time from frozendict import frozendict from threading import Thread from platypush.message.event.http import HttpEvent from platypush.utils import set_thread_name class HttpRequest(object): poll_seconds = 60 timeout = 5 class HttpRequestArguments(object): def __init__(self, url, method='get', *args, **kwargs): self.method = method.lower() self.url = url self.args = args self.kwargs = kwargs def __init__(self, args, bus=None, poll_seconds=None, timeout=None, skip_first_call=True, **kwargs): super().__init__() self.poll_seconds = poll_seconds or self.poll_seconds self.timeout = timeout or self.timeout self.bus = bus self.skip_first_call = skip_first_call self.last_request_timestamp = 0 self.logger = logging.getLogger('platypush') if isinstance(args, self.HttpRequestArguments): self.args = args elif isinstance(args, dict): self.args = self.HttpRequestArguments(**args) else: raise RuntimeError('{} is neither a dictionary nor an HttpRequest') if 'timeout' not in self.args.kwargs: self.args.kwargs['timeout'] = self.timeout self.request_args = { 'method': self.args.method, 'url': self.args.url, **self.args.kwargs } def execute(self): def _thread_func(): set_thread_name('HttpPoll') is_first_call = self.last_request_timestamp == 0 self.last_request_timestamp = time.time() try: method = getattr(requests, self.args.method.lower()) response = method(self.args.url, *self.args.args, **self.args.kwargs) new_items = self.get_new_items(response) if isinstance(new_items, HttpEvent): event = new_items new_items = event.args['response'] else: event = HttpEvent(dict(self), new_items) if new_items and self.bus: if not self.skip_first_call or ( self.skip_first_call and not is_first_call): self.bus.post(event) response.raise_for_status() except Exception as e: self.logger.exception(e) self.logger.warning('Encountered an error while retrieving {}: {}'. format(self.args.url, str(e))) Thread(target=_thread_func, name='HttpPoll').start() def get_new_items(self, response): """ Gets new items out of a response """ raise NotImplementedError("get_new_items must be implemented in a derived class") def __iter__(self): for (key, value) in self.request_args.items(): yield key, value class JsonHttpRequest(HttpRequest): def __init__(self, path=None, *args, **kwargs): super().__init__(*args, **kwargs) self.path = path self.seen_entries = set() def get_new_items(self, response): response = response.json() new_entries = [] if self.path: m = re.match(r'\${\s*(.*)\s*}', self.path) response = eval(m.group(1)) for entry in response: flattened_entry = deep_freeze(entry) if flattened_entry not in self.seen_entries: new_entries.append(entry) self.seen_entries.add(flattened_entry) return new_entries def deep_freeze(x): if isinstance(x, str) or not hasattr(x, "__len__"): return x if hasattr(x, "keys") and hasattr(x, "values"): return frozendict({deep_freeze(k): deep_freeze(v) for k, v in x.items()}) if hasattr(x, "__getitem__"): return tuple(map(deep_freeze, x)) return frozenset(map(deep_freeze, x)) # vim:sw=4:ts=4:et: