Wrapped HTTP requests loop in a try-except block to prevent it from breaking in case of temporary failures

This commit is contained in:
Fabio Manganiello 2018-12-05 15:55:16 +00:00
parent 791c36f5df
commit eb439d227a
1 changed files with 17 additions and 13 deletions

View File

@ -56,22 +56,26 @@ class HttpRequest(object):
is_first_call = self.last_request_timestamp == 0
self.last_request_timestamp = time.time()
method = getattr(requests, self.args.method.lower())
response = method(self.args.url, *self.args.args, **self.args.kwargs)
new_items = self.get_new_items(response)
try:
method = getattr(requests, self.args.method.lower())
response = method(self.args.url, *self.args.args, **self.args.kwargs)
new_items = self.get_new_items(response)
if isinstance(new_items, HttpEvent):
event = new_items
new_items = event.args['response']
else:
event = HttpEvent(dict(self), new_items)
if isinstance(new_items, HttpEvent):
event = new_items
new_items = event.args['response']
else:
event = HttpEvent(dict(self), new_items)
if new_items and self.bus:
if not self.skip_first_call or (
self.skip_first_call and not is_first_call):
self.bus.post(event)
if new_items and self.bus:
if not self.skip_first_call or (
self.skip_first_call and not is_first_call):
self.bus.post(event)
response.raise_for_status()
response.raise_for_status()
except Exception as e:
self.logger.warning('Encountered an error while retrieving {}: {}'.
format(self.args.url, str(e)))
Thread(target=_thread_func).start()