Refactored and improved camera plugin

This commit is contained in:
Fabio Manganiello 2019-12-29 16:28:07 +01:00
parent 663be43f06
commit ba6c890a42
3 changed files with 192 additions and 171 deletions

View file

@ -1,10 +1,7 @@
import os from flask import Response, Blueprint
import shutil from platypush.plugins.camera import CameraPlugin
import tempfile
import time
from flask import Response, Blueprint, send_from_directory
from platypush import Config
from platypush.backend.http.app import template_folder from platypush.backend.http.app import template_folder
from platypush.backend.http.app.utils import authenticate, send_request from platypush.backend.http.app.utils import authenticate, send_request
@ -18,85 +15,60 @@ __routes__ = [
def get_device_id(device_id=None): def get_device_id(device_id=None):
if device_id is None: if device_id is None:
device_id = str(send_request(action='camera.get_default_device_id').output) device_id = int(send_request(action='camera.get_default_device_id').output)
return device_id return device_id
def get_frame_file(device_id=None): def get_camera(device_id=None):
device_id = get_device_id(device_id) device_id = get_device_id(device_id)
was_recording = True camera_conf = Config.get('camera') or {}
frame_file = None camera_conf['device_id'] = device_id
status = send_request(action='camera.status', device_id=device_id).output return CameraPlugin(**camera_conf)
if device_id not in status:
was_recording = False
send_request(action='camera.start_recording',
device_id=device_id)
while not frame_file: def get_frame(device_id=None):
frame_file = send_request(action='camera.status', device_id=device_id). \ cam = get_camera(device_id)
output.get(device_id, {}).get('image_file') with cam:
frame = None
if not frame_file: for _ in range(cam.warmup_frames):
time.sleep(0.1) output = cam.get_stream()
if not was_recording: with output.ready:
with tempfile.NamedTemporaryFile(prefix='camera_capture_', suffix='.jpg', output.ready.wait()
delete=False) as f: frame = output.frame
# stop_recording will delete the temporary frames. Copy the image file print(frame, type(frame))
# to a temporary file before stopping recording
tmp_file = f.name
shutil.copyfile(frame_file, tmp_file) return frame
frame_file = tmp_file
send_request(action='camera.stop_recording', device_id=device_id)
return frame_file
def video_feed(device_id=None): def video_feed(device_id=None):
device_id = get_device_id(device_id) cam = get_camera(device_id)
send_request(action='camera.start_recording', device_id=device_id)
last_frame_file = None
try: with cam:
while True: while True:
frame_file = get_frame_file(device_id) output = cam.get_stream()
if frame_file == last_frame_file: with output.ready:
continue output.ready.wait()
frame = output.frame
with open(frame_file, 'rb') as f: if frame and len(frame):
frame = f.read() yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
last_frame_file = frame_file
finally:
send_request(action='camera.stop_recording', device_id=device_id)
@camera.route('/camera/<device_id>/frame', methods=['GET']) @camera.route('/camera/<device_id>/frame', methods=['GET'])
@authenticate() @authenticate()
def get_camera_frame(device_id): def get_camera_frame(device_id):
frame_file = get_frame_file(device_id) frame = get_frame(device_id)
return send_from_directory(os.path.dirname(frame_file), return Response(frame, mimetype='image/jpeg')
os.path.basename(frame_file))
@camera.route('/camera/frame', methods=['GET']) @camera.route('/camera/frame', methods=['GET'])
@authenticate() @authenticate()
def get_default_camera_frame(): def get_default_camera_frame():
frame_file = get_frame_file() frame = get_frame()
return send_from_directory(os.path.dirname(frame_file), return Response(frame, mimetype='image/jpeg')
os.path.basename(frame_file))
@camera.route('/camera/stream', methods=['GET'])
@authenticate()
def get_default_stream_feed():
return Response(video_feed(),
mimetype='multipart/x-mixed-replace; boundary=frame')
@camera.route('/camera/<device_id>/stream', methods=['GET']) @camera.route('/camera/<device_id>/stream', methods=['GET'])
@ -106,4 +78,11 @@ def get_stream_feed(device_id):
mimetype='multipart/x-mixed-replace; boundary=frame') mimetype='multipart/x-mixed-replace; boundary=frame')
@camera.route('/camera/stream', methods=['GET'])
@authenticate()
def get_default_stream_feed():
return Response(video_feed(),
mimetype='multipart/x-mixed-replace; boundary=frame')
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -1,3 +1,4 @@
import io
import os import os
import re import re
import shutil import shutil
@ -5,8 +6,10 @@ import threading
import time import time
from datetime import datetime from datetime import datetime
from typing import Optional
from platypush.config import Config from platypush.config import Config
from platypush.message import Mapping
from platypush.message.response import Response from platypush.message.response import Response
from platypush.message.event.camera import CameraRecordingStartedEvent, \ from platypush.message.event.camera import CameraRecordingStartedEvent, \
CameraRecordingStoppedEvent, CameraVideoRenderedEvent, \ CameraRecordingStoppedEvent, CameraVideoRenderedEvent, \
@ -15,6 +18,29 @@ from platypush.message.event.camera import CameraRecordingStartedEvent, \
from platypush.plugins import Plugin, action from platypush.plugins import Plugin, action
class StreamingOutput:
def __init__(self):
self.frame = None
self.buffer = io.BytesIO()
self.ready = threading.Condition()
@staticmethod
def is_new_frame(buf):
# JPEG header begin
return buf.startswith(b'\xff\xd8')
def write(self, buf):
if self.is_new_frame(buf):
# New frame, copy the existing buffer's content and notify all clients that it's available
self.buffer.truncate()
with self.ready:
self.frame = self.buffer.getvalue()
self.ready.notify_all()
self.buffer.seek(0)
return self.buffer.write(buf)
class CameraPlugin(Plugin): class CameraPlugin(Plugin):
""" """
Plugin to control generic cameras over OpenCV. Plugin to control generic cameras over OpenCV.
@ -33,6 +59,7 @@ class CameraPlugin(Plugin):
Requires: Requires:
* **opencv** (``pip install opencv-python``) * **opencv** (``pip install opencv-python``)
""" """
_default_warmup_frames = 5 _default_warmup_frames = 5
@ -68,7 +95,8 @@ class CameraPlugin(Plugin):
:param video_type: Default video type to use when exporting captured :param video_type: Default video type to use when exporting captured
frames to camera (default: 0, infers the type from the video file frames to camera (default: 0, infers the type from the video file
extension). See https://docs.opencv.org/4.0.1/dd/d9e/classcv_1_1VideoWriter.html#afec93f94dc6c0b3e28f4dd153bc5a7f0 extension). See
`here <https://docs.opencv.org/4.0.1/dd/d9e/classcv_1_1VideoWriter.html#afec93f94dc6c0b3e28f4dd153bc5a7f0>`_
for a reference on the supported types (e.g. 'MJPEG', 'XVID', 'H264' etc') for a reference on the supported types (e.g. 'MJPEG', 'XVID', 'H264' etc')
:type video_type: str or int :type video_type: str or int
@ -120,7 +148,7 @@ class CameraPlugin(Plugin):
if isinstance(video_type, str): if isinstance(video_type, str):
import cv2 import cv2
self.video_type = cv2.VideoWriter_fourcc(*video_type) self.video_type = cv2.VideoWriter_fourcc(*video_type.upper())
self.sleep_between_frames = sleep_between_frames self.sleep_between_frames = sleep_between_frames
self.max_stored_frames = max_stored_frames self.max_stored_frames = max_stored_frames
@ -134,6 +162,7 @@ class CameraPlugin(Plugin):
self._devices = {} # device_id => VideoCapture map self._devices = {} # device_id => VideoCapture map
self._recording_threads = {} # device_id => Thread map self._recording_threads = {} # device_id => Thread map
self._recording_info = {} # device_id => recording info map self._recording_info = {} # device_id => recording info map
self._output = None
def _init_device(self, device_id, frames_dir=None, **info): def _init_device(self, device_id, frames_dir=None, **info):
import cv2 import cv2
@ -190,7 +219,7 @@ class CameraPlugin(Plugin):
ret = sorted([ ret = sorted([
os.path.join(frames_dir, f) for f in os.listdir(frames_dir) os.path.join(frames_dir, f) for f in os.listdir(frames_dir)
if os.path.isfile(os.path.join(frames_dir, f)) and if os.path.isfile(os.path.join(frames_dir, f)) and
re.search(self._frame_filename_regex, f) re.search(self._frame_filename_regex, f)
]) ])
return ret return ret
@ -243,14 +272,12 @@ class CameraPlugin(Plugin):
frames_dir, n_frames, sleep_between_frames, frames_dir, n_frames, sleep_between_frames,
max_stored_frames, color_transform, video_type, max_stored_frames, color_transform, video_type,
scale_x, scale_y, rotate, flip): scale_x, scale_y, rotate, flip):
import cv2 import cv2
device = self._devices[device_id] device = self._devices[device_id]
color_transform = getattr(cv2, self.color_transform) color_transform = getattr(cv2, color_transform or self.color_transform)
rotation_matrix = None rotation_matrix = None
self._is_recording[device_id].wait() self._is_recording[device_id].wait()
self.logger.info('Starting recording from video device {}'. self.logger.info('Starting recording from video device {}'.format(device_id))
format(device_id))
recording_started_time = time.time() recording_started_time = time.time()
captured_frames = 0 captured_frames = 0
@ -280,8 +307,7 @@ class CameraPlugin(Plugin):
if rotate: if rotate:
rows, cols = frame.shape rows, cols = frame.shape
if not rotation_matrix: if not rotation_matrix:
rotation_matrix = cv2.getRotationMatrix2D( rotation_matrix = cv2.getRotationMatrix2D((cols / 2, rows / 2), rotate, 1)
(cols / 2, rows / 2), rotate, 1)
frame = cv2.warpAffine(frame, rotation_matrix, (cols, rows)) frame = cv2.warpAffine(frame, rotation_matrix, (cols, rows))
@ -294,8 +320,16 @@ class CameraPlugin(Plugin):
frame = cv2.resize(frame, None, fx=scale_x, fy=scale_y, frame = cv2.resize(frame, None, fx=scale_x, fy=scale_y,
interpolation=cv2.INTER_CUBIC) interpolation=cv2.INTER_CUBIC)
self._store_frame_to_file(frame=frame, frames_dir=frames_dir, if self._output:
image_file=image_file) result, frame = cv2.imencode('.jpg', frame)
if not result:
self.logger.warning('Unable to convert frame to JPEG')
continue
self._output.write(frame.tobytes())
elif frames_dir:
self._store_frame_to_file(frame=frame, frames_dir=frames_dir, image_file=image_file)
captured_frames += 1 captured_frames += 1
self.fire_event(CameraFrameCapturedEvent(filename=image_file)) self.fire_event(CameraFrameCapturedEvent(filename=image_file))
@ -326,31 +360,32 @@ class CameraPlugin(Plugin):
return thread return thread
@action @action
def start_recording(self, duration=None, video_file=None, video_type=None, def start_recording(self, duration: Optional[float] = None, video_file: Optional[str] = None,
device_id=None, frames_dir=None, video_type: Optional[str] = None, device_id: Optional[int] = None,
sleep_between_frames=None, max_stored_frames=None, frames_dir: Optional[str] = None, sleep_between_frames: Optional[float] = None,
color_transform=None, scale_x=None, scale_y=None, max_stored_frames: Optional[int] = None, color_transform: Optional[str] = None,
rotate=None, flip=None): scale_x: Optional[float] = None, scale_y: Optional[float] = None,
rotate: Optional[float] = None, flip: Optional[int] = None):
""" """
Start recording Start recording
:param duration: Record duration in seconds (default: None, record until :param duration: Record duration in seconds (default: None, record until
``stop_recording``) ``stop_recording``)
:type duration: float
:param video_file: If set, the stream will be recorded to the specified :param video_file: If set, the stream will be recorded to the specified
video file (default: None) video file (default: None)
:type video_file: str
:param video_type: Overrides the default configured ``video_type`` :param video_type: Overrides the default configured ``video_type``
:type video_file: str
:param device_id, frames_dir, sleep_between_frames, max_stored_frames, :param device_id: Override default device_id
color_transform, scale_x, scale_y, rotate, flip: Set :param frames_dir: Override default frames_dir
these parameters if you want to override the default configured ones. :param sleep_between_frames: Override default sleep_between_frames
:param max_stored_frames: Override default max_stored_frames
:param color_transform: Override default color_transform
:param scale_x: Override default scale_x
:param scale_y: Override default scale_y
:param rotate: Override default rotate
:param flip: Override default flip
""" """
import cv2
device_id = device_id if device_id is not None else self.default_device_id device_id = device_id if device_id is not None else self.default_device_id
if device_id in self._is_recording and \ if device_id in self._is_recording and \
self._is_recording[device_id].is_set(): self._is_recording[device_id].is_set():
@ -360,56 +395,55 @@ class CameraPlugin(Plugin):
recording_started = threading.Event() recording_started = threading.Event()
# noinspection PyUnusedLocal
def on_recording_started(event): def on_recording_started(event):
recording_started.set() recording_started.set()
frames_dir = os.path.abspath(os.path.expanduser(frames_dir)) \ attrs = self._get_attributes(frames_dir=frames_dir, sleep_between_frames=sleep_between_frames,
if frames_dir is not None else self.frames_dir max_stored_frames=max_stored_frames, color_transform=color_transform,
sleep_between_frames = sleep_between_frames if sleep_between_frames \ scale_x=scale_x, scale_y=scale_y, rotate=rotate, flip=flip, video_type=video_type)
is not None else self.sleep_between_frames
max_stored_frames = max_stored_frames if max_stored_frames \
is not None else self.max_stored_frames
color_transform = color_transform if color_transform \
is not None else self.color_transform
scale_x = scale_x if scale_x is not None else self.scale_x
scale_y = scale_y if scale_y is not None else self.scale_y
rotate = rotate if rotate is not None else self.rotate
flip = flip if flip is not None else self.flip
if video_type is not None: # noinspection PyUnresolvedReferences
video_type = cv2.VideoWriter_fourcc(*video_type.upper()) \ if attrs.frames_dir:
if isinstance(video_type, str) else video_type # noinspection PyUnresolvedReferences
else: attrs.frames_dir = os.path.join(attrs.frames_dir, str(device_id))
video_type = self.video_type if video_file:
video_file = os.path.abspath(os.path.expanduser(video_file))
attrs.frames_dir = os.path.join(attrs.frames_dir, 'recording_{}'.format(
datetime.now().strftime('%Y-%m-%d_%H-%M-%S-%f')))
frames_dir = os.path.join(frames_dir, str(device_id)) # noinspection PyUnresolvedReferences
if video_file: self._init_device(device_id,
video_file = os.path.abspath(os.path.expanduser(video_file)) video_file=video_file,
frames_dir = os.path.join(frames_dir, 'recording_{}'.format( video_type=attrs.video_type,
datetime.now().strftime('%Y-%m-%d_%H-%M-%S-%f'))) frames_dir=attrs.frames_dir,
sleep_between_frames=attrs.sleep_between_frames,
self._init_device(device_id, video_file=video_file, max_stored_frames=attrs.max_stored_frames,
video_type=video_type, color_transform=attrs.color_transform,
frames_dir=frames_dir, scale_x=attrs.scale_x,
sleep_between_frames=sleep_between_frames, scale_y=attrs.scale_y,
max_stored_frames=max_stored_frames, rotate=attrs.rotate,
color_transform=color_transform, scale_x=scale_x, flip=attrs.flip)
scale_y=scale_y, rotate=rotate, flip=flip)
self.register_handler(CameraRecordingStartedEvent, on_recording_started) self.register_handler(CameraRecordingStartedEvent, on_recording_started)
# noinspection PyUnresolvedReferences
self._recording_threads[device_id] = threading.Thread( self._recording_threads[device_id] = threading.Thread(
target=self._recording_thread(), kwargs={ target=self._recording_thread(), kwargs={
'duration': duration, 'duration': duration,
'video_file': video_file, 'video_file': video_file,
'video_type': video_type, 'video_type': attrs.video_type,
'image_file': None, 'device_id': device_id, 'image_file': None,
'frames_dir': frames_dir, 'n_frames': None, 'device_id': device_id,
'sleep_between_frames': sleep_between_frames, 'frames_dir': attrs.frames_dir,
'max_stored_frames': max_stored_frames, 'n_frames': None,
'color_transform': color_transform, 'sleep_between_frames': attrs.sleep_between_frames,
'scale_x': scale_x, 'scale_y': scale_y, 'max_stored_frames': attrs.max_stored_frames,
'rotate': rotate, 'flip': flip 'color_transform': attrs.color_transform,
'scale_x': attrs.scale_x,
'scale_y': attrs.scale_y,
'rotate': attrs.rotate,
'flip': attrs.flip,
}) })
self._recording_threads[device_id].start() self._recording_threads[device_id].start()
@ -430,24 +464,52 @@ class CameraPlugin(Plugin):
self._release_device(device_id) self._release_device(device_id)
shutil.rmtree(frames_dir, ignore_errors=True) shutil.rmtree(frames_dir, ignore_errors=True)
def _get_attributes(self, frames_dir=None, warmup_frames=None,
color_transform=None, scale_x=None, scale_y=None,
rotate=None, flip=None, sleep_between_frames=None,
max_stored_frames=None, video_type=None) -> Mapping:
import cv2
warmup_frames = warmup_frames if warmup_frames is not None else self.warmup_frames
frames_dir = os.path.abspath(os.path.expanduser(frames_dir)) if frames_dir is not None else self.frames_dir
sleep_between_frames = sleep_between_frames if sleep_between_frames is not None else self.sleep_between_frames
max_stored_frames = max_stored_frames if max_stored_frames is not None else self.max_stored_frames
color_transform = color_transform if color_transform is not None else self.color_transform
scale_x = scale_x if scale_x is not None else self.scale_x
scale_y = scale_y if scale_y is not None else self.scale_y
rotate = rotate if rotate is not None else self.rotate
flip = flip if flip is not None else self.flip
if video_type is not None:
video_type = cv2.VideoWriter_fourcc(*video_type.upper()) if isinstance(video_type, str) else video_type
else:
video_type = self.video_type
return Mapping(warmup_frames=warmup_frames, frames_dir=frames_dir, sleep_between_frames=sleep_between_frames,
max_stored_frames=max_stored_frames, color_transform=color_transform, scale_x=scale_x,
scale_y=scale_y, rotate=rotate, flip=flip, video_type=video_type)
@action @action
def take_picture(self, image_file, device_id=None, warmup_frames=None, def take_picture(self, image_file: str, device_id: Optional[int] = None, warmup_frames: Optional[int] = None,
color_transform=None, scale_x=None, scale_y=None, color_transform: Optional[str] = None, scale_x: Optional[float] = None,
rotate=None, flip=None): scale_y: Optional[float] = None, rotate: Optional[float] = None, flip: Optional[int] = None):
""" """
Take a picture. Take a picture.
:param image_file: Path where the output image will be stored. :param image_file: Path where the output image will be stored.
:type image_file: str :param device_id: Override default device_id
:param warmup_frames: Override default warmup_frames
:param device_id, warmup_frames, color_transform, scale_x, scale_y, :param color_transform: Override default color_transform
rotate, flip: Overrides the configured default parameters :param scale_x: Override default scale_x
:param scale_y: Override default scale_y
:param rotate: Override default rotate
:param flip: Override default flip
""" """
device_id = device_id if device_id is not None else self.default_device_id device_id = device_id if device_id is not None else self.default_device_id
image_file = os.path.abspath(os.path.expanduser(image_file)) image_file = os.path.abspath(os.path.expanduser(image_file))
picture_taken = threading.Event() picture_taken = threading.Event()
# noinspection PyUnusedLocal
def on_picture_taken(event): def on_picture_taken(event):
picture_taken.set() picture_taken.set()
@ -464,20 +526,13 @@ class CameraPlugin(Plugin):
raise RuntimeError('Recording already in progress and no images ' + raise RuntimeError('Recording already in progress and no images ' +
'have been captured yet') 'have been captured yet')
warmup_frames = warmup_frames if warmup_frames is not None else \ attrs = self._get_attributes(warmup_frames=warmup_frames, color_transform=color_transform, scale_x=scale_x,
self.warmup_frames scale_y=scale_y, rotate=rotate, flip=flip)
color_transform = color_transform if color_transform \
is not None else self.color_transform
scale_x = scale_x if scale_x is not None else self.scale_x
scale_y = scale_y if scale_y is not None else self.scale_y
rotate = rotate if rotate is not None else self.rotate
flip = flip if flip is not None else self.flip
self._init_device(device_id, image_file=image_file, # noinspection PyUnresolvedReferences
warmup_frames=warmup_frames, self._init_device(device_id, image_file=image_file, warmup_frames=attrs.warmup_frames,
color_transform=color_transform, color_transform=attrs.color_transform, scale_x=attrs.scale_x, scale_y=attrs.scale_y,
scale_x=scale_x, scale_y=scale_y, rotate=rotate, rotate=attrs.rotate, flip=attrs.flip)
flip=flip)
self.register_handler(CameraPictureTakenEvent, on_picture_taken) self.register_handler(CameraPictureTakenEvent, on_picture_taken)
self._recording_threads[device_id] = threading.Thread( self._recording_threads[device_id] = threading.Thread(
@ -524,5 +579,18 @@ class CameraPlugin(Plugin):
def get_default_device_id(self): def get_default_device_id(self):
return self.default_device_id return self.default_device_id
def get_stream(self):
return self._output
def __enter__(self):
device_id = self.default_device_id
self._output = StreamingOutput()
self._init_device(device_id=device_id)
self.start_recording(device_id=device_id)
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop_recording(self.default_device_id)
self._output = None
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -2,34 +2,15 @@
.. moduleauthor:: Fabio Manganiello <blacklight86@gmail.com> .. moduleauthor:: Fabio Manganiello <blacklight86@gmail.com>
""" """
import io
import os import os
import socket import socket
import threading import threading
import time import time
from typing import Optional, Union from typing import Optional
from platypush.plugins import action from platypush.plugins import action
from platypush.plugins.camera import CameraPlugin from platypush.plugins.camera import CameraPlugin, StreamingOutput
class StreamingOutput:
def __init__(self):
self.frame = None
self.buffer = io.BytesIO()
self.ready = threading.Condition()
def write(self, buf):
if buf.startswith(b'\xff\xd8'):
# New frame, copy the existing buffer's content and notify all clients that it's available
self.buffer.truncate()
with self.ready:
self.frame = self.buffer.getvalue()
self.ready.notify_all()
self.buffer.seek(0)
return self.buffer.write(buf)
class CameraPiPlugin(CameraPlugin): class CameraPiPlugin(CameraPlugin):
@ -85,7 +66,6 @@ class CameraPiPlugin(CameraPlugin):
self._streaming_thread = None self._streaming_thread = None
self._time_lapse_stop_condition = threading.Condition() self._time_lapse_stop_condition = threading.Condition()
self._recording_stop_condition = threading.Condition() self._recording_stop_condition = threading.Condition()
self._output = None
self._can_stream = False self._can_stream = False
# noinspection PyUnresolvedReferences,PyPackageRequirements # noinspection PyUnresolvedReferences,PyPackageRequirements
@ -200,9 +180,6 @@ class CameraPiPlugin(CameraPlugin):
self._output = StreamingOutput() self._output = StreamingOutput()
camera.start_recording(self._output, format='mjpeg') camera.start_recording(self._output, format='mjpeg')
def get_stream(self):
return self._output
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
self.close() self.close()
@ -501,9 +478,6 @@ class CameraPiPlugin(CameraPlugin):
self.logger.info('Starting streaming on port {}'.format(listen_port)) self.logger.info('Starting streaming on port {}'.format(listen_port))
while self._can_stream: while self._can_stream:
sock = None
stream = None
try: try:
sock = server_socket.accept()[0] sock = server_socket.accept()[0]
stream = sock.makefile('wb') stream = sock.makefile('wb')