[libcamera-devel] [PATCH] py: cam: Network renderer
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Sun Mar 19 14:05:22 CET 2023
Hi Tomi,
Thank you for the patch.
On Sun, Mar 19, 2023 at 01:30:13PM +0200, Tomi Valkeinen via libcamera-devel wrote:
> Here's something I have found useful a few times.
>
> This adds a "tx" renderer to cam.py, which sends the frames over the
> network to a receiver.
>
> It also adds a "cam-rx" tool (non-libcamera based) which receives the
> frames and uses PyQt to show them on the screen, usually ran on a PC.
>
> This is obviously not super efficient, but on the PC side it doesn't
> matter. On the TX side, at least RPi4 seemed to work without noticeable
> lag, but my old 32-bit TI DRA76, when sending three camera streams, the
> performance dropped to ~5fps. Still, I find that more than enough for
> most development work.
>
> This could be extended to also transmit the metadata.
What's the advantage of this approach compared to using GStreamer for
network streaming ? It feels to me that we're reinventing the wheel a
bit here.
> Signed-off-by: Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
> ---
> src/py/cam/cam.py | 4 +
> src/py/cam/cam_tx.py | 94 +++++++++++++
> src/py/examples/cam-rx.py | 155 +++++++++++++++++++++
> src/py/examples/cam_rx_helpers.py | 223 ++++++++++++++++++++++++++++++
> 4 files changed, 476 insertions(+)
> create mode 100644 src/py/cam/cam_tx.py
> create mode 100755 src/py/examples/cam-rx.py
> create mode 100644 src/py/examples/cam_rx_helpers.py
>
> diff --git a/src/py/cam/cam.py b/src/py/cam/cam.py
> index 967a72f5..50f0f8d6 100755
> --- a/src/py/cam/cam.py
> +++ b/src/py/cam/cam.py
> @@ -387,6 +387,7 @@ def main():
> parser.add_argument('--list-controls', action='store_true', help='List cameras controls')
> parser.add_argument('-I', '--info', action='store_true', help='Display information about stream(s)')
> parser.add_argument('-R', '--renderer', default='null', help='Renderer (null, kms, qt, qtgl)')
> + parser.add_argument('--rargs', default='', help='Arguments passed to the renderer (pass --help to see help)')
>
> # per camera options
> parser.add_argument('-C', '--capture', nargs='?', type=int, const=1000000, action=CustomAction, help='Capture until interrupted by user or until CAPTURE frames captured')
> @@ -449,6 +450,9 @@ def main():
> elif args.renderer == 'qtgl':
> import cam_qtgl
> renderer = cam_qtgl.QtRenderer(state)
> + elif args.renderer == 'tx':
> + import cam_tx
> + renderer = cam_tx.TxRenderer(state, args.rargs)
> else:
> print('Bad renderer', args.renderer)
> return -1
> diff --git a/src/py/cam/cam_tx.py b/src/py/cam/cam_tx.py
> new file mode 100644
> index 00000000..3d31c6ef
> --- /dev/null
> +++ b/src/py/cam/cam_tx.py
> @@ -0,0 +1,94 @@
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
> +
> +import argparse
> +import libcamera
> +import libcamera.utils
> +import selectors
> +import socket
> +import struct
> +import sys
> +
> +PORT = 43242
> +
> +# ctx-idx, width, height, format, num-planes, plane1, plane2, plane3, plane4
> +struct_fmt = struct.Struct('<III12pI4I')
> +
> +
> +class TxRenderer:
> + def __init__(self, state, ropts):
> + parser = argparse.ArgumentParser(prog='TxRenderer')
> + parser.add_argument('host', default='localhost', help='Address')
> + args = parser.parse_args(ropts.split(' '))
> +
> + self.host = args.host
> +
> + self.state = state
> +
> + self.cm = state.cm
> + self.contexts = state.contexts
> +
> + self.running = False
> +
> + def setup(self):
> + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
> + sock.connect((self.host, PORT))
> + self.sock = sock
> +
> + buf_mmap_map = {}
> +
> + for ctx in self.contexts:
> + for stream in ctx.streams:
> + for buf in ctx.allocator.buffers(stream):
> + mfb = libcamera.utils.MappedFrameBuffer(buf).mmap()
> + buf_mmap_map[buf] = mfb
> +
> + self.buf_mmap_map = buf_mmap_map
> +
> + def run(self):
> + print('Capturing...')
> +
> + self.running = True
> +
> + sel = selectors.DefaultSelector()
> + sel.register(self.cm.event_fd, selectors.EVENT_READ, self.readcam)
> + sel.register(sys.stdin, selectors.EVENT_READ, self.readkey)
> +
> + print('Press enter to exit')
> +
> + while self.running:
> + events = sel.select()
> + for key, _ in events:
> + callback = key.data
> + callback(key.fileobj)
> +
> + print('Exiting...')
> +
> + def readcam(self, fd):
> + self.running = self.state.event_handler()
> +
> + def readkey(self, fileobj):
> + sys.stdin.readline()
> + self.running = False
> +
> + def request_handler(self, ctx, req):
> + buffers = req.buffers
> +
> + for stream, fb in buffers.items():
> + mfb = self.buf_mmap_map[fb]
> +
> + plane_sizes = [len(p) for p in mfb.planes] + [0] * (4 - len(mfb.planes))
> +
> + stream_config = stream.configuration
> +
> + hdr = struct_fmt.pack(ctx.idx,
> + stream_config.size.width, stream_config.size.height,
> + bytes(str(stream_config.pixel_format), 'ascii'),
> + len(mfb.planes), *plane_sizes)
> +
> + self.sock.sendall(hdr)
> +
> + for p in mfb.planes:
> + self.sock.sendall(p)
> +
> + self.state.request_processed(ctx, req)
> diff --git a/src/py/examples/cam-rx.py b/src/py/examples/cam-rx.py
> new file mode 100755
> index 00000000..a53d59c8
> --- /dev/null
> +++ b/src/py/examples/cam-rx.py
> @@ -0,0 +1,155 @@
> +#!/usr/bin/env python3
> +
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
> +
> +from cam_rx_helpers import data_to_pix
> +from PyQt5 import QtCore, QtWidgets
> +from PyQt5.QtCore import Qt
> +import PyQt5.QtNetwork
> +import struct
> +import sys
> +import traceback
> +
> +PORT = 43242
> +receivers = []
> +
> +struct_fmt = struct.Struct('<III12pI4I')
> +
> +
> +# Loading MJPEG to a QPixmap produces corrupt JPEG data warnings. Ignore these.
> +def qt_message_handler(msg_type, msg_log_context, msg_string):
> + if msg_string.startswith("Corrupt JPEG data"):
> + return
> +
> + # For some reason qInstallMessageHandler returns None, so we won't
> + # call the old handler
> + if old_msg_handler is not None:
> + old_msg_handler(msg_type, msg_log_context, msg_string)
> + else:
> + print(msg_string)
> +
> +
> +old_msg_handler = QtCore.qInstallMessageHandler(qt_message_handler)
> +
> +
> +class Receiver(QtWidgets.QWidget):
> + def __init__(self, socket: PyQt5.QtNetwork.QTcpSocket):
> + super().__init__()
> +
> + self.name = '{}:{}'.format(socket.peerAddress().toString(), socket.peerPort())
> +
> + print('[{}] Accepted new connection'.format(self.name))
> +
> + self.socket = socket
> +
> + self.socket.readyRead.connect(self.on_ready_read)
> + self.socket.disconnected.connect(self.on_disconnected)
> + self.socket.error.connect(self.on_error)
> +
> + self.header_buffer = bytearray()
> + self.data_buffer = bytearray()
> + self.data_size = 0
> +
> + self.state = 0
> +
> + self.resize(1000, 600)
> + self.setAttribute(Qt.WA_ShowWithoutActivating)
> + self.setWindowFlag(Qt.WindowStaysOnTopHint, True)
> +
> + self.gridLayout = QtWidgets.QGridLayout()
> + self.setLayout(self.gridLayout)
> +
> + self.labels = {}
> +
> + self.show()
> + print("done")
> +
> + def on_ready_read(self):
> + while self.socket.bytesAvailable():
> + if self.state == 0:
> + data = self.socket.read(struct_fmt.size - len(self.header_buffer))
> + self.header_buffer.extend(data)
> +
> + if len(self.header_buffer) == struct_fmt.size:
> + self.on_header()
> + else:
> + data = self.socket.read(self.data_size - len(self.data_buffer))
> + self.data_buffer.extend(data)
> +
> + if len(self.data_buffer) == self.data_size:
> + try:
> + self.on_buffers()
> + except Exception:
> + print(traceback.format_exc())
> + qApp.exit(-1)
> + return
> +
> + def on_header(self):
> + self.header_tuple = struct_fmt.unpack_from(self.header_buffer)
> + idx, w, h, fmtstr, num_planes, p0, p1, p2, p3 = self.header_tuple
> + self.data_size = p0 + p1 + p2 + p3
> + self.header_buffer = bytearray()
> +
> + self.state = 1
> +
> + def on_buffers(self):
> + idx, w, h, fmtstr, num_planes, p0, p1, p2, p3 = self.header_tuple
> + fmt = fmtstr.decode('ascii')
> +
> + print('[{}] cam{} {}x{}-{}'.format(self.name, idx, w, h, fmt))
> +
> + if idx not in self.labels:
> + label = QtWidgets.QLabel()
> + label.setSizePolicy(QtWidgets.QSizePolicy.Ignored, QtWidgets.QSizePolicy.Ignored)
> + self.labels[idx] = label
> + self.gridLayout.addWidget(label, self.gridLayout.count() // 2, self.gridLayout.count() % 2)
> +
> + label = self.labels[idx]
> +
> + pix = data_to_pix(fmt, w, h, self.data_buffer)
> +
> + pix = pix.scaled(label.width(), label.height(), Qt.AspectRatioMode.KeepAspectRatio,
> + Qt.TransformationMode.FastTransformation)
> +
> + label.setPixmap(pix)
> +
> + self.data_buffer = bytearray()
> +
> + self.state = 0
> +
> + def on_disconnected(self):
> + print('[{}] Disconnected'.format(self.name))
> + self.close()
> + receivers.remove(self)
> +
> + def on_error(self):
> + print('[{}] Error: {}'.format(self.name, self.socket.errorString()))
> +
> +
> +def new_connection(tcpServer):
> + clientConnection: PyQt5.QtNetwork.QTcpSocket = tcpServer.nextPendingConnection()
> + w = Receiver(clientConnection)
> + receivers.append(w)
> +
> +
> +def readkey():
> + global qApp
> + sys.stdin.readline()
> + qApp.quit()
> +
> +
> +if __name__ == '__main__':
> + global qApp
> +
> + qApp = QtWidgets.QApplication(sys.argv)
> + qApp.setQuitOnLastWindowClosed(False)
> +
> + keynotif = QtCore.QSocketNotifier(sys.stdin.fileno(), QtCore.QSocketNotifier.Read)
> + keynotif.activated.connect(readkey)
> +
> + tcpServer = PyQt5.QtNetwork.QTcpServer(qApp)
> + tcpServer.listen(PyQt5.QtNetwork.QHostAddress('0.0.0.0'), PORT)
> + tcpServer.newConnection.connect(lambda: new_connection(tcpServer))
> +
> + sys.exit(qApp.exec_())
> diff --git a/src/py/examples/cam_rx_helpers.py b/src/py/examples/cam_rx_helpers.py
> new file mode 100644
> index 00000000..293eb63d
> --- /dev/null
> +++ b/src/py/examples/cam_rx_helpers.py
> @@ -0,0 +1,223 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
> +#
> +# Debayering code based on PiCamera documentation
> +
> +from numpy.lib.stride_tricks import as_strided
> +from PyQt5 import QtGui
> +import numpy as np
> +
> +
> +def demosaic(data, r0, g0, g1, b0):
> + # Separate the components from the Bayer data to RGB planes
> +
> + rgb = np.zeros(data.shape + (3,), dtype=data.dtype)
> + rgb[1::2, 0::2, 0] = data[r0[1]::2, r0[0]::2] # Red
> + rgb[0::2, 0::2, 1] = data[g0[1]::2, g0[0]::2] # Green
> + rgb[1::2, 1::2, 1] = data[g1[1]::2, g1[0]::2] # Green
> + rgb[0::2, 1::2, 2] = data[b0[1]::2, b0[0]::2] # Blue
> +
> + # Below we present a fairly naive de-mosaic method that simply
> + # calculates the weighted average of a pixel based on the pixels
> + # surrounding it. The weighting is provided by a byte representation of
> + # the Bayer filter which we construct first:
> +
> + bayer = np.zeros(rgb.shape, dtype=np.uint8)
> + bayer[1::2, 0::2, 0] = 1 # Red
> + bayer[0::2, 0::2, 1] = 1 # Green
> + bayer[1::2, 1::2, 1] = 1 # Green
> + bayer[0::2, 1::2, 2] = 1 # Blue
> +
> + # Allocate an array to hold our output with the same shape as the input
> + # data. After this we define the size of window that will be used to
> + # calculate each weighted average (3x3). Then we pad out the rgb and
> + # bayer arrays, adding blank pixels at their edges to compensate for the
> + # size of the window when calculating averages for edge pixels.
> +
> + output = np.empty(rgb.shape, dtype=rgb.dtype)
> + window = (3, 3)
> + borders = (window[0] - 1, window[1] - 1)
> + border = (borders[0] // 2, borders[1] // 2)
> +
> + rgb = np.pad(rgb, [
> + (border[0], border[0]),
> + (border[1], border[1]),
> + (0, 0),
> + ], 'constant')
> + bayer = np.pad(bayer, [
> + (border[0], border[0]),
> + (border[1], border[1]),
> + (0, 0),
> + ], 'constant')
> +
> + # For each plane in the RGB data, we use a nifty numpy trick
> + # (as_strided) to construct a view over the plane of 3x3 matrices. We do
> + # the same for the bayer array, then use Einstein summation on each
> + # (np.sum is simpler, but copies the data so it's slower), and divide
> + # the results to get our weighted average:
> +
> + for plane in range(3):
> + p = rgb[..., plane]
> + b = bayer[..., plane]
> +
> + pview = as_strided(p, shape=(
> + p.shape[0] - borders[0],
> + p.shape[1] - borders[1]) + window, strides=p.strides * 2)
> + bview = as_strided(b, shape=(
> + b.shape[0] - borders[0],
> + b.shape[1] - borders[1]) + window, strides=b.strides * 2)
> + psum = np.einsum('ijkl->ij', pview)
> + bsum = np.einsum('ijkl->ij', bview)
> + output[..., plane] = psum // bsum
> +
> + return output
> +
> +
> +def convert_raw(data, w, h, fmt):
> + bayer_pattern = fmt[1:5]
> + bitspp = int(fmt[5:])
> +
> + if bitspp == 8:
> + data = data.reshape((h, w))
> + data = data.astype(np.uint16)
> + elif bitspp in [10, 12]:
> + data = data.view(np.uint16)
> + data = data.reshape((h, w))
> + else:
> + raise Exception('Bad bitspp:' + str(bitspp))
> +
> + idx = bayer_pattern.find('R')
> + assert(idx != -1)
> + r0 = (idx % 2, idx // 2)
> +
> + idx = bayer_pattern.find('G')
> + assert(idx != -1)
> + g0 = (idx % 2, idx // 2)
> +
> + idx = bayer_pattern.find('G', idx + 1)
> + assert(idx != -1)
> + g1 = (idx % 2, idx // 2)
> +
> + idx = bayer_pattern.find('B')
> + assert(idx != -1)
> + b0 = (idx % 2, idx // 2)
> +
> + rgb = demosaic(data, r0, g0, g1, b0)
> + rgb = (rgb >> (bitspp - 8)).astype(np.uint8)
> +
> + return rgb
> +
> +
> +def convert_yuv444_to_rgb(yuv):
> + m = np.array([
> + [1.0, 1.0, 1.0],
> + [-0.000007154783816076815, -0.3441331386566162, 1.7720025777816772],
> + [1.4019975662231445, -0.7141380310058594, 0.00001542569043522235]
> + ])
> +
> + rgb = np.dot(yuv, m)
> + rgb[:, :, 0] -= 179.45477266423404
> + rgb[:, :, 1] += 135.45870971679688
> + rgb[:, :, 2] -= 226.8183044444304
> + rgb = rgb.astype(np.uint8)
> +
> + return rgb
> +
> +
> +def convert_yuyv(data, w, h):
> + # YUV422
> + yuyv = data.reshape((h, w // 2 * 4))
> +
> + # YUV444
> + yuv = np.empty((h, w, 3), dtype=np.uint8)
> + yuv[:, :, 0] = yuyv[:, 0::2] # Y
> + yuv[:, :, 1] = yuyv[:, 1::4].repeat(2, axis=1) # U
> + yuv[:, :, 2] = yuyv[:, 3::4].repeat(2, axis=1) # V
> +
> + return convert_yuv444_to_rgb(yuv)
> +
> +
> +def convert_uyvy(data, w, h):
> + # YUV422
> + yuyv = data.reshape((h, w // 2 * 4))
> +
> + # YUV444
> + yuv = np.empty((h, w, 3), dtype=np.uint8)
> + yuv[:, :, 0] = yuyv[:, 1::2] # Y
> + yuv[:, :, 1] = yuyv[:, 0::4].repeat(2, axis=1) # U
> + yuv[:, :, 2] = yuyv[:, 2::4].repeat(2, axis=1) # V
> +
> + return convert_yuv444_to_rgb(yuv)
> +
> +
> +def convert_nv12(data, w, h):
> + plane1 = data[:w * h]
> + plane2 = data[w * h:]
> +
> + y = plane1.reshape((h, w))
> + uv = plane2.reshape((h // 2, w // 2, 2))
> +
> + # YUV444
> + yuv = np.empty((h, w, 3), dtype=np.uint8)
> + yuv[:, :, 0] = y[:, :] # Y
> + yuv[:, :, 1] = uv[:, :, 0].repeat(2, axis=0).repeat(2, axis=1) # U
> + yuv[:, :, 2] = uv[:, :, 1].repeat(2, axis=0).repeat(2, axis=1) # V
> +
> + return convert_yuv444_to_rgb(yuv)
> +
> +
> +def to_rgb(fmt, w, h, data):
> + if fmt == 'YUYV':
> + return convert_yuyv(data, w, h)
> +
> + if fmt == 'UYVY':
> + return convert_uyvy(data, w, h)
> +
> + elif fmt == 'NV12':
> + return convert_nv12(data, w, h)
> +
> + elif fmt == 'RGB888':
> + rgb = data.reshape((h, w, 3))
> + rgb[:, :, [0, 1, 2]] = rgb[:, :, [2, 1, 0]]
> +
> + elif fmt == 'BGR888':
> + rgb = data.reshape((h, w, 3))
> +
> + elif fmt in ['ARGB8888', 'XRGB8888']:
> + rgb = data.reshape((h, w, 4))
> + rgb = np.flip(rgb, axis=2)
> + # drop alpha component
> + rgb = np.delete(rgb, np.s_[0::4], axis=2)
> +
> + elif fmt.startswith('S'):
> + return convert_raw(data, w, h, fmt)
> +
> + else:
> + raise Exception('Unsupported format ' + fmt)
> +
> + return rgb
> +
> +
> +def data_to_rgb(fmt, w, h, data):
> + data = np.frombuffer(data, dtype=np.uint8)
> + rgb = to_rgb(fmt, w, h, data)
> + return rgb
> +
> +
> +def rgb_to_pix(rgb):
> + w = rgb.shape[1]
> + h = rgb.shape[0]
> + qim = QtGui.QImage(rgb, w, h, QtGui.QImage.Format.Format_RGB888)
> + pix = QtGui.QPixmap.fromImage(qim)
> + return pix
> +
> +
> +def data_to_pix(fmt, w, h, data):
> + if fmt == 'MJPEG':
> + pix = QtGui.QPixmap(w, h)
> + pix.loadFromData(data)
> + else:
> + rgb = data_to_rgb(fmt, w, h, data)
> + pix = rgb_to_pix(rgb)
> +
> + return pix
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list