[libcamera-devel] [PATCH] py: cam: Network renderer
Tomi Valkeinen
tomi.valkeinen at ideasonboard.com
Sun Mar 19 12:30:13 CET 2023
Here's something I have found useful a few times.
This adds a "tx" renderer to cam.py, which sends the frames over the
network to a receiver.
It also adds a "cam-rx" tool (non-libcamera based) which receives the
frames and uses PyQt to show them on the screen, usually ran on a PC.
This is obviously not super efficient, but on the PC side it doesn't
matter. On the TX side, at least RPi4 seemed to work without noticeable
lag, but my old 32-bit TI DRA76, when sending three camera streams, the
performance dropped to ~5fps. Still, I find that more than enough for
most development work.
This could be extended to also transmit the metadata.
Signed-off-by: Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
---
src/py/cam/cam.py | 4 +
src/py/cam/cam_tx.py | 94 +++++++++++++
src/py/examples/cam-rx.py | 155 +++++++++++++++++++++
src/py/examples/cam_rx_helpers.py | 223 ++++++++++++++++++++++++++++++
4 files changed, 476 insertions(+)
create mode 100644 src/py/cam/cam_tx.py
create mode 100755 src/py/examples/cam-rx.py
create mode 100644 src/py/examples/cam_rx_helpers.py
diff --git a/src/py/cam/cam.py b/src/py/cam/cam.py
index 967a72f5..50f0f8d6 100755
--- a/src/py/cam/cam.py
+++ b/src/py/cam/cam.py
@@ -387,6 +387,7 @@ def main():
parser.add_argument('--list-controls', action='store_true', help='List cameras controls')
parser.add_argument('-I', '--info', action='store_true', help='Display information about stream(s)')
parser.add_argument('-R', '--renderer', default='null', help='Renderer (null, kms, qt, qtgl)')
+ parser.add_argument('--rargs', default='', help='Arguments passed to the renderer (pass --help to see help)')
# per camera options
parser.add_argument('-C', '--capture', nargs='?', type=int, const=1000000, action=CustomAction, help='Capture until interrupted by user or until CAPTURE frames captured')
@@ -449,6 +450,9 @@ def main():
elif args.renderer == 'qtgl':
import cam_qtgl
renderer = cam_qtgl.QtRenderer(state)
+ elif args.renderer == 'tx':
+ import cam_tx
+ renderer = cam_tx.TxRenderer(state, args.rargs)
else:
print('Bad renderer', args.renderer)
return -1
diff --git a/src/py/cam/cam_tx.py b/src/py/cam/cam_tx.py
new file mode 100644
index 00000000..3d31c6ef
--- /dev/null
+++ b/src/py/cam/cam_tx.py
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
+
+import argparse
+import libcamera
+import libcamera.utils
+import selectors
+import socket
+import struct
+import sys
+
+PORT = 43242
+
+# ctx-idx, width, height, format, num-planes, plane1, plane2, plane3, plane4
+struct_fmt = struct.Struct('<III12pI4I')
+
+
+class TxRenderer:
+ def __init__(self, state, ropts):
+ parser = argparse.ArgumentParser(prog='TxRenderer')
+ parser.add_argument('host', default='localhost', help='Address')
+ args = parser.parse_args(ropts.split(' '))
+
+ self.host = args.host
+
+ self.state = state
+
+ self.cm = state.cm
+ self.contexts = state.contexts
+
+ self.running = False
+
+ def setup(self):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.connect((self.host, PORT))
+ self.sock = sock
+
+ buf_mmap_map = {}
+
+ for ctx in self.contexts:
+ for stream in ctx.streams:
+ for buf in ctx.allocator.buffers(stream):
+ mfb = libcamera.utils.MappedFrameBuffer(buf).mmap()
+ buf_mmap_map[buf] = mfb
+
+ self.buf_mmap_map = buf_mmap_map
+
+ def run(self):
+ print('Capturing...')
+
+ self.running = True
+
+ sel = selectors.DefaultSelector()
+ sel.register(self.cm.event_fd, selectors.EVENT_READ, self.readcam)
+ sel.register(sys.stdin, selectors.EVENT_READ, self.readkey)
+
+ print('Press enter to exit')
+
+ while self.running:
+ events = sel.select()
+ for key, _ in events:
+ callback = key.data
+ callback(key.fileobj)
+
+ print('Exiting...')
+
+ def readcam(self, fd):
+ self.running = self.state.event_handler()
+
+ def readkey(self, fileobj):
+ sys.stdin.readline()
+ self.running = False
+
+ def request_handler(self, ctx, req):
+ buffers = req.buffers
+
+ for stream, fb in buffers.items():
+ mfb = self.buf_mmap_map[fb]
+
+ plane_sizes = [len(p) for p in mfb.planes] + [0] * (4 - len(mfb.planes))
+
+ stream_config = stream.configuration
+
+ hdr = struct_fmt.pack(ctx.idx,
+ stream_config.size.width, stream_config.size.height,
+ bytes(str(stream_config.pixel_format), 'ascii'),
+ len(mfb.planes), *plane_sizes)
+
+ self.sock.sendall(hdr)
+
+ for p in mfb.planes:
+ self.sock.sendall(p)
+
+ self.state.request_processed(ctx, req)
diff --git a/src/py/examples/cam-rx.py b/src/py/examples/cam-rx.py
new file mode 100755
index 00000000..a53d59c8
--- /dev/null
+++ b/src/py/examples/cam-rx.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
+
+from cam_rx_helpers import data_to_pix
+from PyQt5 import QtCore, QtWidgets
+from PyQt5.QtCore import Qt
+import PyQt5.QtNetwork
+import struct
+import sys
+import traceback
+
+PORT = 43242
+receivers = []
+
+struct_fmt = struct.Struct('<III12pI4I')
+
+
+# Loading MJPEG to a QPixmap produces corrupt JPEG data warnings. Ignore these.
+def qt_message_handler(msg_type, msg_log_context, msg_string):
+ if msg_string.startswith("Corrupt JPEG data"):
+ return
+
+ # For some reason qInstallMessageHandler returns None, so we won't
+ # call the old handler
+ if old_msg_handler is not None:
+ old_msg_handler(msg_type, msg_log_context, msg_string)
+ else:
+ print(msg_string)
+
+
+old_msg_handler = QtCore.qInstallMessageHandler(qt_message_handler)
+
+
+class Receiver(QtWidgets.QWidget):
+ def __init__(self, socket: PyQt5.QtNetwork.QTcpSocket):
+ super().__init__()
+
+ self.name = '{}:{}'.format(socket.peerAddress().toString(), socket.peerPort())
+
+ print('[{}] Accepted new connection'.format(self.name))
+
+ self.socket = socket
+
+ self.socket.readyRead.connect(self.on_ready_read)
+ self.socket.disconnected.connect(self.on_disconnected)
+ self.socket.error.connect(self.on_error)
+
+ self.header_buffer = bytearray()
+ self.data_buffer = bytearray()
+ self.data_size = 0
+
+ self.state = 0
+
+ self.resize(1000, 600)
+ self.setAttribute(Qt.WA_ShowWithoutActivating)
+ self.setWindowFlag(Qt.WindowStaysOnTopHint, True)
+
+ self.gridLayout = QtWidgets.QGridLayout()
+ self.setLayout(self.gridLayout)
+
+ self.labels = {}
+
+ self.show()
+ print("done")
+
+ def on_ready_read(self):
+ while self.socket.bytesAvailable():
+ if self.state == 0:
+ data = self.socket.read(struct_fmt.size - len(self.header_buffer))
+ self.header_buffer.extend(data)
+
+ if len(self.header_buffer) == struct_fmt.size:
+ self.on_header()
+ else:
+ data = self.socket.read(self.data_size - len(self.data_buffer))
+ self.data_buffer.extend(data)
+
+ if len(self.data_buffer) == self.data_size:
+ try:
+ self.on_buffers()
+ except Exception:
+ print(traceback.format_exc())
+ qApp.exit(-1)
+ return
+
+ def on_header(self):
+ self.header_tuple = struct_fmt.unpack_from(self.header_buffer)
+ idx, w, h, fmtstr, num_planes, p0, p1, p2, p3 = self.header_tuple
+ self.data_size = p0 + p1 + p2 + p3
+ self.header_buffer = bytearray()
+
+ self.state = 1
+
+ def on_buffers(self):
+ idx, w, h, fmtstr, num_planes, p0, p1, p2, p3 = self.header_tuple
+ fmt = fmtstr.decode('ascii')
+
+ print('[{}] cam{} {}x{}-{}'.format(self.name, idx, w, h, fmt))
+
+ if idx not in self.labels:
+ label = QtWidgets.QLabel()
+ label.setSizePolicy(QtWidgets.QSizePolicy.Ignored, QtWidgets.QSizePolicy.Ignored)
+ self.labels[idx] = label
+ self.gridLayout.addWidget(label, self.gridLayout.count() // 2, self.gridLayout.count() % 2)
+
+ label = self.labels[idx]
+
+ pix = data_to_pix(fmt, w, h, self.data_buffer)
+
+ pix = pix.scaled(label.width(), label.height(), Qt.AspectRatioMode.KeepAspectRatio,
+ Qt.TransformationMode.FastTransformation)
+
+ label.setPixmap(pix)
+
+ self.data_buffer = bytearray()
+
+ self.state = 0
+
+ def on_disconnected(self):
+ print('[{}] Disconnected'.format(self.name))
+ self.close()
+ receivers.remove(self)
+
+ def on_error(self):
+ print('[{}] Error: {}'.format(self.name, self.socket.errorString()))
+
+
+def new_connection(tcpServer):
+ clientConnection: PyQt5.QtNetwork.QTcpSocket = tcpServer.nextPendingConnection()
+ w = Receiver(clientConnection)
+ receivers.append(w)
+
+
+def readkey():
+ global qApp
+ sys.stdin.readline()
+ qApp.quit()
+
+
+if __name__ == '__main__':
+ global qApp
+
+ qApp = QtWidgets.QApplication(sys.argv)
+ qApp.setQuitOnLastWindowClosed(False)
+
+ keynotif = QtCore.QSocketNotifier(sys.stdin.fileno(), QtCore.QSocketNotifier.Read)
+ keynotif.activated.connect(readkey)
+
+ tcpServer = PyQt5.QtNetwork.QTcpServer(qApp)
+ tcpServer.listen(PyQt5.QtNetwork.QHostAddress('0.0.0.0'), PORT)
+ tcpServer.newConnection.connect(lambda: new_connection(tcpServer))
+
+ sys.exit(qApp.exec_())
diff --git a/src/py/examples/cam_rx_helpers.py b/src/py/examples/cam_rx_helpers.py
new file mode 100644
index 00000000..293eb63d
--- /dev/null
+++ b/src/py/examples/cam_rx_helpers.py
@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (C) 2023, Tomi Valkeinen <tomi.valkeinen at ideasonboard.com>
+#
+# Debayering code based on PiCamera documentation
+
+from numpy.lib.stride_tricks import as_strided
+from PyQt5 import QtGui
+import numpy as np
+
+
+def demosaic(data, r0, g0, g1, b0):
+ # Separate the components from the Bayer data to RGB planes
+
+ rgb = np.zeros(data.shape + (3,), dtype=data.dtype)
+ rgb[1::2, 0::2, 0] = data[r0[1]::2, r0[0]::2] # Red
+ rgb[0::2, 0::2, 1] = data[g0[1]::2, g0[0]::2] # Green
+ rgb[1::2, 1::2, 1] = data[g1[1]::2, g1[0]::2] # Green
+ rgb[0::2, 1::2, 2] = data[b0[1]::2, b0[0]::2] # Blue
+
+ # Below we present a fairly naive de-mosaic method that simply
+ # calculates the weighted average of a pixel based on the pixels
+ # surrounding it. The weighting is provided by a byte representation of
+ # the Bayer filter which we construct first:
+
+ bayer = np.zeros(rgb.shape, dtype=np.uint8)
+ bayer[1::2, 0::2, 0] = 1 # Red
+ bayer[0::2, 0::2, 1] = 1 # Green
+ bayer[1::2, 1::2, 1] = 1 # Green
+ bayer[0::2, 1::2, 2] = 1 # Blue
+
+ # Allocate an array to hold our output with the same shape as the input
+ # data. After this we define the size of window that will be used to
+ # calculate each weighted average (3x3). Then we pad out the rgb and
+ # bayer arrays, adding blank pixels at their edges to compensate for the
+ # size of the window when calculating averages for edge pixels.
+
+ output = np.empty(rgb.shape, dtype=rgb.dtype)
+ window = (3, 3)
+ borders = (window[0] - 1, window[1] - 1)
+ border = (borders[0] // 2, borders[1] // 2)
+
+ rgb = np.pad(rgb, [
+ (border[0], border[0]),
+ (border[1], border[1]),
+ (0, 0),
+ ], 'constant')
+ bayer = np.pad(bayer, [
+ (border[0], border[0]),
+ (border[1], border[1]),
+ (0, 0),
+ ], 'constant')
+
+ # For each plane in the RGB data, we use a nifty numpy trick
+ # (as_strided) to construct a view over the plane of 3x3 matrices. We do
+ # the same for the bayer array, then use Einstein summation on each
+ # (np.sum is simpler, but copies the data so it's slower), and divide
+ # the results to get our weighted average:
+
+ for plane in range(3):
+ p = rgb[..., plane]
+ b = bayer[..., plane]
+
+ pview = as_strided(p, shape=(
+ p.shape[0] - borders[0],
+ p.shape[1] - borders[1]) + window, strides=p.strides * 2)
+ bview = as_strided(b, shape=(
+ b.shape[0] - borders[0],
+ b.shape[1] - borders[1]) + window, strides=b.strides * 2)
+ psum = np.einsum('ijkl->ij', pview)
+ bsum = np.einsum('ijkl->ij', bview)
+ output[..., plane] = psum // bsum
+
+ return output
+
+
+def convert_raw(data, w, h, fmt):
+ bayer_pattern = fmt[1:5]
+ bitspp = int(fmt[5:])
+
+ if bitspp == 8:
+ data = data.reshape((h, w))
+ data = data.astype(np.uint16)
+ elif bitspp in [10, 12]:
+ data = data.view(np.uint16)
+ data = data.reshape((h, w))
+ else:
+ raise Exception('Bad bitspp:' + str(bitspp))
+
+ idx = bayer_pattern.find('R')
+ assert(idx != -1)
+ r0 = (idx % 2, idx // 2)
+
+ idx = bayer_pattern.find('G')
+ assert(idx != -1)
+ g0 = (idx % 2, idx // 2)
+
+ idx = bayer_pattern.find('G', idx + 1)
+ assert(idx != -1)
+ g1 = (idx % 2, idx // 2)
+
+ idx = bayer_pattern.find('B')
+ assert(idx != -1)
+ b0 = (idx % 2, idx // 2)
+
+ rgb = demosaic(data, r0, g0, g1, b0)
+ rgb = (rgb >> (bitspp - 8)).astype(np.uint8)
+
+ return rgb
+
+
+def convert_yuv444_to_rgb(yuv):
+ m = np.array([
+ [1.0, 1.0, 1.0],
+ [-0.000007154783816076815, -0.3441331386566162, 1.7720025777816772],
+ [1.4019975662231445, -0.7141380310058594, 0.00001542569043522235]
+ ])
+
+ rgb = np.dot(yuv, m)
+ rgb[:, :, 0] -= 179.45477266423404
+ rgb[:, :, 1] += 135.45870971679688
+ rgb[:, :, 2] -= 226.8183044444304
+ rgb = rgb.astype(np.uint8)
+
+ return rgb
+
+
+def convert_yuyv(data, w, h):
+ # YUV422
+ yuyv = data.reshape((h, w // 2 * 4))
+
+ # YUV444
+ yuv = np.empty((h, w, 3), dtype=np.uint8)
+ yuv[:, :, 0] = yuyv[:, 0::2] # Y
+ yuv[:, :, 1] = yuyv[:, 1::4].repeat(2, axis=1) # U
+ yuv[:, :, 2] = yuyv[:, 3::4].repeat(2, axis=1) # V
+
+ return convert_yuv444_to_rgb(yuv)
+
+
+def convert_uyvy(data, w, h):
+ # YUV422
+ yuyv = data.reshape((h, w // 2 * 4))
+
+ # YUV444
+ yuv = np.empty((h, w, 3), dtype=np.uint8)
+ yuv[:, :, 0] = yuyv[:, 1::2] # Y
+ yuv[:, :, 1] = yuyv[:, 0::4].repeat(2, axis=1) # U
+ yuv[:, :, 2] = yuyv[:, 2::4].repeat(2, axis=1) # V
+
+ return convert_yuv444_to_rgb(yuv)
+
+
+def convert_nv12(data, w, h):
+ plane1 = data[:w * h]
+ plane2 = data[w * h:]
+
+ y = plane1.reshape((h, w))
+ uv = plane2.reshape((h // 2, w // 2, 2))
+
+ # YUV444
+ yuv = np.empty((h, w, 3), dtype=np.uint8)
+ yuv[:, :, 0] = y[:, :] # Y
+ yuv[:, :, 1] = uv[:, :, 0].repeat(2, axis=0).repeat(2, axis=1) # U
+ yuv[:, :, 2] = uv[:, :, 1].repeat(2, axis=0).repeat(2, axis=1) # V
+
+ return convert_yuv444_to_rgb(yuv)
+
+
+def to_rgb(fmt, w, h, data):
+ if fmt == 'YUYV':
+ return convert_yuyv(data, w, h)
+
+ if fmt == 'UYVY':
+ return convert_uyvy(data, w, h)
+
+ elif fmt == 'NV12':
+ return convert_nv12(data, w, h)
+
+ elif fmt == 'RGB888':
+ rgb = data.reshape((h, w, 3))
+ rgb[:, :, [0, 1, 2]] = rgb[:, :, [2, 1, 0]]
+
+ elif fmt == 'BGR888':
+ rgb = data.reshape((h, w, 3))
+
+ elif fmt in ['ARGB8888', 'XRGB8888']:
+ rgb = data.reshape((h, w, 4))
+ rgb = np.flip(rgb, axis=2)
+ # drop alpha component
+ rgb = np.delete(rgb, np.s_[0::4], axis=2)
+
+ elif fmt.startswith('S'):
+ return convert_raw(data, w, h, fmt)
+
+ else:
+ raise Exception('Unsupported format ' + fmt)
+
+ return rgb
+
+
+def data_to_rgb(fmt, w, h, data):
+ data = np.frombuffer(data, dtype=np.uint8)
+ rgb = to_rgb(fmt, w, h, data)
+ return rgb
+
+
+def rgb_to_pix(rgb):
+ w = rgb.shape[1]
+ h = rgb.shape[0]
+ qim = QtGui.QImage(rgb, w, h, QtGui.QImage.Format.Format_RGB888)
+ pix = QtGui.QPixmap.fromImage(qim)
+ return pix
+
+
+def data_to_pix(fmt, w, h, data):
+ if fmt == 'MJPEG':
+ pix = QtGui.QPixmap(w, h)
+ pix.loadFromData(data)
+ else:
+ rgb = data_to_rgb(fmt, w, h, data)
+ pix = rgb_to_pix(rgb)
+
+ return pix
--
2.34.1
More information about the libcamera-devel
mailing list