[PATCH 15 5/7] libcamera: virtual: Add ImageFrameGenerator
Harvey Yang
chenghaoyang at chromium.org
Fri Oct 4 11:59:18 CEST 2024
Besides TestPatternGenerator, this patch adds ImageFrameGenerator that
loads real images (jpg / jpeg for now) as the source and generates
scaled frames.
Signed-off-by: Konami Shu <konamiz at google.com>
Co-developed-by: Harvey Yang <chenghaoyang at chromium.org>
Co-developed-by: Yunke Cao <yunkec at chromium.org>
Co-developed-by: Tomasz Figa <tfiga at chromium.org>
Reviewed-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
---
.../virtual/image_frame_generator.cpp | 179 ++++++++++++++++++
.../pipeline/virtual/image_frame_generator.h | 51 +++++
src/libcamera/pipeline/virtual/meson.build | 4 +
3 files changed, 234 insertions(+)
create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.cpp
create mode 100644 src/libcamera/pipeline/virtual/image_frame_generator.h
diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.cpp b/src/libcamera/pipeline/virtual/image_frame_generator.cpp
new file mode 100644
index 000000000..04382beb7
--- /dev/null
+++ b/src/libcamera/pipeline/virtual/image_frame_generator.cpp
@@ -0,0 +1,179 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2024, Google Inc.
+ *
+ * image_frame_generator.cpp - Derived class of FrameGenerator for
+ * generating frames from images
+ */
+
+#include "image_frame_generator.h"
+
+#include <string>
+
+#include <libcamera/base/file.h>
+#include <libcamera/base/log.h>
+
+#include <libcamera/framebuffer.h>
+
+#include "libcamera/internal/mapped_framebuffer.h"
+
+#include "libyuv/convert.h"
+#include "libyuv/scale.h"
+
+namespace libcamera {
+
+LOG_DECLARE_CATEGORY(Virtual)
+
+/*
+ * Factory function to create an ImageFrameGenerator object.
+ * Read the images and convert them to buffers in NV12 format.
+ * Store the pointers to the buffers to a list (imageFrameDatas)
+ */
+std::unique_ptr<ImageFrameGenerator>
+ImageFrameGenerator::create(ImageFrames &imageFrames)
+{
+ std::unique_ptr<ImageFrameGenerator> imageFrameGenerator =
+ std::make_unique<ImageFrameGenerator>();
+ imageFrameGenerator->imageFrames_ = &imageFrames;
+
+ /*
+ * For each file in the directory, load the image,
+ * convert it to NV12, and store the pointer.
+ */
+ for (unsigned int i = 0; i < imageFrames.number.value_or(1); i++) {
+ std::filesystem::path path;
+ if (!imageFrames.number)
+ /* If the path is to an image */
+ path = imageFrames.path;
+ else
+ /* If the path is to a directory */
+ path = imageFrames.path / (std::to_string(i) + ".jpg");
+
+ File file(path);
+ if (!file.open(File::OpenModeFlag::ReadOnly)) {
+ LOG(Virtual, Error) << "Failed to open image file " << file.fileName()
+ << ": " << strerror(file.error());
+ return nullptr;
+ }
+
+ /* Read the image file to data */
+ auto fileSize = file.size();
+ auto buffer = std::make_unique<uint8_t[]>(fileSize);
+ if (file.read({ buffer.get(), static_cast<size_t>(fileSize) }) != fileSize) {
+ LOG(Virtual, Error) << "Failed to read file " << file.fileName()
+ << ": " << strerror(file.error());
+ return nullptr;
+ }
+
+ /* Get the width and height of the image */
+ int width, height;
+ if (libyuv::MJPGSize(buffer.get(), fileSize, &width, &height)) {
+ LOG(Virtual, Error) << "Failed to get the size of the image file: "
+ << file.fileName();
+ return nullptr;
+ }
+
+ std::unique_ptr<uint8_t[]> dstY =
+ std::make_unique<uint8_t[]>(width * height);
+ std::unique_ptr<uint8_t[]> dstUV =
+ std::make_unique<uint8_t[]>(width * height / 2);
+ int ret = libyuv::MJPGToNV12(buffer.get(), fileSize,
+ dstY.get(), width, dstUV.get(),
+ width, width, height, width, height);
+ if (ret != 0)
+ LOG(Virtual, Error) << "MJPGToNV12() failed with " << ret;
+
+ imageFrameGenerator->imageFrameDatas_.emplace_back(
+ ImageFrameData{ std::move(dstY), std::move(dstUV),
+ Size(width, height) });
+ }
+
+ return imageFrameGenerator;
+}
+
+/*
+ * \var ImageFrameGenerator::frameRepeat
+ * \brief Number of frames to repeat before proceeding to the next frame
+ */
+
+/* Scale the buffers for image frames. */
+void ImageFrameGenerator::configure(const Size &size)
+{
+ /* Reset the source images to prevent multiple configuration calls */
+ scaledFrameDatas_.clear();
+ frameIndex_ = 0;
+ parameter_ = 0;
+
+ for (unsigned int i = 0; i < imageFrames_->number.value_or(1); i++) {
+ /* Scale the imageFrameDatas_ to scaledY and scaledUV */
+ unsigned int halfSizeWidth = (size.width + 1) / 2;
+ unsigned int halfSizeHeight = (size.height + 1) / 2;
+ std::unique_ptr<uint8_t[]> scaledY =
+ std::make_unique<uint8_t[]>(size.width * size.height);
+ std::unique_ptr<uint8_t[]> scaledUV =
+ std::make_unique<uint8_t[]>(halfSizeWidth * halfSizeHeight * 2);
+ auto &src = imageFrameDatas_[i];
+
+ /*
+ * \todo Some platforms might enforce stride due to GPU.
+ * The width needs to be a multiple of the stride to work
+ * properly for now.
+ */
+ libyuv::NV12Scale(src.Y.get(), src.size.width,
+ src.UV.get(), src.size.width,
+ src.size.width, src.size.height,
+ scaledY.get(), size.width, scaledUV.get(), size.width,
+ size.width, size.height, libyuv::FilterMode::kFilterBilinear);
+
+ scaledFrameDatas_.emplace_back(
+ ImageFrameData{ std::move(scaledY), std::move(scaledUV), size });
+ }
+}
+
+int ImageFrameGenerator::generateFrame(const Size &size, const FrameBuffer *buffer)
+{
+ ASSERT(!scaledFrameDatas_.empty());
+
+ MappedFrameBuffer mappedFrameBuffer(buffer, MappedFrameBuffer::MapFlag::Write);
+
+ auto planes = mappedFrameBuffer.planes();
+
+ /* Loop only around the number of images available */
+ frameIndex_ %= imageFrames_->number.value_or(1);
+
+ /* Write the scaledY and scaledUV to the mapped frame buffer */
+ libyuv::NV12Copy(scaledFrameDatas_[frameIndex_].Y.get(), size.width,
+ scaledFrameDatas_[frameIndex_].UV.get(), size.width, planes[0].begin(),
+ size.width, planes[1].begin(), size.width,
+ size.width, size.height);
+
+ /* Proceed to the next image every 4 frames */
+ /* \todo Consider setting the frameRepeat in the config file */
+ parameter_++;
+ if (parameter_ % frameRepeat == 0)
+ frameIndex_++;
+
+ return 0;
+}
+
+/*
+ * \var ImageFrameGenerator::imageFrameDatas_
+ * \brief List of pointers to the not scaled image buffers
+ */
+
+/*
+ * \var ImageFrameGenerator::scaledFrameDatas_
+ * \brief List of pointers to the scaled image buffers
+ */
+
+/*
+ * \var ImageFrameGenerator::imageFrames_
+ * \brief Pointer to the imageFrames_ in VirtualCameraData
+ */
+
+/*
+ * \var ImageFrameGenerator::parameter_
+ * \brief Speed parameter. Change to the next image every parameter_ frames
+ */
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/virtual/image_frame_generator.h b/src/libcamera/pipeline/virtual/image_frame_generator.h
new file mode 100644
index 000000000..a68094a66
--- /dev/null
+++ b/src/libcamera/pipeline/virtual/image_frame_generator.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2024, Google Inc.
+ *
+ * image_frame_generator.h - Derived class of FrameGenerator for
+ * generating frames from images
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <stdint.h>
+#include <sys/types.h>
+
+#include "frame_generator.h"
+
+namespace libcamera {
+
+/* Frame configuration provided by the config file */
+struct ImageFrames {
+ std::filesystem::path path;
+ std::optional<unsigned int> number;
+};
+
+class ImageFrameGenerator : public FrameGenerator
+{
+public:
+ static std::unique_ptr<ImageFrameGenerator> create(ImageFrames &imageFrames);
+
+private:
+ static constexpr unsigned int frameRepeat = 4;
+
+ struct ImageFrameData {
+ std::unique_ptr<uint8_t[]> Y;
+ std::unique_ptr<uint8_t[]> UV;
+ Size size;
+ };
+
+ void configure(const Size &size) override;
+ int generateFrame(const Size &size, const FrameBuffer *buffer) override;
+
+ std::vector<ImageFrameData> imageFrameDatas_;
+ std::vector<ImageFrameData> scaledFrameDatas_;
+ ImageFrames *imageFrames_;
+ unsigned int frameIndex_;
+ unsigned int parameter_;
+};
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/virtual/meson.build b/src/libcamera/pipeline/virtual/meson.build
index 0c537777f..bb38c193c 100644
--- a/src/libcamera/pipeline/virtual/meson.build
+++ b/src/libcamera/pipeline/virtual/meson.build
@@ -1,8 +1,12 @@
# SPDX-License-Identifier: CC0-1.0
libcamera_internal_sources += files([
+ 'image_frame_generator.cpp',
'test_pattern_generator.cpp',
'virtual.cpp',
])
+libjpeg = dependency('libjpeg', required : true)
+
libcamera_deps += [libyuv_dep]
+libcamera_deps += [libjpeg]
--
2.47.0.rc0.187.ge670bccf7e-goog
More information about the libcamera-devel
mailing list