[PATCH v5 3/3] libcamera: android: Add face detection control support
Harvey Yang
chenghaoyang at chromium.org
Mon Sep 23 11:30:46 CEST 2024
From: Harvey Yang <chenghaoyang at chromium.org>
Allow Android HAL adapter to pass the face detection metadata control to
the pipeline and also send face detection metadata to the camera client
if the pipeline generates it.
Signed-off-by: Yudhistira Erlandinata <yerlandinata at chromium.org>
Co-developed-by: Harvey Yang <chenghaoyang at chromium.org>
---
src/android/camera_capabilities.cpp | 45 +++++++++++++++--
src/android/camera_device.cpp | 75 ++++++++++++++++++++++++++++-
2 files changed, 114 insertions(+), 6 deletions(-)
diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
index 71043e12..a89a3115 100644
--- a/src/android/camera_capabilities.cpp
+++ b/src/android/camera_capabilities.cpp
@@ -7,6 +7,8 @@
#include "camera_capabilities.h"
+#include <stdint.h>
+
#include <algorithm>
#include <array>
#include <cmath>
@@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata()
maxFrameDuration_);
/* Statistics static metadata. */
- uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
- staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
- faceDetectMode);
-
int32_t maxFaceCount = 0;
+ auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
+ if (iter != camera_->controls().end()) {
+ const ControlInfo &faceDetectCtrlInfo = iter->second;
+ std::vector<uint8_t> faceDetectModes;
+ bool hasFaceDetection = false;
+
+ for (const auto &value : faceDetectCtrlInfo.values()) {
+ uint8_t mode = value.get<uint8_t>();
+ uint8_t androidMode = 0;
+
+ switch (mode) {
+ case controls::draft::FaceDetectModeOff:
+ androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+ break;
+ case controls::draft::FaceDetectModeSimple:
+ androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
+ hasFaceDetection = true;
+ break;
+ default:
+ LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
+ }
+ faceDetectModes.push_back(androidMode);
+ }
+ if (hasFaceDetection) {
+ /*
+ * \todo Create new libcamera controls to query max
+ * possible faces detected.
+ */
+ maxFaceCount = 10;
+ staticMetadata_->addEntry(
+ ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+ faceDetectModes.data(), faceDetectModes.size());
+ }
+ } else {
+ uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+ staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+ faceDetectMode);
+ }
+
staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
maxFaceCount);
diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index 493f66e7..50caa14a 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -15,6 +15,7 @@
#include <vector>
#include <libcamera/base/log.h>
+#include <libcamera/base/span.h>
#include <libcamera/base/unique_fd.h>
#include <libcamera/base/utils.h>
@@ -22,6 +23,7 @@
#include <libcamera/controls.h>
#include <libcamera/fence.h>
#include <libcamera/formats.h>
+#include <libcamera/geometry.h>
#include <libcamera/property_ids.h>
#include "system/graphics.h"
@@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
controls.set(controls::ScalerCrop, cropRegion);
}
+ if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
+ const uint8_t *data = entry.data.u8;
+ controls.set(controls::draft::FaceDetectMode, data[0]);
+ }
+
if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
const int32_t data = *entry.data.i32;
int32_t testPatternMode = controls::draft::TestPatternModeOff;
@@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
- value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
- resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
+ settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry);
+ resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
+ entry.data.u8, 1);
value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
@@ -1580,6 +1588,69 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
*frameDuration * 1000);
+ const auto &faceDetectRectangles =
+ metadata.get(controls::draft::FaceDetectFaceRectangles);
+ if (faceDetectRectangles) {
+ const Span<const Rectangle> rectangles = *faceDetectRectangles;
+ std::vector<int32_t> flatRectangles;
+ for (const Rectangle &rect : rectangles) {
+ flatRectangles.push_back(rect.x);
+ flatRectangles.push_back(rect.y);
+ flatRectangles.push_back(rect.x + rect.width);
+ flatRectangles.push_back(rect.y + rect.height);
+ }
+ resultMetadata->addEntry(
+ ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
+ }
+
+ const auto &faceDetectFaceScores =
+ metadata.get(controls::draft::FaceDetectFaceScores);
+ if (faceDetectRectangles && faceDetectFaceScores) {
+ const Span<const uint8_t> &scores = *faceDetectFaceScores;
+ if (scores.size() != faceDetectRectangles->size()) {
+ LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
+ << "Expected: "
+ << faceDetectRectangles->size()
+ << ", got:" << scores.size();
+ }
+ resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores);
+ }
+
+ const auto &faceDetectFaceLandmarks =
+ metadata.get(controls::draft::FaceDetectFaceLandmarks);
+ if (faceDetectRectangles && faceDetectFaceScores &&
+ faceDetectFaceLandmarks) {
+ const auto &landmarks = *faceDetectFaceLandmarks;
+ size_t expectedLandmarks = faceDetectRectangles->size() * 3;
+ if (landmarks.size() != expectedLandmarks) {
+ LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
+ << "Expected: " << expectedLandmarks
+ << ", got: " << landmarks.size();
+ }
+
+ std::vector<int32_t> androidLandmarks;
+ for (const auto &landmark : landmarks) {
+ androidLandmarks.push_back(landmark.x);
+ androidLandmarks.push_back(landmark.y);
+ }
+ resultMetadata->addEntry(
+ ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
+ }
+
+ const auto &faceDetectFaceIds =
+ metadata.get(controls::draft::FaceDetectFaceIds);
+ if (faceDetectRectangles && faceDetectFaceScores &&
+ faceDetectFaceLandmarks && faceDetectFaceIds) {
+ const Span<const int32_t> &ids = *faceDetectFaceIds;
+ if (ids.size() != faceDetectRectangles->size()) {
+ LOG(HAL, Error) << "Pipeline returned wrong number of face ids; "
+ << "Expected: "
+ << faceDetectRectangles->size()
+ << ", got:" << ids.size();
+ }
+ resultMetadata->addEntry(ANDROID_STATISTICS_FACE_IDS, ids);
+ }
+
const auto &scalerCrop = metadata.get(controls::ScalerCrop);
if (scalerCrop) {
const Rectangle &crop = *scalerCrop;
--
2.46.0.792.g87dc391469-goog
More information about the libcamera-devel
mailing list