[PATCH v3 3/3] libcamera: android: Add face detection control support

[Harvey Yang]

Allow Android HAL adapter to pass the face detection metadata control to
the pipeline and also send face detection metadata to the camera client
if the pipeline generates it.

Signed-off-by: Yudhistira Erlandinata <yerlandinata at chromium.org>
Co-developed-by: Harvey Yang <chenghaoyang at chromium.org>
---
 src/android/camera_capabilities.cpp | 45 +++++++++++++++++++--
 src/android/camera_device.cpp       | 61 ++++++++++++++++++++++++++++-
 2 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/src/android/camera_capabilities.cpp b/src/android/camera_capabilities.cpp
index 71043e127..a89a3115d 100644
--- a/src/android/camera_capabilities.cpp
+++ b/src/android/camera_capabilities.cpp
@@ -7,6 +7,8 @@
 
 #include "camera_capabilities.h"
 
+#include <stdint.h>
+
 #include <algorithm>
 #include <array>
 #include <cmath>
@@ -1176,11 +1178,46 @@ int CameraCapabilities::initializeStaticMetadata()
 				  maxFrameDuration_);
 
 	/* Statistics static metadata. */
-	uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
-				  faceDetectMode);
-
 	int32_t maxFaceCount = 0;
+	auto iter = camera_->controls().find(controls::draft::FaceDetectMode.id());
+	if (iter != camera_->controls().end()) {
+		const ControlInfo &faceDetectCtrlInfo = iter->second;
+		std::vector<uint8_t> faceDetectModes;
+		bool hasFaceDetection = false;
+
+		for (const auto &value : faceDetectCtrlInfo.values()) {
+			uint8_t mode = value.get<uint8_t>();
+			uint8_t androidMode = 0;
+
+			switch (mode) {
+			case controls::draft::FaceDetectModeOff:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+				break;
+			case controls::draft::FaceDetectModeSimple:
+				androidMode = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;
+				hasFaceDetection = true;
+				break;
+			default:
+				LOG(HAL, Fatal) << "Received invalid face detect mode: " << mode;
+			}
+			faceDetectModes.push_back(androidMode);
+		}
+		if (hasFaceDetection) {
+			/*
+			 * \todo Create new libcamera controls to query max
+			 * possible faces detected.
+			 */
+			maxFaceCount = 10;
+			staticMetadata_->addEntry(
+				ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+				faceDetectModes.data(), faceDetectModes.size());
+		}
+	} else {
+		uint8_t faceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
+		staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES,
+					  faceDetectMode);
+	}
+
 	staticMetadata_->addEntry(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT,
 				  maxFaceCount);
 
diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp
index 493f66e7b..87b087160 100644
--- a/src/android/camera_device.cpp
+++ b/src/android/camera_device.cpp
@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <libcamera/base/log.h>
+#include <libcamera/base/span.h>
 #include <libcamera/base/unique_fd.h>
 #include <libcamera/base/utils.h>
 
@@ -22,6 +23,7 @@
 #include <libcamera/controls.h>
 #include <libcamera/fence.h>
 #include <libcamera/formats.h>
+#include <libcamera/geometry.h>
 #include <libcamera/property_ids.h>
 
 #include "system/graphics.h"
@@ -813,6 +815,11 @@ int CameraDevice::processControls(Camera3RequestDescriptor *descriptor)
 		controls.set(controls::ScalerCrop, cropRegion);
 	}
 
+	if (settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry)) {
+		const uint8_t *data = entry.data.u8;
+		controls.set(controls::draft::FaceDetectMode, data[0]);
+	}
+
 	if (settings.getEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, &entry)) {
 		const int32_t data = *entry.data.i32;
 		int32_t testPatternMode = controls::draft::TestPatternModeOff;
@@ -1540,8 +1547,9 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 	value32 = ANDROID_SENSOR_TEST_PATTERN_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_SENSOR_TEST_PATTERN_MODE, value32);
 
-	value = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
-	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, value);
+	settings.getEntry(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry);
+	resultMetadata->addEntry(ANDROID_STATISTICS_FACE_DETECT_MODE,
+				 entry.data.u8, 1);
 
 	value = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF;
 	resultMetadata->addEntry(ANDROID_STATISTICS_LENS_SHADING_MAP_MODE,
@@ -1580,6 +1588,55 @@ CameraDevice::getResultMetadata(const Camera3RequestDescriptor &descriptor) cons
 		resultMetadata->addEntry(ANDROID_SENSOR_FRAME_DURATION,
 					 *frameDuration * 1000);
 
+	const auto &faceDetectRectangles =
+		metadata.get(controls::draft::FaceDetectFaceRectangles);
+	if (faceDetectRectangles) {
+		const Span<const Rectangle> rectangles = *faceDetectRectangles;
+		std::vector<int32_t> flatRectangles;
+		for (const Rectangle &rect : rectangles) {
+			flatRectangles.push_back(rect.x);
+			flatRectangles.push_back(rect.y);
+			flatRectangles.push_back(rect.x + rect.width);
+			flatRectangles.push_back(rect.y + rect.height);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_RECTANGLES, flatRectangles);
+	}
+
+	const auto &faceDetectFaceScores =
+		metadata.get(controls::draft::FaceDetectFaceScores);
+	if (faceDetectRectangles && faceDetectFaceScores) {
+		const Span<const uint8_t> &scores = *faceDetectFaceScores;
+		if (scores.size() != faceDetectRectangles->size()) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face scores; "
+					<< "Expected: "
+					<< faceDetectRectangles->size()
+					<< ", got:" << scores.size();
+		}
+		resultMetadata->addEntry(ANDROID_STATISTICS_FACE_SCORES, scores);
+	}
+
+	const auto &faceDetectFaceLandmarks =
+		metadata.get(controls::draft::FaceDetectFaceLandmark);
+	if (faceDetectFaceScores && faceDetectRectangles &&
+	    faceDetectFaceLandmarks) {
+		const auto &landmarks = *faceDetectFaceLandmarks;
+		size_t expectedLandmarks = faceDetectRectangles->size() * 3;
+		if (landmarks.size() != expectedLandmarks) {
+			LOG(HAL, Error) << "Pipeline returned wrong number of face landmarks; "
+					<< "Expected: " << expectedLandmarks
+					<< ", got: " << landmarks.size();
+		}
+
+		std::vector<int32_t> androidLandmarks;
+		for (const auto &landmark : landmarks) {
+			androidLandmarks.push_back(landmark.x);
+			androidLandmarks.push_back(landmark.y);
+		}
+		resultMetadata->addEntry(
+			ANDROID_STATISTICS_FACE_LANDMARKS, androidLandmarks);
+	}
+
 	const auto &scalerCrop = metadata.get(controls::ScalerCrop);
 	if (scalerCrop) {
 		const Rectangle &crop = *scalerCrop;
-- 
2.46.0.469.g59c65b2a67-goog