[PATCH v2 3/3] rkisp1: Honor the FrameDurationLimits control
Paul Elder
paul.elder at ideasonboard.com
Fri Feb 21 10:20:45 CET 2025
Add support to rkisp1 for controlling the framerate via the
FrameDurationLimits control.
Signed-off-by: Paul Elder <paul.elder at ideasonboard.com>
Signed-off-by: Kieran Bingham <kieran.bingham at ideasonboard.com>
Reviewed-by: Umang Jain <umang.jain at ideasonboard.com>
---
Changes in v2:
- recover from bitrot
- fix setting frame duration limits in raw mode
---
src/ipa/rkisp1/algorithms/agc.cpp | 60 +++++++++++++++++++-----
src/ipa/rkisp1/algorithms/agc.h | 3 ++
src/ipa/rkisp1/ipa_context.cpp | 16 ++++++-
src/ipa/rkisp1/ipa_context.h | 4 ++
src/ipa/rkisp1/rkisp1.cpp | 2 +
src/libcamera/pipeline/rkisp1/rkisp1.cpp | 1 +
6 files changed, 74 insertions(+), 12 deletions(-)
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp
index 5fece545677e..4e0e3734117e 100644
--- a/src/ipa/rkisp1/algorithms/agc.cpp
+++ b/src/ipa/rkisp1/algorithms/agc.cpp
@@ -190,6 +190,7 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
/* Limit the frame duration to match current initialisation */
ControlInfo &frameDurationLimits = context.ctrlMap[&controls::FrameDurationLimits];
+ context.activeState.agc.minFrameDuration = std::chrono::microseconds(frameDurationLimits.min().get<int64_t>());
context.activeState.agc.maxFrameDuration = std::chrono::microseconds(frameDurationLimits.max().get<int64_t>());
/*
@@ -307,10 +308,21 @@ void Agc::queueRequest(IPAContext &context,
const auto &frameDurationLimits = controls.get(controls::FrameDurationLimits);
if (frameDurationLimits) {
- utils::Duration maxFrameDuration =
- std::chrono::milliseconds((*frameDurationLimits).back());
- agc.maxFrameDuration = maxFrameDuration;
+ /* Limit the control value to the limits in ControlInfo */
+ ControlInfo &limits = context.ctrlMap[&controls::FrameDurationLimits];
+ int64_t minFrameDuration =
+ std::clamp((*frameDurationLimits).front(),
+ limits.min().get<int64_t>(),
+ limits.max().get<int64_t>());
+ int64_t maxFrameDuration =
+ std::clamp((*frameDurationLimits).back(),
+ limits.min().get<int64_t>(),
+ limits.max().get<int64_t>());
+
+ agc.minFrameDuration = std::chrono::microseconds(minFrameDuration);
+ agc.maxFrameDuration = std::chrono::microseconds(maxFrameDuration);
}
+ frameContext.agc.minFrameDuration = agc.minFrameDuration;
frameContext.agc.maxFrameDuration = agc.maxFrameDuration;
}
@@ -387,6 +399,7 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext,
* frameContext.sensor.exposure;
metadata.set(controls::AnalogueGain, frameContext.sensor.gain);
metadata.set(controls::ExposureTime, exposureTime.get<std::micro>());
+ metadata.set(controls::FrameDuration, frameContext.agc.frameDuration.get<std::micro>());
metadata.set(controls::ExposureTimeMode,
frameContext.agc.autoExposureEnabled
? controls::ExposureTimeModeAuto
@@ -396,13 +409,6 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext,
? controls::AnalogueGainModeAuto
: controls::AnalogueGainModeManual);
- /* \todo Use VBlank value calculated from each frame exposure. */
- uint32_t vTotal = context.configuration.sensor.size.height
- + context.configuration.sensor.defVBlank;
- utils::Duration frameDuration = context.configuration.sensor.lineDuration
- * vTotal;
- metadata.set(controls::FrameDuration, frameDuration.get<std::micro>());
-
metadata.set(controls::AeMeteringMode, frameContext.agc.meteringMode);
metadata.set(controls::AeExposureMode, frameContext.agc.exposureMode);
metadata.set(controls::AeConstraintMode, frameContext.agc.constraintMode);
@@ -444,6 +450,27 @@ double Agc::estimateLuminance(double gain) const
return ySum / expMeans_.size() / 255;
}
+/**
+ * \brief Process frame duration and compute vblank
+ * \param[in] context The shared IPA context
+ * \param[in] frameContext The current frame context
+ * \param[in] frameDuration The target frame duration
+ *
+ * Compute and populate vblank from the target frame duration.
+ */
+void Agc::processFrameDuration(IPAContext &context,
+ IPAFrameContext &frameContext,
+ utils::Duration frameDuration)
+{
+ IPACameraSensorInfo &sensorInfo = context.sensorInfo;
+ utils::Duration lineDuration = context.configuration.sensor.lineDuration;
+
+ frameContext.agc.vblank = (frameDuration / lineDuration) - sensorInfo.outputSize.height;
+
+ /* Update frame duration accounting for line length quantization. */
+ frameContext.agc.frameDuration = (sensorInfo.outputSize.height + frameContext.agc.vblank) * lineDuration;
+}
+
/**
* \brief Process RkISP1 statistics, and run AGC operations
* \param[in] context The shared IPA context
@@ -460,6 +487,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
ControlList &metadata)
{
if (!stats) {
+ processFrameDuration(context, frameContext,
+ frameContext.agc.minFrameDuration);
fillMetadata(context, frameContext, metadata);
return;
}
@@ -497,7 +526,9 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
double maxAnalogueGain;
if (frameContext.agc.autoExposureEnabled) {
- minExposureTime = context.configuration.sensor.minExposureTime;
+ minExposureTime = std::clamp(frameContext.agc.minFrameDuration,
+ context.configuration.sensor.minExposureTime,
+ context.configuration.sensor.maxExposureTime);
maxExposureTime = std::clamp(frameContext.agc.maxFrameDuration,
context.configuration.sensor.minExposureTime,
context.configuration.sensor.maxExposureTime);
@@ -541,6 +572,13 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
activeState.agc.automatic.exposure = newExposureTime / lineDuration;
activeState.agc.automatic.gain = aGain;
+ /*
+ * Expand the target frame duration so that we do not run faster than
+ * the minimum frame duration when we have short exposures.
+ */
+ processFrameDuration(context, frameContext,
+ std::max(frameContext.agc.minFrameDuration, newExposureTime));
+
fillMetadata(context, frameContext, metadata);
expMeans_ = {};
}
diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h
index aa86f2c5bc21..62bcde999fe3 100644
--- a/src/ipa/rkisp1/algorithms/agc.h
+++ b/src/ipa/rkisp1/algorithms/agc.h
@@ -50,6 +50,9 @@ private:
void fillMetadata(IPAContext &context, IPAFrameContext &frameContext,
ControlList &metadata);
double estimateLuminance(double gain) const override;
+ void processFrameDuration(IPAContext &context,
+ IPAFrameContext &frameContext,
+ utils::Duration frameDuration);
Span<const uint8_t> expMeans_;
diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
index 261c0472a4fc..99611bd5b390 100644
--- a/src/ipa/rkisp1/ipa_context.cpp
+++ b/src/ipa/rkisp1/ipa_context.cpp
@@ -180,6 +180,9 @@ namespace libcamera::ipa::rkisp1 {
* \var IPAActiveState::agc.meteringMode
* \brief Metering mode as set by the AeMeteringMode control
*
+ * \var IPAActiveState::agc.minFrameDuration
+ * \brief Minimum frame duration as set by the FrameDurationLimits control
+ *
* \var IPAActiveState::agc.maxFrameDuration
* \brief Maximum frame duration as set by the FrameDurationLimits control
*/
@@ -282,7 +285,9 @@ namespace libcamera::ipa::rkisp1 {
* \brief Automatic Gain Control parameters for this frame
*
* The exposure and gain are provided by the AGC algorithm, and are to be
- * applied to the sensor in order to take effect for this frame.
+ * applied to the sensor in order to take effect for this frame. Additionally
+ * the vertical blanking period is determined to maintain a consistent frame
+ * rate matched to the FrameDurationLimits as set by the user.
*
* \var IPAFrameContext::agc.exposure
* \brief Exposure time expressed as a number of lines computed by the algorithm
@@ -292,6 +297,9 @@ namespace libcamera::ipa::rkisp1 {
*
* The gain should be adapted to the sensor specific gain code before applying.
*
+ * \var IPAFrameContext::agc.vblank
+ * \brief Vertical blanking parameter computed by the algorithm
+ *
* \var IPAFrameContext::agc.autoExposureEnabled
* \brief Manual/automatic AGC state (exposure) as set by the ExposureTimeMode control
*
@@ -307,9 +315,15 @@ namespace libcamera::ipa::rkisp1 {
* \var IPAFrameContext::agc.meteringMode
* \brief Metering mode as set by the AeMeteringMode control
*
+ * \var IPAFrameContext::agc.minFrameDuration
+ * \brief Minimum frame duration as set by the FrameDurationLimits control
+ *
* \var IPAFrameContext::agc.maxFrameDuration
* \brief Maximum frame duration as set by the FrameDurationLimits control
*
+ * \var IPAFrameContext::agc.frameDuration
+ * \brief The actual FrameDuration used by the algorithm for the frame
+ *
* \var IPAFrameContext::agc.updateMetering
* \brief Indicate if new ISP AGC metering parameters need to be applied
*
diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
index c765b928a55f..474f7036f003 100644
--- a/src/ipa/rkisp1/ipa_context.h
+++ b/src/ipa/rkisp1/ipa_context.h
@@ -84,6 +84,7 @@ struct IPAActiveState {
controls::AeConstraintModeEnum constraintMode;
controls::AeExposureModeEnum exposureMode;
controls::AeMeteringModeEnum meteringMode;
+ utils::Duration minFrameDuration;
utils::Duration maxFrameDuration;
} agc;
@@ -125,12 +126,15 @@ struct IPAFrameContext : public FrameContext {
struct {
uint32_t exposure;
double gain;
+ uint32_t vblank;
bool autoExposureEnabled;
bool autoGainEnabled;
controls::AeConstraintModeEnum constraintMode;
controls::AeExposureModeEnum exposureMode;
controls::AeMeteringModeEnum meteringMode;
+ utils::Duration minFrameDuration;
utils::Duration maxFrameDuration;
+ utils::Duration frameDuration;
bool updateMetering;
bool autoExposureModeChange;
bool autoGainModeChange;
diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp
index 0e761249d27c..7547d2f274f4 100644
--- a/src/ipa/rkisp1/rkisp1.cpp
+++ b/src/ipa/rkisp1/rkisp1.cpp
@@ -453,10 +453,12 @@ void IPARkISP1::setControls(unsigned int frame)
IPAFrameContext &frameContext = context_.frameContexts.get(frame);
uint32_t exposure = frameContext.agc.exposure;
uint32_t gain = context_.camHelper->gainCode(frameContext.agc.gain);
+ uint32_t vblank = frameContext.agc.vblank;
ControlList ctrls(sensorControls_);
ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure));
ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain));
+ ctrls.set(V4L2_CID_VBLANK, static_cast<int32_t>(vblank));
setSensorControls.emit(frame, ctrls);
}
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index 1ac8d8ae7ed9..52633fe3cb85 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -1325,6 +1325,7 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
std::unordered_map<uint32_t, DelayedControls::ControlParams> params = {
{ V4L2_CID_ANALOGUE_GAIN, { delays.gainDelay, false } },
{ V4L2_CID_EXPOSURE, { delays.exposureDelay, false } },
+ { V4L2_CID_VBLANK, { 1, false } },
};
data->delayedCtrls_ =
--
2.39.2
More information about the libcamera-devel
mailing list