[libcamera-devel] [PATCH v1 6/8] ipa: rkisp1: Introduce AGC
Jean-Michel Hautbois
jeanmichel.hautbois at ideasonboard.com
Fri Nov 19 12:16:52 CET 2021
Now that we have IPAContext and Algorithm, we can implement a simple AGC
based on the IPU3 one. It is very similar, except that there is no
histogram used for an inter quantile mean. The RkISP1 is returning a 5x5
array (for V10) of luminance means. Estimating the relative luminance is
thus a simple mean of all the blocks already calculated by the ISP.
Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
---
src/ipa/rkisp1/algorithms/agc.cpp | 249 ++++++++++++++++++++++++++
src/ipa/rkisp1/algorithms/agc.h | 55 ++++++
src/ipa/rkisp1/algorithms/meson.build | 1 +
src/ipa/rkisp1/ipa_context.cpp | 44 +++++
src/ipa/rkisp1/ipa_context.h | 15 ++
src/ipa/rkisp1/rkisp1.cpp | 72 ++++----
6 files changed, 401 insertions(+), 35 deletions(-)
create mode 100644 src/ipa/rkisp1/algorithms/agc.cpp
create mode 100644 src/ipa/rkisp1/algorithms/agc.h
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp
new file mode 100644
index 00000000..3d4a17ff
--- /dev/null
+++ b/src/ipa/rkisp1/algorithms/agc.cpp
@@ -0,0 +1,249 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2021, Ideas On Board
+ *
+ * agc.cpp - AGC/AEC mean-based control algorithm
+ */
+
+#include "agc.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+
+#include <libcamera/base/log.h>
+
+#include <libcamera/ipa/core_ipa_interface.h>
+
+/**
+ * \file agc.h
+ */
+
+namespace libcamera {
+
+using namespace std::literals::chrono_literals;
+
+namespace ipa::rkisp1::algorithms {
+
+/**
+ * \class Agc
+ * \brief A mean-based auto-exposure algorithm
+ */
+
+LOG_DEFINE_CATEGORY(RkISP1Agc)
+
+/* Limits for analogue gain values */
+static constexpr double kMinAnalogueGain = 1.0;
+static constexpr double kMaxAnalogueGain = 8.0;
+
+/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */
+static constexpr utils::Duration kMaxShutterSpeed = 60ms;
+
+/* Number of frames to wait before calculating stats on minimum exposure */
+static constexpr uint32_t kNumStartupFrames = 10;
+
+/* Maximum luminance used for brightness normalization */
+static constexpr double kMaxLuminance = 255.0;
+
+/*
+ * Normalized luma value target.
+ *
+ * It's a number that's chosen so that, when the camera points at a grey
+ * target, the resulting image brightness is considered right.
+ */
+static constexpr double kNormalizedLumaTarget = 0.4;
+
+Agc::Agc()
+ : frameCount_(0), lineDuration_(0s), minShutterSpeed_(0s),
+ maxShutterSpeed_(0s), filteredExposure_(0s), currentExposure_(0s)
+{
+}
+
+/**
+ * \brief Configure the AGC given a configInfo
+ * \param[in] context The shared IPA context
+ * \param[in] configInfo The IPA configuration data
+ *
+ * \return 0
+ */
+int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo)
+{
+ /* \todo use the IPAContext to provide the limits */
+ lineDuration_ = configInfo.lineLength * 1.0s / configInfo.pixelRate;
+
+ minShutterSpeed_ = context.configuration.agc.minShutterSpeed;
+ maxShutterSpeed_ = std::min(context.configuration.agc.maxShutterSpeed,
+ kMaxShutterSpeed);
+
+ minAnalogueGain_ = std::max(context.configuration.agc.minAnalogueGain, kMinAnalogueGain);
+ maxAnalogueGain_ = std::min(context.configuration.agc.maxAnalogueGain, kMaxAnalogueGain);
+
+ /* Configure the default exposure and gain. */
+ context.frameContext.agc.gain = minAnalogueGain_;
+ context.frameContext.agc.exposure = 10ms / lineDuration_;
+
+ return 0;
+}
+
+/**
+ * \brief Apply a filter on the exposure value to limit the speed of changes
+ */
+void Agc::filterExposure()
+{
+ double speed = 0.2;
+
+ /* Adapt instantly if we are in startup phase */
+ if (frameCount_ < kNumStartupFrames)
+ speed = 1.0;
+
+ if (filteredExposure_ == 0s) {
+ /* DG stands for digital gain.*/
+ filteredExposure_ = currentExposure_;
+ } else {
+ /*
+ * If we are close to the desired result, go faster to avoid making
+ * multiple micro-adjustments.
+ * \todo Make this customisable?
+ */
+ if (filteredExposure_ < 1.2 * currentExposure_ &&
+ filteredExposure_ > 0.8 * currentExposure_)
+ speed = sqrt(speed);
+
+ filteredExposure_ = speed * currentExposure_ +
+ filteredExposure_ * (1.0 - speed);
+ }
+
+ LOG(RkISP1Agc, Debug) << "After filtering, total_exposure " << filteredExposure_;
+}
+
+/**
+ * \brief Estimate the new exposure and gain values
+ * \param[inout] frameContext The shared IPA frame Context
+ * \param[in] currentYGain The gain calculated on the current brightness level
+ */
+void Agc::computeExposure(IPAFrameContext &frameContext, double currentYGain)
+{
+ /* Get the effective exposure and gain applied on the sensor. */
+ uint32_t exposure = frameContext.sensor.exposure;
+ double analogueGain = frameContext.sensor.gain;
+
+ /* Consider within 1% of the target as correctly exposed */
+ if (std::abs(currentYGain - 1.0) < 0.01)
+ LOG(RkISP1Agc, Debug) << "We are well exposed (iqMean = "
+ << currentYGain << ")";
+
+ /* extracted from Rpi::Agc::computeTargetExposure */
+
+ /* Calculate the shutter time in seconds */
+ utils::Duration currentShutter = exposure * lineDuration_;
+
+ /*
+ * Update the exposure value for the next computation using the values
+ * of exposure and gain really used by the sensor.
+ */
+ utils::Duration effectiveExposureValue = currentShutter * analogueGain;
+
+ LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
+ << " Shutter speed " << currentShutter
+ << " Gain " << analogueGain
+ << " Needed ev gain " << currentYGain;
+
+ /*
+ * Calculate the current exposure value for the scene as the latest
+ * exposure value applied multiplied by the new estimated gain.
+ */
+ currentExposure_ = effectiveExposureValue * currentYGain;
+
+ /* Clamp the exposure value to the min and max authorized */
+ utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
+ currentExposure_ = std::min(currentExposure_, maxTotalExposure);
+ LOG(RkISP1Agc, Debug) << "Target total exposure " << currentExposure_
+ << ", maximum is " << maxTotalExposure;
+
+ /* \todo: estimate if we need to desaturate */
+ filterExposure();
+
+ /* Divide the exposure value as new exposure and gain values */
+ utils::Duration exposureValue = filteredExposure_;
+ utils::Duration shutterTime;
+
+ /*
+ * Push the shutter time up to the maximum first, and only then
+ * increase the gain.
+ */
+ shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
+ minShutterSpeed_, maxShutterSpeed_);
+ double stepGain = std::clamp(exposureValue / shutterTime,
+ minAnalogueGain_, maxAnalogueGain_);
+ LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are "
+ << shutterTime << " and "
+ << stepGain;
+
+ /* Update the estimated exposure and gain. */
+ frameContext.agc.exposure = shutterTime / lineDuration_;
+ frameContext.agc.gain = stepGain;
+}
+
+/**
+ * \brief Estimate the average brightness of the frame
+ * \param[in] ae The RkISP1 statistics and ISP results
+ * \param[in] currentYGain The gain calculated on the current brightness level
+ * \return The normalized luma
+ */
+double Agc::computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain)
+{
+ double ySum = 0.0;
+ unsigned int num = 0;
+
+ for (unsigned int aeCell = 0; aeCell < RKISP1_CIF_ISP_AE_MEAN_MAX_V10; aeCell++) {
+ ySum += ae->exp_mean[aeCell] * currentYGain;
+ num++;
+ }
+
+ /* \todo Weight with the AWB gains */
+
+ /* Return the normalized relative luminance. */
+ return ySum / num / kMaxLuminance;
+}
+
+/**
+ * \brief Process RkISP1 statistics, and run AGC operations
+ * \param[in] context The shared IPA context
+ * \param[in] stats The RKIsp1 statistics and ISP results
+ *
+ * Identify the current image brightness, and use that to estimate the optimal
+ * new exposure and gain for the scene.
+ */
+void Agc::process(IPAContext &context, const rkisp1_stat_buffer *stats)
+{
+ const rkisp1_cif_isp_stat *params = &stats->params;
+ ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP);
+
+ const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae;
+
+ double currentYGain = 1.0;
+ double targetY = kNormalizedLumaTarget;
+
+ /*
+ * Do this calculation a few times as brightness increase can be
+ * non-linear when there are saturated regions.
+ */
+ for (int i = 0; i < 8; i++) {
+ double initialY = computeInitialY(ae, currentYGain);
+ double extra_gain = std::min(10.0, targetY / (initialY + .001));
+
+ currentYGain *= extra_gain;
+ LOG(RkISP1Agc, Debug) << "Initial Y " << initialY
+ << " target " << targetY
+ << " gives gain " << currentYGain;
+ if (extra_gain < 1.01)
+ break;
+ }
+
+ computeExposure(context.frameContext, currentYGain);
+ frameCount_++;
+}
+
+} /* namespace ipa::rkisp1::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h
new file mode 100644
index 00000000..fbb8ea98
--- /dev/null
+++ b/src/ipa/rkisp1/algorithms/agc.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2021, Ideas On Board
+ *
+ * agc.h - RkISP1 AGC/AEC mean-based control algorithm
+ */
+#ifndef __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
+#define __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__
+
+#include <linux/rkisp1-config.h>
+
+#include <libcamera/base/utils.h>
+
+#include <libcamera/geometry.h>
+
+#include "algorithm.h"
+
+namespace libcamera {
+
+struct IPACameraSensorInfo;
+
+namespace ipa::rkisp1::algorithms {
+
+class Agc : public Algorithm
+{
+public:
+ Agc();
+ ~Agc() = default;
+
+ int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override;
+ void process(IPAContext &context, const rkisp1_stat_buffer *stats) override;
+
+private:
+ void filterExposure();
+ void computeExposure(IPAFrameContext &frameContext, double currentYGain);
+ double computeInitialY(const rkisp1_cif_isp_ae_stat *ae, double currentYGain);
+
+ uint64_t frameCount_;
+
+ utils::Duration lineDuration_;
+ utils::Duration minShutterSpeed_;
+ utils::Duration maxShutterSpeed_;
+
+ double minAnalogueGain_;
+ double maxAnalogueGain_;
+
+ utils::Duration filteredExposure_;
+ utils::Duration currentExposure_;
+};
+
+} /* namespace ipa::rkisp1::algorithms */
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_RKISP1_ALGORITHMS_AGC_H__ */
diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build
index d98f77e2..05195d99 100644
--- a/src/ipa/rkisp1/algorithms/meson.build
+++ b/src/ipa/rkisp1/algorithms/meson.build
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: CC0-1.0
rkisp1_ipa_algorithms = files([
+ 'agc.cpp',
'algorithm.cpp',
])
diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp
index 819b2c73..16fc248f 100644
--- a/src/ipa/rkisp1/ipa_context.cpp
+++ b/src/ipa/rkisp1/ipa_context.cpp
@@ -55,4 +55,48 @@ namespace libcamera::ipa::rkisp1 {
* are run. This needs to be turned into real per-frame data storage.
*/
+/**
+ * \var IPASessionConfiguration::agc
+ * \brief AGC parameters configuration of the IPA
+ *
+ * \var IPASessionConfiguration::agc.minShutterSpeed
+ * \brief Minimum shutter speed supported with the configured sensor
+ *
+ * \var IPASessionConfiguration::agc.maxShutterSpeed
+ * \brief Maximum shutter speed supported with the configured sensor
+ *
+ * \var IPASessionConfiguration::agc.minAnalogueGain
+ * \brief Minimum analogue gain supported with the configured sensor
+ *
+ * \var IPASessionConfiguration::agc.maxAnalogueGain
+ * \brief Maximum analogue gain supported with the configured sensor
+ */
+
+/**
+ * \var IPAFrameContext::agc
+ * \brief Context for the Automatic Gain Control algorithm
+ *
+ * The exposure and gain determined are expected to be applied to the sensor
+ * at the earliest opportunity.
+ *
+ * \var IPAFrameContext::agc.exposure
+ * \brief Exposure time expressed as a number of lines
+ *
+ * \var IPAFrameContext::agc.gain
+ * \brief Analogue gain multiplier
+ *
+ * The gain should be adapted to the sensor specific gain code before applying.
+ */
+
+/**
+ * \var IPAFrameContext::sensor
+ * \brief Effective sensor values
+ *
+ * \var IPAFrameContext::sensor.exposure
+ * \brief Exposure time expressed as a number of lines
+ *
+ * \var IPAFrameContext::sensor.gain
+ * \brief Analogue gain multiplier
+ */
+
} /* namespace libcamera::ipa::rkisp1 */
diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h
index a1dc39fe..ca48ffc5 100644
--- a/src/ipa/rkisp1/ipa_context.h
+++ b/src/ipa/rkisp1/ipa_context.h
@@ -19,9 +19,24 @@ namespace libcamera {
namespace ipa::rkisp1 {
struct IPASessionConfiguration {
+ struct {
+ utils::Duration minShutterSpeed;
+ utils::Duration maxShutterSpeed;
+ double minAnalogueGain;
+ double maxAnalogueGain;
+ } agc;
};
struct IPAFrameContext {
+ struct {
+ uint32_t exposure;
+ double gain;
+ } agc;
+
+ struct {
+ uint32_t exposure;
+ double gain;
+ } sensor;
};
struct IPAContext {
diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp
index d2f4380a..1783f91f 100644
--- a/src/ipa/rkisp1/rkisp1.cpp
+++ b/src/ipa/rkisp1/rkisp1.cpp
@@ -25,6 +25,7 @@
#include <libcamera/internal/mapped_framebuffer.h>
+#include "algorithms/agc.h"
#include "algorithms/algorithm.h"
#include "libipa/camera_sensor_helper.h"
@@ -32,6 +33,8 @@ namespace libcamera {
LOG_DEFINE_CATEGORY(IPARkISP1)
+using namespace std::literals::chrono_literals;
+
namespace ipa::rkisp1 {
class IPARkISP1 : public IPARkISP1Interface
@@ -77,6 +80,8 @@ private:
unsigned int hwGammaOutMaxSamples_;
unsigned int hwHistogramWeightGridsSize_;
+ utils::Duration lineDuration_;
+
/* Interface to the Camera Helper */
std::unique_ptr<CameraSensorHelper> camHelper_;
@@ -121,6 +126,9 @@ int IPARkISP1::init([[maybe_unused]] const IPASettings &settings,
return -ENODEV;
}
+ /* Construct our Algorithms */
+ algorithms_.push_back(std::make_unique<algorithms::Agc>());
+
return 0;
}
@@ -172,9 +180,29 @@ int IPARkISP1::configure([[maybe_unused]] const IPACameraSensorInfo &info,
<< "Exposure: " << minExposure_ << "-" << maxExposure_
<< " Gain: " << minGain_ << "-" << maxGain_;
+ lineDuration_ = info.lineLength * 1.0s / info.pixelRate;
+
/* Clean context at configuration */
context_ = {};
+ /*
+ * When the AGC computes the new exposure values for a frame, it needs
+ * to know the limits for shutter speed and analogue gain.
+ * As it depends on the sensor, update it with the controls.
+ *
+ * \todo take VBLANK into account for maximum shutter speed
+ */
+ context_.configuration.agc.minShutterSpeed = minExposure_ * lineDuration_;
+ context_.configuration.agc.maxShutterSpeed = maxExposure_ * lineDuration_;
+ context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain_);
+ context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain_);
+
+ for (auto const &algo : algorithms_) {
+ int ret = algo->configure(context_, info);
+ if (ret)
+ return ret;
+ }
+
return 0;
}
@@ -219,6 +247,9 @@ void IPARkISP1::processEvent(const RkISP1Event &event)
reinterpret_cast<rkisp1_stat_buffer *>(
mappedBuffers_.at(bufferId).planes()[0].data());
+ context_.frameContext.sensor.exposure = event.sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>();
+ context_.frameContext.sensor.gain = camHelper_->gain(event.sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>());
+
updateStatistics(frame, stats);
break;
}
@@ -263,44 +294,12 @@ void IPARkISP1::queueRequest(unsigned int frame, rkisp1_params_cfg *params,
void IPARkISP1::updateStatistics(unsigned int frame,
const rkisp1_stat_buffer *stats)
{
- const rkisp1_cif_isp_stat *params = &stats->params;
unsigned int aeState = 0;
- if (stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP) {
- const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae;
-
- const unsigned int target = 60;
+ for (auto const &algo : algorithms_)
+ algo->process(context_, stats);
- unsigned int value = 0;
- unsigned int num = 0;
- for (unsigned int i = 0; i < hwAeMeanMax_; i++) {
- if (ae->exp_mean[i] <= 15)
- continue;
-
- value += ae->exp_mean[i];
- num++;
- }
- value /= num;
-
- double factor = (double)target / value;
-
- if (frame % 3 == 0) {
- double exposure;
-
- exposure = factor * exposure_ * gain_ / minGain_;
- exposure_ = std::clamp<uint64_t>((uint64_t)exposure,
- minExposure_,
- maxExposure_);
-
- exposure = exposure / exposure_ * minGain_;
- gain_ = std::clamp<uint64_t>((uint64_t)exposure,
- minGain_, maxGain_);
-
- setControls(frame + 1);
- }
-
- aeState = fabs(factor - 1.0f) < 0.05f ? 2 : 1;
- }
+ setControls(frame + 1);
metadataReady(frame, aeState);
}
@@ -310,6 +309,9 @@ void IPARkISP1::setControls(unsigned int frame)
RkISP1Action op;
op.op = ActionV4L2Set;
+ exposure_ = context_.frameContext.agc.exposure;
+ gain_ = camHelper_->gainCode(context_.frameContext.agc.gain);
+
ControlList ctrls(ctrls_);
ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure_));
ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain_));
--
2.32.0
More information about the libcamera-devel
mailing list