[libcamera-devel] [PATCH v2 4/4] ipa: ipu3: Introduce a new AGC algorithm
Jean-Michel Hautbois
jeanmichel.hautbois at ideasonboard.com
Fri Aug 27 10:02:27 CEST 2021
The algorithm used until then is a simple one, let's introduce a new
one, based on the one used by the Raspberry Pi code. We can keep both
compiled, and chose to instanciate only one, which demonstrates the
modularity and ease to add functionnalities to the IPA.
This algorithm uses the IPAFrameContext to get the latest AWB gains
applied and use them to estimate the next shutter time and gain values
to set.
For the moment it is not activated as the default algorithm as it may be
a bit unstable. More testing is need ;-).
Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
---
src/ipa/ipu3/algorithms/agc_metering.cpp | 427 +++++++++++++++++++++++
src/ipa/ipu3/algorithms/agc_metering.h | 78 +++++
src/ipa/ipu3/algorithms/meson.build | 1 +
src/ipa/ipu3/ipa_context.h | 6 +
src/ipa/ipu3/ipu3.cpp | 8 +
5 files changed, 520 insertions(+)
create mode 100644 src/ipa/ipu3/algorithms/agc_metering.cpp
create mode 100644 src/ipa/ipu3/algorithms/agc_metering.h
diff --git a/src/ipa/ipu3/algorithms/agc_metering.cpp b/src/ipa/ipu3/algorithms/agc_metering.cpp
new file mode 100644
index 00000000..1dc05082
--- /dev/null
+++ b/src/ipa/ipu3/algorithms/agc_metering.cpp
@@ -0,0 +1,427 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/*
+ * Based on the implementation from the Raspberry Pi IPA,
+ * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd.
+ * Copyright (C) 2021, Google inc.
+ *
+ * agc_metering.cpp - AGC/AEC metering-based control algorithm
+ */
+
+#include "agc_metering.h"
+#include "awb.h"
+
+#include <algorithm>
+#include <cmath>
+#include <numeric>
+#include <stdint.h>
+
+#include <linux/v4l2-controls.h>
+
+#include <libcamera/base/log.h>
+#include <libcamera/base/utils.h>
+
+#include "libipa/histogram.h"
+
+/**
+ * \file agc_metering.h
+ */
+
+namespace libcamera {
+
+using namespace std::literals::chrono_literals;
+
+namespace ipa::ipu3::algorithms {
+
+/**
+ * \class AgcMetering
+ * \brief The class to use the metering-based auto-exposure algorithm
+ *
+ * The metering-based algorithm is calculating an exposure and gain value such
+ * as a given quantity of pixels lie in the top 2% of the histogram. The AWB
+ * gains are also used here, and all cells in the grid are weighted using a
+ * specific metering matrix. The default here is Spot metering.
+ */
+
+LOG_DEFINE_CATEGORY(IPU3AgcMetering)
+
+/* Histogram constants */
+static constexpr uint32_t knumHistogramBins = 256;
+
+/* seems to be a 10-bit pipeline */
+static constexpr uint8_t kPipelineBits = 10;
+
+/* width of the AGC stats grid */
+static constexpr uint32_t kAgcStatsSizeX = 7;
+/* height of the AGC stats grid */
+static constexpr uint32_t kAgcStatsSizeY = 5;
+/* size of the AGC stats grid */
+static constexpr uint32_t kAgcStatsSize = kAgcStatsSizeX * kAgcStatsSizeY;
+
+/**
+ * The AGC algorithm uses region-based metering.
+ * The image is divided up into regions as:
+ *
+ * +--+--------------+--+
+ * |11| 9 |12|
+ * +--+--+--------+--+--+
+ * | | | 3 | | |
+ * | | +--+--+--+ | |
+ * |7 |5 |1 |0 |2 |6 |8 |
+ * | | +--+--+--+ | |
+ * | | | 4 | | |
+ * +--+--+--------+--+--+
+ * |13| 10 |14|
+ * +--+--------------+--+
+ *
+ * The metering-based algorithm is calculating an exposure and gain value such
+ * as a given quantity of weighted pixels lie in the top 2% of the histogram.The
+ * AWB gains applied are also used to estimate the total gain to apply.
+ *
+ * An average luminance value for the image is calculated according to:
+ * \f$Y = \frac{\sum_{i=0}^{i=kNumAgcWeightedZones}{kCenteredWeights_{i}Y_{i}}}
+ * {\sum_{i=0}^{i=kNumAgcWeightedZones}{w_{i}}}\f$
+ */
+
+/* Weight applied on each region */
+static constexpr double kSpotWeights[kNumAgcWeightedZones] = { 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+/* Region number repartition in the image */
+static constexpr uint32_t kAgcStatsRegions[kAgcStatsSize] = {
+ 11, 9, 9, 9, 9, 9, 12,
+ 7, 5, 3, 3, 3, 6, 8,
+ 7, 5, 1, 0, 2, 6, 8,
+ 7, 5, 4, 4, 4, 6, 8,
+ 13, 10, 10, 10, 10, 10, 14
+};
+
+/* Limit the speed of change between two exposure levels */
+static constexpr double kFastReduceThreshold = 0.3;
+
+AgcMetering::AgcMetering()
+ : iqMean_(0.0), prevExposure_(0s), prevExposureNoDg_(0s),
+ currentExposure_(0s), currentExposureNoDg_(0s), currentShutter_(1.0s),
+ currentAnalogueGain_(1.0)
+{
+}
+
+/**
+ * \brief Configure the AGC given a configInfo
+ * \param[in] context The shared IPA context
+ * \param[in] configInfo The IPA configuration data, received from the pipeline
+ * handler
+ *
+ * \return 0
+ */
+int AgcMetering::configure(IPAContext &context, const IPAConfigInfo &configInfo)
+{
+ /* Store the line duration in the IPASessionConfiguration */
+ context.configuration.agc.lineDuration = configInfo.sensorInfo.lineLength
+ * (1.0s / configInfo.sensorInfo.pixelRate);
+
+ /* \todo: those values need to be extracted from a configuration file */
+ shutterConstraints_.push_back(100us);
+ shutterConstraints_.push_back(10ms);
+ shutterConstraints_.push_back(33ms);
+ gainConstraints_.push_back(1.0);
+ gainConstraints_.push_back(4.0);
+ gainConstraints_.push_back(16.0);
+
+ fixedShutter_ = 0s;
+ fixedAnalogueGain_ = 0.0;
+
+ return 0;
+}
+
+/**
+ * \brief Translate the IPU3 statistics to AGC regions
+ * \param[in] stats The statistics buffer coming from the pipeline handler
+ * \param[in] grid The grid used to store the statistics in the IPU3
+ */
+void AgcMetering::generateStats(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid)
+{
+ /* We need to have a AGC grid of kAgcStatsSizeX * kAgcStatsSizeY */
+ uint32_t regionWidth = round(grid.width / static_cast<double>(kAgcStatsSizeX));
+ uint32_t regionHeight = round(grid.height / static_cast<double>(kAgcStatsSizeY));
+ uint32_t hist[knumHistogramBins] = { 0 };
+
+ /* Clear the statistics of the previous frame */
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
+ agcStats_[i].bSum = 0;
+ agcStats_[i].rSum = 0;
+ agcStats_[i].gSum = 0;
+ agcStats_[i].counted = 0;
+ agcStats_[i].total = 0;
+ }
+
+ LOG(IPU3AgcMetering, Debug) << "[" << (int)grid.width
+ << "x" << (int)grid.height << "] cells"
+ << " scaled to [" << regionWidth
+ << "x" << regionHeight << "] AGC regions";
+
+ /*
+ * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid
+ * which is (grid.width x grid.height).
+ */
+ for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) {
+ for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) {
+ uint32_t cellPosition = j * grid.width + i;
+ uint32_t cellX = (cellPosition / regionWidth)
+ % kAgcStatsSizeX;
+ uint32_t cellY = ((cellPosition / grid.width) / regionHeight)
+ % kAgcStatsSizeY;
+
+ uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX];
+ weights_[agcRegionPosition] = kSpotWeights[agcRegionPosition];
+ cellPosition *= sizeof(Ipu3AwbCell);
+
+ /* Cast the initial IPU3 structure to simplify the reading */
+ Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition]));
+ if (currentCell->satRatio == 0) {
+ /* The cell is not saturated, use the current cell */
+ agcStats_[agcRegionPosition].counted++;
+ uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg;
+ hist[greenValue / 2]++;
+ agcStats_[agcRegionPosition].gSum += greenValue / 2;
+ agcStats_[agcRegionPosition].rSum += currentCell->redAvg;
+ agcStats_[agcRegionPosition].bSum += currentCell->blueAvg;
+ }
+ }
+ }
+
+ /* Estimate the quantile mean of the top 2% of the histogram */
+ iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
+}
+
+/**
+ * \brief Apply a filter on the exposure value to limit the speed of changes
+ */
+void AgcMetering::filterExposure()
+{
+ double speed = 0.08;
+ if (prevExposure_ == 0s) {
+ /* DG stands for digital gain.*/
+ prevExposure_ = currentExposure_;
+ prevExposureNoDg_ = currentExposureNoDg_;
+ } else {
+ /*
+ * If we are close to the desired result, go faster to avoid
+ * making multiple micro-adjustments.
+ * \todo: Make this customisable?
+ */
+ if (prevExposure_ < 1.2 * currentExposure_ &&
+ prevExposure_ > 0.8 * currentExposure_)
+ speed = sqrt(speed);
+
+ prevExposure_ = speed * currentExposure_ +
+ prevExposure_ * (1.0 - speed);
+ prevExposureNoDg_ = speed * currentExposureNoDg_ +
+ prevExposureNoDg_ * (1.0 - speed);
+ }
+ /*
+ * We can't let the no_dg exposure deviate too far below the
+ * total exposure, as there might not be enough digital gain available
+ * in the ISP to hide it (which will cause nasty oscillation).
+ * \todo: add the support for digital gain
+ */
+ if (prevExposureNoDg_ <
+ prevExposure_ * kFastReduceThreshold)
+ prevExposureNoDg_ = prevExposure_ * kFastReduceThreshold;
+ LOG(IPU3AgcMetering, Debug) << "After filtering, total_exposure " << prevExposure_;
+}
+
+/**
+ * \brief Estimate the weighted brightness
+ * \param[in] gain The current gain applied
+ * \param[in] context The shared IPA context
+ */
+double AgcMetering::computeInitialY(double gain, IPAContext &context)
+{
+ /*
+ * Note how the calculation below means that equal weights_ give you
+ * "average" metering (i.e. all pixels equally important).
+ */
+ double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0;
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
+ /* We will exclude the saturated pixels from the sum */
+ double counted = agcStats_[i].counted;
+ double rSum = std::min(agcStats_[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ double gSum = std::min(agcStats_[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ double bSum = std::min(agcStats_[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ /* Weight each channel with the selected metering method */
+ redSum += rSum * weights_[i];
+ greenSum += gSum * weights_[i];
+ blueSum += bSum * weights_[i];
+ pixelSum += counted * weights_[i];
+ }
+ /* We don't want to have a division by 0.0 :-) */
+ if (pixelSum == 0.0) {
+ LOG(IPU3AgcMetering, Warning) << "computeInitialY: pixel_sum is zero";
+ return 0;
+ }
+ /*
+ * Estimate the sum of the brightness values, weighted with the gains
+ * applied on the channels in AWB.
+ */
+ double Y_sum = redSum * context.frameContext.awb.gains.red * .299 +
+ greenSum * context.frameContext.awb.gains.green * .587 +
+ blueSum * context.frameContext.awb.gains.blue * .114;
+
+ /* And return the average brightness */
+ return Y_sum / pixelSum / (1 << kPipelineBits);
+}
+
+/**
+ * \brief Compute the exposure value
+ * \param[in] gain The current gain applied
+ */
+void AgcMetering::computeTargetExposure(double gain)
+{
+ currentExposure_ = currentExposureNoDg_ * gain;
+ /* \todo: have a list of shutter speeds */
+ Duration maxShutterSpeed = shutterConstraints_.back();
+ Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back();
+
+ currentExposure_ = std::min(currentExposure_, maxTotalExposure);
+ LOG(IPU3AgcMetering, Debug) << "Target total_exposure " << currentExposure_;
+}
+
+/**
+ * \brief Split exposure value as shutter time and gain
+ */
+void AgcMetering::divideUpExposure()
+{
+ Duration exposureValue = prevExposure_;
+ Duration shutterTime;
+ double analogueGain;
+ shutterTime = shutterConstraints_[0];
+ shutterTime = std::min(shutterTime, shutterConstraints_.back());
+ analogueGain = gainConstraints_[0];
+
+ /**
+ * We have an exposure profile with a list of shutter time and gains
+ * An example is graphed below:
+ *
+ * gain shutter time
+ * (ms)
+ * ^ ^
+ * | |
+ * 8x+---------------------------------------------------------xxxxx+30
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * | xxxxx |
+ * 4x+----------xx--------------------------------------------------+10
+ * | xx |
+ * | xx |
+ * | xx |
+ * | xx |
+ * | xx |
+ * 1x+--xx----------------------------------------------------------+0.1
+ * | x |
+ * |x |
+ * +--------------------------------------------------------------->
+ * total exposure
+ */
+ if (shutterTime * analogueGain < exposureValue) {
+ for (unsigned int stage = 1;
+ stage < gainConstraints_.size(); stage++) {
+ if (fixedShutter_ == 0s) {
+ Duration stageShutter =
+ std::min(shutterConstraints_[stage], shutterConstraints_.back());
+ if (stageShutter * analogueGain >=
+ exposureValue) {
+ shutterTime =
+ exposureValue / analogueGain;
+ break;
+ }
+ shutterTime = stageShutter;
+ }
+ if (fixedAnalogueGain_ == 0.0) {
+ if (gainConstraints_[stage] * shutterTime >= exposureValue) {
+ analogueGain = exposureValue / shutterTime;
+ break;
+ }
+ analogueGain = gainConstraints_[stage];
+ }
+ }
+ }
+ LOG(IPU3AgcMetering, Debug) << "Divided up shutter and gain are "
+ << shutterTime << " and " << analogueGain;
+
+ /* \todo: flickering avoidance ? */
+ filteredShutter_ = shutterTime;
+ filteredAnalogueGain_ = analogueGain;
+}
+
+/**
+ * \brief Calculate the gain for the target value to be in the top 2% of the
+ * histogram
+ * \param[in] currentGain The current gain applied
+ * \param[in] context The shared IPA context
+ */
+void AgcMetering::computeGain(double ¤tGain, IPAContext &context)
+{
+ currentGain = 1.0;
+ /* \todo: the target Y needs to be grabbed from a configuration */
+ double targetY = 0.162;
+ for (int i = 0; i < 8; i++) {
+ double initialY = computeInitialY(currentGain, context);
+ double extra_gain = std::min(10.0, targetY / (initialY + .001));
+
+ currentGain *= extra_gain;
+ LOG(IPU3AgcMetering, Debug) << "Initial Y " << initialY
+ << " target " << targetY
+ << " gives gain " << currentGain;
+ if (extra_gain < 1.01)
+ break;
+ }
+
+ /*
+ * Require the top 2% of pixels to lie at or below 0.5 in the pixel
+ * range (for a range from 0 to 255, it is 205). This lowers the
+ * exposure to stop pixels saturating.
+ */
+ double newGain = (0.5 * knumHistogramBins) / iqMean_;
+ LOG(IPU3AgcMetering, Debug) << "gain: " << currentGain
+ << " new gain: " << newGain;
+ if (newGain > currentGain)
+ currentGain = newGain;
+}
+
+/**
+ * \brief Process IPU3 statistics, and run AGC operations
+ * \param[in] context The shared IPA context
+ * \param[in] stats The IPU3 statistics and ISP results
+ */
+void AgcMetering::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)
+{
+ ASSERT(stats->stats_3a_status.awb_en);
+ generateStats(stats, context.configuration.grid.bdsGrid);
+
+ currentShutter_ = context.frameContext.agc.exposure
+ * context.configuration.agc.lineDuration;
+ currentAnalogueGain_ = context.frameContext.agc.gain;
+
+ /* Estimate the current exposure value */
+ currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_;
+
+ double currentGain = 1.0;
+ computeGain(currentGain, context);
+ computeTargetExposure(currentGain);
+ filterExposure();
+ divideUpExposure();
+
+ context.frameContext.agc.exposure = filteredShutter_
+ / context.configuration.agc.lineDuration;
+ context.frameContext.agc.gain = filteredAnalogueGain_;
+}
+
+} /* namespace ipa::ipu3::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/ipu3/algorithms/agc_metering.h b/src/ipa/ipu3/algorithms/agc_metering.h
new file mode 100644
index 00000000..4fd603e1
--- /dev/null
+++ b/src/ipa/ipu3/algorithms/agc_metering.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Based on the implementation from the Raspberry Pi IPA,
+ * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd.
+ * Copyright (C) 2021, Ideas On Board
+ *
+ * agc_metering.h - IPU3 AGC/AEC control algorithm
+ */
+#ifndef __LIBCAMERA_IPU3_AGC_H__
+#define __LIBCAMERA_IPU3_AGC_H__
+
+#include <linux/intel-ipu3.h>
+
+#include <libcamera/base/utils.h>
+
+#include <libcamera/geometry.h>
+
+#include "algorithm.h"
+#include "awb.h"
+
+namespace libcamera {
+
+struct IPACameraSensorInfo;
+
+namespace ipa::ipu3::algorithms {
+
+using utils::Duration;
+
+/* Number of weighted zones for metering */
+static constexpr uint32_t kNumAgcWeightedZones = 15;
+
+class AgcMetering : public Algorithm
+{
+public:
+ AgcMetering();
+ ~AgcMetering() = default;
+
+ int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
+ void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override;
+
+private:
+ void processBrightness(const ipu3_uapi_stats_3a *stats);
+ void filterExposure();
+ void lockExposureGain(uint32_t &exposure, double &gain);
+ void generateStats(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid);
+ void generateZones(std::vector<RGB> &zones);
+ double computeInitialY(double gain, IPAContext &context);
+ void computeTargetExposure(double currentGain);
+ void divideUpExposure();
+ void computeGain(double ¤tGain, IPAContext &context);
+
+ double weights_[kNumAgcWeightedZones];
+ struct Accumulator agcStats_[kNumAgcWeightedZones];
+
+ double iqMean_;
+
+ Duration prevExposure_;
+ Duration prevExposureNoDg_;
+ Duration currentExposure_;
+ Duration currentExposureNoDg_;
+
+ Duration currentShutter_;
+ std::vector<Duration> shutterConstraints_;
+ Duration fixedShutter_;
+ Duration filteredShutter_;
+
+ double currentAnalogueGain_;
+ std::vector<double> gainConstraints_;
+ double fixedAnalogueGain_;
+ double filteredAnalogueGain_;
+};
+
+} /* namespace ipa::ipu3::algorithms */
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_IPU3_AGC_H__ */
diff --git a/src/ipa/ipu3/algorithms/meson.build b/src/ipa/ipu3/algorithms/meson.build
index 807b53ea..f31b2070 100644
--- a/src/ipa/ipu3/algorithms/meson.build
+++ b/src/ipa/ipu3/algorithms/meson.build
@@ -2,6 +2,7 @@
ipu3_ipa_algorithms = files([
'agc_mean.cpp',
+ 'agc_metering.cpp',
'algorithm.cpp',
'awb.cpp',
'tone_mapping.cpp',
diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h
index 3a292ad7..190a3468 100644
--- a/src/ipa/ipu3/ipa_context.h
+++ b/src/ipa/ipu3/ipa_context.h
@@ -10,6 +10,8 @@
#include <linux/intel-ipu3.h>
+#include <libcamera/base/utils.h>
+
#include <libcamera/geometry.h>
namespace libcamera {
@@ -17,6 +19,10 @@ namespace libcamera {
namespace ipa::ipu3 {
struct IPASessionConfiguration {
+ struct {
+ utils::Duration lineDuration;
+ } agc;
+
struct {
ipu3_uapi_grid_config bdsGrid;
Size bdsOutputSize;
diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
index 6332fc06..be4a082a 100644
--- a/src/ipa/ipu3/ipu3.cpp
+++ b/src/ipa/ipu3/ipu3.cpp
@@ -81,6 +81,14 @@
* are run. This needs to be turned into real per-frame data storage.
*/
+/**
+ * \struct IPASessionConfiguration::agc
+ * \brief AGC configuration of the IPA
+ *
+ * \var IPASessionConfiguration::agc::lineDuration
+ * \brief Duration of one line dependant on the sensor configuration
+ */
+
/**
* \struct IPASessionConfiguration::grid
* \brief Grid configuration of the IPA
--
2.30.2
More information about the libcamera-devel
mailing list