[libcamera-devel] [PATCH v1 6/7] ipa: ipu3: Introduce a new AGC algorithm
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Mon Aug 23 19:20:09 CEST 2021
Hi Jean-Michel,
Thank you for the patch.
On Mon, Aug 23, 2021 at 02:49:36PM +0200, Jean-Michel Hautbois wrote:
> The algorithm used until then is a simple one, let's introduce a new
> one, based on the one used by the Raspberry Pi code. We can keep both
> compiled, and chose to instanciate only one, which demonstrates the
> modularity and ease to add functionnalities to the IPA.
>
> This algorithm uses the IPAFrameContext to get the latest AWB gains
> applied and use them to estimate the next shutter time and gain values
> to set.
>
> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
> ---
> src/ipa/ipu3/algorithms/agc_metering.cpp | 336 +++++++++++++++++++++++
> src/ipa/ipu3/algorithms/agc_metering.h | 81 ++++++
> src/ipa/ipu3/algorithms/awb.cpp | 4 +-
> src/ipa/ipu3/algorithms/meson.build | 1 +
> src/ipa/ipu3/ipa_context.h | 8 +
> src/ipa/ipu3/ipu3.cpp | 12 +-
> 6 files changed, 438 insertions(+), 4 deletions(-)
> create mode 100644 src/ipa/ipu3/algorithms/agc_metering.cpp
> create mode 100644 src/ipa/ipu3/algorithms/agc_metering.h
>
> diff --git a/src/ipa/ipu3/algorithms/agc_metering.cpp b/src/ipa/ipu3/algorithms/agc_metering.cpp
> new file mode 100644
> index 00000000..3479c269
> --- /dev/null
> +++ b/src/ipa/ipu3/algorithms/agc_metering.cpp
> @@ -0,0 +1,336 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Based on the implementation from the Raspberry Pi IPA,
> + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd.
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc_metering.cpp - AGC/AEC control algorithm
> + */
> +
> +#include "agc_metering.h"
> +#include "awb.h"
> +
> +#include <algorithm>
> +#include <cmath>
> +#include <numeric>
> +#include <stdint.h>
> +
> +#include <linux/v4l2-controls.h>
> +
> +#include <libcamera/base/log.h>
> +#include <libcamera/base/utils.h>
> +
> +#include "libipa/histogram.h"
> +
> +namespace libcamera {
> +
> +using namespace std::literals::chrono_literals;
> +
> +namespace ipa::ipu3::algorithms {
> +
> +LOG_DEFINE_CATEGORY(IPU3AgcMetering)
> +
> +/* Histogram constants */
> +static constexpr uint32_t knumHistogramBins = 256;
> +
> +/* seems to be a 10-bit pipeline */
> +static constexpr uint8_t kPipelineBits = 10;
> +
> +/* width of the AGC stats grid */
> +static constexpr uint32_t kAgcStatsSizeX = 7;
> +/* height of the AGC stats grid */
> +static constexpr uint32_t kAgcStatsSizeY = 5;
> +/* size of the AGC stats grid */
> +static constexpr uint32_t kAgcStatsSize = kAgcStatsSizeX * kAgcStatsSizeY;
> +
> +/**
> + * The AGC algorithm uses region-based metering.
> + * The image is divided up into regions as:
> + *
> + * +--+--------------+--+
> + * |11| 9 |12|
> + * +--+--+--------+--+--+
> + * | | | 3 | | |
> + * | | +--+--+--+ | |
> + * |7 |5 |1 |0 |2 |6 |8 |
> + * | | +--+--+--+ | |
> + * | | | 4 | | |
> + * +--+--+--------+--+--+
> + * |13| 10 |14|
> + * +--+--------------+--+
> + * An average luminance value for the image is calculated according to:
> + * \f$Y = \frac{\sum_{i=0}^{i=kNumAgcWeightedZones}{kCenteredWeights_{i}Y_{i}}}
> + * {\sum_{i=0}^{i=kNumAgcWeightedZones}{w_{i}}}\f$
That's a good start, it's only missing the explanation of how the
algorithm works :-) It can be done with an overview here and additional
comments through the code.
> + */
> +
> +/* Weight applied on each region */
> +static constexpr double kCenteredWeights[kNumAgcWeightedZones] = { 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 };
> +/* Region number repartition in the image */
> +static constexpr uint32_t kAgcStatsRegions[kAgcStatsSize] = {
> + 11, 9, 9, 9, 9, 9, 12,
> + 7, 5, 3, 3, 3, 6, 8,
> + 7, 5, 1, 0, 2, 6, 8,
> + 7, 5, 4, 4, 4, 6, 8,
> + 13, 10, 10, 10, 10, 10, 14
Would
11, 9, 9, 9, 9, 9, 12,
7, 5, 3, 3, 3, 6, 8,
7, 5, 1, 0, 2, 6, 8,
7, 5, 4, 4, 4, 6, 8,
13, 10, 10, 10, 10, 10, 14
be more readable ?
> +};
> +
> +AgcMetering::AgcMetering()
> + : iqMean_(0.0), prevExposure_(0s), prevExposureNoDg_(0s),
> + currentExposure_(0s), currentExposureNoDg_(0s), currentShutter_(1.0s),
> + currentAnalogueGain_(1.0)
> +{
> +}
> +
> +int AgcMetering::configure(IPAContext &context, const IPAConfigInfo &configInfo)
> +{
> + context.configuration.agc.lineDuration = configInfo.sensorInfo.lineLength
> + * (1.0s / configInfo.sensorInfo.pixelRate);
> +
> + /* \todo: those values need to be extracted from a configuration file */
> + shutterConstraints_.push_back(100us);
> + shutterConstraints_.push_back(10ms);
> + shutterConstraints_.push_back(33ms);
> + gainConstraints_.push_back(1.0);
> + gainConstraints_.push_back(4.0);
> + gainConstraints_.push_back(16.0);
> +
> + fixedShutter_ = 0s;
> + fixedAnalogueGain_ = 0.0;
> +
> + return 0;
> +}
> +
> +/* Translate the IPU3 statistics into the default statistics region array */
> +void AgcMetering::generateStats(const ipu3_uapi_stats_3a *stats,
> + const ipu3_uapi_grid_config &grid)
> +{
> + uint32_t regionWidth = round(grid.width / static_cast<double>(kAgcStatsSizeX));
> + uint32_t regionHeight = round(grid.height / static_cast<double>(kAgcStatsSizeY));
> + uint32_t hist[knumHistogramBins] = { 0 };
> +
> + LOG(IPU3AgcMetering, Debug) << "[" << (int)grid.width
> + << "x" << (int)grid.height << "] regions"
> + << " scaled to [" << regionWidth
> + << "x" << regionHeight << "] AGC stats";
> +
> + /*
> + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid
> + * which is (grid.width x grid.height).
> + */
> + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) {
> + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) {
> + uint32_t cellPosition = j * grid.width + i;
> + uint32_t cellX = (cellPosition / regionWidth)
> + % kAgcStatsSizeX;
> + uint32_t cellY = ((cellPosition / grid.width) / regionHeight)
> + % kAgcStatsSizeY;
> +
> + uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX];
> + weights_[agcRegionPosition] = kCenteredWeights[agcRegionPosition];
> + cellPosition *= sizeof(Ipu3AwbCell);
> +
> + /* Cast the initial IPU3 structure to simplify the reading */
> + Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition]));
> + if (currentCell->satRatio == 0) {
> + /* The cell is not saturated, use the current cell */
> + agcStats_[agcRegionPosition].counted++;
> + uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg;
> + hist[greenValue / 2]++;
> + agcStats_[agcRegionPosition].gSum += greenValue / 2;
> + agcStats_[agcRegionPosition].rSum += currentCell->redAvg;
> + agcStats_[agcRegionPosition].bSum += currentCell->blueAvg;
> + }
> + }
> + }
> +
> + /* Estimate the quantile mean of the top 2% of the histogram */
> + iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
> +}
> +
> +void AgcMetering::clearStats()
> +{
> + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
> + agcStats_[i].bSum = 0;
> + agcStats_[i].rSum = 0;
> + agcStats_[i].gSum = 0;
> + agcStats_[i].counted = 0;
> + agcStats_[i].uncounted = 0;
> + }
You can move this to the beginning of AgcMetering::generateStats().
> +}
> +
> +void AgcMetering::filterExposure()
> +{
> + double speed = 0.08;
> + if (prevExposure_ == 0s) {
> + /* DG stands for digital gain.*/
> + prevExposure_ = currentExposure_;
> + prevExposureNoDg_ = currentExposureNoDg_;
> + } else {
> + /*
> + * If we are close to the desired result, go faster to avoid
> + * making multiple micro-adjustments.
> + * \ todo: Make this customisable?
Extra space before todo.
> + */
> + if (prevExposure_ < 1.2 * currentExposure_ &&
> + prevExposure_ > 0.8 * currentExposure_)
> + speed = sqrt(speed);
> +
> + prevExposure_ = speed * currentExposure_ +
> + prevExposure_ * (1.0 - speed);
> + prevExposureNoDg_ = speed * currentExposureNoDg_ +
> + prevExposureNoDg_ * (1.0 - speed);
> + }
> + /*
> + * We can't let the no_dg exposure deviate too far below the
> + * total exposure, as there might not be enough digital gain available
> + * in the ISP to hide it (which will cause nasty oscillation).
> + */
> + double fastReduceThreshold = 0.3;
constexpr
> + if (prevExposureNoDg_ <
> + prevExposure_ * fastReduceThreshold)
> + prevExposureNoDg_ = prevExposure_ * fastReduceThreshold;
> + LOG(IPU3AgcMetering, Debug) << "After filtering, total_exposure " << prevExposure_;
> +}
> +
> +double AgcMetering::computeInitialY(double gain)
> +{
> + /*
> + * Note how the calculation below means that equal weights_ give you
> + * "average" metering (i.e. all pixels equally important).
> + */
> + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0;
> + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
> + double counted = agcStats_[i].counted;
> + double rSum = std::min(agcStats_[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted);
> + double gSum = std::min(agcStats_[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted);
> + double bSum = std::min(agcStats_[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted);
> + redSum += rSum * weights_[i];
> + greenSum += gSum * weights_[i];
> + blueSum += bSum * weights_[i];
> + pixelSum += counted * weights_[i];
> + }
> + if (pixelSum == 0.0) {
> + LOG(IPU3AgcMetering, Warning) << "computeInitialY: pixel_sum is zero";
> + return 0;
> + }
> + double Y_sum = redSum * awbStatus_.redGain * .299 +
> + greenSum * awbStatus_.greenGain * .587 +
> + blueSum * awbStatus_.blueGain * .114;
> +
> + return Y_sum / pixelSum / (1 << kPipelineBits);
> +}
> +
> +void AgcMetering::computeTargetExposure(double gain)
> +{
> + currentExposure_ = currentExposureNoDg_ * gain;
> + /* \todo: have a list of shutter speeds */
> + Duration maxShutterSpeed = shutterConstraints_.back();
> + Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back();
> +
> + currentExposure_ = std::min(currentExposure_, maxTotalExposure);
> + LOG(IPU3AgcMetering, Debug) << "Target total_exposure " << currentExposure_;
> +}
> +
> +void AgcMetering::divideUpExposure()
> +{
> + Duration exposureValue = prevExposure_;
> + Duration shutterTime;
> + double analogueGain;
> + shutterTime = shutterConstraints_[0];
> + shutterTime = std::min(shutterTime, shutterConstraints_.back());
> + analogueGain = gainConstraints_[0];
> +
> + if (shutterTime * analogueGain < exposureValue) {
> + for (unsigned int stage = 1;
> + stage < gainConstraints_.size(); stage++) {
> + if (fixedShutter_ == 0s) {
> + Duration stageShutter =
> + std::min(shutterConstraints_[stage], shutterConstraints_.back());
> + if (stageShutter * analogueGain >=
> + exposureValue) {
> + shutterTime =
> + exposureValue / analogueGain;
> + break;
> + }
> + shutterTime = stageShutter;
> + }
> + if (fixedAnalogueGain_ == 0.0) {
> + if (gainConstraints_[stage] * shutterTime >= exposureValue) {
> + analogueGain = exposureValue / shutterTime;
> + break;
> + }
> + analogueGain = gainConstraints_[stage];
> + }
> + }
> + }
> + LOG(IPU3AgcMetering, Debug) << "Divided up shutter and gain are "
> + << shutterTime << " and " << analogueGain;
> +
> + /* \todo: flickering avoidance ? */
> + filteredShutter_ = shutterTime;
> + filteredAnalogueGain_ = analogueGain;
> +}
> +
> +void AgcMetering::computeGain(double ¤tGain)
> +{
> + currentGain = 1.0;
> + /* \todo: the target Y needs to be grabbed from a configuration */
> + double targetY = 0.162;
> + for (int i = 0; i < 8; i++) {
> + double initialY = computeInitialY(currentGain);
> + double extra_gain = std::min(10.0, targetY / (initialY + .001));
> +
> + currentGain *= extra_gain;
> + LOG(IPU3AgcMetering, Debug) << "Initial Y " << initialY
> + << " target " << targetY
> + << " gives gain " << currentGain;
> + if (extra_gain < 1.01)
> + break;
> + }
> +
> + /*
> + * Require the top 2% of pixels to lie at or below 0.8 in the pixel
> + * range (for a range from 0 to 255, it is 205). This lowers the
> + * exposure to stop pixels saturating.
> + */
> + double newGain = (0.8 * knumHistogramBins) / iqMean_;
> + LOG(IPU3AgcMetering, Debug) << "gain: " << currentGain
> + << " new gain: " << newGain;
> +
> + /* Are we at the upper bound ? */
> + if (newGain < currentGain)
> + currentGain = newGain;
> +}
> +
> +void AgcMetering::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)
> +{
> + ASSERT(stats->stats_3a_status.awb_en);
> + clearStats();
> + generateStats(stats, context.configuration.grid.bdsGrid);
> +
> + currentShutter_ = context.frameContext.agc.exposure
> + * context.configuration.agc.lineDuration;
> + currentAnalogueGain_ = context.frameContext.agc.gain;
> +
> + /* Estimate the current exposure value */
> + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_;
> +
> + /* Get the current awb gains from IPAFrameContext */
> + awbStatus_.redGain = context.frameContext.awb.gains.red;
> + awbStatus_.greenGain = context.frameContext.awb.gains.green;
> + awbStatus_.blueGain = context.frameContext.awb.gains.blue;
> +
> + double currentGain = 1.0;
> + computeGain(currentGain);
> + computeTargetExposure(currentGain);
> + filterExposure();
> + divideUpExposure();
> +
> + context.frameContext.agc.exposure = filteredShutter_
> + / context.configuration.agc.lineDuration;
> + context.frameContext.agc.gain = filteredAnalogueGain_;
> +}
> +
> +} /* namespace ipa::ipu3::algorithms */
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/ipu3/algorithms/agc_metering.h b/src/ipa/ipu3/algorithms/agc_metering.h
> new file mode 100644
> index 00000000..c10846e7
> --- /dev/null
> +++ b/src/ipa/ipu3/algorithms/agc_metering.h
> @@ -0,0 +1,81 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Based on the implementation from the Raspberry Pi IPA,
> + * Copyright (C) 2019-2021, Raspberry Pi (Trading) Ltd.
> + * Copyright (C) 2021, Ideas On Board
> + *
> + * agc_metering.h - IPU3 AGC/AEC control algorithm
> + */
> +#ifndef __LIBCAMERA_IPU3_AGC_H__
> +#define __LIBCAMERA_IPU3_AGC_H__
> +
> +#include <linux/intel-ipu3.h>
> +
> +#include <libcamera/base/utils.h>
> +
> +#include <libcamera/geometry.h>
> +
> +#include "algorithm.h"
> +#include "awb.h"
> +
> +namespace libcamera {
> +
> +struct IPACameraSensorInfo;
> +
> +namespace ipa::ipu3::algorithms {
> +
> +using utils::Duration;
> +
> +/* Number of weighted zones for metering */
> +static constexpr uint32_t kNumAgcWeightedZones = 15;
> +
> +class AgcMetering : public Algorithm
> +{
> +public:
> + AgcMetering();
> + ~AgcMetering() = default;
> +
> + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
> + void process(IPAContext &context, const ipu3_uapi_stats_3a *stats) override;
> +
> +private:
> + void processBrightness(const ipu3_uapi_stats_3a *stats);
> + void filterExposure();
> + void lockExposureGain(uint32_t &exposure, double &gain);
> + void generateStats(const ipu3_uapi_stats_3a *stats,
> + const ipu3_uapi_grid_config &grid);
> + void clearStats();
> + void generateZones(std::vector<RGB> &zones);
> + double computeInitialY(double gain);
> + void computeTargetExposure(double currentGain);
> + void divideUpExposure();
> + void computeGain(double ¤tGain);
> +
> + double weights_[kNumAgcWeightedZones];
> + struct IspStatsRegion agcStats_[kNumAgcWeightedZones];
> +
> + double iqMean_;
> +
> + Duration prevExposure_;
> + Duration prevExposureNoDg_;
> + Duration currentExposure_;
> + Duration currentExposureNoDg_;
> +
> + Duration currentShutter_;
> + std::vector<Duration> shutterConstraints_;
> + Duration fixedShutter_;
> + Duration filteredShutter_;
> +
> + double currentAnalogueGain_;
> + std::vector<double> gainConstraints_;
> + double fixedAnalogueGain_;
> + double filteredAnalogueGain_;
> +
> + struct AwbStatus awbStatus_;
> +};
> +
> +} /* namespace ipa::ipu3::algorithms */
> +
> +} /* namespace libcamera */
> +
> +#endif /* __LIBCAMERA_IPU3_AGC_H__ */
> diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp
> index 8e4230b5..294871b1 100644
> --- a/src/ipa/ipu3/algorithms/awb.cpp
> +++ b/src/ipa/ipu3/algorithms/awb.cpp
> @@ -138,7 +138,7 @@ static const struct ipu3_uapi_ccm_mat_config imguCssCcmDefault = {
> };
>
> /* Minimum level of green in a given zone */
> -static constexpr uint32_t kMinGreenLevelInZone = 16;
> +static constexpr uint32_t kMinGreenLevelInZone = 32;
Why ?
>
> Awb::Awb()
> : Algorithm()
> @@ -213,7 +213,7 @@ void Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats,
> * for it to be relevant.
> * \todo This proportion could be configured.
> */
> - minZonesCounted_ = (regionWidth * regionHeight) * 80 / 100;
> + minZonesCounted_ = (regionWidth * regionHeight) * 50 / 100;
Same here.
>
> /*
> * Generate a (kAwbStatsSizeX x kAwbStatsSizeY) array from the IPU3 grid which is
> diff --git a/src/ipa/ipu3/algorithms/meson.build b/src/ipa/ipu3/algorithms/meson.build
> index 807b53ea..f31b2070 100644
> --- a/src/ipa/ipu3/algorithms/meson.build
> +++ b/src/ipa/ipu3/algorithms/meson.build
> @@ -2,6 +2,7 @@
>
> ipu3_ipa_algorithms = files([
> 'agc_mean.cpp',
> + 'agc_metering.cpp',
> 'algorithm.cpp',
> 'awb.cpp',
> 'tone_mapping.cpp',
> diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h
> index 9d9444dc..0a987da4 100644
> --- a/src/ipa/ipu3/ipa_context.h
> +++ b/src/ipa/ipu3/ipa_context.h
> @@ -10,13 +10,21 @@
>
> #include <linux/intel-ipu3.h>
>
> +#include <libcamera/base/utils.h>
> +
> #include <libcamera/geometry.h>
>
> namespace libcamera {
>
> namespace ipa::ipu3 {
>
> +using utils::Duration;
> +
> struct IPASessionConfiguration {
> + struct {
> + Duration lineDuration;
Or
utils::Duration lineDuration;
and no using directive.
On a side note, Duration should likely be moved out from utils.h, it's
not a base helper.
> + } agc;
> +
> struct {
> ipu3_uapi_grid_config bdsGrid;
> Size bdsOutputSize;
> diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
> index b73c4f7b..1425b344 100644
> --- a/src/ipa/ipu3/ipu3.cpp
> +++ b/src/ipa/ipu3/ipu3.cpp
> @@ -29,7 +29,7 @@
>
> #include "libcamera/internal/mapped_framebuffer.h"
>
> -#include "algorithms/agc_mean.h"
> +#include "algorithms/agc_metering.h"
> #include "algorithms/algorithm.h"
> #include "algorithms/awb.h"
> #include "algorithms/tone_mapping.h"
> @@ -81,6 +81,14 @@
> * are run. This needs to be turned into real per-frame data storage.
> */
>
> +/**
> + * \struct IPASessionConfiguration::agc
> + * \brief AGC configuration of the IPA
> + *
> + * \var IPASessionConfiguration::agc::lineDuration
> + * \brief Duration of one line dependant on the sensor configuration
> + */
> +
> /**
> * \struct IPASessionConfiguration::grid
> * \brief Grid configuration of the IPA
> @@ -266,7 +274,7 @@ int IPAIPU3::init(const IPASettings &settings,
> *ipaControls = ControlInfoMap(std::move(controls), controls::controls);
>
> /* Construct our Algorithms */
> - algorithms_.push_back(std::make_unique<algorithms::AgcMean>());
> + algorithms_.push_back(std::make_unique<algorithms::AgcMetering>());
> algorithms_.push_back(std::make_unique<algorithms::Awb>());
> algorithms_.push_back(std::make_unique<algorithms::ToneMapping>());
>
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list