[libcamera-devel] [PATCH v2 07/14] ipa: ipu3: agc: Improve gain calculation
Jean-Michel Hautbois
jeanmichel.hautbois at ideasonboard.com
Wed Nov 10 20:58:54 CET 2021
When an image is partially saturated, its brightness is not increasing
linearly when the shutter time or gain increases. It is a big issue with
a backlight as the algorithm is fading to darkness right now.
Introduce a function to estimate the brightness of the frame, based on
the current exposure/gain and loop on it several times to estimate it
again and approach the non linear function.
Inspired-by: 7de5506c30b3 ("libcamera: src: ipa: raspberrypi: agc: Improve gain update calculation for partly saturated images")
Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
---
src/ipa/ipu3/algorithms/agc.cpp | 103 +++++++++++++++++++++++++++++++-
src/ipa/ipu3/algorithms/agc.h | 6 +-
2 files changed, 105 insertions(+), 4 deletions(-)
diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp
index 119a7938..ee37a9d5 100644
--- a/src/ipa/ipu3/algorithms/agc.cpp
+++ b/src/ipa/ipu3/algorithms/agc.cpp
@@ -67,6 +67,9 @@ static constexpr uint32_t kMinCellsPerZoneRatio = 255 * 20 / 100;
/* Number of frames to wait before calculating stats on minimum exposure */
static constexpr uint32_t kNumStartupFrames = 10;
+/* Maximum luminance used for brightness normalization */
+static constexpr uint32_t kMaxLuminance = 255;
+
Agc::Agc()
: frameCount_(0), iqMean_(0.0), lineDuration_(0s), minExposureLines_(0),
maxExposureLines_(0), filteredExposure_(0s), currentExposure_(0s),
@@ -186,10 +189,16 @@ void Agc::filterExposure()
* \brief Estimate the new exposure and gain values
* \param[inout] exposure The exposure value reference as a number of lines
* \param[inout] gain The gain reference to be updated
+ * \param[in] currentYGain The gain calculated on the current brightness level
*/
-void Agc::computeExposure(uint32_t &exposure, double &analogueGain)
+void Agc::computeExposure(uint32_t &exposure, double &analogueGain, double currentYGain)
{
- /* Estimate the gain needed to have the proportion wanted */
+ /*
+ * Estimate the gain needed to have the proportion of pixels in a given
+ * range wanted. iqMean_ returns the mean value of the top 2% of the
+ * cumulative histogram, and we want it to be as close as possible to a
+ * configured target.
+ */
double evGain = kEvGainTarget * knumHistogramBins / iqMean_;
if (std::abs(evGain - 1.0) < 0.01) {
@@ -199,6 +208,7 @@ void Agc::computeExposure(uint32_t &exposure, double &analogueGain)
}
/* extracted from Rpi::Agc::computeTargetExposure */
+
/* Calculate the shutter time in seconds */
utils::Duration currentShutter = exposure * lineDuration_;
LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
@@ -206,6 +216,14 @@ void Agc::computeExposure(uint32_t &exposure, double &analogueGain)
<< " Gain " << analogueGain
<< " Needed ev gain " << evGain;
+ if (evGain < currentYGain)
+ evGain = currentYGain;
+
+ /* Consider within 1% of the target as correctly exposed */
+ if (std::abs(evGain - 1.0) < 0.01)
+ LOG(IPU3Agc, Debug) << "We are well exposed (iqMean = "
+ << iqMean_ << ")";
+
/*
* Calculate the current exposure value for the scene as the latest
* exposure value applied multiplied by the new estimated gain.
@@ -253,6 +271,57 @@ void Agc::computeExposure(uint32_t &exposure, double &analogueGain)
prevExposureValue_ = shutterTime * analogueGain;
}
+/**
+ * \brief Estimate the average brightness of the frame
+ * \param[in] context The shared IPA context
+ * \param[in] grid The grid used to store the statistics in the IPU3
+ * \param[in] stats The IPU3 statistics and ISP results
+ * \param[in] currentYGain The gain calculated on the current brightness level
+ * \return The normalized luma
+ *
+ * Luma is the weighted sum of gamma-compressed R′G′B′ components of a color
+ * video. The luma values are normalized as 0.0 to 1.0, with 1.0 being a
+ * theoretical perfect reflector of 100% reference white. We use the Rec. 601
+ * luma here.
+ *
+ * More detailed information can be found in:
+ * https://en.wikipedia.org/wiki/Luma_(video)
+ */
+double Agc::computeInitialY(IPAFrameContext &frameContext,
+ const ipu3_uapi_grid_config &grid,
+ const ipu3_uapi_stats_3a *stats,
+ double currentYGain)
+{
+ double redSum = 0, greenSum = 0, blueSum = 0;
+
+ for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
+ for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
+ uint32_t cellPosition = cellY * stride_ + cellX;
+
+ const ipu3_uapi_awb_set_item *cell =
+ reinterpret_cast<const ipu3_uapi_awb_set_item *>(
+ &stats->awb_raw_buffer.meta_data[cellPosition]
+ );
+
+ redSum += cell->R_avg * currentYGain;
+ greenSum += (cell->Gr_avg + cell->Gb_avg) / 2 * currentYGain;
+ blueSum += cell->B_avg * currentYGain;
+ }
+ }
+
+ /*
+ * Estimate the sum of the brightness values, weighted with the gains
+ * applied on the channels in AWB as the Rec. 601 luma.
+ */
+ double Y_sum = redSum * frameContext.awb.gains.red * .299 +
+ greenSum * frameContext.awb.gains.green * .587 +
+ blueSum * frameContext.awb.gains.blue * .114;
+
+ /* Return the normalized relative luminance. */
+ return Y_sum / (grid.height * grid.width) / kMaxLuminance;
+}
+
+
/**
* \brief Process IPU3 statistics, and run AGC operations
* \param[in] context The shared IPA context
@@ -267,7 +336,35 @@ void Agc::process(IPAContext &context, const ipu3_uapi_stats_3a *stats)
uint32_t &exposure = context.frameContext.agc.exposure;
double &analogueGain = context.frameContext.agc.gain;
measureBrightness(stats, context.configuration.grid.bdsGrid);
- computeExposure(exposure, analogueGain);
+
+ double currentYGain = 1.0;
+ /*
+ * Normalized luma value target.
+ *
+ * It's a number that's chosen so that, when the camera points at a grey
+ * target, the resulting image brightness is considered right.
+ */
+ double targetY = 0.16;
+
+ /*
+ * Do this calculation a few times as brightness increase can be
+ * non-linear when there are saturated regions.
+ */
+ for (int i = 0; i < 8; i++) {
+ double initialY = computeInitialY(context.frameContext,
+ context.configuration.grid.bdsGrid,
+ stats, currentYGain);
+ double extra_gain = std::min(10.0, targetY / (initialY + .001));
+
+ currentYGain *= extra_gain;
+ LOG(IPU3Agc, Debug) << "Initial Y " << initialY
+ << " target " << targetY
+ << " gives gain " << currentYGain;
+ if (extra_gain < 1.01)
+ break;
+ }
+
+ computeExposure(exposure, analogueGain, currentYGain);
frameCount_++;
}
diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h
index 69e0b831..0a9152a9 100644
--- a/src/ipa/ipu3/algorithms/agc.h
+++ b/src/ipa/ipu3/algorithms/agc.h
@@ -34,7 +34,11 @@ private:
void measureBrightness(const ipu3_uapi_stats_3a *stats,
const ipu3_uapi_grid_config &grid);
void filterExposure();
- void computeExposure(uint32_t &exposure, double &gain);
+ void computeExposure(uint32_t &exposure, double &gain, double currentYGain);
+ double computeInitialY(IPAFrameContext &frameContext,
+ const ipu3_uapi_grid_config &grid,
+ const ipu3_uapi_stats_3a *stats,
+ double currentYGain);
uint64_t frameCount_;
uint64_t lastFrame_;
--
2.32.0
More information about the libcamera-devel
mailing list