[PATCH v2 6/8] ipa: ipu3: Remove bespoke AGC functions from IPU3
Jacopo Mondi
jacopo.mondi at ideasonboard.com
Mon Apr 22 12:36:39 CEST 2024
Hi Dan
On Wed, Apr 17, 2024 at 02:15:34PM +0100, Daniel Scally wrote:
> Now that the IPU3's Agc is derived from MeanLuminanceAgc we can
> delete all the unecessary bespoke functions.
>
> Reviewed-by: Stefan Klug <stefan.klug at ideasonboard.com>
> Signed-off-by: Daniel Scally <dan.scally at ideasonboard.com>
> ---
> Changes in v2:
>
> - Kept the documentation for estimateLuminance()
>
> src/ipa/ipu3/algorithms/agc.cpp | 227 +-------------------------------
> src/ipa/ipu3/algorithms/agc.h | 13 --
> 2 files changed, 3 insertions(+), 237 deletions(-)
>
> diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp
> index 3b9761bd..46fc3b33 100644
> --- a/src/ipa/ipu3/algorithms/agc.cpp
> +++ b/src/ipa/ipu3/algorithms/agc.cpp
> @@ -125,8 +125,6 @@ int Agc::configure(IPAContext &context,
> activeState.agc.gain = minAnalogueGain_;
> activeState.agc.exposure = 10ms / configuration.sensor.lineDuration;
>
> - frameCount_ = 0;
> -
> context.activeState.agc.constraintMode = constraintModes().begin()->first;
> context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first;
>
> @@ -139,42 +137,6 @@ int Agc::configure(IPAContext &context,
> return 0;
> }
>
> -/**
> - * \brief Estimate the mean value of the top 2% of the histogram
> - * \param[in] stats The statistics computed by the ImgU
> - * \param[in] grid The grid used to store the statistics in the IPU3
> - * \return The mean value of the top 2% of the histogram
> - */
> -double Agc::measureBrightness(const ipu3_uapi_stats_3a *stats,
> - const ipu3_uapi_grid_config &grid) const
> -{
> - /* Initialise the histogram array */
> - uint32_t hist[knumHistogramBins] = { 0 };
> -
> - for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
> - for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
> - uint32_t cellPosition = cellY * stride_ + cellX;
> -
> - const ipu3_uapi_awb_set_item *cell =
> - reinterpret_cast<const ipu3_uapi_awb_set_item *>(
> - &stats->awb_raw_buffer.meta_data[cellPosition]
> - );
> -
> - uint8_t gr = cell->Gr_avg;
> - uint8_t gb = cell->Gb_avg;
> - /*
> - * Store the average green value to estimate the
> - * brightness. Even the overexposed pixels are
> - * taken into account.
> - */
> - hist[(gr + gb) / 2]++;
> - }
> - }
> -
> - /* Estimate the quantile mean of the top 2% of the histogram. */
> - return Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
> -}
> -
> Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats,
> const ipu3_uapi_grid_config &grid)
> {
> @@ -208,123 +170,9 @@ Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats,
> return Histogram(Span<uint32_t>(hist));
> }
>
> -/**
> - * \brief Apply a filter on the exposure value to limit the speed of changes
> - * \param[in] exposureValue The target exposure from the AGC algorithm
> - *
> - * The speed of the filter is adaptive, and will produce the target quicker
> - * during startup, or when the target exposure is within 20% of the most recent
> - * filter output.
> - *
> - * \return The filtered exposure
> - */
> -utils::Duration Agc::filterExposure(utils::Duration exposureValue)
> -{
> - double speed = 0.2;
> -
> - /* Adapt instantly if we are in startup phase. */
> - if (frameCount_ < kNumStartupFrames)
> - speed = 1.0;
> -
> - /*
> - * If we are close to the desired result, go faster to avoid making
> - * multiple micro-adjustments.
> - * \todo Make this customisable?
> - */
> - if (filteredExposure_ < 1.2 * exposureValue &&
> - filteredExposure_ > 0.8 * exposureValue)
> - speed = sqrt(speed);
> -
> - filteredExposure_ = speed * exposureValue +
> - filteredExposure_ * (1.0 - speed);
> -
> - LOG(IPU3Agc, Debug) << "After filtering, exposure " << filteredExposure_;
> -
> - return filteredExposure_;
> -}
> -
> -/**
> - * \brief Estimate the new exposure and gain values
> - * \param[inout] frameContext The shared IPA frame Context
> - * \param[in] yGain The gain calculated based on the relative luminance target
> - * \param[in] iqMeanGain The gain calculated based on the relative luminance target
> - */
> -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext,
> - double yGain, double iqMeanGain)
> -{
> - const IPASessionConfiguration &configuration = context.configuration;
> - /* Get the effective exposure and gain applied on the sensor. */
> - uint32_t exposure = frameContext.sensor.exposure;
> - double analogueGain = frameContext.sensor.gain;
> -
> - /* Use the highest of the two gain estimates. */
> - double evGain = std::max(yGain, iqMeanGain);
> -
> - /* Consider within 1% of the target as correctly exposed */
> - if (utils::abs_diff(evGain, 1.0) < 0.01)
> - LOG(IPU3Agc, Debug) << "We are well exposed (evGain = "
> - << evGain << ")";
> -
> - /* extracted from Rpi::Agc::computeTargetExposure */
> -
> - /* Calculate the shutter time in seconds */
> - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration;
> -
> - /*
> - * Update the exposure value for the next computation using the values
> - * of exposure and gain really used by the sensor.
> - */
> - utils::Duration effectiveExposureValue = currentShutter * analogueGain;
> -
> - LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
> - << " Shutter speed " << currentShutter
> - << " Gain " << analogueGain
> - << " Needed ev gain " << evGain;
> -
> - /*
> - * Calculate the current exposure value for the scene as the latest
> - * exposure value applied multiplied by the new estimated gain.
> - */
> - utils::Duration exposureValue = effectiveExposureValue * evGain;
> -
> - /* Clamp the exposure value to the min and max authorized */
> - utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
> - exposureValue = std::min(exposureValue, maxTotalExposure);
> - LOG(IPU3Agc, Debug) << "Target total exposure " << exposureValue
> - << ", maximum is " << maxTotalExposure;
> -
> - /*
> - * Filter the exposure.
> - * \todo estimate if we need to desaturate
> - */
> - exposureValue = filterExposure(exposureValue);
> -
> - /*
> - * Divide the exposure value as new exposure and gain values.
> - *
> - * Push the shutter time up to the maximum first, and only then
> - * increase the gain.
> - */
> - utils::Duration shutterTime =
> - std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
> - minShutterSpeed_, maxShutterSpeed_);
> - double stepGain = std::clamp(exposureValue / shutterTime,
> - minAnalogueGain_, maxAnalogueGain_);
> - LOG(IPU3Agc, Debug) << "Divided up shutter and gain are "
> - << shutterTime << " and "
> - << stepGain;
> -}
> -
> /**
> * \brief Estimate the relative luminance of the frame with a given gain
> - * \param[in] frameContext The shared IPA frame context
> - * \param[in] grid The grid used to store the statistics in the IPU3
> - * \param[in] stats The IPU3 statistics and ISP results
> - * \param[in] gain The gain to apply to the frame
> - * \return The relative luminance
> - *
> - * This function estimates the average relative luminance of the frame that
> - * would be output by the sensor if an additional \a gain was applied.
> + * \param[in] gain The gain to apply in estimating luminance
> *
> * The estimation is based on the AWB statistics for the current frame. Red,
> * green and blue averages for all cells are first multiplied by the gain, and
> @@ -339,42 +187,9 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext,
> *
> * More detailed information can be found in:
> * https://en.wikipedia.org/wiki/Relative_luminance
> + *
> + * @return The relative luminance of the frame
While this is valid Doxygen, it's usually "\return"
With this fixed
Reviewed-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
Thanks
j
> */
> -double Agc::estimateLuminance(IPAActiveState &activeState,
> - const ipu3_uapi_grid_config &grid,
> - const ipu3_uapi_stats_3a *stats,
> - double gain)
> -{
> - double redSum = 0, greenSum = 0, blueSum = 0;
> -
> - /* Sum the per-channel averages, saturated to 255. */
> - for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
> - for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
> - uint32_t cellPosition = cellY * stride_ + cellX;
> -
> - const ipu3_uapi_awb_set_item *cell =
> - reinterpret_cast<const ipu3_uapi_awb_set_item *>(
> - &stats->awb_raw_buffer.meta_data[cellPosition]
> - );
> - const uint8_t G_avg = (cell->Gr_avg + cell->Gb_avg) / 2;
> -
> - redSum += std::min(cell->R_avg * gain, 255.0);
> - greenSum += std::min(G_avg * gain, 255.0);
> - blueSum += std::min(cell->B_avg * gain, 255.0);
> - }
> - }
> -
> - /*
> - * Apply the AWB gains to approximate colours correctly, use the Rec.
> - * 601 formula to calculate the relative luminance, and normalize it.
> - */
> - double ySum = redSum * activeState.awb.gains.red * 0.299
> - + greenSum * activeState.awb.gains.green * 0.587
> - + blueSum * activeState.awb.gains.blue * 0.114;
> -
> - return ySum / (grid.height * grid.width) / 255;
> -}
> -
> double Agc::estimateLuminance(double gain)
> {
> double redSum = 0, greenSum = 0, blueSum = 0;
> @@ -408,42 +223,6 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
> const ipu3_uapi_stats_3a *stats,
> ControlList &metadata)
> {
> - /*
> - * Estimate the gain needed to have the proportion of pixels in a given
> - * desired range. iqMean is the mean value of the top 2% of the
> - * cumulative histogram, and we want it to be as close as possible to a
> - * configured target.
> - */
> - double iqMean = measureBrightness(stats, context.configuration.grid.bdsGrid);
> - double iqMeanGain = kEvGainTarget * knumHistogramBins / iqMean;
> -
> - /*
> - * Estimate the gain needed to achieve a relative luminance target. To
> - * account for non-linearity caused by saturation, the value needs to be
> - * estimated in an iterative process, as multiplying by a gain will not
> - * increase the relative luminance by the same factor if some image
> - * regions are saturated.
> - */
> - double yGain = 1.0;
> - double yTarget = kRelativeLuminanceTarget;
> -
> - for (unsigned int i = 0; i < 8; i++) {
> - double yValue = estimateLuminance(context.activeState,
> - context.configuration.grid.bdsGrid,
> - stats, yGain);
> - double extraGain = std::min(10.0, yTarget / (yValue + .001));
> -
> - yGain *= extraGain;
> - LOG(IPU3Agc, Debug) << "Y value: " << yValue
> - << ", Y target: " << yTarget
> - << ", gives gain " << yGain;
> - if (extraGain < 1.01)
> - break;
> - }
> -
> - computeExposure(context, frameContext, yGain, iqMeanGain);
> - frameCount_++;
> -
> Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid);
> rGain_ = context.activeState.awb.gains.red;
> gGain_ = context.activeState.awb.gains.blue;
> diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h
> index 40f32188..945d1846 100644
> --- a/src/ipa/ipu3/algorithms/agc.h
> +++ b/src/ipa/ipu3/algorithms/agc.h
> @@ -38,29 +38,16 @@ public:
> ControlList &metadata) override;
>
> private:
> - double measureBrightness(const ipu3_uapi_stats_3a *stats,
> - const ipu3_uapi_grid_config &grid) const;
> - utils::Duration filterExposure(utils::Duration currentExposure);
> - void computeExposure(IPAContext &context, IPAFrameContext &frameContext,
> - double yGain, double iqMeanGain);
> - double estimateLuminance(IPAActiveState &activeState,
> - const ipu3_uapi_grid_config &grid,
> - const ipu3_uapi_stats_3a *stats,
> - double gain);
> double estimateLuminance(double gain) override;
> Histogram parseStatistics(const ipu3_uapi_stats_3a *stats,
> const ipu3_uapi_grid_config &grid);
>
> - uint64_t frameCount_;
> -
> utils::Duration minShutterSpeed_;
> utils::Duration maxShutterSpeed_;
>
> double minAnalogueGain_;
> double maxAnalogueGain_;
>
> - utils::Duration filteredExposure_;
> -
> uint32_t stride_;
> double rGain_;
> double gGain_;
> --
> 2.34.1
>
More information about the libcamera-devel
mailing list