[libcamera-devel] [PATCH v1 7/7] ipa: ipu3: Implement a new AGC algorithm
David Plowman
david.plowman at raspberrypi.com
Tue Jun 29 15:22:25 CEST 2021
Hi Jean-Michel
Thanks for your work - very happy to see our code being useful elsewhere!
One minor thing - would it be ok to include a Raspberry Pi copyright
alongside the IdeasonBoard one?
Thanks!
David
On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautbois <
jeanmichel.hautbois at ideasonboard.com> wrote:
> This one comes from RPi for most if it, except that we are not
> exchanging any metadata between algorithms for now.
> When process() is called, the current analogue gain and shutter time are
> calculated. The AWB stats from IPU3 are then parsed to generate new
> statistics dedicated to AGC. This new grid is used to estimate the
> luminance and each region is weighted. A default centered metering is
> used as is should be the most used one.
>
> After calculating weighted regions, analogue gain and shutter time are
> divided up and the values are then sent back to the IPAIPU3.
>
> Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
> ---
> src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++--------------
> src/ipa/ipu3/ipu3_agc.h | 23 +++-
> 2 files changed, 196 insertions(+), 103 deletions(-)
>
> diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp
> index 042d67fa..0d421404 100644
> --- a/src/ipa/ipu3/ipu3_agc.cpp
> +++ b/src/ipa/ipu3/ipu3_agc.cpp
> @@ -27,37 +27,19 @@ namespace ipa::ipu3 {
>
> LOG_DEFINE_CATEGORY(IPU3Agc)
>
> -/* Number of frames to wait before calculating stats on minimum exposure
> */
> -static constexpr uint32_t kInitialFrameMinAECount = 4;
> -/* Number of frames to wait between new gain/exposure estimations */
> -static constexpr uint32_t kFrameSkipCount = 6;
> -
> -/* Maximum ISO value for analogue gain */
> -static constexpr uint32_t kMinISO = 100;
> -static constexpr uint32_t kMaxISO = 1500;
> -
> -/* Maximum analogue gain value
> - * \todo grab it from a camera helper */
> -static constexpr uint32_t kMinGain = kMinISO / 100;
> -static constexpr uint32_t kMaxGain = kMaxISO / 100;
> -
> -/* \todo use calculated value based on sensor */
> -static constexpr uint32_t kMinExposure = 1;
> -static constexpr uint32_t kMaxExposure = 1976;
> -
> /* Histogram constants */
> static constexpr uint32_t knumHistogramBins = 256;
> -static constexpr double kEvGainTarget = 0.5;
>
> -/* A cell is 8 bytes and contains averages for RGB values and saturation
> ratio */
> -static constexpr uint8_t kCellSize = 8;
> +/* seems to be a 8-bit pipeline */
> +static constexpr uint8_t kPipelineBits = 8;
>
> IPU3Agc::IPU3Agc()
> : frameCount_(0), lastFrame_(0), converged_(false),
> updateControls_(false), iqMean_(0.0), gamma_(1.0),
> lineDuration_(0s), maxExposureTime_(0s),
> prevExposure_(0s), prevExposureNoDg_(0s),
> - currentExposure_(0s), currentExposureNoDg_(0s)
> + currentExposure_(0s), currentExposureNoDg_(0s),
> + currentShutter_(1.0s), currentAnalogueGain_(1.0)
> {
> }
>
> @@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config
> &bdsGrid, const IPAConfigI
> }
> minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);
> maxGain_ = itGain->second.max().get<int32_t>();
> +
> + /* \todo: those values need to be extracted from a configuration
> file */
> + shutterConstraints_.push_back(100us);
> + shutterConstraints_.push_back(10ms);
> + shutterConstraints_.push_back(33ms);
> + gainConstraints_.push_back(1.0);
> + gainConstraints_.push_back(4.0);
> + gainConstraints_.push_back(16.0);
> +
> + fixedShutter_ = 0s;
> + fixedAnalogueGain_ = 0.0;
> }
>
> -void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats)
> +/* Translate the IPU3 statistics into the default statistics region array
> */
> +void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats)
> {
> - const struct ipu3_uapi_grid_config statsAeGrid =
> stats->stats_4a_config.awb_config.grid;
> - Rectangle aeRegion = { statsAeGrid.x_start,
> - statsAeGrid.y_start,
> - static_cast<unsigned int>(statsAeGrid.x_end
> - statsAeGrid.x_start) + 1,
> - static_cast<unsigned int>(statsAeGrid.y_end
> - statsAeGrid.y_start) + 1 };
> - Point topleft = aeRegion.topLeft();
> - int topleftX = topleft.x >> aeGrid_.block_width_log2;
> - int topleftY = topleft.y >> aeGrid_.block_height_log2;
> -
> - /* Align to the grid cell width and height */
> - uint32_t startX = topleftX << aeGrid_.block_width_log2;
> - uint32_t startY = topleftY * aeGrid_.width <<
> aeGrid_.block_width_log2;
> - uint32_t endX = (startX + (aeRegion.size().width >>
> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2;
> - uint32_t i, j;
> - uint32_t count = 0;
> -
> + uint32_t regionWidth = round(aeGrid_.width /
> static_cast<double>(kAgcStatsSizeX));
> + uint32_t regionHeight = round(aeGrid_.height /
> static_cast<double>(kAgcStatsSizeY));
> uint32_t hist[knumHistogramBins] = { 0 };
> - for (j = topleftY;
> - j < topleftY + (aeRegion.size().height >>
> aeGrid_.block_height_log2);
> - j++) {
> - for (i = startX + startY; i < endX + startY; i +=
> kCellSize) {
> - /*
> - * The grid width (and maybe height) is not
> reliable.
> - * We observed a bit shift which makes the value
> 160 to be 32 in the stats grid.
> - * Use the one passed at init time.
> - */
> - if (stats->awb_raw_buffer.meta_data[i + 4 + j *
> aeGrid_.width] == 0) {
> - uint8_t Gr =
> stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width];
> - uint8_t Gb =
> stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width];
> - hist[(Gr + Gb) / 2]++;
> - count++;
> +
> + LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" <<
> (int)aeGrid_.height << "] regions"
> + << " scaled to [" << regionWidth << "x" <<
> regionHeight << "] AGC stats";
> +
> + /*
> + * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the
> IPU3 grid which is
> + * (aeGrid_.width x aeGrid_.height).
> + */
> + for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) {
> + for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth;
> i++) {
> + uint32_t cellPosition = j * aeGrid_.width + i;
> + uint32_t cellX = (cellPosition / regionWidth) %
> kAgcStatsSizeX;
> + uint32_t cellY = ((cellPosition / aeGrid_.width) /
> regionHeight) % kAgcStatsSizeY;
> +
> + uint32_t agcRegionPosition =
> kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX];
> + weights_[agcRegionPosition] =
> kCenteredWeights[agcRegionPosition];
> + cellPosition *= sizeof(Ipu3AwbCell);
> +
> + /* Cast the initial IPU3 structure to simplify the
> reading */
> + Ipu3AwbCell *currentCell =
> reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t
> *>(&stats->awb_raw_buffer.meta_data[cellPosition]));
> + if (currentCell->satRatio == 0) {
> + /* The cell is not saturated, use the
> current cell */
> + agcStats_[agcRegionPosition].counted++;
> + uint32_t greenValue =
> currentCell->greenRedAvg + currentCell->greenBlueAvg;
> + hist[greenValue / 2]++;
> + agcStats_[agcRegionPosition].gSum +=
> greenValue / 2;
> + agcStats_[agcRegionPosition].rSum +=
> currentCell->redAvg;
> + agcStats_[agcRegionPosition].bSum +=
> currentCell->blueAvg;
> }
> }
> }
>
> - /* Limit the gamma effect for now */
> - gamma_ = 1.1;
> -
> /* Estimate the quantile mean of the top 2% of the histogram */
> iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98,
> 1.0);
> }
>
> +void IPU3Agc::clearStats()
> +{
> + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
> + agcStats_[i].bSum = 0;
> + agcStats_[i].rSum = 0;
> + agcStats_[i].gSum = 0;
> + agcStats_[i].counted = 0;
> + agcStats_[i].uncounted = 0;
> + }
> +
> + awb_.blueGain = 1.0;
> + awb_.greenGain = 1.0;
> + awb_.redGain = 1.0;
> +}
> +
> void IPU3Agc::filterExposure()
> {
> - double speed = 0.2;
> + double speed = 0.08;
> if (prevExposure_ == 0s) {
> /* DG stands for digital gain.*/
> prevExposure_ = currentExposure_;
> @@ -156,65 +162,131 @@ void IPU3Agc::filterExposure()
> * total exposure, as there might not be enough digital gain
> available
> * in the ISP to hide it (which will cause nasty oscillation).
> */
> - double fastReduceThreshold = 0.4;
> + double fastReduceThreshold = 0.3;
> if (prevExposureNoDg_ <
> prevExposure_ * fastReduceThreshold)
> prevExposureNoDg_ = prevExposure_ * fastReduceThreshold;
> LOG(IPU3Agc, Debug) << "After filtering, total_exposure " <<
> prevExposure_;
> }
>
> -void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain)
> +double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const
> &awb,
> + double weights[], double gain)
> {
> - updateControls_ = false;
> + /* Note how the calculation below means that equal weights give you
> + * "average" metering (i.e. all pixels equally important). */
> + double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0;
> + for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
> + double counted = regions[i].counted;
> + double rSum = std::min(regions[i].rSum * gain, ((1 <<
> kPipelineBits) - 1) * counted);
> + double gSum = std::min(regions[i].gSum * gain, ((1 <<
> kPipelineBits) - 1) * counted);
> + double bSum = std::min(regions[i].bSum * gain, ((1 <<
> kPipelineBits) - 1) * counted);
> + redSum += rSum * weights[i];
> + greenSum += gSum * weights[i];
> + blueSum += bSum * weights[i];
> + pixelSum += counted * weights[i];
> + }
> + if (pixelSum == 0.0) {
> + LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is
> zero";
> + return 0;
> + }
> + double Y_sum = redSum * awb.redGain * .299 +
> + greenSum * awb.greenGain * .587 +
> + blueSum * awb.blueGain * .114;
>
> - /* Algorithm initialization should wait for first valid frames */
> - /* \todo - have a number of frames given by DelayedControls ?
> - * - implement a function for IIR */
> - if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ -
> lastFrame_ < kFrameSkipCount))
> - return;
> + return Y_sum / pixelSum / (1 << kPipelineBits);
> +}
>
> - /* Are we correctly exposed ? */
> - if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) {
> - LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = "
> << iqMean_;
> - converged_ = true;
> - } else {
> - double newGain = kEvGainTarget * knumHistogramBins /
> iqMean_;
> -
> - /* extracted from Rpi::Agc::computeTargetExposure */
> - libcamera::utils::Duration currentShutter = exposure *
> lineDuration_;
> - currentExposureNoDg_ = currentShutter * gain;
> - LOG(IPU3Agc, Debug) << "Actual total exposure " <<
> currentExposureNoDg_
> - << " Shutter speed " << currentShutter
> - << " Gain " << gain;
> - currentExposure_ = currentExposureNoDg_ * newGain;
> - libcamera::utils::Duration maxTotalExposure =
> maxExposureTime_ * kMaxGain;
> - currentExposure_ = std::min(currentExposure_,
> maxTotalExposure);
> - LOG(IPU3Agc, Debug) << "Target total exposure " <<
> currentExposure_;
> -
> - /* \todo: estimate if we need to desaturate */
> - filterExposure();
> -
> - libcamera::utils::Duration newExposure = 0.0s;
> - if (currentShutter < maxExposureTime_) {
> - exposure =
> std::clamp(static_cast<uint32_t>(exposure * currentExposure_ /
> currentExposureNoDg_), kMinExposure, kMaxExposure);
> - newExposure = currentExposure_ / exposure;
> - gain = std::clamp(static_cast<uint32_t>(gain *
> currentExposure_ / newExposure), kMinGain, kMaxGain);
> - updateControls_ = true;
> - } else if (currentShutter >= maxExposureTime_) {
> - gain = std::clamp(static_cast<uint32_t>(gain *
> currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain);
> - newExposure = currentExposure_ / gain;
> - exposure =
> std::clamp(static_cast<uint32_t>(exposure * currentExposure_ /
> newExposure), kMinExposure, kMaxExposure);
> - updateControls_ = true;
> +void IPU3Agc::computeTargetExposure(double gain)
> +{
> + currentExposure_ = currentExposureNoDg_ * gain;
> + /* \todo: have a list of shutter speeds */
> + Duration maxShutterSpeed = shutterConstraints_.back();
> + Duration maxTotalExposure = maxShutterSpeed *
> gainConstraints_.back();
> +
> + currentExposure_ = std::min(currentExposure_, maxTotalExposure);
> + LOG(IPU3Agc, Debug) << "Target total_exposure " <<
> currentExposure_;
> +}
> +
> +void IPU3Agc::divideUpExposure()
> +{
> + Duration exposureValue = prevExposure_;
> + Duration shutterTime;
> + double analogueGain;
> + shutterTime = shutterConstraints_[0];
> + shutterTime = std::min(shutterTime, shutterConstraints_.back());
> + analogueGain = gainConstraints_[0];
> +
> + if (shutterTime * analogueGain < exposureValue) {
> + for (unsigned int stage = 1;
> + stage < gainConstraints_.size(); stage++) {
> + if (fixedShutter_ == 0s) {
> + Duration stageShutter =
> +
> std::min(shutterConstraints_[stage], shutterConstraints_.back());
> + if (stageShutter * analogueGain >=
> + exposureValue) {
> + shutterTime =
> + exposureValue /
> analogueGain;
> + break;
> + }
> + shutterTime = stageShutter;
> + }
> + if (fixedAnalogueGain_ == 0.0) {
> + if (gainConstraints_[stage] * shutterTime
> >= exposureValue) {
> + analogueGain = exposureValue /
> shutterTime;
> + break;
> + }
> + analogueGain = gainConstraints_[stage];
> + }
> }
> - LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure *
> lineDuration_ << " and gain " << gain;
> }
> - lastFrame_ = frameCount_;
> + LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " <<
> shutterTime << " and "
> + << analogueGain;
> +
> + /* \todo: flickering avoidance ? */
> + filteredShutter_ = shutterTime;
> + filteredAnalogueGain_ = analogueGain;
> +}
> +
> +void IPU3Agc::computeGain(double ¤tGain)
> +{
> + currentGain = 1.0;
> + /* \todo: the target Y needs to be grabbed from a configuration */
> + double targetY = 0.162;
> + for (int i = 0; i < 8; i++) {
> + double initialY = computeInitialY(agcStats_, awb_,
> weights_, currentGain);
> + double extra_gain = std::min(10.0, targetY / (initialY +
> .001));
> +
> + currentGain *= extra_gain;
> + LOG(IPU3Agc, Debug) << "Initial Y " << initialY << "
> target " << targetY
> + << " gives gain " << currentGain;
> + if (extra_gain < 1.01)
> + break;
> + }
> +
> + double newGain = 128 / iqMean_;
> + LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " <<
> newGain;
> }
>
> -void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t
> &exposure, double &gain)
> +void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t
> &exposure, double &analogueGain)
> {
> - processBrightness(stats);
> - lockExposureGain(exposure, gain);
> + ASSERT(stats->stats_3a_status.awb_en);
> + clearStats();
> + generateStats(stats);
> + currentShutter_ = exposure * lineDuration_;
> + /* \todo: the gain needs to be calculated based on sensor
> informations */
> + currentAnalogueGain_ = analogueGain;
> + currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_;
> +
> + double currentGain = 1;
> + computeGain(currentGain);
> + computeTargetExposure(currentGain);
> + filterExposure();
> + divideUpExposure();
> +
> + exposure = filteredShutter_ / lineDuration_;
> + analogueGain = filteredAnalogueGain_;
> +
> + updateControls_ = true;
> frameCount_++;
> }
>
> diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h
> index ce43c534..f1b1157b 100644
> --- a/src/ipa/ipu3/ipu3_agc.h
> +++ b/src/ipa/ipu3/ipu3_agc.h
> @@ -35,8 +35,8 @@ public:
> IPU3Agc();
> ~IPU3Agc() = default;
>
> - void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure,
> double &gain);
> void initialise(struct ipu3_uapi_grid_config &bdsGrid, const
> IPAConfigInfo &configInfo);
> + void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure,
> double &analogueGain);
> bool converged() { return converged_; }
> bool updateControls() { return updateControls_; }
> /* \todo Use a metadata exchange between IPAs */
> @@ -46,6 +46,17 @@ private:
> void processBrightness(const ipu3_uapi_stats_3a *stats);
> void filterExposure();
> void lockExposureGain(uint32_t &exposure, double &gain);
> + void generateStats(const ipu3_uapi_stats_3a *stats);
> + void clearStats();
> + void generateZones(std::vector<RGB> &zones);
> + double computeInitialY(IspStatsRegion regions[], AwbStatus const
> &awb, double weights[], double gain);
> + void computeTargetExposure(double currentGain);
> + void divideUpExposure();
> + void computeGain(double ¤tGain);
> +
> + AwbStatus awb_;
> + double weights_[kNumAgcWeightedZones];
> + IspStatsRegion agcStats_[kNumAgcWeightedZones];
>
> struct ipu3_uapi_grid_config aeGrid_;
> ControlInfoMap ctrls_;
> @@ -72,6 +83,16 @@ private:
> Duration prevExposureNoDg_;
> Duration currentExposure_;
> Duration currentExposureNoDg_;
> +
> + Duration currentShutter_;
> + std::vector<Duration> shutterConstraints_;
> + Duration fixedShutter_;
> + Duration filteredShutter_;
> +
> + double currentAnalogueGain_;
> + std::vector<double> gainConstraints_;
> + double fixedAnalogueGain_;
> + double filteredAnalogueGain_;
> };
>
> } /* namespace ipa::ipu3 */
> --
> 2.30.2
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.libcamera.org/pipermail/libcamera-devel/attachments/20210629/36577802/attachment-0001.htm>
More information about the libcamera-devel
mailing list