<div dir="ltr">Hi Jean-Michel<div><br></div><div>Thanks for your work - very happy to see our code being useful elsewhere!</div><div><br></div><div>One minor thing - would it be ok to include a Raspberry Pi copyright alongside the IdeasonBoard one?</div><div><br></div><div>Thanks!</div><div>David</div><div><br></div></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Mon, 28 Jun 2021 at 21:23, Jean-Michel Hautbois <<a href="mailto:jeanmichel.hautbois@ideasonboard.com">jeanmichel.hautbois@ideasonboard.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">This one comes from RPi for most if it, except that we are not<br>
exchanging any metadata between algorithms for now.<br>
When process() is called, the current analogue gain and shutter time are<br>
calculated. The AWB stats from IPU3 are then parsed to generate new<br>
statistics dedicated to AGC. This new grid is used to estimate the<br>
luminance and each region is weighted. A default centered metering is<br>
used as is should be the most used one.<br>
<br>
After calculating weighted regions, analogue gain and shutter time are<br>
divided up and the values are then sent back to the IPAIPU3.<br>
<br>
Signed-off-by: Jean-Michel Hautbois <<a href="mailto:jeanmichel.hautbois@ideasonboard.com" target="_blank">jeanmichel.hautbois@ideasonboard.com</a>><br>
---<br>
src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++--------------<br>
src/ipa/ipu3/ipu3_agc.h | 23 +++-<br>
2 files changed, 196 insertions(+), 103 deletions(-)<br>
<br>
diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp<br>
index 042d67fa..0d421404 100644<br>
--- a/src/ipa/ipu3/ipu3_agc.cpp<br>
+++ b/src/ipa/ipu3/ipu3_agc.cpp<br>
@@ -27,37 +27,19 @@ namespace ipa::ipu3 {<br>
<br>
LOG_DEFINE_CATEGORY(IPU3Agc)<br>
<br>
-/* Number of frames to wait before calculating stats on minimum exposure */<br>
-static constexpr uint32_t kInitialFrameMinAECount = 4;<br>
-/* Number of frames to wait between new gain/exposure estimations */<br>
-static constexpr uint32_t kFrameSkipCount = 6;<br>
-<br>
-/* Maximum ISO value for analogue gain */<br>
-static constexpr uint32_t kMinISO = 100;<br>
-static constexpr uint32_t kMaxISO = 1500;<br>
-<br>
-/* Maximum analogue gain value<br>
- * \todo grab it from a camera helper */<br>
-static constexpr uint32_t kMinGain = kMinISO / 100;<br>
-static constexpr uint32_t kMaxGain = kMaxISO / 100;<br>
-<br>
-/* \todo use calculated value based on sensor */<br>
-static constexpr uint32_t kMinExposure = 1;<br>
-static constexpr uint32_t kMaxExposure = 1976;<br>
-<br>
/* Histogram constants */<br>
static constexpr uint32_t knumHistogramBins = 256;<br>
-static constexpr double kEvGainTarget = 0.5;<br>
<br>
-/* A cell is 8 bytes and contains averages for RGB values and saturation ratio */<br>
-static constexpr uint8_t kCellSize = 8;<br>
+/* seems to be a 8-bit pipeline */<br>
+static constexpr uint8_t kPipelineBits = 8;<br>
<br>
IPU3Agc::IPU3Agc()<br>
: frameCount_(0), lastFrame_(0), converged_(false),<br>
updateControls_(false), iqMean_(0.0), gamma_(1.0),<br>
lineDuration_(0s), maxExposureTime_(0s),<br>
prevExposure_(0s), prevExposureNoDg_(0s),<br>
- currentExposure_(0s), currentExposureNoDg_(0s)<br>
+ currentExposure_(0s), currentExposureNoDg_(0s),<br>
+ currentShutter_(1.0s), currentAnalogueGain_(1.0)<br>
{<br>
}<br>
<br>
@@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigI<br>
}<br>
minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);<br>
maxGain_ = itGain->second.max().get<int32_t>();<br>
+<br>
+ /* \todo: those values need to be extracted from a configuration file */<br>
+ shutterConstraints_.push_back(100us);<br>
+ shutterConstraints_.push_back(10ms);<br>
+ shutterConstraints_.push_back(33ms);<br>
+ gainConstraints_.push_back(1.0);<br>
+ gainConstraints_.push_back(4.0);<br>
+ gainConstraints_.push_back(16.0);<br>
+<br>
+ fixedShutter_ = 0s;<br>
+ fixedAnalogueGain_ = 0.0;<br>
}<br>
<br>
-void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats)<br>
+/* Translate the IPU3 statistics into the default statistics region array */<br>
+void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats)<br>
{<br>
- const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid;<br>
- Rectangle aeRegion = { statsAeGrid.x_start,<br>
- statsAeGrid.y_start,<br>
- static_cast<unsigned int>(statsAeGrid.x_end - statsAeGrid.x_start) + 1,<br>
- static_cast<unsigned int>(statsAeGrid.y_end - statsAeGrid.y_start) + 1 };<br>
- Point topleft = aeRegion.topLeft();<br>
- int topleftX = topleft.x >> aeGrid_.block_width_log2;<br>
- int topleftY = topleft.y >> aeGrid_.block_height_log2;<br>
-<br>
- /* Align to the grid cell width and height */<br>
- uint32_t startX = topleftX << aeGrid_.block_width_log2;<br>
- uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2;<br>
- uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2;<br>
- uint32_t i, j;<br>
- uint32_t count = 0;<br>
-<br>
+ uint32_t regionWidth = round(aeGrid_.width / static_cast<double>(kAgcStatsSizeX));<br>
+ uint32_t regionHeight = round(aeGrid_.height / static_cast<double>(kAgcStatsSizeY));<br>
uint32_t hist[knumHistogramBins] = { 0 };<br>
- for (j = topleftY;<br>
- j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2);<br>
- j++) {<br>
- for (i = startX + startY; i < endX + startY; i += kCellSize) {<br>
- /*<br>
- * The grid width (and maybe height) is not reliable.<br>
- * We observed a bit shift which makes the value 160 to be 32 in the stats grid.<br>
- * Use the one passed at init time.<br>
- */<br>
- if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) {<br>
- uint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width];<br>
- uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width];<br>
- hist[(Gr + Gb) / 2]++;<br>
- count++;<br>
+<br>
+ LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" << (int)aeGrid_.height << "] regions"<br>
+ << " scaled to [" << regionWidth << "x" << regionHeight << "] AGC stats";<br>
+<br>
+ /*<br>
+ * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid which is<br>
+ * (aeGrid_.width x aeGrid_.height).<br>
+ */<br>
+ for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) {<br>
+ for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) {<br>
+ uint32_t cellPosition = j * aeGrid_.width + i;<br>
+ uint32_t cellX = (cellPosition / regionWidth) % kAgcStatsSizeX;<br>
+ uint32_t cellY = ((cellPosition / aeGrid_.width) / regionHeight) % kAgcStatsSizeY;<br>
+<br>
+ uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX];<br>
+ weights_[agcRegionPosition] = kCenteredWeights[agcRegionPosition];<br>
+ cellPosition *= sizeof(Ipu3AwbCell);<br>
+<br>
+ /* Cast the initial IPU3 structure to simplify the reading */<br>
+ Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition]));<br>
+ if (currentCell->satRatio == 0) {<br>
+ /* The cell is not saturated, use the current cell */<br>
+ agcStats_[agcRegionPosition].counted++;<br>
+ uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg;<br>
+ hist[greenValue / 2]++;<br>
+ agcStats_[agcRegionPosition].gSum += greenValue / 2;<br>
+ agcStats_[agcRegionPosition].rSum += currentCell->redAvg;<br>
+ agcStats_[agcRegionPosition].bSum += currentCell->blueAvg;<br>
}<br>
}<br>
}<br>
<br>
- /* Limit the gamma effect for now */<br>
- gamma_ = 1.1;<br>
-<br>
/* Estimate the quantile mean of the top 2% of the histogram */<br>
iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);<br>
}<br>
<br>
+void IPU3Agc::clearStats()<br>
+{<br>
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {<br>
+ agcStats_[i].bSum = 0;<br>
+ agcStats_[i].rSum = 0;<br>
+ agcStats_[i].gSum = 0;<br>
+ agcStats_[i].counted = 0;<br>
+ agcStats_[i].uncounted = 0;<br>
+ }<br>
+<br>
+ awb_.blueGain = 1.0;<br>
+ awb_.greenGain = 1.0;<br>
+ awb_.redGain = 1.0;<br>
+}<br>
+<br>
void IPU3Agc::filterExposure()<br>
{<br>
- double speed = 0.2;<br>
+ double speed = 0.08;<br>
if (prevExposure_ == 0s) {<br>
/* DG stands for digital gain.*/<br>
prevExposure_ = currentExposure_;<br>
@@ -156,65 +162,131 @@ void IPU3Agc::filterExposure()<br>
* total exposure, as there might not be enough digital gain available<br>
* in the ISP to hide it (which will cause nasty oscillation).<br>
*/<br>
- double fastReduceThreshold = 0.4;<br>
+ double fastReduceThreshold = 0.3;<br>
if (prevExposureNoDg_ <<br>
prevExposure_ * fastReduceThreshold)<br>
prevExposureNoDg_ = prevExposure_ * fastReduceThreshold;<br>
LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_;<br>
}<br>
<br>
-void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain)<br>
+double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const &awb,<br>
+ double weights[], double gain)<br>
{<br>
- updateControls_ = false;<br>
+ /* Note how the calculation below means that equal weights give you<br>
+ * "average" metering (i.e. all pixels equally important). */<br>
+ double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0;<br>
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {<br>
+ double counted = regions[i].counted;<br>
+ double rSum = std::min(regions[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted);<br>
+ double gSum = std::min(regions[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted);<br>
+ double bSum = std::min(regions[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted);<br>
+ redSum += rSum * weights[i];<br>
+ greenSum += gSum * weights[i];<br>
+ blueSum += bSum * weights[i];<br>
+ pixelSum += counted * weights[i];<br>
+ }<br>
+ if (pixelSum == 0.0) {<br>
+ LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is zero";<br>
+ return 0;<br>
+ }<br>
+ double Y_sum = redSum * awb.redGain * .299 +<br>
+ greenSum * awb.greenGain * .587 +<br>
+ blueSum * awb.blueGain * .114;<br>
<br>
- /* Algorithm initialization should wait for first valid frames */<br>
- /* \todo - have a number of frames given by DelayedControls ?<br>
- * - implement a function for IIR */<br>
- if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount))<br>
- return;<br>
+ return Y_sum / pixelSum / (1 << kPipelineBits);<br>
+}<br>
<br>
- /* Are we correctly exposed ? */<br>
- if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) {<br>
- LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_;<br>
- converged_ = true;<br>
- } else {<br>
- double newGain = kEvGainTarget * knumHistogramBins / iqMean_;<br>
-<br>
- /* extracted from Rpi::Agc::computeTargetExposure */<br>
- libcamera::utils::Duration currentShutter = exposure * lineDuration_;<br>
- currentExposureNoDg_ = currentShutter * gain;<br>
- LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_<br>
- << " Shutter speed " << currentShutter<br>
- << " Gain " << gain;<br>
- currentExposure_ = currentExposureNoDg_ * newGain;<br>
- libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain;<br>
- currentExposure_ = std::min(currentExposure_, maxTotalExposure);<br>
- LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_;<br>
-<br>
- /* \todo: estimate if we need to desaturate */<br>
- filterExposure();<br>
-<br>
- libcamera::utils::Duration newExposure = 0.0s;<br>
- if (currentShutter < maxExposureTime_) {<br>
- exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure);<br>
- newExposure = currentExposure_ / exposure;<br>
- gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / newExposure), kMinGain, kMaxGain);<br>
- updateControls_ = true;<br>
- } else if (currentShutter >= maxExposureTime_) {<br>
- gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain);<br>
- newExposure = currentExposure_ / gain;<br>
- exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure);<br>
- updateControls_ = true;<br>
+void IPU3Agc::computeTargetExposure(double gain)<br>
+{<br>
+ currentExposure_ = currentExposureNoDg_ * gain;<br>
+ /* \todo: have a list of shutter speeds */<br>
+ Duration maxShutterSpeed = shutterConstraints_.back();<br>
+ Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back();<br>
+<br>
+ currentExposure_ = std::min(currentExposure_, maxTotalExposure);<br>
+ LOG(IPU3Agc, Debug) << "Target total_exposure " << currentExposure_;<br>
+}<br>
+<br>
+void IPU3Agc::divideUpExposure()<br>
+{<br>
+ Duration exposureValue = prevExposure_;<br>
+ Duration shutterTime;<br>
+ double analogueGain;<br>
+ shutterTime = shutterConstraints_[0];<br>
+ shutterTime = std::min(shutterTime, shutterConstraints_.back());<br>
+ analogueGain = gainConstraints_[0];<br>
+<br>
+ if (shutterTime * analogueGain < exposureValue) {<br>
+ for (unsigned int stage = 1;<br>
+ stage < gainConstraints_.size(); stage++) {<br>
+ if (fixedShutter_ == 0s) {<br>
+ Duration stageShutter =<br>
+ std::min(shutterConstraints_[stage], shutterConstraints_.back());<br>
+ if (stageShutter * analogueGain >=<br>
+ exposureValue) {<br>
+ shutterTime =<br>
+ exposureValue / analogueGain;<br>
+ break;<br>
+ }<br>
+ shutterTime = stageShutter;<br>
+ }<br>
+ if (fixedAnalogueGain_ == 0.0) {<br>
+ if (gainConstraints_[stage] * shutterTime >= exposureValue) {<br>
+ analogueGain = exposureValue / shutterTime;<br>
+ break;<br>
+ }<br>
+ analogueGain = gainConstraints_[stage];<br>
+ }<br>
}<br>
- LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain;<br>
}<br>
- lastFrame_ = frameCount_;<br>
+ LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " << shutterTime << " and "<br>
+ << analogueGain;<br>
+<br>
+ /* \todo: flickering avoidance ? */<br>
+ filteredShutter_ = shutterTime;<br>
+ filteredAnalogueGain_ = analogueGain;<br>
+}<br>
+<br>
+void IPU3Agc::computeGain(double ¤tGain)<br>
+{<br>
+ currentGain = 1.0;<br>
+ /* \todo: the target Y needs to be grabbed from a configuration */<br>
+ double targetY = 0.162;<br>
+ for (int i = 0; i < 8; i++) {<br>
+ double initialY = computeInitialY(agcStats_, awb_, weights_, currentGain);<br>
+ double extra_gain = std::min(10.0, targetY / (initialY + .001));<br>
+<br>
+ currentGain *= extra_gain;<br>
+ LOG(IPU3Agc, Debug) << "Initial Y " << initialY << " target " << targetY<br>
+ << " gives gain " << currentGain;<br>
+ if (extra_gain < 1.01)<br>
+ break;<br>
+ }<br>
+<br>
+ double newGain = 128 / iqMean_;<br>
+ LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " << newGain;<br>
}<br>
<br>
-void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain)<br>
+void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain)<br>
{<br>
- processBrightness(stats);<br>
- lockExposureGain(exposure, gain);<br>
+ ASSERT(stats->stats_3a_status.awb_en);<br>
+ clearStats();<br>
+ generateStats(stats);<br>
+ currentShutter_ = exposure * lineDuration_;<br>
+ /* \todo: the gain needs to be calculated based on sensor informations */<br>
+ currentAnalogueGain_ = analogueGain;<br>
+ currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_;<br>
+<br>
+ double currentGain = 1;<br>
+ computeGain(currentGain);<br>
+ computeTargetExposure(currentGain);<br>
+ filterExposure();<br>
+ divideUpExposure();<br>
+<br>
+ exposure = filteredShutter_ / lineDuration_;<br>
+ analogueGain = filteredAnalogueGain_;<br>
+<br>
+ updateControls_ = true;<br>
frameCount_++;<br>
}<br>
<br>
diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h<br>
index ce43c534..f1b1157b 100644<br>
--- a/src/ipa/ipu3/ipu3_agc.h<br>
+++ b/src/ipa/ipu3/ipu3_agc.h<br>
@@ -35,8 +35,8 @@ public:<br>
IPU3Agc();<br>
~IPU3Agc() = default;<br>
<br>
- void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain);<br>
void initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigInfo &configInfo);<br>
+ void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain);<br>
bool converged() { return converged_; }<br>
bool updateControls() { return updateControls_; }<br>
/* \todo Use a metadata exchange between IPAs */<br>
@@ -46,6 +46,17 @@ private:<br>
void processBrightness(const ipu3_uapi_stats_3a *stats);<br>
void filterExposure();<br>
void lockExposureGain(uint32_t &exposure, double &gain);<br>
+ void generateStats(const ipu3_uapi_stats_3a *stats);<br>
+ void clearStats();<br>
+ void generateZones(std::vector<RGB> &zones);<br>
+ double computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, double weights[], double gain);<br>
+ void computeTargetExposure(double currentGain);<br>
+ void divideUpExposure();<br>
+ void computeGain(double ¤tGain);<br>
+<br>
+ AwbStatus awb_;<br>
+ double weights_[kNumAgcWeightedZones];<br>
+ IspStatsRegion agcStats_[kNumAgcWeightedZones];<br>
<br>
struct ipu3_uapi_grid_config aeGrid_;<br>
ControlInfoMap ctrls_;<br>
@@ -72,6 +83,16 @@ private:<br>
Duration prevExposureNoDg_;<br>
Duration currentExposure_;<br>
Duration currentExposureNoDg_;<br>
+<br>
+ Duration currentShutter_;<br>
+ std::vector<Duration> shutterConstraints_;<br>
+ Duration fixedShutter_;<br>
+ Duration filteredShutter_;<br>
+<br>
+ double currentAnalogueGain_;<br>
+ std::vector<double> gainConstraints_;<br>
+ double fixedAnalogueGain_;<br>
+ double filteredAnalogueGain_;<br>
};<br>
<br>
} /* namespace ipa::ipu3 */<br>
-- <br>
2.30.2<br>
<br>
</blockquote></div>