[libcamera-devel] [PATCH v1 7/7] ipa: ipu3: Implement a new AGC algorithm
Jean-Michel Hautbois
jeanmichel.hautbois at ideasonboard.com
Mon Jun 28 22:22:55 CEST 2021
This one comes from RPi for most if it, except that we are not
exchanging any metadata between algorithms for now.
When process() is called, the current analogue gain and shutter time are
calculated. The AWB stats from IPU3 are then parsed to generate new
statistics dedicated to AGC. This new grid is used to estimate the
luminance and each region is weighted. A default centered metering is
used as is should be the most used one.
After calculating weighted regions, analogue gain and shutter time are
divided up and the values are then sent back to the IPAIPU3.
Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
---
src/ipa/ipu3/ipu3_agc.cpp | 276 ++++++++++++++++++++++++--------------
src/ipa/ipu3/ipu3_agc.h | 23 +++-
2 files changed, 196 insertions(+), 103 deletions(-)
diff --git a/src/ipa/ipu3/ipu3_agc.cpp b/src/ipa/ipu3/ipu3_agc.cpp
index 042d67fa..0d421404 100644
--- a/src/ipa/ipu3/ipu3_agc.cpp
+++ b/src/ipa/ipu3/ipu3_agc.cpp
@@ -27,37 +27,19 @@ namespace ipa::ipu3 {
LOG_DEFINE_CATEGORY(IPU3Agc)
-/* Number of frames to wait before calculating stats on minimum exposure */
-static constexpr uint32_t kInitialFrameMinAECount = 4;
-/* Number of frames to wait between new gain/exposure estimations */
-static constexpr uint32_t kFrameSkipCount = 6;
-
-/* Maximum ISO value for analogue gain */
-static constexpr uint32_t kMinISO = 100;
-static constexpr uint32_t kMaxISO = 1500;
-
-/* Maximum analogue gain value
- * \todo grab it from a camera helper */
-static constexpr uint32_t kMinGain = kMinISO / 100;
-static constexpr uint32_t kMaxGain = kMaxISO / 100;
-
-/* \todo use calculated value based on sensor */
-static constexpr uint32_t kMinExposure = 1;
-static constexpr uint32_t kMaxExposure = 1976;
-
/* Histogram constants */
static constexpr uint32_t knumHistogramBins = 256;
-static constexpr double kEvGainTarget = 0.5;
-/* A cell is 8 bytes and contains averages for RGB values and saturation ratio */
-static constexpr uint8_t kCellSize = 8;
+/* seems to be a 8-bit pipeline */
+static constexpr uint8_t kPipelineBits = 8;
IPU3Agc::IPU3Agc()
: frameCount_(0), lastFrame_(0), converged_(false),
updateControls_(false), iqMean_(0.0), gamma_(1.0),
lineDuration_(0s), maxExposureTime_(0s),
prevExposure_(0s), prevExposureNoDg_(0s),
- currentExposure_(0s), currentExposureNoDg_(0s)
+ currentExposure_(0s), currentExposureNoDg_(0s),
+ currentShutter_(1.0s), currentAnalogueGain_(1.0)
{
}
@@ -83,55 +65,79 @@ void IPU3Agc::initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigI
}
minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);
maxGain_ = itGain->second.max().get<int32_t>();
+
+ /* \todo: those values need to be extracted from a configuration file */
+ shutterConstraints_.push_back(100us);
+ shutterConstraints_.push_back(10ms);
+ shutterConstraints_.push_back(33ms);
+ gainConstraints_.push_back(1.0);
+ gainConstraints_.push_back(4.0);
+ gainConstraints_.push_back(16.0);
+
+ fixedShutter_ = 0s;
+ fixedAnalogueGain_ = 0.0;
}
-void IPU3Agc::processBrightness(const ipu3_uapi_stats_3a *stats)
+/* Translate the IPU3 statistics into the default statistics region array */
+void IPU3Agc::generateStats(const ipu3_uapi_stats_3a *stats)
{
- const struct ipu3_uapi_grid_config statsAeGrid = stats->stats_4a_config.awb_config.grid;
- Rectangle aeRegion = { statsAeGrid.x_start,
- statsAeGrid.y_start,
- static_cast<unsigned int>(statsAeGrid.x_end - statsAeGrid.x_start) + 1,
- static_cast<unsigned int>(statsAeGrid.y_end - statsAeGrid.y_start) + 1 };
- Point topleft = aeRegion.topLeft();
- int topleftX = topleft.x >> aeGrid_.block_width_log2;
- int topleftY = topleft.y >> aeGrid_.block_height_log2;
-
- /* Align to the grid cell width and height */
- uint32_t startX = topleftX << aeGrid_.block_width_log2;
- uint32_t startY = topleftY * aeGrid_.width << aeGrid_.block_width_log2;
- uint32_t endX = (startX + (aeRegion.size().width >> aeGrid_.block_width_log2)) << aeGrid_.block_width_log2;
- uint32_t i, j;
- uint32_t count = 0;
-
+ uint32_t regionWidth = round(aeGrid_.width / static_cast<double>(kAgcStatsSizeX));
+ uint32_t regionHeight = round(aeGrid_.height / static_cast<double>(kAgcStatsSizeY));
uint32_t hist[knumHistogramBins] = { 0 };
- for (j = topleftY;
- j < topleftY + (aeRegion.size().height >> aeGrid_.block_height_log2);
- j++) {
- for (i = startX + startY; i < endX + startY; i += kCellSize) {
- /*
- * The grid width (and maybe height) is not reliable.
- * We observed a bit shift which makes the value 160 to be 32 in the stats grid.
- * Use the one passed at init time.
- */
- if (stats->awb_raw_buffer.meta_data[i + 4 + j * aeGrid_.width] == 0) {
- uint8_t Gr = stats->awb_raw_buffer.meta_data[i + 0 + j * aeGrid_.width];
- uint8_t Gb = stats->awb_raw_buffer.meta_data[i + 3 + j * aeGrid_.width];
- hist[(Gr + Gb) / 2]++;
- count++;
+
+ LOG(IPU3Agc, Debug) << "[" << (int)aeGrid_.width << "x" << (int)aeGrid_.height << "] regions"
+ << " scaled to [" << regionWidth << "x" << regionHeight << "] AGC stats";
+
+ /*
+ * Generate a (kAgcStatsSizeX x kAgcStatsSizeY) array from the IPU3 grid which is
+ * (aeGrid_.width x aeGrid_.height).
+ */
+ for (unsigned int j = 0; j < kAgcStatsSizeY * regionHeight; j++) {
+ for (unsigned int i = 0; i < kAgcStatsSizeX * regionWidth; i++) {
+ uint32_t cellPosition = j * aeGrid_.width + i;
+ uint32_t cellX = (cellPosition / regionWidth) % kAgcStatsSizeX;
+ uint32_t cellY = ((cellPosition / aeGrid_.width) / regionHeight) % kAgcStatsSizeY;
+
+ uint32_t agcRegionPosition = kAgcStatsRegions[cellY * kAgcStatsSizeX + cellX];
+ weights_[agcRegionPosition] = kCenteredWeights[agcRegionPosition];
+ cellPosition *= sizeof(Ipu3AwbCell);
+
+ /* Cast the initial IPU3 structure to simplify the reading */
+ Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition]));
+ if (currentCell->satRatio == 0) {
+ /* The cell is not saturated, use the current cell */
+ agcStats_[agcRegionPosition].counted++;
+ uint32_t greenValue = currentCell->greenRedAvg + currentCell->greenBlueAvg;
+ hist[greenValue / 2]++;
+ agcStats_[agcRegionPosition].gSum += greenValue / 2;
+ agcStats_[agcRegionPosition].rSum += currentCell->redAvg;
+ agcStats_[agcRegionPosition].bSum += currentCell->blueAvg;
}
}
}
- /* Limit the gamma effect for now */
- gamma_ = 1.1;
-
/* Estimate the quantile mean of the top 2% of the histogram */
iqMean_ = Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
}
+void IPU3Agc::clearStats()
+{
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
+ agcStats_[i].bSum = 0;
+ agcStats_[i].rSum = 0;
+ agcStats_[i].gSum = 0;
+ agcStats_[i].counted = 0;
+ agcStats_[i].uncounted = 0;
+ }
+
+ awb_.blueGain = 1.0;
+ awb_.greenGain = 1.0;
+ awb_.redGain = 1.0;
+}
+
void IPU3Agc::filterExposure()
{
- double speed = 0.2;
+ double speed = 0.08;
if (prevExposure_ == 0s) {
/* DG stands for digital gain.*/
prevExposure_ = currentExposure_;
@@ -156,65 +162,131 @@ void IPU3Agc::filterExposure()
* total exposure, as there might not be enough digital gain available
* in the ISP to hide it (which will cause nasty oscillation).
*/
- double fastReduceThreshold = 0.4;
+ double fastReduceThreshold = 0.3;
if (prevExposureNoDg_ <
prevExposure_ * fastReduceThreshold)
prevExposureNoDg_ = prevExposure_ * fastReduceThreshold;
LOG(IPU3Agc, Debug) << "After filtering, total_exposure " << prevExposure_;
}
-void IPU3Agc::lockExposureGain(uint32_t &exposure, double &gain)
+double IPU3Agc::computeInitialY(IspStatsRegion regions[], AwbStatus const &awb,
+ double weights[], double gain)
{
- updateControls_ = false;
+ /* Note how the calculation below means that equal weights give you
+ * "average" metering (i.e. all pixels equally important). */
+ double redSum = 0, greenSum = 0, blueSum = 0, pixelSum = 0;
+ for (unsigned int i = 0; i < kNumAgcWeightedZones; i++) {
+ double counted = regions[i].counted;
+ double rSum = std::min(regions[i].rSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ double gSum = std::min(regions[i].gSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ double bSum = std::min(regions[i].bSum * gain, ((1 << kPipelineBits) - 1) * counted);
+ redSum += rSum * weights[i];
+ greenSum += gSum * weights[i];
+ blueSum += bSum * weights[i];
+ pixelSum += counted * weights[i];
+ }
+ if (pixelSum == 0.0) {
+ LOG(IPU3Agc, Warning) << "computeInitialY: pixel_sum is zero";
+ return 0;
+ }
+ double Y_sum = redSum * awb.redGain * .299 +
+ greenSum * awb.greenGain * .587 +
+ blueSum * awb.blueGain * .114;
- /* Algorithm initialization should wait for first valid frames */
- /* \todo - have a number of frames given by DelayedControls ?
- * - implement a function for IIR */
- if ((frameCount_ < kInitialFrameMinAECount) || (frameCount_ - lastFrame_ < kFrameSkipCount))
- return;
+ return Y_sum / pixelSum / (1 << kPipelineBits);
+}
- /* Are we correctly exposed ? */
- if (std::abs(iqMean_ - kEvGainTarget * knumHistogramBins) <= 1) {
- LOG(IPU3Agc, Debug) << "!!! Good exposure with iqMean = " << iqMean_;
- converged_ = true;
- } else {
- double newGain = kEvGainTarget * knumHistogramBins / iqMean_;
-
- /* extracted from Rpi::Agc::computeTargetExposure */
- libcamera::utils::Duration currentShutter = exposure * lineDuration_;
- currentExposureNoDg_ = currentShutter * gain;
- LOG(IPU3Agc, Debug) << "Actual total exposure " << currentExposureNoDg_
- << " Shutter speed " << currentShutter
- << " Gain " << gain;
- currentExposure_ = currentExposureNoDg_ * newGain;
- libcamera::utils::Duration maxTotalExposure = maxExposureTime_ * kMaxGain;
- currentExposure_ = std::min(currentExposure_, maxTotalExposure);
- LOG(IPU3Agc, Debug) << "Target total exposure " << currentExposure_;
-
- /* \todo: estimate if we need to desaturate */
- filterExposure();
-
- libcamera::utils::Duration newExposure = 0.0s;
- if (currentShutter < maxExposureTime_) {
- exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / currentExposureNoDg_), kMinExposure, kMaxExposure);
- newExposure = currentExposure_ / exposure;
- gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / newExposure), kMinGain, kMaxGain);
- updateControls_ = true;
- } else if (currentShutter >= maxExposureTime_) {
- gain = std::clamp(static_cast<uint32_t>(gain * currentExposure_ / currentExposureNoDg_), kMinGain, kMaxGain);
- newExposure = currentExposure_ / gain;
- exposure = std::clamp(static_cast<uint32_t>(exposure * currentExposure_ / newExposure), kMinExposure, kMaxExposure);
- updateControls_ = true;
+void IPU3Agc::computeTargetExposure(double gain)
+{
+ currentExposure_ = currentExposureNoDg_ * gain;
+ /* \todo: have a list of shutter speeds */
+ Duration maxShutterSpeed = shutterConstraints_.back();
+ Duration maxTotalExposure = maxShutterSpeed * gainConstraints_.back();
+
+ currentExposure_ = std::min(currentExposure_, maxTotalExposure);
+ LOG(IPU3Agc, Debug) << "Target total_exposure " << currentExposure_;
+}
+
+void IPU3Agc::divideUpExposure()
+{
+ Duration exposureValue = prevExposure_;
+ Duration shutterTime;
+ double analogueGain;
+ shutterTime = shutterConstraints_[0];
+ shutterTime = std::min(shutterTime, shutterConstraints_.back());
+ analogueGain = gainConstraints_[0];
+
+ if (shutterTime * analogueGain < exposureValue) {
+ for (unsigned int stage = 1;
+ stage < gainConstraints_.size(); stage++) {
+ if (fixedShutter_ == 0s) {
+ Duration stageShutter =
+ std::min(shutterConstraints_[stage], shutterConstraints_.back());
+ if (stageShutter * analogueGain >=
+ exposureValue) {
+ shutterTime =
+ exposureValue / analogueGain;
+ break;
+ }
+ shutterTime = stageShutter;
+ }
+ if (fixedAnalogueGain_ == 0.0) {
+ if (gainConstraints_[stage] * shutterTime >= exposureValue) {
+ analogueGain = exposureValue / shutterTime;
+ break;
+ }
+ analogueGain = gainConstraints_[stage];
+ }
}
- LOG(IPU3Agc, Debug) << "Adjust exposure " << exposure * lineDuration_ << " and gain " << gain;
}
- lastFrame_ = frameCount_;
+ LOG(IPU3Agc, Debug) << "Divided up shutter and gain are " << shutterTime << " and "
+ << analogueGain;
+
+ /* \todo: flickering avoidance ? */
+ filteredShutter_ = shutterTime;
+ filteredAnalogueGain_ = analogueGain;
+}
+
+void IPU3Agc::computeGain(double ¤tGain)
+{
+ currentGain = 1.0;
+ /* \todo: the target Y needs to be grabbed from a configuration */
+ double targetY = 0.162;
+ for (int i = 0; i < 8; i++) {
+ double initialY = computeInitialY(agcStats_, awb_, weights_, currentGain);
+ double extra_gain = std::min(10.0, targetY / (initialY + .001));
+
+ currentGain *= extra_gain;
+ LOG(IPU3Agc, Debug) << "Initial Y " << initialY << " target " << targetY
+ << " gives gain " << currentGain;
+ if (extra_gain < 1.01)
+ break;
+ }
+
+ double newGain = 128 / iqMean_;
+ LOG(IPU3Agc, Debug) << "gain: " << currentGain << " new gain: " << newGain;
}
-void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain)
+void IPU3Agc::process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain)
{
- processBrightness(stats);
- lockExposureGain(exposure, gain);
+ ASSERT(stats->stats_3a_status.awb_en);
+ clearStats();
+ generateStats(stats);
+ currentShutter_ = exposure * lineDuration_;
+ /* \todo: the gain needs to be calculated based on sensor informations */
+ currentAnalogueGain_ = analogueGain;
+ currentExposureNoDg_ = currentShutter_ * currentAnalogueGain_;
+
+ double currentGain = 1;
+ computeGain(currentGain);
+ computeTargetExposure(currentGain);
+ filterExposure();
+ divideUpExposure();
+
+ exposure = filteredShutter_ / lineDuration_;
+ analogueGain = filteredAnalogueGain_;
+
+ updateControls_ = true;
frameCount_++;
}
diff --git a/src/ipa/ipu3/ipu3_agc.h b/src/ipa/ipu3/ipu3_agc.h
index ce43c534..f1b1157b 100644
--- a/src/ipa/ipu3/ipu3_agc.h
+++ b/src/ipa/ipu3/ipu3_agc.h
@@ -35,8 +35,8 @@ public:
IPU3Agc();
~IPU3Agc() = default;
- void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &gain);
void initialise(struct ipu3_uapi_grid_config &bdsGrid, const IPAConfigInfo &configInfo);
+ void process(const ipu3_uapi_stats_3a *stats, uint32_t &exposure, double &analogueGain);
bool converged() { return converged_; }
bool updateControls() { return updateControls_; }
/* \todo Use a metadata exchange between IPAs */
@@ -46,6 +46,17 @@ private:
void processBrightness(const ipu3_uapi_stats_3a *stats);
void filterExposure();
void lockExposureGain(uint32_t &exposure, double &gain);
+ void generateStats(const ipu3_uapi_stats_3a *stats);
+ void clearStats();
+ void generateZones(std::vector<RGB> &zones);
+ double computeInitialY(IspStatsRegion regions[], AwbStatus const &awb, double weights[], double gain);
+ void computeTargetExposure(double currentGain);
+ void divideUpExposure();
+ void computeGain(double ¤tGain);
+
+ AwbStatus awb_;
+ double weights_[kNumAgcWeightedZones];
+ IspStatsRegion agcStats_[kNumAgcWeightedZones];
struct ipu3_uapi_grid_config aeGrid_;
ControlInfoMap ctrls_;
@@ -72,6 +83,16 @@ private:
Duration prevExposureNoDg_;
Duration currentExposure_;
Duration currentExposureNoDg_;
+
+ Duration currentShutter_;
+ std::vector<Duration> shutterConstraints_;
+ Duration fixedShutter_;
+ Duration filteredShutter_;
+
+ double currentAnalogueGain_;
+ std::vector<double> gainConstraints_;
+ double fixedAnalogueGain_;
+ double filteredAnalogueGain_;
};
} /* namespace ipa::ipu3 */
--
2.30.2
More information about the libcamera-devel
mailing list