[libcamera-devel] [PATCH 2/2] ipa: ipu3: Use metadata and improve the doc
Jean-Michel Hautbois
jeanmichel.hautbois at ideasonboard.com
Mon Jul 12 15:16:30 CEST 2021
Using the metadata to exchange the status of the awb algorithm helps to
simplify the interface. The structures used by the AWB statistics are in
ipu3_awb.h and documented.
A bit of the doc has been improved in this same patch.
There is one metadata variable which will be used and set in the AGC
algorithm in a near future, which is the agcGamma. Use it as if it exists, the
Metadata class won't find it and awb will default to a 1.0 value.
Doing it now is convenient to have nice process() and updateParamaters() calls.
Signed-off-by: Jean-Michel Hautbois <jeanmichel.hautbois at ideasonboard.com>
---
src/ipa/ipu3/ipu3.cpp | 8 ++-
src/ipa/ipu3/ipu3_awb.cpp | 116 ++++++++++++++++++++++++++++----------
src/ipa/ipu3/ipu3_awb.h | 34 ++++++-----
3 files changed, 111 insertions(+), 47 deletions(-)
diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
index 091856f5..1a3d98e9 100644
--- a/src/ipa/ipu3/ipu3.cpp
+++ b/src/ipa/ipu3/ipu3.cpp
@@ -80,6 +80,8 @@ private:
std::unique_ptr<IPU3Agc> agcAlgo_;
/* Interface to the Camera Helper */
std::unique_ptr<CameraSensorHelper> camHelper_;
+ /* Metadata storage */
+ Metadata metadata_;
/* Local parameter storage */
struct ipu3_uapi_params params_;
@@ -277,7 +279,7 @@ void IPAIPU3::processControls([[maybe_unused]] unsigned int frame,
void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)
{
if (agcAlgo_->updateControls())
- awbAlgo_->updateWbParameters(params_, agcAlgo_->gamma());
+ awbAlgo_->updateWbParameters(params_, &metadata_);
*params = params_;
@@ -297,8 +299,10 @@ void IPAIPU3::parseStatistics(unsigned int frame,
agcAlgo_->process(stats, exposure_, gain);
gain_ = camHelper_->gainCode(gain);
- awbAlgo_->calculateWBGains(stats);
+ /* Calculate the AWB gains */
+ awbAlgo_->process(stats, &metadata_);
+ /* Update the exposure and gains on sensor side */
if (agcAlgo_->updateControls())
setControls(frame);
diff --git a/src/ipa/ipu3/ipu3_awb.cpp b/src/ipa/ipu3/ipu3_awb.cpp
index 9b409c8f..d441e835 100644
--- a/src/ipa/ipu3/ipu3_awb.cpp
+++ b/src/ipa/ipu3/ipu3_awb.cpp
@@ -18,29 +18,54 @@ namespace ipa::ipu3 {
LOG_DEFINE_CATEGORY(IPU3Awb)
-static constexpr uint32_t kMinZonesCounted = 16;
-static constexpr uint32_t kMinGreenLevelInZone = 32;
+/**
+ * The Grey World algorithm assumes that the scene, in average, is neutral grey.
+ * Reference: Lam, Edmund & Fung, George. (2008). Automatic White Balancing in
+ * Digital Photography. 10.1201/9781420054538.ch10.
+ *
+ * The IPU3 is generating statistics from the Bayer Demosaic Scaler output
+ * into a grid defined in the ipu3_uapi_awb_config_s structure.
+ *
+ * For example, when the BDS output is 2592x1944 then the grid is calculated to be:
+ * 81*30 with a cell beeing of size 32*64.
+ * We then have an average of 2048 R, G and B pixels per cell.
+ *
+ * The AWB algorithm could use those variable grid sizes as an input, but it would
+ * make it a bit more complex. In order to have something consistent with what is
+ * done on RPi, fix a default grid size to kAwbStatsSizeX x kAwbStatsSizeY.
+ *
+ * Before calculating the gains, we will convert the statistics to go from the BDS
+ * grid configuration to this intern grid size in generateAwbStats.
+ * When the stats are converted, the saturation flag in the initial grid is used to
+ * decide if the zone is saturated or not, making the zone relevant or not.
+ *
+ * The Grey World algorithm will then estimate the red and blue gains to apply, and
+ * send those back through metadata.
+ */
/**
- * \struct IspStatsRegion
+ * \struct StatsRegion
* \brief RGB statistics for a given region
*
- * The IspStatsRegion structure is intended to abstract the ISP specific
- * statistics and use an agnostic algorithm to compute AWB.
+ * The StatsRegion structure is intended to abstract the ISP specific
+ * statistics to compute AWB. The Grey World algorithm uses an average
+ * for a specific counted pixels. When a specific zone in the scene is
+ * saturated, we want to exclude it from the calculation, and consider
+ * it as an outlier.
*
- * \var IspStatsRegion::counted
- * \brief Number of pixels used to calculate the sums
+ * \var StatsRegion::counted
+ * \brief Number of unsatured pixels used to calculate the sums
*
- * \var IspStatsRegion::uncounted
- * \brief Remaining number of pixels in the region
+ * \var StatsRegion::uncounted
+ * \brief Remaining number of pixels in the region (ie saturated)
*
- * \var IspStatsRegion::rSum
+ * \var StatsRegion::rSum
* \brief Sum of the red values in the region
*
- * \var IspStatsRegion::gSum
+ * \var StatsRegion::gSum
* \brief Sum of the green values in the region
*
- * \var IspStatsRegion::bSum
+ * \var StatsRegion::bSum
* \brief Sum of the blue values in the region
*/
@@ -48,26 +73,35 @@ static constexpr uint32_t kMinGreenLevelInZone = 32;
* \struct AwbStatus
* \brief AWB parameters calculated
*
- * The AwbStatus structure is intended to store the AWB
- * parameters calculated by the algorithm
+ * The AwbStatus structure is intended to store the AWB parameters
+ * calculated by the algorithm, and shared through the metadata
+ * object, for other algorithms
*
* \var AwbStatus::temperatureK
- * \brief Color temperature calculated
+ * \brief Color temperature calculated, in Kelvin
*
* \var AwbStatus::redGain
- * \brief Gain calculated for the red channel
+ * \brief Gain calculated for the red channel. This is a floating-point
+ * value used as a multiplier on the ISP side
*
* \var AwbStatus::greenGain
- * \brief Gain calculated for the green channel
+ * \brief Gain calculated for the green channel. This is a floating-point
+ * value used as a multiplier on the ISP side
*
* \var AwbStatus::blueGain
- * \brief Gain calculated for the blue channel
+ * \brief Gain calculated for the blue channel. This is a floating-point
+ * value used as a multiplier on the ISP side
*/
/**
* \struct Ipu3AwbCell
* \brief Memory layout for each cell in AWB metadata
*
+ * This is the internal layout on IPU3 for one cell of AWB statistics.
+ * There is ipu3_uapi_awb_config_s->grid.width * 2^block_width_log2 per
+ * ipu3_uapi_awb_config_s->grid.height * 2^block_height_log2 cells for
+ * one frame statistics.
+ *
* The Ipu3AwbCell structure is used to get individual values
* such as red average or saturation ratio in a particular cell.
*
@@ -84,7 +118,8 @@ static constexpr uint32_t kMinGreenLevelInZone = 32;
* \brief Green average for blue lines
*
* \var Ipu3AwbCell::satRatio
- * \brief Saturation ratio in the cell
+ * \brief Saturation ratio in the cell. It depends on the rgbs_thr_* values, and
+ * will be set if rgbs_thr_b has IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT bit set.
*
* \var Ipu3AwbCell::padding
* \brief array of unused bytes for padding
@@ -159,6 +194,9 @@ const struct ipu3_uapi_gamma_corr_lut imguCssGammaLut = { {
7807, 7871, 7935, 7999, 8063, 8127, 8191
} };
+/* Minimum level of green (on a 8 bits base) in a given zone */
+static constexpr uint32_t kMinGreenLevelInZone = 16;
+
IPU3Awb::IPU3Awb()
: Algorithm()
{
@@ -166,6 +204,7 @@ IPU3Awb::IPU3Awb()
asyncResults_.greenGain = 1.0;
asyncResults_.redGain = 1.0;
asyncResults_.temperatureK = 4500;
+ minZonesCounted_ = 0;
}
IPU3Awb::~IPU3Awb()
@@ -202,7 +241,7 @@ void IPU3Awb::initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, st
params.acc_param.gamma.gc_lut = imguCssGammaLut;
params.acc_param.gamma.gc_ctrl.enable = 1;
- zones_.reserve(kAwbStatsSizeX * kAwbStatsSizeY);
+ zones_.reserve(kAwbStatsSize);
}
/**
@@ -238,10 +277,10 @@ uint32_t IPU3Awb::estimateCCT(double red, double green, double blue)
/* Generate an RGB vector with the average values for each region */
void IPU3Awb::generateZones(std::vector<RGB> &zones)
{
- for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) {
+ for (unsigned int i = 0; i < kAwbStatsSize; i++) {
RGB zone;
double counted = awbStats_[i].counted;
- if (counted >= kMinZonesCounted) {
+ if (counted >= minZonesCounted_) {
zone.G = awbStats_[i].gSum / counted;
if (zone.G >= kMinGreenLevelInZone) {
zone.R = awbStats_[i].rSum / counted;
@@ -258,6 +297,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats)
uint32_t regionWidth = round(awbGrid_.width / static_cast<double>(kAwbStatsSizeX));
uint32_t regionHeight = round(awbGrid_.height / static_cast<double>(kAwbStatsSizeY));
+ minZonesCounted_ = ((regionWidth * regionHeight) * 4) / 5;
/*
* Generate a (kAwbStatsSizeX x kAwbStatsSizeY) array from the IPU3 grid which is
* (awbGrid_.width x awbGrid_.height).
@@ -269,7 +309,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats)
uint32_t cellY = ((cellPosition / awbGrid_.width) / regionHeight) % kAwbStatsSizeY;
uint32_t awbRegionPosition = cellY * kAwbStatsSizeX + cellX;
- cellPosition *= 8;
+ cellPosition *= sizeof(Ipu3AwbCell);
/* Cast the initial IPU3 structure to simplify the reading */
Ipu3AwbCell *currentCell = reinterpret_cast<Ipu3AwbCell *>(const_cast<uint8_t *>(&stats->awb_raw_buffer.meta_data[cellPosition]));
@@ -287,7 +327,7 @@ void IPU3Awb::generateAwbStats(const ipu3_uapi_stats_3a *stats)
void IPU3Awb::clearAwbStats()
{
- for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) {
+ for (unsigned int i = 0; i < kAwbStatsSize; i++) {
awbStats_[i].bSum = 0;
awbStats_[i].rSum = 0;
awbStats_[i].gSum = 0;
@@ -344,24 +384,38 @@ void IPU3Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats)
generateAwbStats(stats);
generateZones(zones_);
LOG(IPU3Awb, Debug) << "Valid zones: " << zones_.size();
- if (zones_.size() > 10) {
+
+ /* We need at least 5% of valid zones to estimate the gain correction */
+ if (zones_.size() > kAwbStatsSize / 20) {
awbGreyWorld();
LOG(IPU3Awb, Debug) << "Gain found for red: " << asyncResults_.redGain
<< " and for blue: " << asyncResults_.blueGain;
}
}
-void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma)
+void IPU3Awb::process(const ipu3_uapi_stats_3a *stats, Metadata *imageMetadata)
+{
+ calculateWBGains(stats);
+ /* We need to update the AWB status, to give back the gains */
+ imageMetadata->set("awb.status", asyncResults_);
+}
+
+void IPU3Awb::updateWbParameters(ipu3_uapi_params ¶ms, Metadata *imageMetadata)
{
/*
* Green gains should not be touched and considered 1.
* Default is 16, so do not change it at all.
- * 4096 is the value for a gain of 1.0
+ * 8192 is the value for a gain of 1.0
*/
- params.acc_param.bnr.wb_gains.gr = 16;
- params.acc_param.bnr.wb_gains.r = 4096 * asyncResults_.redGain;
- params.acc_param.bnr.wb_gains.b = 4096 * asyncResults_.blueGain;
- params.acc_param.bnr.wb_gains.gb = 16;
+ params.acc_param.bnr.wb_gains.gr = 8192;
+ params.acc_param.bnr.wb_gains.r = 8192 * asyncResults_.redGain;
+ params.acc_param.bnr.wb_gains.b = 8192 * asyncResults_.blueGain;
+ params.acc_param.bnr.wb_gains.gb = 8192;
+
+ /* When the AGC algorithm has run, it may have set a new gamma */
+ double agcGamma = 1.0;
+ if (imageMetadata->get("agc.gamma", agcGamma) != 0)
+ LOG(IPU3Awb, Debug) << "Awb: no gamma found, defaulted to 1.0";
LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK
<< " and gamma calculated: " << agcGamma;
diff --git a/src/ipa/ipu3/ipu3_awb.h b/src/ipa/ipu3/ipu3_awb.h
index 122cf68c..cf12032d 100644
--- a/src/ipa/ipu3/ipu3_awb.h
+++ b/src/ipa/ipu3/ipu3_awb.h
@@ -14,14 +14,18 @@
#include <libcamera/geometry.h>
#include "libipa/algorithm.h"
+#include "libipa/metadata.h"
namespace libcamera {
namespace ipa::ipu3 {
-/* Region size for the statistics generation algorithm */
+/* Width of the AWB regions used for Grey World calculation */
static constexpr uint32_t kAwbStatsSizeX = 16;
+/* Height of the AWB regions used for Grey World calculation */
static constexpr uint32_t kAwbStatsSizeY = 12;
+/* Total size of the AWB regions used for Grey World calculation */
+static constexpr uint32_t kAwbStatsSize = kAwbStatsSizeX * kAwbStatsSizeY;
class IPU3Awb : public Algorithm
{
@@ -30,17 +34,8 @@ public:
~IPU3Awb();
void initialise(ipu3_uapi_params ¶ms, const Size &bdsOutputSize, struct ipu3_uapi_grid_config &bdsGrid);
- void calculateWBGains(const ipu3_uapi_stats_3a *stats);
- void updateWbParameters(ipu3_uapi_params ¶ms, double agcGamma);
-
- struct Ipu3AwbCell {
- unsigned char greenRedAvg;
- unsigned char redAvg;
- unsigned char blueAvg;
- unsigned char greenBlueAvg;
- unsigned char satRatio;
- unsigned char padding[3];
- } __attribute__((packed));
+ void process(const ipu3_uapi_stats_3a *stats, Metadata *imageMetadata);
+ void updateWbParameters(ipu3_uapi_params ¶ms, Metadata *imageMetadata);
/* \todo Make these three structs available to all the ISPs ? */
struct RGB {
@@ -56,7 +51,7 @@ public:
}
};
- struct IspStatsRegion {
+ struct StatsRegion {
unsigned int counted;
unsigned int uncounted;
unsigned long long rSum;
@@ -71,18 +66,29 @@ public:
double blueGain;
};
+ struct Ipu3AwbCell {
+ unsigned char greenRedAvg;
+ unsigned char redAvg;
+ unsigned char blueAvg;
+ unsigned char greenBlueAvg;
+ unsigned char satRatio;
+ unsigned char padding[3];
+ };
+
private:
void generateZones(std::vector<RGB> &zones);
void generateAwbStats(const ipu3_uapi_stats_3a *stats);
void clearAwbStats();
void awbGreyWorld();
uint32_t estimateCCT(double red, double green, double blue);
+ void calculateWBGains(const ipu3_uapi_stats_3a *stats);
struct ipu3_uapi_grid_config awbGrid_;
std::vector<RGB> zones_;
- IspStatsRegion awbStats_[kAwbStatsSizeX * kAwbStatsSizeY];
+ StatsRegion awbStats_[kAwbStatsSize];
AwbStatus asyncResults_;
+ uint32_t minZonesCounted_;
};
} /* namespace ipa::ipu3 */
--
2.30.2
More information about the libcamera-devel
mailing list