[PATCH 08/10] ipa: mali-c55: Add AWB Algorithm
Daniel Scally
dan.scally at ideasonboard.com
Thu Jun 13 15:26:00 CEST 2024
Add a simple grey-world auto white balance algorithm to the mali-c55
IPA.
Acked-by: Nayden Kanchev <nayden.kanchev at arm.com>
Co-developed-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
Signed-off-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
Signed-off-by: Daniel Scally <dan.scally at ideasonboard.com>
---
src/ipa/mali-c55/algorithms/awb.cpp | 231 ++++++++++++++++++++++++
src/ipa/mali-c55/algorithms/awb.h | 38 ++++
src/ipa/mali-c55/algorithms/meson.build | 1 +
src/ipa/mali-c55/ipa_context.h | 10 +
4 files changed, 280 insertions(+)
create mode 100644 src/ipa/mali-c55/algorithms/awb.cpp
create mode 100644 src/ipa/mali-c55/algorithms/awb.h
diff --git a/src/ipa/mali-c55/algorithms/awb.cpp b/src/ipa/mali-c55/algorithms/awb.cpp
new file mode 100644
index 00000000..5ab3dcba
--- /dev/null
+++ b/src/ipa/mali-c55/algorithms/awb.cpp
@@ -0,0 +1,231 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2024, Ideas On Board Oy
+ *
+ * awb.cpp - Mali C55 grey world auto white balance algorithm
+ */
+
+#include "awb.h"
+
+#include <math.h>
+
+#include <libcamera/base/log.h>
+
+#include <libcamera/control_ids.h>
+
+namespace libcamera {
+
+namespace ipa::mali_c55::algorithms {
+
+LOG_DEFINE_CATEGORY(MaliC55Awb)
+
+/* Number of frames at which we should run AWB at full speed */
+static constexpr uint32_t kNumStartupFrames = 4;
+
+Awb::Awb()
+{
+}
+
+int Awb::configure([[maybe_unused]] IPAContext &context,
+ [[maybe_unused]] const IPACameraSensorInfo &configInfo)
+{
+ /*
+ * Initially we have no idea what the colour balance will be like, so
+ * for the first frame we will make no assumptions and leave the R/B
+ * channels unmodified.
+ */
+ context.activeState.awb.rGain = 1.0;
+ context.activeState.awb.bGain = 1.0;
+
+ return 0;
+}
+
+void Awb::fillGainsParamBlock(mali_c55_params_block_header *block, IPAContext &context,
+ IPAFrameContext &frameContext)
+{
+ struct mali_c55_params_awb_gains *gains =
+ reinterpret_cast<struct mali_c55_params_awb_gains *>(block);
+
+ gains->header.type = MALI_C55_PARAM_BLOCK_AWB_GAINS;
+ gains->header.enabled = true;
+ gains->header.size = sizeof(struct mali_c55_params_awb_gains);
+
+ double rGain = context.activeState.awb.rGain;
+ double bGain = context.activeState.awb.bGain;
+
+ /*
+ * The gains here map as follows:
+ * gain00 = R
+ * gain01 = Gr
+ * gain10 = Gb
+ * gain11 = B
+ *
+ * This holds true regardless of the bayer order of the input data, as
+ * the mapping is done internally in the ISP.
+ */
+ gains->gain00 = int(rGain * pow(2, 8));
+ gains->gain01 = 256; /* Otherwise known as 1.0 */
+ gains->gain10 = 256;
+ gains->gain11 = int(bGain * pow(2, 8));
+
+ frameContext.awb.rGain = rGain;
+ frameContext.awb.bGain = bGain;
+}
+
+void Awb::fillConfigParamBlock(mali_c55_params_block_header *block)
+{
+ struct mali_c55_params_awb_config *config =
+ reinterpret_cast<struct mali_c55_params_awb_config *>(block);
+
+ config->header.type = MALI_C55_PARAM_BLOCK_AWB_CONFIG;
+ config->header.enabled = true;
+ config->header.size = sizeof(struct mali_c55_params_awb_config);
+
+ /* Tap the stats after the purple fringe block */
+ config->tap_point = MALI_C55_AWB_STATS_TAP_PF;
+
+ /* Get R/G and B/G ratios as statistics */
+ config->stats_mode = MALI_C55_AWB_MODE_RGBG;
+
+ /* Default white level */
+ config->white_level = 1023;
+
+ /* Default black level */
+ config->black_level = 0;
+
+ /*
+ * By default pixels are included who's colour ratios are bounded in a
+ * region (on a cr ratio x cb ratio graph) defined by four points:
+ * (0.25, 0.25)
+ * (0.25, 1.99609375)
+ * (1.99609375, 1.99609375)
+ * (1.99609375, 0.25)
+ *
+ * The ratios themselves are stored in Q4.8 format.
+ *
+ * \todo should these perhaps be tunable?
+ */
+ config->cr_max = 511;
+ config->cr_min = 64;
+ config->cb_max = 511;
+ config->cb_min = 64;
+
+ /* We use the full 15x15 zoning scheme */
+ config->nodes_used_horiz = 15;
+ config->nodes_used_vert = 15;
+
+ /*
+ * We set the trimming boundaries equivalent to the main boundaries. In
+ * other words; no trimming.
+ */
+ config->cr_high = 511;
+ config->cr_low = 64;
+ config->cb_high = 511;
+ config->cb_low = 64;
+
+}
+
+void Awb::prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext, mali_c55_params_block_header *block)
+{
+ fillGainsParamBlock(block, context, frameContext);
+
+ if (frame > 0)
+ return;
+
+ char *params = reinterpret_cast<char *>(block);
+
+ block = reinterpret_cast<mali_c55_params_block_header *>
+ (params + sizeof(struct mali_c55_params_awb_gains));
+
+ fillConfigParamBlock(block);
+}
+
+void Awb::process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext, const mali_c55_stats_buffer *stats,
+ [[maybe_unused]] ControlList &metadata)
+{
+ const struct mali_c55_awb_average_ratios *awb_ratios = stats->awb_ratios;
+
+ /*
+ * The ISP produces average R:G and B:G ratios for zones. We take the
+ * average of all the zones with data and simply invert them to provide
+ * gain figures that we can apply to approximate a grey world.
+ */
+ unsigned int counted_zones = 0;
+ double rgSum = 0, bgSum = 0;
+
+ for (unsigned int i = 0; i < 225; i++) {
+ if (!awb_ratios[i].num_pixels)
+ continue;
+
+ /*
+ * The statistics are in Q4.8 format, so we convert to double
+ * here.
+ */
+ rgSum += (awb_ratios[i].avg_rg_gr * pow(2, -8));
+ bgSum += (awb_ratios[i].avg_bg_br * pow(2, -8));
+ counted_zones++;
+ }
+
+ /*
+ * Sometimes the first frame's statistics have no valid pixels, in which
+ * case we'll just assume a grey world until they say otherwise.
+ */
+ double rgAvg, bgAvg;
+ if (!counted_zones) {
+ rgAvg = 1.0;
+ bgAvg = 1.0;
+ } else {
+ rgAvg = rgSum / counted_zones;
+ bgAvg = bgSum / counted_zones;
+ }
+
+ /*
+ * The statistics are generated _after_ white balancing is performed in
+ * the ISP. To get the true ratio we therefore have to adjust the stats
+ * figure by the gains that were applied when the statistics for this
+ * frame were generated.
+ */
+ double rRatio = rgAvg / frameContext.awb.rGain;
+ double bRatio = bgAvg / frameContext.awb.bGain;
+
+ /*
+ * And then we can simply invert the ratio to find the gain we should
+ * apply.
+ */
+ double rGain = 1 / rRatio;
+ double bGain = 1 / bRatio;
+
+ /*
+ * Running at full speed, this algorithm results in oscillations in the
+ * colour balance. To remove those we dampen the speed at which it makes
+ * changes in gain, unless we're in the startup phase in which case we
+ * want to fix the miscolouring as quickly as possible.
+ */
+ double speed = frame < kNumStartupFrames ? 1.0 : 0.2;
+ rGain = speed * rGain + context.activeState.awb.rGain * (1.0 - speed);
+ bGain = speed * bGain + context.activeState.awb.bGain * (1.0 - speed);
+
+ context.activeState.awb.rGain = rGain;
+ context.activeState.awb.bGain = bGain;
+
+ metadata.set(controls::ColourGains, {
+ static_cast<float>(frameContext.awb.rGain),
+ static_cast<float>(frameContext.awb.bGain),
+ });
+
+ LOG(MaliC55Awb, Debug) << "For frame number " << frame << ": "
+ << "Average R/G Ratio: " << rgAvg
+ << ", Average B/G Ratio: " << bgAvg
+ << "\nrGain applied to this frame: " << frameContext.awb.rGain
+ << ", bGain applied to this frame: " << frameContext.awb.bGain
+ << "\nrGain to apply: " << context.activeState.awb.rGain
+ << ", bGain to apply: " << context.activeState.awb.bGain;
+}
+
+REGISTER_IPA_ALGORITHM(Awb, "Awb")
+
+} /* namespace ipa::mali_c55::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/mali-c55/algorithms/awb.h b/src/ipa/mali-c55/algorithms/awb.h
new file mode 100644
index 00000000..4836585d
--- /dev/null
+++ b/src/ipa/mali-c55/algorithms/awb.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2024, Ideas on Board Oy
+ *
+ * awb.h - Mali C55 grey world auto white balance algorithm
+ */
+
+#include "algorithm.h"
+#include "ipa_context.h"
+
+namespace libcamera {
+
+namespace ipa::mali_c55::algorithms {
+
+class Awb : public Algorithm
+{
+public:
+ Awb();
+ ~Awb() = default;
+
+ int configure(IPAContext &context,
+ const IPACameraSensorInfo &configInfo) override;
+ void prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ mali_c55_params_block_header *block) override;
+ void process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const mali_c55_stats_buffer *stats,
+ ControlList &metadata) override;
+private:
+ void fillGainsParamBlock(mali_c55_params_block_header *block,
+ IPAContext &context, IPAFrameContext &frameContext);
+ void fillConfigParamBlock(mali_c55_params_block_header *block);
+};
+
+} /* namespace ipa::mali_c55::algorithms */
+
+} /* namespace libcamera */
diff --git a/src/ipa/mali-c55/algorithms/meson.build b/src/ipa/mali-c55/algorithms/meson.build
index 96808431..f11791aa 100644
--- a/src/ipa/mali-c55/algorithms/meson.build
+++ b/src/ipa/mali-c55/algorithms/meson.build
@@ -2,5 +2,6 @@
mali_c55_ipa_algorithms = files([
'agc.cpp',
+ 'awb.cpp',
'blc.cpp',
])
diff --git a/src/ipa/mali-c55/ipa_context.h b/src/ipa/mali-c55/ipa_context.h
index 73a7cd78..105e5776 100644
--- a/src/ipa/mali-c55/ipa_context.h
+++ b/src/ipa/mali-c55/ipa_context.h
@@ -50,6 +50,11 @@ struct IPAActiveState {
uint32_t exposureMode;
uint32_t temperatureK;
} agc;
+
+ struct {
+ double rGain;
+ double bGain;
+ } awb;
};
struct IPAFrameContext : public FrameContext {
@@ -58,6 +63,11 @@ struct IPAFrameContext : public FrameContext {
double sensorGain;
double ispGain;
} agc;
+
+ struct {
+ double rGain;
+ double bGain;
+ } awb;
};
struct IPAContext {
--
2.30.2
More information about the libcamera-devel
mailing list