<div dir="ltr"><div dir="ltr">Hi Jean-Michel,<div><br></div><div>Thank you for your work.</div></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, 23 Mar 2022 at 16:01, Jean-Michel Hautbois via libcamera-devel <<a href="mailto:libcamera-devel@lists.libcamera.org" target="_blank">libcamera-devel@lists.libcamera.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Now that the ancillary links are plumbed and we can set the lens<br>
position, implement a contrast-based algorithm for RPi. This algorithm<br>
is adapted from the one proposed for the IPU3 IPA.<br>
<br>
It is currently taking all the regions and tries to make the focus on<br>
the global scene in a first attempt.<br>
<br>
Signed-off-by: Jean-Michel Hautbois <<a href="mailto:jeanmichel.hautbois@ideasonboard.com" target="_blank">jeanmichel.hautbois@ideasonboard.com</a>><br>
---<br>
.../raspberrypi/controller/af_algorithm.hpp | 20 ++<br>
src/ipa/raspberrypi/controller/af_status.h | 31 +++<br>
src/ipa/raspberrypi/controller/focus_status.h | 3 +<br>
src/ipa/raspberrypi/controller/iob/af.cpp | 231 ++++++++++++++++++<br>
src/ipa/raspberrypi/controller/iob/af.h | 55 +++++<br>
src/ipa/raspberrypi/meson.build | 1 +<br>
6 files changed, 341 insertions(+)<br>
create mode 100644 src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
create mode 100644 src/ipa/raspberrypi/controller/af_status.h<br>
create mode 100644 src/ipa/raspberrypi/controller/iob/af.cpp<br>
create mode 100644 src/ipa/raspberrypi/controller/iob/af.h<br>
<br>
diff --git a/src/ipa/raspberrypi/controller/af_algorithm.hpp b/src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
new file mode 100644<br>
index 00000000..553a37e1<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
@@ -0,0 +1,20 @@<br>
+/* SPDX-License-Identifier: BSD-2-Clause */<br>
+/*<br>
+ * Copyright (C) 2019, Raspberry Pi (Trading) Limited<br></blockquote><div><br></div><div>2022 :-)</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ *<br>
+ * af_algorithm.hpp - autofocus control algorithm interface<br>
+ */<br>
+#pragma once<br>
+<br>
+#include "algorithm.hpp"<br>
+<br>
+namespace RPiController {<br>
+<br>
+class AfAlgorithm : public Algorithm<br>
+{<br>
+public:<br>
+ AfAlgorithm(Controller *controller) : Algorithm(controller) {}<br>
+ // An af algorithm must provide the following:<br>
+};<br>
+<br>
+} // namespace RPiController<br>
diff --git a/src/ipa/raspberrypi/controller/af_status.h b/src/ipa/raspberrypi/controller/af_status.h<br>
new file mode 100644<br>
index 00000000..835e1e2f<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/af_status.h<br>
@@ -0,0 +1,31 @@<br>
+/* SPDX-License-Identifier: BSD-2-Clause */<br>
+/*<br>
+ * Copyright (C) 2020, Raspberry Pi (Trading) Limited<br></blockquote><div><br></div><div>2022</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af_status.h - autofocus measurement status<br>
+ */<br>
+#pragma once<br>
+<br>
+#include <linux/bcm2835-isp.h><br></blockquote><div><br></div><div>This header should not be included here.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+/*<br>
+ * The focus algorithm should post the following structure into the image's<br>
+ * "af.status" metadata.<br>
+ */<br>
+<br>
+#ifdef __cplusplus<br>
+extern "C" {<br>
+#endif<br></blockquote><div><br></div><div>You can remove the extern "C", this is a throwback from the time this file would be</div><div>linked with a C library.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+struct AfStatus {<br>
+ unsigned int num;<br>
+ uint32_t focus_measures[FOCUS_REGIONS];<br>
+ bool stable;<br>
+ uint32_t focus;<br>
+ double maxVariance;<br>
+};<br></blockquote><div><br></div><div>I wonder if all these are actually needed in AfStatus? The idea of the Controller metadata is to provide the IPA with all that it needs to set the lens control.</div><div>maxVariance is probably not needed by the IPA, and num/focus_measure is passed out through FocusStatus, see below.</div><div><br></div><div>The bool stable is ok for now, but I expect it needs to change to an enum showing focussing states very soon.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+#ifdef __cplusplus<br>
+}<br>
+#endif<br>
diff --git a/src/ipa/raspberrypi/controller/focus_status.h b/src/ipa/raspberrypi/controller/focus_status.h<br>
index ace2fe2c..8122df4b 100644<br>
--- a/src/ipa/raspberrypi/controller/focus_status.h<br>
+++ b/src/ipa/raspberrypi/controller/focus_status.h<br>
@@ -19,6 +19,9 @@ extern "C" {<br>
struct FocusStatus {<br>
unsigned int num;<br>
uint32_t focus_measures[FOCUS_REGIONS];<br>
+ bool stable;<br>
+ uint32_t focus;<br>
+ double maxVariance;<br></blockquote><div><br></div><div>I don't think we want to include these additions in FocusStatus. This struct is only used to report the focus FoM values back to the application.</div><div>For focus control, you can use the AfStatus struct above.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
};<br>
<br>
#ifdef __cplusplus<br>
diff --git a/src/ipa/raspberrypi/controller/iob/af.cpp b/src/ipa/raspberrypi/controller/iob/af.cpp<br>
new file mode 100644<br>
index 00000000..dc5258ba<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/iob/af.cpp<br>
@@ -0,0 +1,231 @@<br>
+/* SPDX-License-Identifier: LGPL-2.1-or-later */<br>
+/*<br>
+ * Copyright (C) 2021, Red Hat<br>
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af.cpp - automatic contrast-based focus algorithm<br>
+ */<br>
+#include <cmath><br>
+<br>
+#include <stdint.h><br>
+<br>
+#include <libcamera/base/log.h><br>
+<br>
+#include "af.h"<br>
+<br>
+using namespace RPiController;<br>
+using namespace libcamera;<br>
+<br>
+LOG_DEFINE_CATEGORY(IoBAf)<br>
+<br>
+#define NAME "<a href="http://iob.af" rel="noreferrer" target="_blank">iob.af</a>"<br>
+<br>
+/*<br>
+ * Maximum focus steps of the VCM control<br>
+ * \todo should be obtained from the VCM driver<br>
+ */<br>
+static constexpr uint32_t kMaxFocusSteps = 1023;<br>
+<br>
+/* Minimum focus step for searching appropriate focus */<br>
+static constexpr uint32_t kCoarseSearchStep = 30;<br>
+static constexpr uint32_t kFineSearchStep = 1;<br>
+<br>
+/* Max ratio of variance change, 0.0 < kMaxChange < 1.0 */<br>
+static constexpr double kMaxChange = 0.5;<br>
+<br>
+/* The numbers of frame to be ignored, before performing focus scan. */<br>
+static constexpr uint32_t kIgnoreFrame = 10;<br>
+<br>
+/* Fine scan range 0 < kFineRange < 1 */<br>
+static constexpr double kFineRange = 0.05;<br>
+<br>
+Af::Af(Controller *controller)<br>
+ : AfAlgorithm(controller), focus_(0), bestFocus_(0), ignoreCounter_(0),<br>
+ currentVariance_(0.0), previousVariance_(0.0), maxStep_(0),<br>
+ coarseCompleted_(false), fineCompleted_(false)<br>
+{<br>
+}<br>
+<br>
+char const *Af::Name() const<br>
+{<br>
+ return NAME;<br>
+}<br>
+<br>
+void Af::Initialise()<br>
+{<br>
+ status_.focus = 0.0;<br>
+ status_.maxVariance = 0.0;<br>
+ status_.stable = false;<br>
+}<br>
+<br>
+void Af::Prepare(Metadata *image_metadata)<br>
+{<br>
+ image_metadata->Set("af.status", status_);<br>
+}<br>
+<br>
+double Af::estimateVariance()<br>
+{<br>
+ unsigned int i;<br>
+ double mean;<br>
+ uint64_t total = 0;<br>
+ double var_sum = 0.0;<br>
+<br>
+ /* Compute the mean value. */<br>
+ for (i = 0; i < FOCUS_REGIONS; i++)<br>
+ total += status_.focus_measures[i];<br>
+ mean = total / FOCUS_REGIONS;<br>
+<br>
+ /* Compute the sum of the squared variance. */<br>
+ for (i = 0; i < FOCUS_REGIONS; i++)<br>
+ var_sum += std::pow(status_.focus_measures[i] - mean, 2);<br>
+<br>
+ return var_sum / FOCUS_REGIONS;<br>
+}<br>
+<br>
+bool Af::afNeedIgnoreFrame()<br>
+{<br>
+ if (ignoreCounter_ == 0)<br>
+ return false;<br>
+ else<br>
+ ignoreCounter_--;<br>
+ return true;<br>
+}<br>
+<br>
+void Af::afCoarseScan()<br>
+{<br>
+ if (coarseCompleted_)<br>
+ return;<br>
+<br>
+ if (afNeedIgnoreFrame())<br>
+ return;<br>
+<br>
+ if (afScan(kCoarseSearchStep)) {<br>
+ coarseCompleted_ = true;<br>
+ status_.maxVariance = 0;<br>
+ focus_ = status_.focus - (status_.focus * kFineRange);<br>
+ status_.focus = focus_;<br>
+ previousVariance_ = 0;<br>
+ maxStep_ = std::clamp(focus_ + static_cast<uint32_t>((focus_ * kFineRange)),<br>
+ 0U, kMaxFocusSteps);<br>
+ }<br>
+}<br>
+<br>
+void Af::afFineScan()<br>
+{<br>
+ if (!coarseCompleted_)<br>
+ return;<br>
+<br>
+ if (afNeedIgnoreFrame())<br>
+ return;<br>
+<br>
+ if (afScan(kFineSearchStep)) {<br>
+ status_.stable = true;<br>
+ fineCompleted_ = true;<br>
+ }<br>
+}<br>
+<br>
+bool Af::afScan(uint32_t minSteps)<br>
+{<br>
+ if (focus_ > maxStep_) {<br>
+ /* If the max step is reached, move lens to the position. */<br>
+ status_.focus = bestFocus_;<br>
+ return true;<br>
+ } else {<br>
+ /*<br>
+ * Find the maximum of the variance by estimating its<br>
+ * derivative. If the direction changes, it means we have passed<br>
+ * a maximum one step before.<br>
+ */<br>
+ if ((currentVariance_ - status_.maxVariance) >=<br>
+ -(status_.maxVariance * 0.1)) {<br>
+ /*<br>
+ * Positive and zero derivative:<br>
+ * The variance is still increasing. The focus could be<br>
+ * increased for the next comparison. Also, the max<br>
+ * variance and previous focus value are updated.<br>
+ */<br>
+ bestFocus_ = focus_;<br>
+ focus_ += minSteps;<br>
+ status_.focus = focus_;<br>
+ status_.maxVariance = currentVariance_;<br>
+ } else {<br>
+ /*<br>
+ * Negative derivative:<br>
+ * The variance starts to decrease which means the maximum<br>
+ * variance is found. Set focus step to previous good one<br>
+ * then return immediately.<br>
+ */<br>
+ status_.focus = bestFocus_;<br>
+ return true;<br>
+ }<br>
+ }<br>
+<br>
+ previousVariance_ = currentVariance_;<br>
+ LOG(IoBAf, Debug) << " Previous step is "<br>
+ << bestFocus_<br>
+ << " Current step is "<br>
+ << focus_;<br>
+ return false;<br>
+}<br>
+<br>
+void Af::afReset()<br>
+{<br>
+ if (afNeedIgnoreFrame())<br>
+ return;<br>
+<br>
+ status_.maxVariance = 0;<br>
+ status_.focus = 0;<br>
+ focus_ = 0;<br>
+ status_.stable = false;<br>
+ ignoreCounter_ = kIgnoreFrame;<br>
+ previousVariance_ = 0.0;<br>
+ coarseCompleted_ = false;<br>
+ fineCompleted_ = false;<br>
+ maxStep_ = kMaxFocusSteps;<br>
+}<br>
+<br>
+bool Af::afIsOutOfFocus()<br>
+{<br>
+ const uint32_t diff_var = std::abs(currentVariance_ -<br>
+ status_.maxVariance);<br>
+ const double var_ratio = diff_var / status_.maxVariance;<br>
+ LOG(IoBAf, Debug) << "Variance change rate: "<br>
+ << var_ratio<br>
+ << " Current VCM step: "<br>
+ << status_.focus;<br>
+ if (var_ratio > kMaxChange)<br>
+ return true;<br>
+ else<br>
+ return false;<br>
+}<br>
+<br>
+void Af::Process(StatisticsPtr &stats, Metadata *image_metadata)<br>
+{<br>
+ unsigned int i;<br>
+ image_metadata->Get("af.status", status_);<br></blockquote><div><br></div><div>I'm a bit unsure of this one. Perhaps I don't understand the code well enough, but do you need to do this? "af.status" gets set in Af::Prepare() from status_, then here we set status_ from "af.status". Nothing will have changed status_ between the two calls, so maybe this is unnecessary?</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+ /* Use the second filter results only, and cache those. */<br>
+ for (i = 0; i < FOCUS_REGIONS; i++)<br>
+ status_.focus_measures[i] = stats->focus_stats[i].contrast_val[1][1]<br>
+ / stats->focus_stats[i].contrast_val_num[1][1];<br>
+ status_.num = i;<br>
+<br>
+ currentVariance_ = estimateVariance();<br>
+<br>
+ if (!status_.stable) {<br>
+ afCoarseScan();<br>
+ afFineScan();<br>
+ } else {<br>
+ if (afIsOutOfFocus())<br>
+ afReset();<br>
+ else<br>
+ ignoreCounter_ = kIgnoreFrame;<br>
+ }<br>
+}<br>
+<br>
+/* Register algorithm with the system. */<br>
+static Algorithm *Create(Controller *controller)<br>
+{<br>
+ return new Af(controller);<br>
+}<br>
+static RegisterAlgorithm reg(NAME, &Create);<br>
diff --git a/src/ipa/raspberrypi/controller/iob/af.h b/src/ipa/raspberrypi/controller/iob/af.h<br>
new file mode 100644<br>
index 00000000..45c9711f<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/iob/af.h<br>
@@ -0,0 +1,55 @@<br>
+/* SPDX-License-Identifier: LGPL-2.1-or-later */<br>
+/*<br>
+ * Copyright (C) 2021, Red Hat<br>
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af.h - automatic contrast-based focus algorithm<br>
+ */<br>
+#pragma once<br>
+<br>
+#include "../af_algorithm.hpp"<br>
+#include "../af_status.h"<br>
+#include "../metadata.hpp"<br>
+<br>
+namespace RPiController {<br>
+<br>
+class Af : public AfAlgorithm<br></blockquote><div><br></div><div>It may be practical to put class Af in a separate (iob?) namespace to avoid symbol clashes. Once RPi have an algorithm implementation, we will have to enclose our class Af in a rpi namespace or equivalent. What do folks think?</div><div><br></div><div>Regards,</div><div>Naush</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+{<br>
+public:<br>
+ Af(Controller *controller);<br>
+ char const *Name() const override;<br>
+ void Initialise() override;<br>
+ void Prepare(Metadata *image_metadata) override;<br>
+ void Process(StatisticsPtr &stats, Metadata *image_metadata) override;<br>
+private:<br>
+ double estimateVariance();<br>
+ bool afNeedIgnoreFrame();<br>
+ void afCoarseScan();<br>
+ void afFineScan();<br>
+ bool afScan(uint32_t minSteps);<br>
+ void afReset();<br>
+ bool afIsOutOfFocus();<br>
+<br>
+ AfStatus status_;<br>
+<br>
+ /* VCM step configuration. It is the current setting of the VCM step. */<br>
+ uint32_t focus_;<br>
+ /* The best VCM step. It is a local optimum VCM step during scanning. */<br>
+ uint32_t bestFocus_;<br>
+<br>
+ /* The frames ignored before starting measuring. */<br>
+ uint32_t ignoreCounter_;<br>
+<br>
+ /* Current AF statistic variance. */<br>
+ double currentVariance_;<br>
+ /* It is used to determine the derivative during scanning */<br>
+ double previousVariance_;<br>
+ /* The designated maximum range of focus scanning. */<br>
+ uint32_t maxStep_;<br>
+ /* If the coarse scan completes, it is set to true. */<br>
+ bool coarseCompleted_;<br>
+ /* If the fine scan completes, it is set to true. */<br>
+ bool fineCompleted_;<br>
+};<br>
+<br>
+} /* namespace RPiController */<br>
diff --git a/src/ipa/raspberrypi/meson.build b/src/ipa/raspberrypi/meson.build<br>
index 32897e07..37068ecc 100644<br>
--- a/src/ipa/raspberrypi/meson.build<br>
+++ b/src/ipa/raspberrypi/meson.build<br>
@@ -28,6 +28,7 @@ rpi_ipa_sources = files([<br>
'controller/controller.cpp',<br>
'controller/histogram.cpp',<br>
'controller/algorithm.cpp',<br>
+ 'controller/iob/af.cpp',<br>
'controller/rpi/alsc.cpp',<br>
'controller/rpi/awb.cpp',<br>
'controller/rpi/sharpen.cpp',<br>
-- <br>
2.32.0<br>
<br>
</blockquote></div></div>