<div dir="ltr"><div dir="ltr">Hi Jean-Michel,<div><br></div><div>Thank you for your work.</div></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, 23 Mar 2022 at 16:01, Jean-Michel Hautbois via libcamera-devel <<a href="mailto:libcamera-devel@lists.libcamera.org" target="_blank">libcamera-devel@lists.libcamera.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Now that the ancillary links are plumbed and we can set the lens<br>
position, implement a contrast-based algorithm for RPi. This algorithm<br>
is adapted from the one proposed for the IPU3 IPA.<br>
<br>
It is currently taking all the regions and tries to make the focus on<br>
the global scene in a first attempt.<br>
<br>
Signed-off-by: Jean-Michel Hautbois <<a href="mailto:jeanmichel.hautbois@ideasonboard.com" target="_blank">jeanmichel.hautbois@ideasonboard.com</a>><br>
---<br>
 .../raspberrypi/controller/af_algorithm.hpp   |  20 ++<br>
 src/ipa/raspberrypi/controller/af_status.h    |  31 +++<br>
 src/ipa/raspberrypi/controller/focus_status.h |   3 +<br>
 src/ipa/raspberrypi/controller/iob/af.cpp     | 231 ++++++++++++++++++<br>
 src/ipa/raspberrypi/controller/iob/af.h       |  55 +++++<br>
 src/ipa/raspberrypi/meson.build               |   1 +<br>
 6 files changed, 341 insertions(+)<br>
 create mode 100644 src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
 create mode 100644 src/ipa/raspberrypi/controller/af_status.h<br>
 create mode 100644 src/ipa/raspberrypi/controller/iob/af.cpp<br>
 create mode 100644 src/ipa/raspberrypi/controller/iob/af.h<br>
<br>
diff --git a/src/ipa/raspberrypi/controller/af_algorithm.hpp b/src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
new file mode 100644<br>
index 00000000..553a37e1<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/af_algorithm.hpp<br>
@@ -0,0 +1,20 @@<br>
+/* SPDX-License-Identifier: BSD-2-Clause */<br>
+/*<br>
+ * Copyright (C) 2019, Raspberry Pi (Trading) Limited<br></blockquote><div><br></div><div>2022 :-)</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ *<br>
+ * af_algorithm.hpp - autofocus control algorithm interface<br>
+ */<br>
+#pragma once<br>
+<br>
+#include "algorithm.hpp"<br>
+<br>
+namespace RPiController {<br>
+<br>
+class AfAlgorithm : public Algorithm<br>
+{<br>
+public:<br>
+       AfAlgorithm(Controller *controller) : Algorithm(controller) {}<br>
+       // An af algorithm must provide the following:<br>
+};<br>
+<br>
+} // namespace RPiController<br>
diff --git a/src/ipa/raspberrypi/controller/af_status.h b/src/ipa/raspberrypi/controller/af_status.h<br>
new file mode 100644<br>
index 00000000..835e1e2f<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/af_status.h<br>
@@ -0,0 +1,31 @@<br>
+/* SPDX-License-Identifier: BSD-2-Clause */<br>
+/*<br>
+ * Copyright (C) 2020, Raspberry Pi (Trading) Limited<br></blockquote><div><br></div><div>2022</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af_status.h - autofocus measurement status<br>
+ */<br>
+#pragma once<br>
+<br>
+#include <linux/bcm2835-isp.h><br></blockquote><div><br></div><div>This header should not be included here.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+/*<br>
+ * The focus algorithm should post the following structure into the image's<br>
+ * "af.status" metadata.<br>
+ */<br>
+<br>
+#ifdef __cplusplus<br>
+extern "C" {<br>
+#endif<br></blockquote><div><br></div><div>You can remove the extern "C", this is a throwback from the time this file would be</div><div>linked with a C library.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+struct AfStatus {<br>
+       unsigned int num;<br>
+       uint32_t focus_measures[FOCUS_REGIONS];<br>
+       bool stable;<br>
+       uint32_t focus;<br>
+       double maxVariance;<br>
+};<br></blockquote><div><br></div><div>I wonder if all these are actually needed in AfStatus?  The idea of the Controller metadata is to provide the IPA with all that it needs to set the lens control.</div><div>maxVariance is probably not needed by the IPA, and num/focus_measure is passed out through FocusStatus, see below.</div><div><br></div><div>The bool stable is ok for now, but I expect it needs to change to an enum showing focussing states very soon.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+#ifdef __cplusplus<br>
+}<br>
+#endif<br>
diff --git a/src/ipa/raspberrypi/controller/focus_status.h b/src/ipa/raspberrypi/controller/focus_status.h<br>
index ace2fe2c..8122df4b 100644<br>
--- a/src/ipa/raspberrypi/controller/focus_status.h<br>
+++ b/src/ipa/raspberrypi/controller/focus_status.h<br>
@@ -19,6 +19,9 @@ extern "C" {<br>
 struct FocusStatus {<br>
        unsigned int num;<br>
        uint32_t focus_measures[FOCUS_REGIONS];<br>
+       bool stable;<br>
+       uint32_t focus;<br>
+       double maxVariance;<br></blockquote><div><br></div><div>I don't think we want to include these additions in  FocusStatus.  This struct is only used to report the focus FoM values back to the application.</div><div>For focus control, you can use the AfStatus struct above.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
 };<br>
<br>
 #ifdef __cplusplus<br>
diff --git a/src/ipa/raspberrypi/controller/iob/af.cpp b/src/ipa/raspberrypi/controller/iob/af.cpp<br>
new file mode 100644<br>
index 00000000..dc5258ba<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/iob/af.cpp<br>
@@ -0,0 +1,231 @@<br>
+/* SPDX-License-Identifier: LGPL-2.1-or-later */<br>
+/*<br>
+ * Copyright (C) 2021, Red Hat<br>
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af.cpp - automatic contrast-based focus algorithm<br>
+ */<br>
+#include <cmath><br>
+<br>
+#include <stdint.h><br>
+<br>
+#include <libcamera/base/log.h><br>
+<br>
+#include "af.h"<br>
+<br>
+using namespace RPiController;<br>
+using namespace libcamera;<br>
+<br>
+LOG_DEFINE_CATEGORY(IoBAf)<br>
+<br>
+#define NAME "<a href="http://iob.af" rel="noreferrer" target="_blank">iob.af</a>"<br>
+<br>
+/*<br>
+ * Maximum focus steps of the VCM control<br>
+ * \todo should be obtained from the VCM driver<br>
+ */<br>
+static constexpr uint32_t kMaxFocusSteps = 1023;<br>
+<br>
+/* Minimum focus step for searching appropriate focus */<br>
+static constexpr uint32_t kCoarseSearchStep = 30;<br>
+static constexpr uint32_t kFineSearchStep = 1;<br>
+<br>
+/* Max ratio of variance change, 0.0 < kMaxChange < 1.0 */<br>
+static constexpr double kMaxChange = 0.5;<br>
+<br>
+/* The numbers of frame to be ignored, before performing focus scan. */<br>
+static constexpr uint32_t kIgnoreFrame = 10;<br>
+<br>
+/* Fine scan range 0 < kFineRange < 1 */<br>
+static constexpr double kFineRange = 0.05;<br>
+<br>
+Af::Af(Controller *controller)<br>
+       : AfAlgorithm(controller), focus_(0), bestFocus_(0), ignoreCounter_(0),<br>
+         currentVariance_(0.0), previousVariance_(0.0), maxStep_(0),<br>
+         coarseCompleted_(false), fineCompleted_(false)<br>
+{<br>
+}<br>
+<br>
+char const *Af::Name() const<br>
+{<br>
+       return NAME;<br>
+}<br>
+<br>
+void Af::Initialise()<br>
+{<br>
+       status_.focus = 0.0;<br>
+       status_.maxVariance = 0.0;<br>
+       status_.stable = false;<br>
+}<br>
+<br>
+void Af::Prepare(Metadata *image_metadata)<br>
+{<br>
+       image_metadata->Set("af.status", status_);<br>
+}<br>
+<br>
+double Af::estimateVariance()<br>
+{<br>
+       unsigned int i;<br>
+       double mean;<br>
+       uint64_t total = 0;<br>
+       double var_sum = 0.0;<br>
+<br>
+       /* Compute the mean value. */<br>
+       for (i = 0; i < FOCUS_REGIONS; i++)<br>
+               total += status_.focus_measures[i];<br>
+       mean = total / FOCUS_REGIONS;<br>
+<br>
+       /* Compute the sum of the squared variance. */<br>
+       for (i = 0; i < FOCUS_REGIONS; i++)<br>
+               var_sum += std::pow(status_.focus_measures[i] - mean, 2);<br>
+<br>
+       return var_sum / FOCUS_REGIONS;<br>
+}<br>
+<br>
+bool Af::afNeedIgnoreFrame()<br>
+{<br>
+       if (ignoreCounter_ == 0)<br>
+               return false;<br>
+       else<br>
+               ignoreCounter_--;<br>
+       return true;<br>
+}<br>
+<br>
+void Af::afCoarseScan()<br>
+{<br>
+       if (coarseCompleted_)<br>
+               return;<br>
+<br>
+       if (afNeedIgnoreFrame())<br>
+               return;<br>
+<br>
+       if (afScan(kCoarseSearchStep)) {<br>
+               coarseCompleted_ = true;<br>
+               status_.maxVariance = 0;<br>
+               focus_ = status_.focus - (status_.focus * kFineRange);<br>
+               status_.focus = focus_;<br>
+               previousVariance_ = 0;<br>
+               maxStep_ = std::clamp(focus_ + static_cast<uint32_t>((focus_ * kFineRange)),<br>
+                                     0U, kMaxFocusSteps);<br>
+       }<br>
+}<br>
+<br>
+void Af::afFineScan()<br>
+{<br>
+       if (!coarseCompleted_)<br>
+               return;<br>
+<br>
+       if (afNeedIgnoreFrame())<br>
+               return;<br>
+<br>
+       if (afScan(kFineSearchStep)) {<br>
+               status_.stable = true;<br>
+               fineCompleted_ = true;<br>
+       }<br>
+}<br>
+<br>
+bool Af::afScan(uint32_t minSteps)<br>
+{<br>
+       if (focus_ > maxStep_) {<br>
+               /* If the max step is reached, move lens to the position. */<br>
+               status_.focus = bestFocus_;<br>
+               return true;<br>
+       } else {<br>
+               /*<br>
+                * Find the maximum of the variance by estimating its<br>
+                * derivative. If the direction changes, it means we have passed<br>
+                * a maximum one step before.<br>
+                */<br>
+               if ((currentVariance_ - status_.maxVariance) >=<br>
+                   -(status_.maxVariance * 0.1)) {<br>
+                       /*<br>
+                        * Positive and zero derivative:<br>
+                        * The variance is still increasing. The focus could be<br>
+                        * increased for the next comparison. Also, the max<br>
+                        * variance and previous focus value are updated.<br>
+                        */<br>
+                       bestFocus_ = focus_;<br>
+                       focus_ += minSteps;<br>
+                       status_.focus = focus_;<br>
+                       status_.maxVariance = currentVariance_;<br>
+               } else {<br>
+                       /*<br>
+                        * Negative derivative:<br>
+                        * The variance starts to decrease which means the maximum<br>
+                        * variance is found. Set focus step to previous good one<br>
+                        * then return immediately.<br>
+                        */<br>
+                       status_.focus = bestFocus_;<br>
+                       return true;<br>
+               }<br>
+       }<br>
+<br>
+       previousVariance_ = currentVariance_;<br>
+       LOG(IoBAf, Debug) << " Previous step is "<br>
+                         << bestFocus_<br>
+                         << " Current step is "<br>
+                         << focus_;<br>
+       return false;<br>
+}<br>
+<br>
+void Af::afReset()<br>
+{<br>
+       if (afNeedIgnoreFrame())<br>
+               return;<br>
+<br>
+       status_.maxVariance = 0;<br>
+       status_.focus = 0;<br>
+       focus_ = 0;<br>
+       status_.stable = false;<br>
+       ignoreCounter_ = kIgnoreFrame;<br>
+       previousVariance_ = 0.0;<br>
+       coarseCompleted_ = false;<br>
+       fineCompleted_ = false;<br>
+       maxStep_ = kMaxFocusSteps;<br>
+}<br>
+<br>
+bool Af::afIsOutOfFocus()<br>
+{<br>
+       const uint32_t diff_var = std::abs(currentVariance_ -<br>
+                                          status_.maxVariance);<br>
+       const double var_ratio = diff_var / status_.maxVariance;<br>
+       LOG(IoBAf, Debug) << "Variance change rate: "<br>
+                         << var_ratio<br>
+                         << " Current VCM step: "<br>
+                         << status_.focus;<br>
+       if (var_ratio > kMaxChange)<br>
+               return true;<br>
+       else<br>
+               return false;<br>
+}<br>
+<br>
+void Af::Process(StatisticsPtr &stats, Metadata *image_metadata)<br>
+{<br>
+       unsigned int i;<br>
+       image_metadata->Get("af.status", status_);<br></blockquote><div><br></div><div>I'm a bit unsure of this one.  Perhaps I don't understand the code well enough, but do you need to do this?  "af.status" gets set in Af::Prepare() from status_, then here we set status_ from "af.status".  Nothing will have changed status_ between the two calls, so maybe this is unnecessary?</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+<br>
+       /* Use the second filter results only, and cache those. */<br>
+       for (i = 0; i < FOCUS_REGIONS; i++)<br>
+               status_.focus_measures[i] = stats->focus_stats[i].contrast_val[1][1]<br>
+                                         / stats->focus_stats[i].contrast_val_num[1][1];<br>
+       status_.num = i;<br>
+<br>
+       currentVariance_ = estimateVariance();<br>
+<br>
+       if (!status_.stable) {<br>
+               afCoarseScan();<br>
+               afFineScan();<br>
+       } else {<br>
+               if (afIsOutOfFocus())<br>
+                       afReset();<br>
+               else<br>
+                       ignoreCounter_ = kIgnoreFrame;<br>
+       }<br>
+}<br>
+<br>
+/* Register algorithm with the system. */<br>
+static Algorithm *Create(Controller *controller)<br>
+{<br>
+       return new Af(controller);<br>
+}<br>
+static RegisterAlgorithm reg(NAME, &Create);<br>
diff --git a/src/ipa/raspberrypi/controller/iob/af.h b/src/ipa/raspberrypi/controller/iob/af.h<br>
new file mode 100644<br>
index 00000000..45c9711f<br>
--- /dev/null<br>
+++ b/src/ipa/raspberrypi/controller/iob/af.h<br>
@@ -0,0 +1,55 @@<br>
+/* SPDX-License-Identifier: LGPL-2.1-or-later */<br>
+/*<br>
+ * Copyright (C) 2021, Red Hat<br>
+ * Copyright (C) 2022, Ideas On Board<br>
+ *<br>
+ * af.h - automatic contrast-based focus algorithm<br>
+ */<br>
+#pragma once<br>
+<br>
+#include "../af_algorithm.hpp"<br>
+#include "../af_status.h"<br>
+#include "../metadata.hpp"<br>
+<br>
+namespace RPiController {<br>
+<br>
+class Af : public AfAlgorithm<br></blockquote><div><br></div><div>It may be practical to put class Af in a separate (iob?) namespace to avoid symbol clashes.  Once RPi have an algorithm implementation, we will have to enclose our class Af in a rpi namespace or equivalent.  What do folks think?</div><div><br></div><div>Regards,</div><div>Naush</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
+{<br>
+public:<br>
+       Af(Controller *controller);<br>
+       char const *Name() const override;<br>
+       void Initialise() override;<br>
+       void Prepare(Metadata *image_metadata) override;<br>
+       void Process(StatisticsPtr &stats, Metadata *image_metadata) override;<br>
+private:<br>
+       double estimateVariance();<br>
+       bool afNeedIgnoreFrame();<br>
+       void afCoarseScan();<br>
+       void afFineScan();<br>
+       bool afScan(uint32_t minSteps);<br>
+       void afReset();<br>
+       bool afIsOutOfFocus();<br>
+<br>
+       AfStatus status_;<br>
+<br>
+       /* VCM step configuration. It is the current setting of the VCM step. */<br>
+       uint32_t focus_;<br>
+       /* The best VCM step. It is a local optimum VCM step during scanning. */<br>
+       uint32_t bestFocus_;<br>
+<br>
+       /* The frames ignored before starting measuring. */<br>
+       uint32_t ignoreCounter_;<br>
+<br>
+       /* Current AF statistic variance. */<br>
+       double currentVariance_;<br>
+       /* It is used to determine the derivative during scanning */<br>
+       double previousVariance_;<br>
+       /* The designated maximum range of focus scanning. */<br>
+       uint32_t maxStep_;<br>
+       /* If the coarse scan completes, it is set to true. */<br>
+       bool coarseCompleted_;<br>
+       /* If the fine scan completes, it is set to true. */<br>
+       bool fineCompleted_;<br>
+};<br>
+<br>
+} /* namespace RPiController */<br>
diff --git a/src/ipa/raspberrypi/meson.build b/src/ipa/raspberrypi/meson.build<br>
index 32897e07..37068ecc 100644<br>
--- a/src/ipa/raspberrypi/meson.build<br>
+++ b/src/ipa/raspberrypi/meson.build<br>
@@ -28,6 +28,7 @@ rpi_ipa_sources = files([<br>
     'controller/controller.cpp',<br>
     'controller/histogram.cpp',<br>
     'controller/algorithm.cpp',<br>
+    'controller/iob/af.cpp',<br>
     'controller/rpi/alsc.cpp',<br>
     'controller/rpi/awb.cpp',<br>
     'controller/rpi/sharpen.cpp',<br>
-- <br>
2.32.0<br>
<br>
</blockquote></div></div>