[libcamera-devel] [PATCH v2 09/10] ipa: raspberrypi: Generalise the autofocus algorithm

Jacopo Mondi jacopo.mondi at ideasonboard.com
Thu Mar 30 20:16:09 CEST 2023


Hi Nick

On Mon, Mar 27, 2023 at 01:20:29PM +0100, Naushir Patuck via libcamera-devel wrote:
> From: Nick Hollinghurst <nick.hollinghurst at raspberrypi.com>
>
> Remove any hard-coded assumptions about the target hardware platform
> from the autofocus algorithm. Instead, use the "target" string provided
> by the camera tuning config and generalised statistics structures to
> determing parameters such as grid and region sizes.
>
> Additionally, PDAF statistics are represented by a generalised region
> statistics structure to be device agnostic.
>
> These changes also require the autofocus algorithm to initialise
> region weights on the first frame's prepare()/process() call rather
> than during initialisation.
>
> Signed-off-by: Nick Hollinghurst <nick.hollinghurst at raspberrypi.com>
> Signed-off-by: Naushir Patuck <naush at raspberrypi.com>
> Tested-by: Naushir Patuck <naush at raspberrypi.com>

I won't pretend I've actually gone through the calculations, but the
few comments I had on v1 have been clarified

Reviewed-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>

Thanks
  j

> ---
>  src/ipa/raspberrypi/cam_helper_imx708.cpp  |  23 ++-
>  src/ipa/raspberrypi/controller/pdaf_data.h |  21 +--
>  src/ipa/raspberrypi/controller/rpi/af.cpp  | 176 +++++++++++----------
>  src/ipa/raspberrypi/controller/rpi/af.h    |  28 ++--
>  4 files changed, 133 insertions(+), 115 deletions(-)
>
> diff --git a/src/ipa/raspberrypi/cam_helper_imx708.cpp b/src/ipa/raspberrypi/cam_helper_imx708.cpp
> index 1f213d3c0833..641ba18f4b9d 100644
> --- a/src/ipa/raspberrypi/cam_helper_imx708.cpp
> +++ b/src/ipa/raspberrypi/cam_helper_imx708.cpp
> @@ -69,11 +69,14 @@ private:
>  	/* Largest long exposure scale factor given as a left shift on the frame length. */
>  	static constexpr int longExposureShiftMax = 7;
>
> +	static constexpr int pdafStatsRows = 12;
> +	static constexpr int pdafStatsCols = 16;
> +
>  	void populateMetadata(const MdParser::RegisterMap &registers,
>  			      Metadata &metadata) const override;
>
>  	static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp,
> -				  PdafData &pdaf);
> +				  PdafRegions &pdaf);
>
>  	bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp);
>  	void putAGCStatistics(StatisticsPtr stats);
> @@ -120,11 +123,11 @@ void CamHelperImx708::prepare(libcamera::Span<const uint8_t> buffer, Metadata &m
>  	size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3;
>
>  	if (buffer.size() > 2 * bytesPerLine) {
> -		PdafData pdaf;
> +		PdafRegions pdaf;
>  		if (parsePdafData(&buffer[2 * bytesPerLine],
>  				  buffer.size() - 2 * bytesPerLine,
>  				  mode_.bitdepth, pdaf))
> -			metadata.set("pdaf.data", pdaf);
> +			metadata.set("pdaf.regions", pdaf);
>  	}
>
>  	/* Parse AE-HIST data where present */
> @@ -239,7 +242,7 @@ void CamHelperImx708::populateMetadata(const MdParser::RegisterMap &registers,
>  }
>
>  bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len,
> -				    unsigned bpp, PdafData &pdaf)
> +				    unsigned bpp, PdafRegions &pdaf)
>  {
>  	size_t step = bpp >> 1; /* bytes per PDAF grid entry */
>
> @@ -248,13 +251,17 @@ bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len,
>  		return false;
>  	}
>
> +	pdaf.init({ pdafStatsCols, pdafStatsRows });
> +
>  	ptr += 2 * step;
> -	for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) {
> -		for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) {
> +	for (unsigned i = 0; i < pdafStatsRows; ++i) {
> +		for (unsigned j = 0; j < pdafStatsCols; ++j) {
>  			unsigned c = (ptr[0] << 3) | (ptr[1] >> 5);
>  			int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2);
> -			pdaf.conf[i][j] = c;
> -			pdaf.phase[i][j] = c ? p : 0;
> +			PdafData pdafData;
> +			pdafData.conf = c;
> +			pdafData.phase = c ? p : 0;
> +			pdaf.set(libcamera::Point(j, i), { pdafData, 1, 0 });
>  			ptr += step;
>  		}
>  	}
> diff --git a/src/ipa/raspberrypi/controller/pdaf_data.h b/src/ipa/raspberrypi/controller/pdaf_data.h
> index 03c00d72c0e8..ae6ab996ded0 100644
> --- a/src/ipa/raspberrypi/controller/pdaf_data.h
> +++ b/src/ipa/raspberrypi/controller/pdaf_data.h
> @@ -2,20 +2,23 @@
>  /*
>   * Copyright (C) 2022, Raspberry Pi Ltd
>   *
> - * pdaf_data.h - PDAF Metadata; for now this is
> - * largely based on IMX708's PDAF "Type 1" output.
> + * pdaf_data.h - PDAF Metadata
>   */
>  #pragma once
>
>  #include <stdint.h>
>
> -#define PDAF_DATA_ROWS 12
> -#define PDAF_DATA_COLS 16
> +#include "region_stats.h"
>
> -struct PdafData {
> -	/* Confidence values, in raster order, in arbitrary units */
> -	uint16_t conf[PDAF_DATA_ROWS][PDAF_DATA_COLS];
> +namespace RPiController {
>
> -	/* Phase error, in raster order, in s11 Q4 format (S.6.4) */
> -	int16_t phase[PDAF_DATA_ROWS][PDAF_DATA_COLS];
> +struct PdafData {
> +	/* Confidence, in arbitrary units */
> +	uint16_t conf;
> +	/* Phase error, in s16 Q4 format (S.11.4) */
> +	int16_t phase;
>  };
> +
> +using PdafRegions = RegionStats<PdafData>;
> +
> +}; // namespace RPiController
> diff --git a/src/ipa/raspberrypi/controller/rpi/af.cpp b/src/ipa/raspberrypi/controller/rpi/af.cpp
> index a623651875f2..ed0c8a94d064 100644
> --- a/src/ipa/raspberrypi/controller/rpi/af.cpp
> +++ b/src/ipa/raspberrypi/controller/rpi/af.cpp
> @@ -174,9 +174,8 @@ Af::Af(Controller *controller)
>  	  statsRegion_(0, 0, 0, 0),
>  	  windows_(),
>  	  useWindows_(false),
> -	  phaseWeights_{},
> -	  contrastWeights_{},
> -	  sumWeights_(0),
> +	  phaseWeights_(),
> +	  contrastWeights_(),
>  	  scanState_(ScanState::Idle),
>  	  initted_(false),
>  	  ftarget_(-1.0),
> @@ -190,7 +189,15 @@ Af::Af(Controller *controller)
>  	  scanData_(),
>  	  reportState_(AfState::Idle)
>  {
> -	scanData_.reserve(24);
> +	/*
> +	 * Reserve space for data, to reduce memory fragmentation. It's too early
> +	 * to query the size of the PDAF (from camera) and Contrast (from ISP)
> +	 * statistics, but these are plausible upper bounds.
> +	 */
> +	phaseWeights_.w.reserve(16 * 12);
> +	contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width *
> +				   getHardwareConfig().focusRegions.height);
> +	scanData_.reserve(32);
>  }
>
>  Af::~Af()
> @@ -226,7 +233,7 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met
>  			  << statsRegion_.y << ','
>  			  << statsRegion_.width << ','
>  			  << statsRegion_.height;
> -	computeWeights();
> +	invalidateWeights();
>
>  	if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) {
>  		/*
> @@ -239,111 +246,99 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met
>  	skipCount_ = cfg_.skipFrames;
>  }
>
> -void Af::computeWeights()
> +void Af::computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols)
>  {
> -	constexpr int MaxCellWeight = 240 / (int)MaxWindows;
> +	wgts->rows = rows;
> +	wgts->cols = cols;
> +	wgts->sum = 0;
> +	wgts->w.resize(rows * cols);
> +	std::fill(wgts->w.begin(), wgts->w.end(), 0);
>
> -	sumWeights_ = 0;
> -	for (int i = 0; i < PDAF_DATA_ROWS; ++i)
> -		std::fill(phaseWeights_[i], phaseWeights_[i] + PDAF_DATA_COLS, 0);
> -
> -	if (useWindows_ &&
> -	    statsRegion_.width >= PDAF_DATA_COLS && statsRegion_.height >= PDAF_DATA_ROWS) {
> +	if (rows > 0 && cols > 0 && useWindows_ &&
> +	    statsRegion_.height >= rows && statsRegion_.width >= cols) {
>  		/*
>  		 * Here we just merge all of the given windows, weighted by area.
>  		 * \todo Perhaps a better approach might be to find the phase in each
>  		 * window and choose either the closest or the highest-confidence one?
> -		 *
> -		 * Using mostly "int" arithmetic, because Rectangle has signed x, y
> +		 * Ensure weights sum to less than (1<<16). 46080 is a "round number"
> +		 * below 65536, for better rounding when window size is a simple
> +		 * fraction of image dimensions.
>  		 */
> -		int cellH = (int)(statsRegion_.height / PDAF_DATA_ROWS);
> -		int cellW = (int)(statsRegion_.width / PDAF_DATA_COLS);
> -		int cellA = cellH * cellW;
> +		const unsigned maxCellWeight = 46080u / (MaxWindows * rows * cols);
> +		const unsigned cellH = statsRegion_.height / rows;
> +		const unsigned cellW = statsRegion_.width / cols;
> +		const unsigned cellA = cellH * cellW;
>
>  		for (auto &w : windows_) {
> -			for (int i = 0; i < PDAF_DATA_ROWS; ++i) {
> -				int y0 = std::max(statsRegion_.y + cellH * i, w.y);
> -				int y1 = std::min(statsRegion_.y + cellH * (i + 1), w.y + (int)(w.height));
> +			for (unsigned r = 0; r < rows; ++r) {
> +				int y0 = std::max(statsRegion_.y + (int)(cellH * r), w.y);
> +				int y1 = std::min(statsRegion_.y + (int)(cellH * (r + 1)),
> +						  w.y + (int)(w.height));
>  				if (y0 >= y1)
>  					continue;
>  				y1 -= y0;
> -				for (int j = 0; j < PDAF_DATA_COLS; ++j) {
> -					int x0 = std::max(statsRegion_.x + cellW * j, w.x);
> -					int x1 = std::min(statsRegion_.x + cellW * (j + 1), w.x + (int)(w.width));
> +				for (unsigned c = 0; c < cols; ++c) {
> +					int x0 = std::max(statsRegion_.x + (int)(cellW * c), w.x);
> +					int x1 = std::min(statsRegion_.x + (int)(cellW * (c + 1)),
> +							  w.x + (int)(w.width));
>  					if (x0 >= x1)
>  						continue;
> -					int a = y1 * (x1 - x0);
> -					a = (MaxCellWeight * a + cellA - 1) / cellA;
> -					phaseWeights_[i][j] += a;
> -					sumWeights_ += a;
> +					unsigned a = y1 * (x1 - x0);
> +					a = (maxCellWeight * a + cellA - 1) / cellA;
> +					wgts->w[r * cols + c] += a;
> +					wgts->sum += a;
>  				}
>  			}
>  		}
>  	}
>
> -	if (sumWeights_ == 0) {
> -		/*
> -		 * Default AF window is the middle 1/2 width of the middle 1/3 height
> -		 * since this maps nicely to both PDAF (16x12) and Focus (4x3) grids.
> -		 */
> -		for (int i = PDAF_DATA_ROWS / 3; i < 2 * PDAF_DATA_ROWS / 3; ++i) {
> -			for (int j = PDAF_DATA_COLS / 4; j < 3 * PDAF_DATA_COLS / 4; ++j) {
> -				phaseWeights_[i][j] = MaxCellWeight;
> -				sumWeights_ += MaxCellWeight;
> +	if (wgts->sum == 0) {
> +		/* Default AF window is the middle 1/2 width of the middle 1/3 height */
> +		for (unsigned r = rows / 3; r < rows - rows / 3; ++r) {
> +			for (unsigned c = cols / 4; c < cols - cols / 4; ++c) {
> +				wgts->w[r * cols + c] = 1;
> +				wgts->sum += 1;
>  			}
>  		}
>  	}
> +}
>
> -	/* Scale from PDAF to Focus Statistics grid (which has fixed size 4x3) */
> -	constexpr int FocusStatsRows = 3;
> -	constexpr int FocusStatsCols = 4;
> -	static_assert(FOCUS_REGIONS == FocusStatsRows * FocusStatsCols);
> -	static_assert(PDAF_DATA_ROWS % FocusStatsRows == 0);
> -	static_assert(PDAF_DATA_COLS % FocusStatsCols == 0);
> -	constexpr int YFactor = PDAF_DATA_ROWS / FocusStatsRows;
> -	constexpr int XFactor = PDAF_DATA_COLS / FocusStatsCols;
> -
> -	LOG(RPiAf, Debug) << "Recomputed weights:";
> -	for (int i = 0; i < FocusStatsRows; ++i) {
> -		for (int j = 0; j < FocusStatsCols; ++j) {
> -			unsigned w = 0;
> -			for (int y = 0; y < YFactor; ++y)
> -				for (int x = 0; x < XFactor; ++x)
> -					w += phaseWeights_[YFactor * i + y][XFactor * j + x];
> -			contrastWeights_[FocusStatsCols * i + j] = w;
> -		}
> -		LOG(RPiAf, Debug) << "   "
> -				  << contrastWeights_[FocusStatsCols * i + 0] << " "
> -				  << contrastWeights_[FocusStatsCols * i + 1] << " "
> -				  << contrastWeights_[FocusStatsCols * i + 2] << " "
> -				  << contrastWeights_[FocusStatsCols * i + 3];
> -	}
> +void Af::invalidateWeights()
> +{
> +	phaseWeights_.sum = 0;
> +	contrastWeights_.sum = 0;
>  }
>
> -bool Af::getPhase(PdafData const &data, double &phase, double &conf) const
> +bool Af::getPhase(PdafRegions const &regions, double &phase, double &conf)
>  {
> +	libcamera::Size size = regions.size();
> +	if (size.height != phaseWeights_.rows || size.width != phaseWeights_.cols ||
> +	    phaseWeights_.sum == 0) {
> +		LOG(RPiAf, Debug) << "Recompute Phase weights " << size.width << 'x' << size.height;
> +		computeWeights(&phaseWeights_, size.height, size.width);
> +	}
> +
>  	uint32_t sumWc = 0;
>  	int64_t sumWcp = 0;
> -
> -	for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) {
> -		for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) {
> -			if (phaseWeights_[i][j]) {
> -				uint32_t c = data.conf[i][j];
> -				if (c >= cfg_.confThresh) {
> -					if (c > cfg_.confClip)
> -						c = cfg_.confClip;
> -					c -= (cfg_.confThresh >> 2);
> -					sumWc += phaseWeights_[i][j] * c;
> -					c -= (cfg_.confThresh >> 2);
> -					sumWcp += phaseWeights_[i][j] * data.phase[i][j] * (int64_t)c;
> -				}
> +	for (unsigned i = 0; i < regions.numRegions(); ++i) {
> +		unsigned w = phaseWeights_.w[i];
> +		if (w) {
> +			const PdafData &data = regions.get(i).val;
> +			unsigned c = data.conf;
> +			if (c >= cfg_.confThresh) {
> +				if (c > cfg_.confClip)
> +					c = cfg_.confClip;
> +				c -= (cfg_.confThresh >> 2);
> +				sumWc += w * c;
> +				c -= (cfg_.confThresh >> 2);
> +				sumWcp += (int64_t)(w * c) * (int64_t)data.phase;
>  			}
>  		}
>  	}
>
> -	if (0 < sumWeights_ && sumWeights_ <= sumWc) {
> +	if (0 < phaseWeights_.sum && phaseWeights_.sum <= sumWc) {
>  		phase = (double)sumWcp / (double)sumWc;
> -		conf = (double)sumWc / (double)sumWeights_;
> +		conf = (double)sumWc / (double)phaseWeights_.sum;
>  		return true;
>  	} else {
>  		phase = 0.0;
> @@ -352,14 +347,21 @@ bool Af::getPhase(PdafData const &data, double &phase, double &conf) const
>  	}
>  }
>
> -double Af::getContrast(const FocusRegions &focusStats) const
> +double Af::getContrast(const FocusRegions &focusStats)
>  {
> -	uint32_t sumWc = 0;
> +	libcamera::Size size = focusStats.size();
> +	if (size.height != contrastWeights_.rows ||
> +	    size.width != contrastWeights_.cols || contrastWeights_.sum == 0) {
> +		LOG(RPiAf, Debug) << "Recompute Contrast weights "
> +				  << size.width << 'x' << size.height;
> +		computeWeights(&contrastWeights_, size.height, size.width);
> +	}
>
> +	uint64_t sumWc = 0;
>  	for (unsigned i = 0; i < focusStats.numRegions(); ++i)
> -		sumWc += contrastWeights_[i] * focusStats.get(i).val;
> +		sumWc += contrastWeights_.w[i] * focusStats.get(i).val;
>
> -	return (sumWeights_ == 0) ? 0.0 : (double)sumWc / (double)sumWeights_;
> +	return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0;
>  }
>
>  void Af::doPDAF(double phase, double conf)
> @@ -623,14 +625,14 @@ void Af::prepare(Metadata *imageMetadata)
>
>  	if (initted_) {
>  		/* Get PDAF from the embedded metadata, and run AF algorithm core */
> -		PdafData data;
> +		PdafRegions regions;
>  		double phase = 0.0, conf = 0.0;
>  		double oldFt = ftarget_;
>  		double oldFs = fsmooth_;
>  		ScanState oldSs = scanState_;
>  		uint32_t oldSt = stepCount_;
> -		if (imageMetadata->get("pdaf.data", data) == 0)
> -			getPhase(data, phase, conf);
> +		if (imageMetadata->get("pdaf.regions", regions) == 0)
> +			getPhase(regions, phase, conf);
>  		doAF(prevContrast_, phase, conf);
>  		updateLensPosition();
>  		LOG(RPiAf, Debug) << std::fixed << std::setprecision(2)
> @@ -691,7 +693,7 @@ void Af::setMetering(bool mode)
>  {
>  	if (useWindows_ != mode) {
>  		useWindows_ = mode;
> -		computeWeights();
> +		invalidateWeights();
>  	}
>  }
>
> @@ -708,7 +710,9 @@ void Af::setWindows(libcamera::Span<libcamera::Rectangle const> const &wins)
>  		if (windows_.size() >= MaxWindows)
>  			break;
>  	}
> -	computeWeights();
> +
> +	if (useWindows_)
> +		invalidateWeights();
>  }
>
>  bool Af::setLensPosition(double dioptres, int *hwpos)
> diff --git a/src/ipa/raspberrypi/controller/rpi/af.h b/src/ipa/raspberrypi/controller/rpi/af.h
> index 7959371baf64..b479feb88c39 100644
> --- a/src/ipa/raspberrypi/controller/rpi/af.h
> +++ b/src/ipa/raspberrypi/controller/rpi/af.h
> @@ -11,12 +11,6 @@
>  #include "../pdaf_data.h"
>  #include "../pwl.h"
>
> -/*
> - * \todo FOCUS_REGIONS is taken from bcm2835-isp.h, but should be made as a
> - * generic RegionStats structure.
> - */
> -#define FOCUS_REGIONS 12
> -
>  /*
>   * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF.
>   *
> @@ -121,9 +115,20 @@ private:
>  		double conf;
>  	};
>
> -	void computeWeights();
> -	bool getPhase(PdafData const &data, double &phase, double &conf) const;
> -	double getContrast(const FocusRegions &focusStats) const;
> +	struct RegionWeights {
> +		unsigned rows;
> +		unsigned cols;
> +		uint32_t sum;
> +		std::vector<uint16_t> w;
> +
> +		RegionWeights()
> +			: rows(0), cols(0), sum(0), w() {}
> +	};
> +
> +	void computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols);
> +	void invalidateWeights();
> +	bool getPhase(PdafRegions const &regions, double &phase, double &conf);
> +	double getContrast(const FocusRegions &focusStats);
>  	void doPDAF(double phase, double conf);
>  	bool earlyTerminationByPhase(double phase);
>  	double findPeak(unsigned index) const;
> @@ -143,9 +148,8 @@ private:
>  	libcamera::Rectangle statsRegion_;
>  	std::vector<libcamera::Rectangle> windows_;
>  	bool useWindows_;
> -	uint8_t phaseWeights_[PDAF_DATA_ROWS][PDAF_DATA_COLS];
> -	uint16_t contrastWeights_[FOCUS_REGIONS];
> -	uint32_t sumWeights_;
> +	RegionWeights phaseWeights_;
> +	RegionWeights contrastWeights_;
>
>  	/* Working state. */
>  	ScanState scanState_;
> --
> 2.34.1
>


More information about the libcamera-devel mailing list