[libcamera-devel] [PATCH] libcamera: ipa: raspberrypi: Add support for imx519 sensor
David Plowman
david.plowman at raspberrypi.com
Mon Oct 11 10:16:30 CEST 2021
Hi Lee
Thank you very much for your contribution, we're very pleased to see
others making use of the Pi's camera system in this way!
On Mon, 11 Oct 2021 at 05:06, Arducam info <info at arducam.com> wrote:
>
> The necessary tuning file and CamHelper is added for the imx519 sensor.
>
> The imx519 is a 16MP rolling shutter sensor. To enable
> it, please add
>
> dtoverlay=imx519
>
> to the /boot/config.txt file and reboot the Pi.
>
> Signed-off-by: Lee Jackson <info at arducam.com>
> ---
> src/ipa/raspberrypi/cam_helper_imx519.cpp | 185 +++++++++++++++
> src/ipa/raspberrypi/data/imx519.json | 272 ++++++++++++++++++++++
> src/ipa/raspberrypi/data/meson.build | 1 +
> src/ipa/raspberrypi/meson.build | 1 +
> 4 files changed, 459 insertions(+)
> create mode 100644 src/ipa/raspberrypi/cam_helper_imx519.cpp
> create mode 100644 src/ipa/raspberrypi/data/imx519.json
>
> diff --git a/src/ipa/raspberrypi/cam_helper_imx519.cpp b/src/ipa/raspberrypi/cam_helper_imx519.cpp
> new file mode 100644
> index 00000000..eaf24982
> --- /dev/null
> +++ b/src/ipa/raspberrypi/cam_helper_imx519.cpp
> @@ -0,0 +1,185 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> + * Based on cam_helper_imx477.cpp
> + * Copyright (C) 2020, Raspberry Pi (Trading) Limited
> + *
> + * cam_helper_imx519.cpp - camera helper for imx519 sensor
> + * Copyright (C) 2021, Arducam Technology co., Ltd.
> + */
> +
> +#include <assert.h>
> +#include <cmath>
> +#include <stddef.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +
> +#include <libcamera/base/log.h>
> +
> +#include "cam_helper.hpp"
> +#include "md_parser.hpp"
> +
> +using namespace RPiController;
> +using namespace libcamera;
> +using libcamera::utils::Duration;
> +
> +namespace libcamera {
> +LOG_DECLARE_CATEGORY(IPARPI)
> +}
> +
> +/*
> + * We care about two gain registers and a pair of exposure registers. Their
> + * I2C addresses from the Sony IMX519 datasheet:
> + */
> +constexpr uint32_t expHiReg = 0x0202;
> +constexpr uint32_t expLoReg = 0x0203;
> +constexpr uint32_t gainHiReg = 0x0204;
> +constexpr uint32_t gainLoReg = 0x0205;
> +constexpr uint32_t frameLengthHiReg = 0x0340;
> +constexpr uint32_t frameLengthLoReg = 0x0341;
> +constexpr std::initializer_list<uint32_t> registerList =
> + { expHiReg, expLoReg, gainHiReg, gainLoReg, frameLengthHiReg, frameLengthLoReg };
> +
> +class CamHelperImx519 : public CamHelper
> +{
> +public:
> + CamHelperImx519();
> + uint32_t GainCode(double gain) const override;
> + double Gain(uint32_t gain_code) const override;
> + void Prepare(libcamera::Span<const uint8_t> buffer, Metadata &metadata) override;
> + uint32_t GetVBlanking(Duration &exposure, Duration minFrameDuration,
> + Duration maxFrameDuration) const override;
> + void GetDelays(int &exposure_delay, int &gain_delay,
> + int &vblank_delay) const override;
> + bool SensorEmbeddedDataPresent() const override;
> +
> +private:
> + /*
> + * Smallest difference between the frame length and integration time,
> + * in units of lines.
> + */
> + static constexpr int frameIntegrationDiff = 32;
> + /* Maximum frame length allowable for long exposure calculations. */
> + static constexpr int frameLengthMax = 0xffdc;
> + /* Largest long exposure scale factor given as a left shift on the frame length. */
> + static constexpr int longExposureShiftMax = 7;
> +
> + void PopulateMetadata(const MdParser::RegisterMap ®isters,
> + Metadata &metadata) const override;
> +};
> +
> +CamHelperImx519::CamHelperImx519()
> + : CamHelper(std::make_unique<MdParserSmia>(registerList), frameIntegrationDiff)
> +{
> +}
> +
> +uint32_t CamHelperImx519::GainCode(double gain) const
> +{
> + return static_cast<uint32_t>(1024 - 1024 / gain);
> +}
> +
> +double CamHelperImx519::Gain(uint32_t gain_code) const
> +{
> + return 1024.0 / (1024 - gain_code);
> +}
> +
> +void CamHelperImx519::Prepare(libcamera::Span<const uint8_t> buffer, Metadata &metadata)
> +{
> + MdParser::RegisterMap registers;
> + DeviceStatus deviceStatus;
> +
> + if (metadata.Get("device.status", deviceStatus)) {
> + LOG(IPARPI, Error) << "DeviceStatus not found from DelayedControls";
> + return;
> + }
> +
> + parseEmbeddedData(buffer, metadata);
> +
> + /*
> + * The DeviceStatus struct is first populated with values obtained from
> + * DelayedControls. If this reports frame length is > frameLengthMax,
> + * it means we are using a long exposure mode. Since the long exposure
> + * scale factor is not returned back through embedded data, we must rely
> + * on the existing exposure lines and frame length values returned by
> + * DelayedControls.
> + *
> + * Otherwise, all values are updated with what is reported in the
> + * embedded data.
> + */
> + if (deviceStatus.frame_length > frameLengthMax) {
> + DeviceStatus parsedDeviceStatus;
> +
> + metadata.Get("device.status", parsedDeviceStatus);
> + parsedDeviceStatus.shutter_speed = deviceStatus.shutter_speed;
> + parsedDeviceStatus.frame_length = deviceStatus.frame_length;
> + metadata.Set("device.status", parsedDeviceStatus);
> +
> + LOG(IPARPI, Debug) << "Metadata updated for long exposure: "
> + << parsedDeviceStatus;
> + }
> +}
> +
> +uint32_t CamHelperImx519::GetVBlanking(Duration &exposure,
> + Duration minFrameDuration,
> + Duration maxFrameDuration) const
> +{
> + uint32_t frameLength, exposureLines;
> + unsigned int shift = 0;
> +
> + frameLength = mode_.height + CamHelper::GetVBlanking(exposure, minFrameDuration,
> + maxFrameDuration);
> + /*
> + * Check if the frame length calculated needs to be setup for long
> + * exposure mode. This will require us to use a long exposure scale
> + * factor provided by a shift operation in the sensor.
> + */
> + while (frameLength > frameLengthMax) {
> + if (++shift > longExposureShiftMax) {
> + shift = longExposureShiftMax;
> + frameLength = frameLengthMax;
> + break;
> + }
> + frameLength >>= 1;
> + }
> +
> + if (shift) {
> + /* Account for any rounding in the scaled frame length value. */
> + frameLength <<= shift;
> + exposureLines = ExposureLines(exposure);
> + exposureLines = std::min(exposureLines, frameLength - frameIntegrationDiff);
> + exposure = Exposure(exposureLines);
> + }
> +
> + return frameLength - mode_.height;
> +}
> +
> +void CamHelperImx519::GetDelays(int &exposure_delay, int &gain_delay,
> + int &vblank_delay) const
> +{
> + exposure_delay = 2;
> + gain_delay = 2;
> + vblank_delay = 3;
> +}
> +
> +bool CamHelperImx519::SensorEmbeddedDataPresent() const
> +{
> + return true;
> +}
> +
> +void CamHelperImx519::PopulateMetadata(const MdParser::RegisterMap ®isters,
> + Metadata &metadata) const
> +{
> + DeviceStatus deviceStatus;
> +
> + deviceStatus.shutter_speed = Exposure(registers.at(expHiReg) * 256 + registers.at(expLoReg));
> + deviceStatus.analogue_gain = Gain(registers.at(gainHiReg) * 256 + registers.at(gainLoReg));
> + deviceStatus.frame_length = registers.at(frameLengthHiReg) * 256 + registers.at(frameLengthLoReg);
> +
> + metadata.Set("device.status", deviceStatus);
> +}
> +
> +static CamHelper *Create()
> +{
> + return new CamHelperImx519();
> +}
> +
> +static RegisterCamHelper reg("imx519", &Create);
> diff --git a/src/ipa/raspberrypi/data/imx519.json b/src/ipa/raspberrypi/data/imx519.json
> new file mode 100644
> index 00000000..39e788ad
> --- /dev/null
> +++ b/src/ipa/raspberrypi/data/imx519.json
> @@ -0,0 +1,272 @@
> +{
> + "rpi.black_level":
> + {
> + "black_level": 4096
> + },
> + "rpi.dpc":
> + {
> + },
> + "rpi.lux":
> + {
> + "reference_shutter_speed": 10812,
> + "reference_gain": 2.0,
> + "reference_aperture": 1.0,
> + "reference_lux": 1250,
> + "reference_Y": 12737
> + },
> + "rpi.noise":
> + {
> + "reference_constant": 0,
> + "reference_slope": 2.91
> + },
> + "rpi.geq":
> + {
> + "offset": 444,
> + "slope": 0.00731
> + },
> + "rpi.sdn":
> + {
> + "deviation":3.7,
> + "strength":0.75
> + },
> + "rpi.awb":
> + {
> + "priors":
> + [
> + {
> + "lux": 0, "prior":
> + [
> + 2000, 1.0, 3000, 0.0, 13000, 0.0
> + ]
> + },
> + {
> + "lux": 800, "prior":
> + [
> + 2000, 0.0, 6000, 2.0, 13000, 2.0
> + ]
> + },
> + {
> + "lux": 1500, "prior":
> + [
> + 2000, 0.0, 4000, 1.0, 6000, 6.0, 6500, 7.0, 7000, 1.0, 13000, 1.0
> + ]
> + }
> + ],
> + "modes":
> + {
> + "auto":
> + {
> + "lo": 2500,
> + "hi": 8000
> + },
> + "incandescent":
> + {
> + "lo": 2500,
> + "hi": 3000
> + },
> + "tungsten":
> + {
> + "lo": 3000,
> + "hi": 3500
> + },
> + "fluorescent":
> + {
> + "lo": 4000,
> + "hi": 4700
> + },
> + "indoor":
> + {
> + "lo": 3000,
> + "hi": 5000
> + },
> + "daylight":
> + {
> + "lo": 5500,
> + "hi": 6500
> + },
> + "cloudy":
> + {
> + "lo": 7000,
> + "hi": 8600
> + }
> + },
> + "bayes": 0
Just a small thing, but if we're going with the simple grey world
algorithm it might be worth deleting the other AWB parameters as they
don't do anything.
And a few final questions:
* Are you planning to upstream an open source driver for the imx519?
The preference within libcamera (someone will correct me if I'm
wrong!) is to support sensors where code and drivers are open.
* If you're planning to open-source the driver we'd be pleased to
receive pull requests into our own Raspberry Pi kernel distribution.
* I assume this is a fairly standard Bayer sensor? If you have a
working open source driver and can send me a module then I'd be very
happy to help you further with the json tuning file.
Thanks and best regards
David
> +
> + },
> + "rpi.agc":
> + {
> + "metering_modes":
> + {
> + "centre-weighted":
> + {
> + "weights":
> + [
> + 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0
> + ]
> + },
> + "spot":
> + {
> + "weights":
> + [
> + 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
> + ]
> + },
> + "matrix":
> + {
> + "weights":
> + [
> + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
> + ]
> + }
> + },
> + "exposure_modes":
> + {
> + "normal":
> + {
> + "shutter":
> + [
> + 100, 10000, 30000, 60000, 120000
> + ],
> + "gain":
> + [
> + 1.0, 2.0, 4.0, 6.0, 12.0
> + ]
> + },
> + "short":
> + {
> + "shutter":
> + [
> + 100, 5000, 10000, 20000, 120000
> + ],
> + "gain":
> + [
> + 1.0, 2.0, 4.0, 6.0, 12.0
> + ]
> + }
> + },
> + "constraint_modes":
> + {
> + "normal":
> + [
> + {
> + "bound": "LOWER", "q_lo": 0.98, "q_hi": 1.0, "y_target":
> + [
> + 0, 0.5, 1000, 0.5
> + ]
> + }
> + ],
> + "highlight":
> + [
> + {
> + "bound": "LOWER", "q_lo": 0.98, "q_hi": 1.0, "y_target":
> + [
> + 0, 0.5, 1000, 0.5
> + ]
> + },
> + {
> + "bound": "UPPER", "q_lo": 0.98, "q_hi": 1.0, "y_target":
> + [
> + 0, 0.8, 1000, 0.8
> + ]
> + }
> + ]
> + },
> + "y_target":
> + [
> + 0, 0.16, 1000, 0.165, 10000, 0.17
> + ]
> + },
> + "rpi.alsc":
> + {
> + "omega": 1.3,
> + "n_iter": 100,
> + "luminance_strength": 0.5,
> + "calibrations_Cr":
> + [
> + {
> + "ct": 6500, "table":
> + [
> + 2.544, 2.541, 2.531, 2.518, 2.506, 2.495, 2.489, 2.489, 2.489, 2.491, 2.502, 2.511, 2.524, 2.534, 2.542, 2.547,
> + 2.541, 2.532, 2.521, 2.506, 2.493, 2.479, 2.469, 2.462, 2.462, 2.473, 2.488, 2.502, 2.511, 2.527, 2.536, 2.542,
> + 2.531, 2.523, 2.506, 2.492, 2.474, 2.458, 2.444, 2.435, 2.435, 2.446, 2.466, 2.481, 2.498, 2.511, 2.526, 2.536,
> + 2.523, 2.513, 2.492, 2.474, 2.458, 2.438, 2.418, 2.408, 2.408, 2.421, 2.443, 2.463, 2.481, 2.498, 2.517, 2.526,
> + 2.513, 2.502, 2.481, 2.459, 2.438, 2.418, 2.397, 2.379, 2.379, 2.399, 2.421, 2.443, 2.463, 2.486, 2.506, 2.517,
> + 2.506, 2.492, 2.469, 2.445, 2.423, 2.397, 2.377, 2.368, 2.369, 2.379, 2.399, 2.427, 2.451, 2.474, 2.495, 2.511,
> + 2.506, 2.492, 2.469, 2.445, 2.423, 2.397, 2.372, 2.364, 2.364, 2.374, 2.399, 2.427, 2.449, 2.474, 2.495, 2.511,
> + 2.506, 2.493, 2.474, 2.451, 2.426, 2.401, 2.382, 2.368, 2.369, 2.382, 2.405, 2.431, 2.449, 2.476, 2.497, 2.509,
> + 2.504, 2.493, 2.475, 2.452, 2.431, 2.412, 2.401, 2.382, 2.382, 2.403, 2.416, 2.438, 2.457, 2.477, 2.497, 2.505,
> + 2.499, 2.492, 2.474, 2.457, 2.441, 2.431, 2.412, 2.404, 2.404, 2.416, 2.432, 2.447, 2.459, 2.478, 2.495, 2.501,
> + 2.499, 2.488, 2.473, 2.463, 2.453, 2.441, 2.431, 2.425, 2.425, 2.432, 2.445, 2.459, 2.466, 2.478, 2.491, 2.495,
> + 2.499, 2.481, 2.472, 2.466, 2.458, 2.449, 2.441, 2.439, 2.439, 2.444, 2.454, 2.464, 2.469, 2.477, 2.484, 2.491
> + ]
> + }
> + ],
> + "calibrations_Cb":
> + [
> + {
> + "ct": 6500, "table":
> + [
> + 1.352, 1.352, 1.349, 1.348, 1.347, 1.346, 1.347, 1.349, 1.351, 1.351, 1.352, 1.353, 1.353, 1.355, 1.358, 1.359,
> + 1.359, 1.355, 1.352, 1.349, 1.348, 1.348, 1.348, 1.351, 1.352, 1.352, 1.352, 1.354, 1.356, 1.358, 1.361, 1.366,
> + 1.364, 1.359, 1.355, 1.352, 1.349, 1.349, 1.351, 1.353, 1.353, 1.353, 1.355, 1.356, 1.358, 1.362, 1.366, 1.371,
> + 1.371, 1.365, 1.359, 1.355, 1.352, 1.352, 1.353, 1.357, 1.357, 1.356, 1.356, 1.359, 1.362, 1.366, 1.371, 1.377,
> + 1.375, 1.371, 1.365, 1.362, 1.359, 1.359, 1.359, 1.364, 1.364, 1.364, 1.364, 1.365, 1.367, 1.371, 1.377, 1.382,
> + 1.379, 1.375, 1.371, 1.367, 1.367, 1.367, 1.369, 1.373, 1.374, 1.374, 1.373, 1.373, 1.374, 1.377, 1.382, 1.386,
> + 1.383, 1.378, 1.374, 1.372, 1.372, 1.372, 1.373, 1.375, 1.378, 1.379, 1.379, 1.379, 1.379, 1.381, 1.386, 1.389,
> + 1.384, 1.379, 1.376, 1.374, 1.373, 1.373, 1.374, 1.379, 1.379, 1.379, 1.379, 1.379, 1.381, 1.382, 1.386, 1.394,
> + 1.386, 1.379, 1.376, 1.375, 1.374, 1.374, 1.374, 1.378, 1.378, 1.378, 1.378, 1.379, 1.381, 1.383, 1.386, 1.394,
> + 1.385, 1.379, 1.376, 1.375, 1.374, 1.374, 1.375, 1.378, 1.378, 1.378, 1.378, 1.379, 1.381, 1.383, 1.387, 1.393,
> + 1.379, 1.378, 1.375, 1.374, 1.373, 1.374, 1.375, 1.377, 1.378, 1.378, 1.378, 1.379, 1.381, 1.382, 1.386, 1.388,
> + 1.379, 1.376, 1.374, 1.372, 1.372, 1.374, 1.375, 1.377, 1.378, 1.378, 1.377, 1.378, 1.379, 1.382, 1.384, 1.387
> + ]
> + }
> + ],
> + "luminance_lut":
> + [
> + 3.257, 3.091, 2.639, 2.301, 2.009, 1.795, 1.742, 1.742, 1.742, 1.742, 1.798, 2.011, 2.309, 2.649, 3.109, 3.251,
> + 3.091, 2.703, 2.311, 2.059, 1.791, 1.631, 1.529, 1.482, 1.482, 1.533, 1.635, 1.791, 2.068, 2.313, 2.719, 3.109,
> + 2.707, 2.424, 2.059, 1.791, 1.626, 1.429, 1.329, 1.282, 1.282, 1.333, 1.433, 1.629, 1.791, 2.068, 2.438, 2.719,
> + 2.555, 2.233, 1.889, 1.626, 1.429, 1.321, 1.176, 1.126, 1.126, 1.181, 1.327, 1.433, 1.629, 1.894, 2.241, 2.538,
> + 2.482, 2.108, 1.775, 1.518, 1.321, 1.176, 1.112, 1.011, 1.012, 1.122, 1.181, 1.327, 1.523, 1.779, 2.113, 2.462,
> + 2.478, 2.048, 1.719, 1.465, 1.266, 1.112, 1.009, 1.001, 1.007, 1.012, 1.122, 1.274, 1.469, 1.722, 2.051, 2.454,
> + 2.478, 2.048, 1.719, 1.465, 1.266, 1.112, 1.004, 1.001, 1.001, 1.011, 1.122, 1.274, 1.469, 1.722, 2.051, 2.454,
> + 2.478, 2.099, 1.768, 1.511, 1.312, 1.163, 1.108, 1.004, 1.008, 1.112, 1.171, 1.321, 1.515, 1.767, 2.097, 2.454,
> + 2.541, 2.217, 1.877, 1.614, 1.417, 1.312, 1.163, 1.112, 1.112, 1.171, 1.317, 1.421, 1.616, 1.874, 2.214, 2.516,
> + 2.686, 2.404, 2.046, 1.779, 1.614, 1.417, 1.317, 1.267, 1.267, 1.319, 1.421, 1.614, 1.773, 2.039, 2.399, 2.669,
> + 3.063, 2.678, 2.292, 2.046, 1.779, 1.622, 1.521, 1.469, 1.469, 1.519, 1.622, 1.773, 2.038, 2.277, 2.669, 3.049,
> + 3.224, 3.063, 2.616, 2.285, 1.995, 1.781, 1.725, 1.723, 1.723, 1.723, 1.777, 1.984, 2.271, 2.598, 3.049, 3.187
> + ],
> + "sigma": 0.005,
> + "sigma_Cb": 0.005
> + },
> + "rpi.contrast":
> + {
> + "ce_enable": 1,
> + "gamma_curve":
> + [
> + 0, 0, 1024, 5040, 2048, 9338, 3072, 12356, 4096, 15312, 5120, 18051, 6144, 20790, 7168, 23193,
> + 8192, 25744, 9216, 27942, 10240, 30035, 11264, 32005, 12288, 33975, 13312, 35815, 14336, 37600, 15360, 39168,
> + 16384, 40642, 18432, 43379, 20480, 45749, 22528, 47753, 24576, 49621, 26624, 51253, 28672, 52698, 30720, 53796,
> + 32768, 54876, 36864, 57012, 40960, 58656, 45056, 59954, 49152, 61183, 53248, 62355, 57344, 63419, 61440, 64476,
> + 65535, 65535
> + ]
> + },
> + "rpi.ccm":
> + {
> + "ccms":
> + [
> + {
> + "ct": 5500, "ccm":
> + [
> + 1.40348, -0.45861, 0.05512, -0.24419, 1.25488, -0.01068, -0.08564, -0.31571, 1.40135
> + ]
> + }
> + ]
> + },
> + "rpi.sharpen":
> + {
> + "threshold":1.5,
> + "strength":0.5,
> + "limit":0.2
> + }
> +}
> diff --git a/src/ipa/raspberrypi/data/meson.build b/src/ipa/raspberrypi/data/meson.build
> index 2def75cb..e84cd099 100644
> --- a/src/ipa/raspberrypi/data/meson.build
> +++ b/src/ipa/raspberrypi/data/meson.build
> @@ -7,6 +7,7 @@ conf_files = files([
> 'imx378.json',
> 'imx477.json',
> 'imx477_noir.json',
> + 'imx519.json',
> 'ov5647.json',
> 'ov5647_noir.json',
> 'ov9281.json',
> diff --git a/src/ipa/raspberrypi/meson.build b/src/ipa/raspberrypi/meson.build
> index 1af31e4a..176055f4 100644
> --- a/src/ipa/raspberrypi/meson.build
> +++ b/src/ipa/raspberrypi/meson.build
> @@ -22,6 +22,7 @@ rpi_ipa_sources = files([
> 'cam_helper_imx219.cpp',
> 'cam_helper_imx290.cpp',
> 'cam_helper_imx477.cpp',
> + 'cam_helper_imx519.cpp',
> 'cam_helper_ov9281.cpp',
> 'controller/controller.cpp',
> 'controller/histogram.cpp',
> --
> 2.17.1
>
More information about the libcamera-devel
mailing list