[libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for v4l2_compat

[Niklas Söderlund]

Hi Paul,

Thanks for your work.

On 2020-06-24 23:07:52 +0900, Paul Elder wrote:
> Test the V4L2 compatibility layer by running v4l2-compliance -s on every
> /dev/video* device.

I wonder if this is not a tad to aggressive. Imagine the test being run 
on a device where one video node is not covered by a libcamera pipeline 
and also does not pass v4l2-compliance. Would this not lead to the 
libcamera test suite to fail?

Would it be possible you think to check the driver name of the video 
device and check it against an whitelist (vimc, vivid, ?) before running 
the v4l2-compliance suite?

You can get the driver name from

    v4l2-ctl -D -d /dev/videoX

> 
> Signed-off-by: Paul Elder <paul.elder at ideasonboard.com>
> ---
>  test/meson.build                     |   1 +
>  test/v4l2_compat/meson.build         |  10 +++
>  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++
>  3 files changed, 136 insertions(+)
>  create mode 100644 test/v4l2_compat/meson.build
>  create mode 100755 test/v4l2_compat/v4l2_compat_test.py
> 
> diff --git a/test/meson.build b/test/meson.build
> index a868813..591920f 100644
> --- a/test/meson.build
> +++ b/test/meson.build
> @@ -12,6 +12,7 @@ subdir('pipeline')
>  subdir('process')
>  subdir('serialization')
>  subdir('stream')
> +subdir('v4l2_compat')
>  subdir('v4l2_subdevice')
>  subdir('v4l2_videodevice')
>  
> diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build
> new file mode 100644
> index 0000000..a67aac4
> --- /dev/null
> +++ b/test/v4l2_compat/meson.build
> @@ -0,0 +1,10 @@
> +# SPDX-License-Identifier: CC0-1.0
> +
> +pymod = import('python')
> +py3 = pymod.find_installation('python3')
> +
> +v4l2_compat_test = files('v4l2_compat_test.py')
> +
> +test('v4l2_compat_test', py3,
> +     args : v4l2_compat_test,
> +     suite : 'v4l2_compat')
> diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py
> new file mode 100755
> index 0000000..f56db4e
> --- /dev/null
> +++ b/test/v4l2_compat/v4l2_compat_test.py
> @@ -0,0 +1,125 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2019, Google Inc.
> +#
> +# Author: Paul Elder <paul.elder at ideasonboard.com>
> +#
> +# v4l2_compat_test.py - Test the V4L2 compatibility layer
> +
> +import os
> +import re
> +from shutil import which
> +import subprocess
> +import sys
> +
> +TestPass = 0
> +TestFail = -1
> +TestSkip = 77
> +
> +
> +supported_pipelines = [
> +    "uvcvideo",
> +    "vimc",
> +]
> +
> +
> +def find_file(name, path):
> +    for root, dirs, files in os.walk(path):
> +        if name in files:
> +            return os.path.join(root, name)
> +
> +
> +def grep(exp, arr):
> +    return [s for s in arr if re.search(exp, s)]
> +
> +
> +def run_with_stdout(cmd, args="", env={}):
> +    try:
> +        with open(os.devnull, 'w') as devnull:
> +            output = subprocess.check_output(f"{cmd} {args}", shell=True, env=env, stderr=devnull)
> +    except subprocess.CalledProcessError as err:
> +        output = err.output
> +    return output.decode("utf-8").split("\n")
> +
> +
> +def extract_result(result):
> +    res = result.split(", ")
> +    ret = {}
> +    ret["total"]     = int(res[0].split(": ")[-1])
> +    ret["succeeded"] = int(res[1].split(": ")[-1])
> +    ret["failed"]    = int(res[2].split(": ")[-1])
> +    ret["warnings"]  = int(res[3].split(": ")[-1])
> +    ret["device"]    = res[0].split()[4].strip(":")
> +    ret["driver"]    = res[0].split()[2]
> +    return ret
> +
> +
> +def print_output_arr(output_arr):
> +    print("\n".join(output_arr))
> +
> +
> +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):
> +    output = run_with_stdout(v4l2_compliance, f"-s -d {device}", {"LD_PRELOAD": v4l2_compat})
> +    result = extract_result(output[-2])
> +    if result["driver"] != "libcamera":
> +        return TestSkip
> +
> +    if result['failed'] == 0:
> +        return TestPass
> +
> +    # vimc will fail s_fmt because it only supports framesizes that are
> +    # multiples of 3
> +    if base_driver == "vimc" and result["failed"] <= 1:
> +        failures = grep("fail", output)
> +        if re.search("S_FMT cannot handle an invalid format", failures[0]) is None:
> +            print_output_arr(output)
> +            return TestFail
> +        return TestPass
> +
> +    print_output_arr(output)
> +    return TestFail
> +
> +
> +def main():
> +    v4l2_compliance = which("v4l2-compliance")
> +    if v4l2_compliance is None:
> +        print("v4l2-compliance is not available")
> +        return TestSkip
> +
> +    v4l2_ctl = which("v4l2-ctl")
> +    if v4l2_ctl is None:
> +        print("v4l2-ctl is not available")
> +        return TestSkip
> +
> +    v4l2_compat = find_file("v4l2-compat.so", ".")
> +    if v4l2_compat is None:
> +        print("v4l2-compat.so is not built")
> +        return TestSkip
> +
> +    dev_nodes = grep("video", os.listdir("/dev"))
> +    dev_nodes = list(map(lambda s: f"/dev/{s}", dev_nodes))
> +    if len(dev_nodes) == 0:
> +        print("no video nodes available to test with")
> +        return TestSkip
> +
> +    failed = []
> +    for device in dev_nodes:
> +        out = run_with_stdout(v4l2_ctl, f"-D -d {device}")
> +        driver = grep("Driver name", out)[0].split(":")[-1].strip()
> +        if driver not in supported_pipelines:
> +            continue
> +
> +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)
> +        if ret == TestFail:
> +            failed.append(device)
> +
> +    if len(failed) > 0:
> +        print(f"Failed {len(failed)} tests:")
> +        for device in failed:
> +            print(f"- {device}")
> +
> +    return TestPass if not failed else TestFail
> +
> +
> +if __name__ == '__main__':
> +    sys.exit(main())
> -- 
> 2.27.0
> 
> _______________________________________________
> libcamera-devel mailing list
> libcamera-devel at lists.libcamera.org
> https://lists.libcamera.org/listinfo/libcamera-devel

-- 
Regards,
Niklas Söderlund