[libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for v4l2_compat

[Paul Elder]

Hi Niklas,

Thank you for the comments.

On Wed, Jun 24, 2020 at 07:38:04PM +0200, Niklas Söderlund wrote:
> Hi Paul,
> 
> Thanks for your work.
> 
> On 2020-06-24 23:07:52 +0900, Paul Elder wrote:
> > Test the V4L2 compatibility layer by running v4l2-compliance -s on every
> > /dev/video* device.
> 
> I wonder if this is not a tad to aggressive. Imagine the test being run 
> on a device where one video node is not covered by a libcamera pipeline 
> and also does not pass v4l2-compliance. Would this not lead to the 
> libcamera test suite to fail?
> 
> Would it be possible you think to check the driver name of the video 
> device and check it against an whitelist (vimc, vivid, ?) before running 
> the v4l2-compliance suite?
> 
> You can get the driver name from
> 
>     v4l2-ctl -D -d /dev/videoX

Yes, as Laurent pointed out, I do do this already :)

There was one part though, that I ran v4l2-compliance -s and then
checked if libcamera actually intercepted it... in v2 I've moved this
over to v4l2-ctl.


Thanks,

Paul

> > 
> > Signed-off-by: Paul Elder <paul.elder at ideasonboard.com>
> > ---
> >  test/meson.build                     |   1 +
> >  test/v4l2_compat/meson.build         |  10 +++
> >  test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++
> >  3 files changed, 136 insertions(+)
> >  create mode 100644 test/v4l2_compat/meson.build
> >  create mode 100755 test/v4l2_compat/v4l2_compat_test.py
> > 
> > diff --git a/test/meson.build b/test/meson.build
> > index a868813..591920f 100644
> > --- a/test/meson.build
> > +++ b/test/meson.build
> > @@ -12,6 +12,7 @@ subdir('pipeline')
> >  subdir('process')
> >  subdir('serialization')
> >  subdir('stream')
> > +subdir('v4l2_compat')
> >  subdir('v4l2_subdevice')
> >  subdir('v4l2_videodevice')
> >  
> > diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build
> > new file mode 100644
> > index 0000000..a67aac4
> > --- /dev/null
> > +++ b/test/v4l2_compat/meson.build
> > @@ -0,0 +1,10 @@
> > +# SPDX-License-Identifier: CC0-1.0
> > +
> > +pymod = import('python')
> > +py3 = pymod.find_installation('python3')
> > +
> > +v4l2_compat_test = files('v4l2_compat_test.py')
> > +
> > +test('v4l2_compat_test', py3,
> > +     args : v4l2_compat_test,
> > +     suite : 'v4l2_compat')
> > diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py
> > new file mode 100755
> > index 0000000..f56db4e
> > --- /dev/null
> > +++ b/test/v4l2_compat/v4l2_compat_test.py
> > @@ -0,0 +1,125 @@
> > +#!/usr/bin/env python3
> > +# SPDX-License-Identifier: GPL-2.0-or-later
> > +# Copyright (C) 2019, Google Inc.
> > +#
> > +# Author: Paul Elder <paul.elder at ideasonboard.com>
> > +#
> > +# v4l2_compat_test.py - Test the V4L2 compatibility layer
> > +
> > +import os
> > +import re
> > +from shutil import which
> > +import subprocess
> > +import sys
> > +
> > +TestPass = 0
> > +TestFail = -1
> > +TestSkip = 77
> > +
> > +
> > +supported_pipelines = [
> > +    "uvcvideo",
> > +    "vimc",
> > +]
> > +
> > +
> > +def find_file(name, path):
> > +    for root, dirs, files in os.walk(path):
> > +        if name in files:
> > +            return os.path.join(root, name)
> > +
> > +
> > +def grep(exp, arr):
> > +    return [s for s in arr if re.search(exp, s)]
> > +
> > +
> > +def run_with_stdout(cmd, args="", env={}):
> > +    try:
> > +        with open(os.devnull, 'w') as devnull:
> > +            output = subprocess.check_output(f"{cmd} {args}", shell=True, env=env, stderr=devnull)
> > +    except subprocess.CalledProcessError as err:
> > +        output = err.output
> > +    return output.decode("utf-8").split("\n")
> > +
> > +
> > +def extract_result(result):
> > +    res = result.split(", ")
> > +    ret = {}
> > +    ret["total"]     = int(res[0].split(": ")[-1])
> > +    ret["succeeded"] = int(res[1].split(": ")[-1])
> > +    ret["failed"]    = int(res[2].split(": ")[-1])
> > +    ret["warnings"]  = int(res[3].split(": ")[-1])
> > +    ret["device"]    = res[0].split()[4].strip(":")
> > +    ret["driver"]    = res[0].split()[2]
> > +    return ret
> > +
> > +
> > +def print_output_arr(output_arr):
> > +    print("\n".join(output_arr))
> > +
> > +
> > +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):
> > +    output = run_with_stdout(v4l2_compliance, f"-s -d {device}", {"LD_PRELOAD": v4l2_compat})
> > +    result = extract_result(output[-2])
> > +    if result["driver"] != "libcamera":
> > +        return TestSkip
> > +
> > +    if result['failed'] == 0:
> > +        return TestPass
> > +
> > +    # vimc will fail s_fmt because it only supports framesizes that are
> > +    # multiples of 3
> > +    if base_driver == "vimc" and result["failed"] <= 1:
> > +        failures = grep("fail", output)
> > +        if re.search("S_FMT cannot handle an invalid format", failures[0]) is None:
> > +            print_output_arr(output)
> > +            return TestFail
> > +        return TestPass
> > +
> > +    print_output_arr(output)
> > +    return TestFail
> > +
> > +
> > +def main():
> > +    v4l2_compliance = which("v4l2-compliance")
> > +    if v4l2_compliance is None:
> > +        print("v4l2-compliance is not available")
> > +        return TestSkip
> > +
> > +    v4l2_ctl = which("v4l2-ctl")
> > +    if v4l2_ctl is None:
> > +        print("v4l2-ctl is not available")
> > +        return TestSkip
> > +
> > +    v4l2_compat = find_file("v4l2-compat.so", ".")
> > +    if v4l2_compat is None:
> > +        print("v4l2-compat.so is not built")
> > +        return TestSkip
> > +
> > +    dev_nodes = grep("video", os.listdir("/dev"))
> > +    dev_nodes = list(map(lambda s: f"/dev/{s}", dev_nodes))
> > +    if len(dev_nodes) == 0:
> > +        print("no video nodes available to test with")
> > +        return TestSkip
> > +
> > +    failed = []
> > +    for device in dev_nodes:
> > +        out = run_with_stdout(v4l2_ctl, f"-D -d {device}")
> > +        driver = grep("Driver name", out)[0].split(":")[-1].strip()
> > +        if driver not in supported_pipelines:
> > +            continue
> > +
> > +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)
> > +        if ret == TestFail:
> > +            failed.append(device)
> > +
> > +    if len(failed) > 0:
> > +        print(f"Failed {len(failed)} tests:")
> > +        for device in failed:
> > +            print(f"- {device}")
> > +
> > +    return TestPass if not failed else TestFail
> > +
> > +
> > +if __name__ == '__main__':
> > +    sys.exit(main())
> > -- 
> > 2.27.0
> > 
> > _______________________________________________
> > libcamera-devel mailing list
> > libcamera-devel at lists.libcamera.org
> > https://lists.libcamera.org/listinfo/libcamera-devel
> 
> -- 
> Regards,
> Niklas Söderlund