[libcamera-devel] [PATCH v3] tests: v4l2_compat: Add test for v4l2_compat

[Laurent Pinchart]

Hi Paul,

Thank you for the patch.

On Tue, Jun 30, 2020 at 12:15:29AM +0900, Paul Elder wrote:
> Test the V4L2 compatibility layer by running v4l2-compliance -s on every
> /dev/video* device.
> 
> Signed-off-by: Paul Elder <paul.elder at ideasonboard.com>
> Reviewed-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> 
> ---
> Changes in v3:
> - use argparse
> - add argument to test all cameras, and test only one camera per driver
>   by default
> - print the devices that we're testing and if each succeeded or failed
> 
> Note that as of v2, the tests will fail if the tester has a camera
> supported by libcamera that has unsupported formats, since they will
> cause a floating point exception. So I don't think this should be merged
> until that is fixed, otherwise we might get nasty test failures in
> bisection.

Is this fixed by the PixelFormatInfo series you've just sent, or are we
still missing something ?

> Changes in v2:
> - change all strings to single-quote
> - simplify meson.build
> - get path to v4l2-compat.so from cli arg, rather than running custom find()
>   - remove find_file()
> - extend test timeout to 60 seconds (from default of 30)
> - don't run v4l2-compliance subprocesses in shell
> - check if v4l2-compliance runs are killed by signal (eg. the known
>   SIGABRT due to floating point exception)
> - move the check to see if libcamera supports the camera to v4l2-ctl
>   instead of v4l2-compliance (in v1 we only checked if the pipeline was
>   supported with v4l2-ctl)
> ---
>  test/meson.build                     |   1 +
>  test/v4l2_compat/meson.build         |  10 ++
>  test/v4l2_compat/v4l2_compat_test.py | 143 +++++++++++++++++++++++++++
>  3 files changed, 154 insertions(+)
>  create mode 100644 test/v4l2_compat/meson.build
>  create mode 100755 test/v4l2_compat/v4l2_compat_test.py
> 
> diff --git a/test/meson.build b/test/meson.build
> index 7808a26..f41d6e7 100644
> --- a/test/meson.build
> +++ b/test/meson.build
> @@ -12,6 +12,7 @@ subdir('pipeline')
>  subdir('process')
>  subdir('serialization')
>  subdir('stream')
> +subdir('v4l2_compat')
>  subdir('v4l2_subdevice')
>  subdir('v4l2_videodevice')
>  
> diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build
> new file mode 100644
> index 0000000..5b29de7
> --- /dev/null
> +++ b/test/v4l2_compat/meson.build
> @@ -0,0 +1,10 @@
> +# SPDX-License-Identifier: CC0-1.0
> +
> +if is_variable('v4l2_compat')
> +    v4l2_compat_test = files('v4l2_compat_test.py')
> +
> +    test('v4l2_compat_test', v4l2_compat_test,
> +         args : v4l2_compat,
> +         suite : 'v4l2_compat',
> +         timeout : 60)
> +endif
> diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py
> new file mode 100755
> index 0000000..fd76647
> --- /dev/null
> +++ b/test/v4l2_compat/v4l2_compat_test.py
> @@ -0,0 +1,143 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2020, Google Inc.
> +#
> +# Author: Paul Elder <paul.elder at ideasonboard.com>
> +#
> +# v4l2_compat_test.py - Test the V4L2 compatibility layer
> +
> +import argparse
> +import glob
> +import os
> +import re
> +import shutil
> +import signal
> +import subprocess
> +import sys
> +
> +TestPass = 0
> +TestFail = -1
> +TestSkip = 77
> +
> +
> +supported_pipelines = [
> +    'uvcvideo',
> +    'vimc',
> +]
> +
> +
> +def grep(exp, arr):
> +    return [s for s in arr if re.search(exp, s)]
> +
> +
> +def run_with_stdout(*args, env={}):
> +    try:
> +        with open(os.devnull, 'w') as devnull:
> +            output = subprocess.check_output(args, env=env, stderr=devnull)
> +        ret = 0
> +    except subprocess.CalledProcessError as err:
> +        output = err.output
> +        ret = err.returncode
> +    return ret, output.decode('utf-8').split('\n')
> +
> +
> +def extract_result(result):
> +    res = result.split(', ')
> +    ret = {}
> +    ret['total']     = int(res[0].split(': ')[-1])
> +    ret['succeeded'] = int(res[1].split(': ')[-1])
> +    ret['failed']    = int(res[2].split(': ')[-1])
> +    ret['warnings']  = int(res[3].split(': ')[-1])
> +    ret['device']    = res[0].split()[4].strip(':')
> +    ret['driver']    = res[0].split()[2]
> +    return ret
> +
> +
> +def print_output_arr(output_arr):
> +    print('\n'.join(output_arr))
> +
> +
> +def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):
> +    ret, output = run_with_stdout(v4l2_compliance, '-s', '-d', device, env={'LD_PRELOAD': v4l2_compat})
> +    if ret < 0:
> +        print_output_arr(output)
> +        print(f'Test for {device} terminated due to signal {signal.Signals(-ret).name}')
> +        return TestFail
> +
> +    result = extract_result(output[-2])
> +    if result['failed'] == 0:
> +        return TestPass
> +
> +    # vimc will fail s_fmt because it only supports framesizes that are
> +    # multiples of 3
> +    if base_driver == 'vimc' and result['failed'] == 1:
> +        failures = grep('fail', output)
> +        if re.search('S_FMT cannot handle an invalid format', failures[0]) is None:
> +            print_output_arr(output)
> +            return TestFail
> +        return TestPass
> +
> +    print_output_arr(output)
> +    return TestFail
> +
> +
> +def main(argv):
> +    parser = argparse.ArgumentParser()
> +    parser.add_argument('-a', '--all', action='store_true',
> +                        help='Test all available cameras')
> +    parser.add_argument('v4l2_compat', type=str,
> +                        help='Path to v4l2-compat.so')
> +    args = parser.parse_args(argv[1:])
> +
> +    v4l2_compat = args.v4l2_compat
> +
> +    v4l2_compliance = shutil.which('v4l2-compliance')
> +    if v4l2_compliance is None:
> +        print('v4l2-compliance is not available')
> +        return TestSkip
> +
> +    v4l2_ctl = shutil.which('v4l2-ctl')
> +    if v4l2_ctl is None:
> +        print('v4l2-ctl is not available')
> +        return TestSkip
> +
> +    dev_nodes = glob.glob('/dev/video*')
> +    if len(dev_nodes) == 0:
> +        print('no video nodes available to test with')
> +        return TestSkip
> +
> +    failed = []
> +    drivers_tested = {driver : False for driver in supported_pipelines}

    drivers_tested = {}

> +    for device in dev_nodes:
> +        _, out = run_with_stdout(v4l2_ctl, '-D', '-d', device, env={'LD_PRELOAD': v4l2_compat})
> +        driver = grep('Driver name', out)[0].split(':')[-1].strip()
> +        if driver != "libcamera":
> +            continue
> +
> +        _, out = run_with_stdout(v4l2_ctl, '-D', '-d', device)
> +        driver = grep('Driver name', out)[0].split(':')[-1].strip()
> +        if driver not in supported_pipelines:
> +            continue
> +
> +        if (not args.all) and drivers_tested[driver]:

        if not args.all and driver in drivers_tested:

> +            continue
> +
> +        print(f'Testing {device} with {driver} driver...', end=' ')

Maybe

        print(f'Testing {device} with {driver} driver... ', end='')

Reviewed-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>

> +        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)
> +        if ret == TestFail:
> +            failed.append(device)
> +            print('failed')
> +        else:
> +            print('success')
> +        drivers_tested[driver] = True
> +
> +    if len(failed) > 0:
> +        print(f'Failed {len(failed)} tests:')
> +        for device in failed:
> +            print(f'- {device}')
> +
> +    return TestPass if not failed else TestFail
> +
> +
> +if __name__ == '__main__':
> +    sys.exit(main(sys.argv))

-- 
Regards,

Laurent Pinchart