[libcamera-devel] [RFC PATCH] tests: v4l2_compat: Add test for v4l2_compat

[Paul Elder]

Test the V4L2 compatibility layer by running v4l2-compliance -s on every
/dev/video* device.

Signed-off-by: Paul Elder <paul.elder at ideasonboard.com>
---
 test/meson.build                     |   1 +
 test/v4l2_compat/meson.build         |  10 +++
 test/v4l2_compat/v4l2_compat_test.py | 125 +++++++++++++++++++++++++++
 3 files changed, 136 insertions(+)
 create mode 100644 test/v4l2_compat/meson.build
 create mode 100755 test/v4l2_compat/v4l2_compat_test.py

diff --git a/test/meson.build b/test/meson.build
index a868813..591920f 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -12,6 +12,7 @@ subdir('pipeline')
 subdir('process')
 subdir('serialization')
 subdir('stream')
+subdir('v4l2_compat')
 subdir('v4l2_subdevice')
 subdir('v4l2_videodevice')
 
diff --git a/test/v4l2_compat/meson.build b/test/v4l2_compat/meson.build
new file mode 100644
index 0000000..a67aac4
--- /dev/null
+++ b/test/v4l2_compat/meson.build
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: CC0-1.0
+
+pymod = import('python')
+py3 = pymod.find_installation('python3')
+
+v4l2_compat_test = files('v4l2_compat_test.py')
+
+test('v4l2_compat_test', py3,
+     args : v4l2_compat_test,
+     suite : 'v4l2_compat')
diff --git a/test/v4l2_compat/v4l2_compat_test.py b/test/v4l2_compat/v4l2_compat_test.py
new file mode 100755
index 0000000..f56db4e
--- /dev/null
+++ b/test/v4l2_compat/v4l2_compat_test.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2019, Google Inc.
+#
+# Author: Paul Elder <paul.elder at ideasonboard.com>
+#
+# v4l2_compat_test.py - Test the V4L2 compatibility layer
+
+import os
+import re
+from shutil import which
+import subprocess
+import sys
+
+TestPass = 0
+TestFail = -1
+TestSkip = 77
+
+
+supported_pipelines = [
+    "uvcvideo",
+    "vimc",
+]
+
+
+def find_file(name, path):
+    for root, dirs, files in os.walk(path):
+        if name in files:
+            return os.path.join(root, name)
+
+
+def grep(exp, arr):
+    return [s for s in arr if re.search(exp, s)]
+
+
+def run_with_stdout(cmd, args="", env={}):
+    try:
+        with open(os.devnull, 'w') as devnull:
+            output = subprocess.check_output(f"{cmd} {args}", shell=True, env=env, stderr=devnull)
+    except subprocess.CalledProcessError as err:
+        output = err.output
+    return output.decode("utf-8").split("\n")
+
+
+def extract_result(result):
+    res = result.split(", ")
+    ret = {}
+    ret["total"]     = int(res[0].split(": ")[-1])
+    ret["succeeded"] = int(res[1].split(": ")[-1])
+    ret["failed"]    = int(res[2].split(": ")[-1])
+    ret["warnings"]  = int(res[3].split(": ")[-1])
+    ret["device"]    = res[0].split()[4].strip(":")
+    ret["driver"]    = res[0].split()[2]
+    return ret
+
+
+def print_output_arr(output_arr):
+    print("\n".join(output_arr))
+
+
+def test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, base_driver):
+    output = run_with_stdout(v4l2_compliance, f"-s -d {device}", {"LD_PRELOAD": v4l2_compat})
+    result = extract_result(output[-2])
+    if result["driver"] != "libcamera":
+        return TestSkip
+
+    if result['failed'] == 0:
+        return TestPass
+
+    # vimc will fail s_fmt because it only supports framesizes that are
+    # multiples of 3
+    if base_driver == "vimc" and result["failed"] <= 1:
+        failures = grep("fail", output)
+        if re.search("S_FMT cannot handle an invalid format", failures[0]) is None:
+            print_output_arr(output)
+            return TestFail
+        return TestPass
+
+    print_output_arr(output)
+    return TestFail
+
+
+def main():
+    v4l2_compliance = which("v4l2-compliance")
+    if v4l2_compliance is None:
+        print("v4l2-compliance is not available")
+        return TestSkip
+
+    v4l2_ctl = which("v4l2-ctl")
+    if v4l2_ctl is None:
+        print("v4l2-ctl is not available")
+        return TestSkip
+
+    v4l2_compat = find_file("v4l2-compat.so", ".")
+    if v4l2_compat is None:
+        print("v4l2-compat.so is not built")
+        return TestSkip
+
+    dev_nodes = grep("video", os.listdir("/dev"))
+    dev_nodes = list(map(lambda s: f"/dev/{s}", dev_nodes))
+    if len(dev_nodes) == 0:
+        print("no video nodes available to test with")
+        return TestSkip
+
+    failed = []
+    for device in dev_nodes:
+        out = run_with_stdout(v4l2_ctl, f"-D -d {device}")
+        driver = grep("Driver name", out)[0].split(":")[-1].strip()
+        if driver not in supported_pipelines:
+            continue
+
+        ret = test_v4l2_compliance(v4l2_compliance, v4l2_compat, device, driver)
+        if ret == TestFail:
+            failed.append(device)
+
+    if len(failed) > 0:
+        print(f"Failed {len(failed)} tests:")
+        for device in failed:
+            print(f"- {device}")
+
+    return TestPass if not failed else TestFail
+
+
+if __name__ == '__main__':
+    sys.exit(main())
-- 
2.27.0