[PATCH v3 3/7] utils: Add script to generate control_ids_debug.yaml

[Kieran Bingham]

Quoting Stefan Klug (2024-10-08 16:29:41)
> For flexible debugging it is helpful to minimize the roundtrip time.
> This script parses the source tree and looks for usages of
> 
> set<type>(controls::debug::Something,
> 
> and adds (or removes) the controls as necessary from the yaml
> description. It is meant to be used during development to ease the
> creation of the correct yaml entries.
 

Should we include this in the build steps for debug builds? or would
automating this be problematic? I guess that's where it would then
introduce the ruyaml as a build time dependency which perhaps we don't
want yet.

As this is a new script to assist development, it can't introduce
regressions, and can be worked on on top. So the only thing I care
specifically about is that the formatter is clean :-)


Reviewed-by: Kieran Bingham <kieran.bingham at ideasonboard.com>


> Signed-off-by: Stefan Klug <stefan.klug at ideasonboard.com>
> 
> ---
> Changes in v3:
> - Remove superfluous comma and args variable
> - Default to ruamel.yaml instead of ruyaml
> - Small fixes on layout/comments
> 
> Changes in v2:
> - Search only until the first comma of the set() call, to allow
>   linebreaks there.
> - Support ruamel.yaml as fallback
> - Rename output to ctrl_file
> - Add "generated by" comment in yaml file
> ---
>  utils/gen-debug-controls.py | 162 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 162 insertions(+)
>  create mode 100755 utils/gen-debug-controls.py
> 
> diff --git a/utils/gen-debug-controls.py b/utils/gen-debug-controls.py
> new file mode 100755
> index 000000000000..025850731c0b
> --- /dev/null
> +++ b/utils/gen-debug-controls.py
> @@ -0,0 +1,162 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (C) 2024, Google Inc.
> +#
> +# Author: Stefan Klug <stefan.klug at ideasonboard.com>
> +#
> +# This script looks for occurrences of the debug metadata controls in the source
> +# tree and updates src/libcamera/control_ids_debug.yaml accordingly. It is meant
> +# to be used during development to ease updating of the yaml file while
> +# debugging.
> +
> +import argparse
> +import logging
> +import os
> +import re
> +import sys
> +from dataclasses import dataclass
> +from pathlib import Path
> +
> +logger = logging.getLogger(__name__)
> +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
> +
> +try:
> +    import ruamel.yaml as ruyaml
> +except:
> +    logger.error(
> +        f'Failed to import ruamel.yaml. Please install the ruamel.yaml package.')
> +    sys.exit(1)
> +
> + at dataclass
> +class FoundMatch:
> +    file: os.PathLike
> +    whole_match: str
> +    line: int
> +    type: str
> +    name: str
> +    size: str = None
> +
> +
> +def get_control_name(control):
> +    k = list(control.keys())
> +    if len(k) != 1:
> +        raise Exception(f"Can't handle control entry with {len(k)} keys")
> +    return k[0]
> +
> +
> +def find_debug_controls(dir):
> +    extensions = ['.cpp', '.h']
> +    files = [p for p in dir.rglob('*') if p.suffix in extensions]
> +
> +    # The following regex was tested on
> +    # set<Span<type>>( controls::debug::something , static_cast<type>(var) )
> +    # set<>( controls::debug::something , static_cast<type>(var) )
> +    # set( controls::debug::something , static_cast<type> (var) )
> +    exp = re.compile(r'set'  # set function
> +                     r'(?:\<((?:[^)(])*)\>)?'  # followed by a optional template param
> +                     r'\(\s*controls::debug::(\w+)\s*,'  # referencing a debug control
> +                     )
> +    matches = []
> +    for p in files:
> +        with p.open('r') as f:
> +            for idx, line in enumerate(f):
> +                match = exp.search(line)
> +                if match:
> +                    m = FoundMatch(file=p, line=idx, type=match.group(1),
> +                                   name=match.group(2), whole_match=match.group(0))
> +                    if m.type is not None and m.type.startswith('Span'):
> +                        # Simple span type detection treating the last word
> +                        # inside <> as type.
> +                        r = re.match(r'Span<(?:.*\s+)(.*)>', m.type)
> +                        m.type = r.group(1)
> +                        m.size = '[n]'
> +                    matches.append(m)
> +    return matches
> +
> +
> +def main(argv):
> +    parser = argparse.ArgumentParser(
> +        description='Automatically updates control_ids_debug.yaml')
> +    parser.parse_args(argv[1:])
> +
> +    yaml = ruyaml.YAML()
> +    root_dir = Path(__file__).resolve().parent.parent
> +    ctrl_file = root_dir.joinpath('src/libcamera/control_ids_debug.yaml')
> +
> +    matches = find_debug_controls(root_dir.joinpath('src'))
> +
> +    doc = yaml.load(ctrl_file)
> +
> +    controls = doc['controls']
> +
> +    # Create a map of names in the existing yaml for easier updating.
> +    controls_map = {}
> +    for control in controls:
> +        for k, v in control.items():
> +            controls_map[k] = v
> +
> +    obsolete_names = list(controls_map.keys())
> +
> +    for m in matches:
> +        if not m.type:
> +            p = m.file.relative_to(Path.cwd(), walk_up=True)
> +            logger.warning(
> +                f'{p}:{m.line + 1}: Failed to deduce type from {m.whole_match} ... skipping')
> +            continue
> +
> +        p = m.file.relative_to(root_dir)
> +        desc = {'type': m.type,
> +                'description': f'Debug control {m.name} found in {p}:{m.line}'}
> +        if m.size is not None:
> +            desc['size'] = m.size
> +
> +        if m.name in controls_map:
> +            # Can't use == for modified check because of the special yaml dicts.
> +            update_needed = False
> +            if list(controls_map[m.name].keys()) != list(desc.keys()):
> +                update_needed = True
> +            else:
> +                for k, v in controls_map[m.name].items():
> +                    if v != desc[k]:
> +                        update_needed = True
> +                        break
> +
> +            if update_needed:
> +                logger.info(f"Update control '{m.name}'")
> +                controls_map[m.name].clear()
> +                controls_map[m.name].update(desc)
> +
> +            obsolete_names.remove(m.name)
> +        else:
> +            logger.info(f"Add control '{m.name}'")
> +            insert_before = len(controls)
> +            for idx, control in enumerate(controls):
> +                if get_control_name(control).lower() > m.name.lower():
> +                    insert_before = idx
> +                    break
> +            controls.insert(insert_before, {m.name: desc})
> +
> +    # Remove elements from controls without recreating the list (to keep
> +    # comments etc.).
> +    idx = 0
> +    while idx < len(controls):
> +        name = get_control_name(controls[idx])
> +        if name in obsolete_names:
> +            logger.info(f"Remove control '{name}'")
> +            controls.pop(idx)
> +        else:
> +            idx += 1
> +
> +    with ctrl_file.open('w') as f:
> +        # Ruyaml looses the header.
> +        f.write(("# SPDX-License-Identifier: LGPL-2.1-or-later\n"
> +                 "#\n"
> +                 "# This file was generated by utils/gen-debug-controls.py\n"
> +                 "#\n"))
> +        yaml.dump(doc, f)
> +
> +    return 0
> +
> +
> +if __name__ == '__main__':
> +    sys.exit(main(sys.argv))
> -- 
> 2.43.0
>