src.fairreckitlib.evaluation.pipeline.evaluation_config_parsing

  1"""This module contains a parser for the evaluation and metric configuration(s).
  2
  3Functions:
  4
  5    parse_evaluation_config: parse all metric configurations.
  6
  7This program has been developed by students from the bachelor Computer Science at
  8Utrecht University within the Software Project course.
  9© Copyright Utrecht University (Department of Information and Computing Sciences)
 10"""
 11
 12from typing import Any, Dict, List
 13
 14from ...core.config.config_factories import GroupFactory
 15from ...core.config.config_value_param import ConfigNumberParam
 16from ...core.core_constants import KEY_TOP_K
 17from ...core.events.event_dispatcher import EventDispatcher
 18from ...core.parsing.parse_config_object import parse_config_object_list
 19from ...core.parsing.parse_event import ON_PARSE, ParseEventArgs
 20from ...data.filter.filter_config_parsing import parse_data_subset_config
 21from ...data.set.dataset_registry import DataRegistry
 22from ..evaluation_factory import KEY_EVALUATION
 23from ..metrics.metric_constants import KEY_METRIC_PARAM_K, KEY_METRIC_SUBGROUP
 24from .evaluation_config import MetricConfig
 25
 26
 27def parse_evaluation_config(
 28        data_registry: DataRegistry,
 29        data_filter_factory: GroupFactory,
 30        experiment_config: Dict[str, Any],
 31        metric_category_factory: GroupFactory,
 32        event_dispatcher: EventDispatcher) -> List[MetricConfig]:
 33    """Parse all metric configurations.
 34
 35    Args:
 36        data_registry: the data registry with available dataset matrices.
 37        data_filter_factory: the dataset filter group factory.
 38        experiment_config: the experiment's total configuration.
 39        metric_category_factory: the metric factory containing grouped available metrics.
 40        event_dispatcher: to dispatch the parse event on failure.
 41
 42    Returns:
 43        a list of parsed MetricConfig's which is possibly empty.
 44    """
 45    # evaluation is not mandatory
 46    if not KEY_EVALUATION in experiment_config:
 47        return []
 48
 49    eval_config = experiment_config[KEY_EVALUATION]
 50
 51    top_k_param = {}
 52    # override top K param according to experiment configuration
 53    if experiment_config.get(KEY_TOP_K, False):
 54        top_k_param[KEY_METRIC_PARAM_K] = ConfigNumberParam(
 55            KEY_METRIC_PARAM_K,
 56            int,
 57            experiment_config[KEY_TOP_K],
 58            (1, experiment_config[KEY_TOP_K])
 59        )
 60
 61    # parse metric configurations as objects
 62    parsed_config_objs = parse_config_object_list(
 63        KEY_EVALUATION,
 64        'metric',
 65        eval_config,
 66        metric_category_factory,
 67        event_dispatcher,
 68        params=top_k_param
 69    )
 70
 71    # convert object to metric configurations
 72    metric_config_list = []
 73    for (metric, metric_config) in parsed_config_objs:
 74        metric_subgroup = None
 75
 76        # attempt to parse metric subgroup
 77        if KEY_METRIC_SUBGROUP in metric_config:
 78            metric_data_subset, _ = parse_data_subset_config(
 79                metric_config[KEY_METRIC_SUBGROUP],
 80                data_registry,
 81                data_filter_factory,
 82                event_dispatcher,
 83                data_parent_name='metric \'' + metric.name + '\'',
 84                required=False
 85            )
 86
 87            # skip on failure or no filter passes
 88            if metric_data_subset is not None and len(metric_data_subset.filter_passes) == 0:
 89                event_dispatcher.dispatch(ParseEventArgs(
 90                    ON_PARSE,
 91                    'PARSE WARNING: metric \'' + metric.name +
 92                    '\' data filter passes are empty, skipping...'
 93                ))
 94
 95            metric_subgroup = metric_data_subset
 96
 97        metric_config_list.append(MetricConfig(
 98            metric.name,
 99            metric.params,
100            metric_subgroup
101        ))
102
103    return metric_config_list