src.fairreckitlib.evaluation.pipeline.evaluation_run

 1"""This module contains functionality that wraps running the evaluation pipeline multiple times.
 2
 3Classes:
 4
 5    EvaluationPipelineConfig: configuration class to run the evaluation pipelines.
 6
 7Functions:
 8
 9    run_evaluation_pipelines: run (multiple) pipelines for specified metric configurations.
10
11This program has been developed by students from the bachelor Computer Science at
12Utrecht University within the Software Project course.
13© Copyright Utrecht University (Department of Information and Computing Sciences)
14"""
15
16from dataclasses import dataclass
17import os
18from typing import List, Callable
19
20from ...core.config.config_factories import GroupFactory
21from ...core.core_constants import MODEL_RATINGS_FILE
22from ...core.events.event_dispatcher import EventDispatcher
23from ...data.data_transition import DataTransition
24from .evaluation_config import MetricConfig
25from .evaluation_pipeline import EvaluationPipeline, EvaluationSetPaths
26
27
28@dataclass
29class EvaluationPipelineConfig:
30    """Evaluation Pipeline Configuration.
31
32    output_dir: the directory to store the output.
33    data_transition: data input.
34    data_filter_factory: the factory with available filters for all dataset-matrix pairs.
35    eval_type_factory: the factory with available metric category factories.
36    metric_config_list: list of metric configurations to compute.
37    """
38
39    model_dirs: List[str]
40    data_transition: DataTransition
41    data_filter_factory: GroupFactory
42    eval_type_factory: GroupFactory
43    metric_config_list: List[MetricConfig]
44
45
46def run_evaluation_pipelines(
47        pipeline_config: EvaluationPipelineConfig,
48        event_dispatcher: EventDispatcher,
49        is_running: Callable[[], bool]) -> None:
50    """Run several evaluation pipelines according to the specified eval pipeline configuration.
51
52    Args:
53        pipeline_config: the configuration on how to run the evaluation pipelines.
54        event_dispatcher: used to dispatch eval/IO events when running the evaluation pipelines.
55        is_running: function that returns whether the pipelines
56            are still running. Stops early when False is returned.
57    """
58    data_transition = pipeline_config.data_transition
59
60    eval_pipeline = EvaluationPipeline(
61        data_transition.dataset,
62        pipeline_config.data_filter_factory,
63        pipeline_config.eval_type_factory,
64        event_dispatcher
65    )
66
67    # remove any metrics that have a subgroup that is not related to the data transition
68    metric_config_list = [metric for metric in pipeline_config.metric_config_list
69                          if metric.subgroup is None or \
70                          metric.subgroup.dataset == data_transition.dataset.get_name() and \
71                          metric.subgroup.matrix == data_transition.matrix_name]
72
73    for model_dir in pipeline_config.model_dirs:
74        rating_set_path = os.path.join(model_dir, MODEL_RATINGS_FILE)
75        output_path = os.path.join(model_dir, 'evaluations.json')
76
77        try:
78            eval_pipeline.run(
79                output_path,
80                EvaluationSetPaths(
81                    rating_set_path,
82                    data_transition.train_set_path,
83                    data_transition.test_set_path
84                ),
85                metric_config_list,
86                is_running
87            )
88        except FileNotFoundError:
89            continue