src.fairreckitlib.data.set.processor.dataset_processor_base

  1"""This module contains the base functionality shared by all dataset processors.
  2
  3Classes:
  4
  5    DatasetProcessorBase: the base class for dataset processors.
  6
  7This program has been developed by students from the bachelor Computer Science at
  8Utrecht University within the Software Project course.
  9© Copyright Utrecht University (Department of Information and Computing Sciences)
 10"""
 11
 12from abc import ABCMeta, abstractmethod
 13from typing import Callable, Dict, List, Optional, Tuple
 14
 15from ..dataset_config import DatasetConfig, DatasetMatrixConfig, DatasetTableConfig
 16
 17
 18class DatasetProcessorBase(metaclass=ABCMeta):
 19    """DataProcessor base class for all FairRecKit datasets.
 20
 21    Datasets are preprocessed so that they will be of a recognized standard format
 22    on the other side. A configuration file is produced in the resulting dataset
 23    directory that stores the metadata for achieving this. For further information
 24    it is advised to take a look at the Dataset(Config) class.
 25
 26    The dataset configuration mainly consists of:
 27
 28    1) event tables: contain user event tables that can be used to construct a matrix of.
 29    2) matrix tables: contain available matrices associated with the dataset.
 30    3) (other) tables: contain the shared tables associated with the dataset.
 31
 32    For each of these three categories an abstract function is exposed in order to retrieve
 33    (table name, table configuration processor) tuples. The tables names are expected to be
 34    unique across all categories. The table configuration processors are allowed to return
 35    None on failure and will be excluded from the final configuration. Moreover, tables that
 36    do not contain any records are excluded as well.
 37    The base dataset processor handles the processing logic. It needs to produce at least
 38    one valid event table or one valid matrix configuration to be successful, concluding
 39    that remaining tables are optional.
 40
 41    Abstract methods:
 42
 43    get_event_configs
 44    get_matrix_configs
 45    get_table_configs
 46
 47    Public methods:
 48
 49    run
 50    """
 51
 52    def __init__(self, dataset_dir: str, dataset_name: str):
 53        """Construct the base DatasetProcessor.
 54
 55        Args:
 56            dataset_dir: path of the dataset directory.
 57            dataset_name: name of the dataset (processor).
 58        """
 59        self.dataset_dir = dataset_dir
 60        self.dataset_name = dataset_name
 61
 62    @abstractmethod
 63    def get_event_configs(self) -> List[Tuple[str, Callable[[], Optional[DatasetTableConfig]]]]:
 64        """Get event table configuration processors.
 65
 66        Returns:
 67            a list of tuples consisting of the event table name and the event table processor.
 68        """
 69        raise NotImplementedError()
 70
 71    @abstractmethod
 72    def get_matrix_configs(self) -> List[Tuple[str, Callable[[], Optional[DatasetMatrixConfig]]]]:
 73        """Get matrix configuration processors.
 74
 75        Returns:
 76            a list of tuples consisting of the matrix name and the matrix processor.
 77        """
 78        raise NotImplementedError()
 79
 80    @abstractmethod
 81    def get_table_configs(self) -> List[Tuple[str, Callable[[], Optional[DatasetTableConfig]]]]:
 82        """Get table configuration processors.
 83
 84        Returns:
 85            a list of tuples consisting of the table name and the table processor.
 86        """
 87        raise NotImplementedError()
 88
 89    def run_event_table_processors(self) -> Dict[str, DatasetTableConfig]:
 90        """Run the dataset's event table processors.
 91
 92        Returns:
 93            a dictionary with valid event table name-configuration pairs.
 94        """
 95        dataset_events = {}
 96        for table_name, process_config in self.get_event_configs():
 97            config = process_config()
 98            if config is not None and config.num_records > 0:
 99                dataset_events[table_name] = config
100
101        return dataset_events
102
103    def run_matrix_table_processors(self) -> Dict[str, DatasetMatrixConfig]:
104        """Run the dataset's matrix processors.
105
106        Returns:
107            a dictionary with valid matrix name-configuration pairs.
108        """
109        dataset_matrices = {}
110        for matrix_name, process_config in self.get_matrix_configs():
111            config = process_config()
112            if config is not None and config.table.num_records > 0:
113                dataset_matrices[matrix_name] = config
114
115        return dataset_matrices
116
117    def run_table_processors(self) -> Dict[str, DatasetTableConfig]:
118        """Run the dataset's additional table processors.
119
120        Returns:
121            a dictionary with valid table name-configuration pairs.
122        """
123        dataset_tables = {}
124        for table_name, process_config in self.get_table_configs():
125            config = process_config()
126            if config is not None and config.num_records > 0:
127                dataset_tables[table_name] = config
128
129        return dataset_tables
130
131    def run(self) -> Optional[DatasetConfig]:
132        """Run the dataset configuration processor.
133
134        Returns:
135            the dataset configuration or None on failure.
136        """
137        dataset_events = self.run_event_table_processors()
138        dataset_matrices = self.run_matrix_table_processors()
139        if len(dataset_events) == 0 and len(dataset_matrices) == 0:
140            return None
141
142        return DatasetConfig(
143            self.dataset_name,
144            dataset_events,
145            dataset_matrices,
146            self.run_table_processors()
147        )