src.fairreckitlib.data.set.dataset_config_parser

  1"""This module contains the parser for the dataset configuration and parser utility functions.
  2
  3Classes:
  4
  5    DatasetParser: dataset configuration parser.
  6
  7Functions:
  8
  9    parse_file_name: parse a file name from a configuration and verify existence on disk.
 10    parse_float: parse floating-point value from a configuration.
 11    parse_int: parse integer value from a configuration.
 12    parse_optional_bool: parse optional boolean value from a configuration.
 13    parse_optional_string: parse optional string value from a configuration.
 14    parse_rating_matrix: parse rating matrix configuration.
 15    parse_string: parse a string value from a configuration.
 16    parse_string_list: parse a list of strings from a configuration.
 17
 18This program has been developed by students from the bachelor Computer Science at
 19Utrecht University within the Software Project course.
 20© Copyright Utrecht University (Department of Information and Computing Sciences)
 21"""
 22import os.path
 23from typing import Any, Dict, List, Optional, Tuple
 24
 25from ...core.core_constants import KEY_NAME
 26from ...core.events.event_dispatcher import EventDispatcher
 27from ...core.io.io_utility import load_yml
 28from ...core.parsing.parse_assert import \
 29    assert_is_type, assert_is_key_in_dict, assert_is_one_of_list
 30from ...core.parsing.parse_event import ON_PARSE, ParseEventArgs, print_parse_event
 31from .dataset_constants import KEY_DATASET, KEY_EVENTS, KEY_MATRICES, KEY_TABLES
 32from .dataset_constants import KEY_MATRIX, KEY_IDX_ITEM, KEY_IDX_USER
 33from .dataset_constants import KEY_RATING_MIN, KEY_RATING_MAX, KEY_RATING_TYPE
 34from .dataset_constants import TABLE_KEY, TABLE_PRIMARY_KEY, TABLE_FOREIGN_KEYS, TABLE_COLUMNS
 35from .dataset_constants import TABLE_FILE, TABLE_COMPRESSION, TABLE_ENCODING
 36from .dataset_constants import TABLE_HEADER, TABLE_NUM_RECORDS, TABLE_SEP
 37from .dataset_config import DatasetIndexConfig, DatasetMatrixConfig, RatingMatrixConfig
 38from .dataset_config import DatasetConfig, DatasetFileConfig, DatasetTableConfig, FileOptionsConfig
 39from .dataset_config import DATASET_RATINGS_EXPLICIT, DATASET_RATINGS_IMPLICIT
 40
 41VALID_SEPARATORS = [',', '|']
 42VALID_COMPRESSIONS = ['bz2']
 43VALID_ENCODINGS = ['utf-8', 'ISO-8859-1']
 44
 45class DatasetConfigParser:
 46    """Dataset Configuration Parser.
 47
 48    Public methods:
 49
 50    parse_dataset_config
 51    parse_dataset_config_from_yml
 52    """
 53
 54    def __init__(self, verbose: bool):
 55        """Construct the DatasetConfigParser.
 56
 57        Args:
 58            verbose: whether the parser should give verbose output.
 59        """
 60        self.verbose = verbose
 61        handle_parse_event = lambda parser, args: \
 62            print_parse_event(args) if parser.verbose else None
 63
 64        self.event_dispatcher = EventDispatcher()
 65        self.event_dispatcher.add_listener(ON_PARSE, self, (handle_parse_event, None))
 66
 67    def parse_dataset_config(
 68            self,
 69            data_dir: str,
 70            dataset_config: Dict[str, Any],
 71            available_datasets: List[str]) -> Optional[DatasetConfig]:
 72        """Parse a dataset configuration.
 73
 74        Args:
 75            data_dir: the directory where the dataset is stored.
 76            dataset_config: the dataset configuration.
 77            available_datasets: a list of already available datasets.
 78
 79        Returns:
 80            the parsed configuration or None on failure.
 81        """
 82        # attempt to parse the name of the dataset
 83        dataset_name = parse_string(
 84            dataset_config,
 85            KEY_DATASET,
 86            self.event_dispatcher
 87        )
 88        if dataset_name is None:
 89            return None
 90
 91        # verify that the dataset name is not already present
 92        if dataset_name in available_datasets:
 93            self.event_dispatcher.dispatch(ParseEventArgs(
 94                ON_PARSE,
 95                'PARSE ERROR: dataset already exists: ' + dataset_name
 96            ))
 97            return None
 98
 99        # attempt to parse the dataset (event) tables
100        events = self.parse_dataset_events(data_dir, dataset_config)
101
102        # attempt to parse the dataset (matrix) tables
103        matrices = self.parse_dataset_matrices(data_dir, dataset_config)
104
105        # attempt to parse the dataset (other) tables
106        tables = self.parse_dataset_tables(data_dir, dataset_config)
107
108        return DatasetConfig(
109            dataset_name,
110            events,
111            matrices,
112            tables
113        )
114
115    def parse_dataset_config_from_yml(
116            self,
117            data_dir: str,
118            file_name: str,
119            available_datasets: List[str]) -> Optional[DatasetConfig]:
120        """Parse a dataset configuration.
121
122        Args:
123            data_dir: the directory where the dataset is stored.
124            file_name: the name of the yml file with extension.
125            available_datasets: a list of already available datasets.
126
127        Returns:
128            the parsed configuration or None on failure.
129        """
130        return self.parse_dataset_config(
131            data_dir,
132            load_yml(os.path.join(data_dir, file_name)),
133            available_datasets
134        )
135
136    def parse_dataset_events(
137            self,
138            data_dir: str,
139            dataset_config: Dict[str, Any],) -> Dict[str, DatasetTableConfig]:
140        """Parse dataset event tables from the configuration.
141
142        Args:
143            data_dir: the directory where the dataset is stored.
144            dataset_config: the dataset configuration.
145
146        Returns:
147            a dictionary with parsed event table configurations.
148        """
149        events = {}
150        if dataset_config.get(KEY_EVENTS) is not None:
151            if assert_is_type(
152                dataset_config[KEY_EVENTS],
153                dict,
154                self.event_dispatcher,
155                'PARSE WARNING: dataset events invalid value'
156            ):
157                for table_name, table_config in dataset_config[KEY_EVENTS].items():
158                    config = self.parse_dataset_table_config(data_dir, table_config)
159                    if config is None:
160                        continue
161
162                    events[table_name] = config
163
164        return events
165
166    def parse_dataset_matrices(
167            self,
168            data_dir: str,
169            dataset_config: Dict[str, Any]) -> Dict[str, DatasetMatrixConfig]:
170        """Parse dataset matrices from the configuration.
171
172        Args:
173            data_dir: the directory where the dataset is stored.
174            dataset_config: the dataset configuration.
175
176        Returns:
177            a dictionary with parsed matrix configurations.
178        """
179        matrices = {}
180        if dataset_config.get(KEY_MATRICES) is not None:
181            if assert_is_type(
182                dataset_config[KEY_MATRICES],
183                dict,
184                self.event_dispatcher,
185                'PARSE WARNING: dataset matrices invalid value'
186            ):
187                for matrix_name, matrix_config in dataset_config[KEY_MATRICES].items():
188                    config = self.parse_dataset_matrix_config(
189                        data_dir,
190                        matrix_config
191                    )
192                    if config is None:
193                        continue
194
195                    matrices[matrix_name] = config
196
197        return matrices
198
199    def parse_dataset_tables(
200            self,
201            data_dir: str,
202            dataset_config: Dict[str, Any],) -> Dict[str, DatasetTableConfig]:
203        """Parse dataset tables from the configuration.
204
205        Args:
206            data_dir: the directory where the dataset is stored.
207            dataset_config: the dataset configuration.
208
209        Returns:
210            a dictionary with parsed table configurations.
211        """
212        tables = {}
213        if dataset_config.get(KEY_TABLES) is not None:
214            if assert_is_type(
215                dataset_config[KEY_TABLES],
216                dict,
217                self.event_dispatcher,
218                'PARSE WARNING: dataset tables invalid value'
219            ):
220                for table_name, table_config in dataset_config[KEY_TABLES].items():
221                    config = self.parse_dataset_table_config(data_dir, table_config)
222                    if config is None:
223                        continue
224
225                    tables[table_name] = config
226
227        return tables
228
229    def parse_file_options_config(
230            self,
231            file_config: Dict[str, Any]) -> Optional[FileOptionsConfig]:
232        """Parse a dataset file configuration.
233
234        Args:
235            file_config: the dataset file configuration.
236
237        Returns:
238            the parsed configuration or None on failure.
239        """
240        # attempt to parse the optional separator string
241        success, file_sep = parse_optional_string(
242            file_config,
243            TABLE_SEP,
244            VALID_SEPARATORS,
245            self.event_dispatcher
246        )
247        if not success:
248            return None
249
250        # attempt to parse the optional compression string
251        success, file_compression = parse_optional_string(
252            file_config,
253            TABLE_COMPRESSION,
254            VALID_COMPRESSIONS,
255            self.event_dispatcher
256        )
257        if not success:
258            return None
259
260        # attempt to parse the optional encoding string
261        success, file_encoding = parse_optional_string(
262            file_config,
263            TABLE_ENCODING,
264            VALID_ENCODINGS,
265            self.event_dispatcher
266        )
267        if not success:
268            return None
269
270        # attempt to parse the optional header boolean
271        success, file_header = parse_optional_bool(
272            file_config,
273            TABLE_HEADER,
274            self.event_dispatcher
275        )
276        if not success:
277            return None
278
279        return FileOptionsConfig(
280            file_sep,
281            file_compression,
282            file_encoding,
283            file_header
284        )
285
286    def parse_dataset_file_config(
287            self,
288            data_dir: str,
289            file_config: Dict[str, Any]) -> Optional[DatasetFileConfig]:
290        """Parse a dataset file configuration.
291
292        Args:
293            data_dir: the directory where the file is stored.
294            file_config: the dataset file configuration.
295
296        Returns:
297            the parsed configuration or None on failure.
298        """
299        # attempt to parse the (required) file name
300        success, file_name = parse_file_name(
301            data_dir,
302            file_config,
303            KEY_NAME,
304            self.event_dispatcher
305        )
306        if not success:
307            return None
308
309        # attempt to parse the file options
310        file_options = self.parse_file_options_config(file_config)
311        if file_options is None:
312            return None
313
314        return DatasetFileConfig(file_name, file_options)
315
316    def parse_dataset_index_config(
317            self,
318            data_dir: str,
319            index_config: Dict[str, Any]) -> Optional[DatasetIndexConfig]:
320        """Parse a dataset matrix' user/item index configuration.
321
322        Args:
323            data_dir: the directory where the file is stored.
324            index_config: the dataset matrix index configuration.
325
326        Returns:
327            the parsed configuration or None on failure.
328        """
329        # attempt to parse (optional) file name
330        success, file_name = parse_file_name(
331            data_dir,
332            index_config,
333            TABLE_FILE,
334            self.event_dispatcher,
335            required=False
336        )
337        if not success:
338            return None
339
340        # attempt to parse the key that is associated with the index
341        file_key = parse_string(
342            index_config,
343            TABLE_KEY,
344            self.event_dispatcher
345        )
346        if file_key is None:
347            return None
348
349        # attempt to parse the number of records in the file
350        num_records = parse_int(
351            index_config,
352            TABLE_NUM_RECORDS,
353            self.event_dispatcher
354        )
355        if num_records is None:
356            return None
357
358        return DatasetIndexConfig(file_name, file_key, num_records)
359
360    def parse_dataset_matrix_config(
361            self,
362            data_dir: str,
363            matrix_config: Dict[str, Any]) -> Optional[DatasetMatrixConfig]:
364        """Parse a dataset matrix configuration.
365
366        Args:
367            data_dir: the directory where the dataset matrix is stored.
368            matrix_config: the dataset matrix configuration.
369
370        Returns:
371            the parsed configuration or None on failure.
372        """
373        # attempt to parse the matrix table
374        matrix_table = self.parse_dataset_table_config(data_dir,
375                                                       matrix_config.get(KEY_MATRIX, {}))
376        if matrix_table is None:
377            return None
378
379        # attempt to parse the matrix users
380        matrix_users = self.parse_dataset_index_config(data_dir,
381                                                       matrix_config.get(KEY_IDX_USER, {}))
382        if matrix_users is None:
383            return None
384
385        # attempt to parse the matrix items
386        matrix_items = self.parse_dataset_index_config(data_dir,
387                                                       matrix_config.get(KEY_IDX_ITEM, {}))
388        if matrix_items is None:
389            return None
390
391        # attempt to parse the matrix ratings
392        matrix_ratings = parse_rating_matrix(
393            matrix_config,
394            self.event_dispatcher
395        )
396        if matrix_ratings is None:
397            return None
398
399        return DatasetMatrixConfig(
400            matrix_table,
401            matrix_ratings,
402            matrix_users,
403            matrix_items
404        )
405
406    def parse_dataset_table_config(
407            self,
408            data_dir: str,
409            table_config: Dict[str, Any]) -> Optional[DatasetTableConfig]:
410        """Parse a dataset table configuration.
411
412        Args:
413            data_dir: the directory where the table is stored.
414            table_config: the dataset table configuration.
415
416        Returns:
417            the parsed configuration or None on failure.
418        """
419        file_config = self.parse_dataset_file_config(data_dir, table_config.get(TABLE_FILE, {}))
420        if file_config is None:
421            return None
422
423        table_primary_key = parse_string_list(
424            table_config,
425            TABLE_PRIMARY_KEY,
426            1,
427            self.event_dispatcher
428        )
429        if table_primary_key is None:
430            return None
431
432        table_foreign_keys = None
433        if TABLE_FOREIGN_KEYS in table_config:
434            table_foreign_keys = parse_string_list(
435                table_config,
436                TABLE_FOREIGN_KEYS,
437                0,
438                self.event_dispatcher
439            )
440
441        table_columns = parse_string_list(
442            table_config,
443            TABLE_COLUMNS,
444            1,
445            self.event_dispatcher
446        )
447        if table_columns is None:
448            return None
449
450        table_num_records = parse_int(
451            table_config,
452            TABLE_NUM_RECORDS,
453            self.event_dispatcher
454        )
455        if table_num_records is None:
456            return None
457
458        return DatasetTableConfig(
459            table_primary_key,
460            table_foreign_keys,
461            table_columns,
462            table_num_records,
463            file_config
464        )
465
466
467def parse_file_name(
468        data_dir: str,
469        file_config: Dict[str, Any],
470        file_key: str,
471        event_dispatcher: EventDispatcher,
472        *,
473        required: bool=True) -> Tuple[bool, Optional[str]]:
474    """Parse the file name from the configuration.
475
476    In addition, when the file name is parsed correctly it is checked
477    for existence in the specified data directory.
478
479    Args:
480        data_dir: the directory where the file is stored.
481        file_config: the configuration dictionary to parse from.
482        file_key: the key in the configuration that contains the file name.
483        event_dispatcher: to dispatch the parse event on failure.
484        required: whether the parsing is required to succeed.
485
486    Returns:
487        whether the parsing succeeded and the parsed file name or None on failure.
488    """
489    if required and not assert_is_key_in_dict(
490        file_key,
491        file_config,
492        event_dispatcher,
493        'PARSE ERROR: file configuration missing key \'' + file_key + '\''
494    ): return False, None
495
496    file_name = file_config.get(file_key)
497    if required and file_name is None:
498        event_dispatcher.dispatch(ParseEventArgs(
499            ON_PARSE,
500            'PARSE ERROR: file configuration missing value for \'' + file_key + '\''
501        ))
502        return False, None
503
504    if file_name is not None:
505        if not assert_is_type(
506            file_name,
507            str,
508            event_dispatcher,
509            'PARSE ERROR: file configuration contains invalid name'
510        ): return False, None
511
512        file_path = os.path.join(data_dir, file_name)
513        if not os.path.isfile(file_path):
514            event_dispatcher.dispatch(ParseEventArgs(
515                ON_PARSE,
516                'PARSE ERROR: file configuration file name does not exist: ' + file_path
517            ))
518            return False, None
519
520    return True, file_name
521
522
523def parse_float(
524        config: Dict[str, Any],
525        float_key: str,
526        event_dispatcher: EventDispatcher) -> Optional[float]:
527    """Parse a float-point value from the configuration.
528
529    Args:
530        config: the configuration dictionary to parse from.
531        float_key: the key in the configuration that contains the float-point value.
532        event_dispatcher: to dispatch the parse event on failure.
533
534    Returns:
535        the parsed float-point value or None on failure.
536    """
537    if not assert_is_key_in_dict(
538        float_key,
539        config,
540        event_dispatcher,
541        'PARSE ERROR: configuration contains invalid \'' + float_key + '\' value'
542    ): return None
543
544    float_value = config[float_key]
545
546    if not assert_is_type(
547        float_value,
548        float,
549        event_dispatcher,
550        'PARSE ERROR: configuration contains invalid \'' + float_key + '\''
551    ): return None
552
553    return float_value
554
555
556def parse_int(
557        config: Dict[str, Any],
558        int_key: str,
559        event_dispatcher: EventDispatcher) -> Optional[int]:
560    """Parse an integer value from the configuration.
561
562    The integer is expected to be greater than zero to be parsed successfully.
563
564    Args:
565        config: the configuration dictionary to parse from.
566        int_key: the key in the configuration that contains the integer value.
567        event_dispatcher: to dispatch the parse event on failure.
568
569    Returns:
570        the parsed integer value or None on failure.
571    """
572    if not assert_is_key_in_dict(
573        int_key,
574        config,
575        event_dispatcher,
576        'PARSE ERROR: configuration contains invalid \'' + int_key + '\' value'
577    ): return None
578
579    int_value = config[int_key]
580
581    if isinstance(int_value, bool):
582        event_dispatcher.dispatch(ParseEventArgs(
583            ON_PARSE,
584            'PARSE ERROR: configuration contains invalid \'' + int_key + '\'',
585            expected_type=int,
586            actual_type=bool
587        ))
588        return None
589
590    if not assert_is_type(
591        int_value,
592        int,
593        event_dispatcher,
594        'PARSE ERROR: configuration contains invalid \'' + int_key + '\''
595    ): return None
596
597    if int_value <= 0:
598        event_dispatcher.dispatch(ParseEventArgs(
599            ON_PARSE,
600        'PARSE ERROR: configuration contains invalid \'' + int_key + '\' less than or equal to zero'
601        ))
602        return None
603
604    return int_value
605
606
607def parse_optional_bool(
608        config: Dict[str, Any],
609        bool_key: str,
610        event_dispatcher: EventDispatcher) -> Tuple[bool, Optional[bool]]:
611    """Parse an optional boolean from the configuration.
612
613    Args:
614        config: the configuration dictionary to parse from.
615        bool_key: the key in the configuration that contains the boolean.
616        event_dispatcher: to dispatch the parse event on failure.
617
618    Returns:
619        whether the parsing succeeded and the optional boolean value.
620    """
621    bool_value = config.get(bool_key)
622    if bool_value is not None:
623        if not assert_is_type(
624            bool_value,
625            bool,
626            event_dispatcher,
627            'PARSE ERROR: configuration contains invalid ' + bool_key + ' value'
628        ): return False, None
629    else:
630        bool_value = False
631
632    return True, bool_value
633
634
635def parse_optional_string(
636        config: Dict[str, Any],
637        string_key: str,
638        string_options: List[str],
639        event_dispatcher: EventDispatcher) -> Tuple[bool, Optional[str]]:
640    """Parse an optional string from a list of valid values from the configuration.
641
642    Args:
643        config: the configuration dictionary to parse from.
644        string_key: the key in the configuration that contains the string.
645        string_options: the options that are available for the string that is being parsed.
646        event_dispatcher: to dispatch the parse event on failure.
647
648    Returns:
649        whether the parsing succeeded and the optional string value.
650    """
651    string_value = config.get(string_key)
652    if string_value is not None:
653        if not assert_is_type(
654            string_value,
655            str,
656            event_dispatcher,
657            'PARSE ERROR: configuration contains invalid \'' + string_key + '\' value'
658        ): return False, None
659
660        if not assert_is_one_of_list(
661            string_value,
662            string_options,
663            event_dispatcher,
664            'PARSE ERROR: configuration contains invalid \'' + string_key + '\''
665        ): return False, None
666
667    return True, string_value
668
669
670def parse_rating_matrix(
671        matrix_config: Dict[str, Any],
672        event_dispatcher: EventDispatcher) -> Optional[RatingMatrixConfig]:
673    """Parse a rating matrix from the configuration.
674
675    Args:
676        matrix_config: the matrix configuration dictionary to parse from.
677        event_dispatcher: to dispatch the parse event on failure.
678
679    Returns:
680        the parsed string or None on failure.
681    """
682    rating_min = parse_float(
683        matrix_config,
684        KEY_RATING_MIN,
685        event_dispatcher
686    )
687    if rating_min is None:
688        return None
689
690    if rating_min <= 0.0:
691        event_dispatcher.dispatch(ParseEventArgs(
692            ON_PARSE,
693            'PARSE ERROR: matrix configuration contains minimum rating greater than zero'
694        ))
695        return None
696
697    rating_max = parse_float(
698        matrix_config,
699        KEY_RATING_MAX,
700        event_dispatcher
701    )
702    if rating_max is None:
703        return None
704
705    if rating_max < rating_min:
706        event_dispatcher.dispatch(ParseEventArgs(
707            ON_PARSE,
708            'PARSE ERROR: matrix configuration contains maximum rating less than minimum rating'
709        ))
710        return None
711
712    rating_type = parse_string(
713        matrix_config,
714        KEY_RATING_TYPE,
715        event_dispatcher,
716        one_of_list=[DATASET_RATINGS_EXPLICIT, DATASET_RATINGS_IMPLICIT]
717    )
718
719    if rating_type is None:
720        return None
721
722    return RatingMatrixConfig(rating_min, rating_max, rating_type)
723
724
725def parse_string(
726        config: Dict[str, Any],
727        string_key: str,
728        event_dispatcher: EventDispatcher,
729        *,
730        one_of_list: List[str]=None) -> Optional[str]:
731    """Parse a string from the configuration.
732
733    Args:
734        config: the configuration dictionary to parse from.
735        string_key: the key in the configuration that contains the string.
736        event_dispatcher: to dispatch the parse event on failure.
737        one_of_list: when not None the string is to be expected one of the specified list.
738
739    Returns:
740        the parsed string or None on failure.
741    """
742    if not assert_is_key_in_dict(
743        string_key,
744        config,
745        event_dispatcher,
746        'PARSE ERROR: configuration contains invalid \'' + string_key + '\' value'
747    ): return None
748
749    string_value = config[string_key]
750
751    if not assert_is_type(
752        string_value,
753        str,
754        event_dispatcher,
755        'PARSE ERROR: configuration contains invalid \'' + string_key + '\''
756    ): return None
757
758    if one_of_list is not None:
759        if not assert_is_one_of_list(
760            string_value,
761            one_of_list,
762            event_dispatcher,
763            'PARSE ERROR: configuration contains invalid \'' + string_key + '\''
764        ): return None
765
766    return string_value
767
768
769def parse_string_list(
770        config: Dict[str, Any],
771        string_list_key: str,
772        min_list_length: int,
773        event_dispatcher: EventDispatcher) -> Optional[List[str]]:
774    """Parse a list of strings from the configuration.
775
776    Args:
777        config: the configuration dictionary to parse from.
778        string_list_key: the key in the configuration that contains the string list.
779        min_list_length: the minimum length of the list to succeed.
780        event_dispatcher: to dispatch the parse event on failure.
781
782    Returns:
783        the parsed string list or None on failure.
784    """
785    if not assert_is_key_in_dict(
786        string_list_key,
787        config,
788        event_dispatcher,
789        'PARSE ERROR: configuration contains invalid \'' + string_list_key + '\' value'
790    ): return None
791
792    string_list = config[string_list_key]
793
794    if not assert_is_type(
795        string_list,
796        list,
797        event_dispatcher,
798        'PARSE ERROR: configuration contains invalid \'' + string_list_key + '\''
799    ): return None
800
801    result_strings = []
802    for string in string_list:
803        if not assert_is_type(
804            string,
805            str,
806            event_dispatcher,
807            'PARSE ERROR: configuration list \'' + string_list_key + '\' contains invalid value'
808        ): return None
809
810        result_strings.append(string)
811
812    if len(result_strings) < min_list_length:
813        event_dispatcher.dispatch(ParseEventArgs(
814            ON_PARSE,
815            'PARSE ERROR: configuration list \'' + string_list_key + '\' contains too few values'
816        ))
817        return None
818
819    return result_strings