Skip to content

dataset

dataset

Module for dataset

Classes

Dataset

Bases: ABC

Dataset to load, transform, shuffle the data before training

Functions
__getitem__ abstractmethod 
__getitem__(index)

Returns the data of the item/batch at index

Source code in niceml/data/datasets/dataset.py 
36
37
38
39
@abstractmethod
def __getitem__(self, index: int):
    """Returns the data of the item/batch at index"""
    pass
__len__ abstractmethod 
__len__()

Returns the number of batches/items

Source code in niceml/data/datasets/dataset.py 
45
46
47
48
@abstractmethod
def __len__(self):
    """Returns the number of batches/items"""
    pass
get_data_by_key abstractmethod 
get_data_by_key(data_key)

Returns the data by the key (identifier of the data)

Source code in niceml/data/datasets/dataset.py 
54
55
56
@abstractmethod
def get_data_by_key(self, data_key):
    """Returns the data by the key (identifier of the data)"""
get_datainfo abstractmethod 
get_datainfo(batch_index)

returns the datainfo for the batch at index

Source code in niceml/data/datasets/dataset.py 
41
42
43
@abstractmethod
def get_datainfo(self, batch_index: int) -> List[DataInfo]:
    """returns the datainfo for the batch at index"""
get_dataset_stats 
get_dataset_stats()

Returns the dataset stats

Source code in niceml/data/datasets/dataset.py 
50
51
52
def get_dataset_stats(self) -> dict:
    """Returns the dataset stats"""
    return dict(size=self.get_item_count())
get_item_count abstractmethod 
get_item_count()

Returns the current count of items in the dataset

Source code in niceml/data/datasets/dataset.py 
14
15
16
@abstractmethod
def get_item_count(self) -> int:
    """Returns the current count of items in the dataset"""
get_items_per_epoch abstractmethod 
get_items_per_epoch()

Returns the items per epoch

Source code in niceml/data/datasets/dataset.py 
18
19
20
@abstractmethod
def get_items_per_epoch(self) -> int:
    """Returns the items per epoch"""
get_set_name abstractmethod 
get_set_name()

Returns the name of the set e.g. train

Source code in niceml/data/datasets/dataset.py 
22
23
24
@abstractmethod
def get_set_name(self) -> str:
    """Returns the name of the set e.g. train"""
initialize abstractmethod 
initialize(data_description, exp_context)

Initializes with the data description and context

Source code in niceml/data/datasets/dataset.py 
26
27
28
29
30
@abstractmethod
def initialize(
    self, data_description: DataDescription, exp_context: ExperimentContext
):
    """Initializes with the data description and context"""
iter_with_info 
iter_with_info()

Iterates over the dataset and adds the data_info to each data

Source code in niceml/data/datasets/dataset.py 
32
33
34
def iter_with_info(self) -> Iterable:
    """Iterates over the dataset and adds the data_info to each data"""
    return DataIterator(self)