Skip to content

localexperimentcache

localexperimentcache

Implementation of the ExperimentCache interface for local experiments.

Classes

ExperimentCache

Bases: ABC

Interface for the experiment cache

Functions
__contains__ abstractmethod 
__contains__(exp_id)

Checks if the experiment is in the cache

Source code in niceml/experiments/localexperimentcache.py 
28
29
30
@abstractmethod
def __contains__(self, exp_id: str) -> bool:
    """Checks if the experiment is in the cache"""
get_exp_count_in_cache abstractmethod 
get_exp_count_in_cache()

Returns the amount of cached experiments

Source code in niceml/experiments/localexperimentcache.py 
32
33
34
@abstractmethod
def get_exp_count_in_cache(self) -> int:
    """Returns the amount of cached experiments"""
load_exp_info abstractmethod 
load_exp_info(exp_id)

Loads the experiment info from the cache

Source code in niceml/experiments/localexperimentcache.py 
45
46
47
@abstractmethod
def load_exp_info(self, exp_id: str) -> ExperimentInfo:
    """Loads the experiment info from the cache"""
load_experiment abstractmethod 
load_experiment(exp_id, image_loader=None, df_loader=None)

Loads the experiment from the cache

Source code in niceml/experiments/localexperimentcache.py 
36
37
38
39
40
41
42
43
@abstractmethod
def load_experiment(
    self,
    exp_id: str,
    image_loader: Optional[ImageLoader] = None,
    df_loader: Optional[DfLoader] = None,
) -> ExperimentData:
    """Loads the experiment from the cache"""
save_experiment abstractmethod 
save_experiment(exp_data)

Saves the experiment to the cache

Source code in niceml/experiments/localexperimentcache.py 
49
50
51
@abstractmethod
def save_experiment(self, exp_data: ExperimentData):
    """Saves the experiment to the cache"""
should_reload 
should_reload(experiment_info)

Checks if the experiment should be reloaded

Source code in niceml/experiments/localexperimentcache.py 
53
54
55
56
57
58
def should_reload(self, experiment_info: ExperimentInfo) -> bool:
    """Checks if the experiment should be reloaded"""
    if experiment_info.short_id not in self:
        return True
    loaded_exp_info = self.load_exp_info(experiment_info.short_id)
    return experiment_info.is_modified(loaded_exp_info)

LocalExperimentCache 

LocalExperimentCache(store_folder)

Bases: ExperimentCache

Implementation of the ExperimentCache interface for local disk experiments.

Factory method for the local experiment cache

Source code in niceml/experiments/localexperimentcache.py 
117
118
119
120
121
def __init__(self, store_folder: str):
    """Factory method for the local experiment cache"""
    self.store_folder = store_folder
    if not isdir(self.store_folder):
        makedirs(self.store_folder, exist_ok=True)
Functions
__contains__ 
__contains__(exp_id)

Checks if the experiment is in the cache

Source code in niceml/experiments/localexperimentcache.py 
123
124
125
126
127
128
129
def __contains__(self, exp_id: str) -> bool:
    """Checks if the experiment is in the cache"""
    try:
        self.find_folder_name_of_exp_id(exp_id)
        return True
    except ExpIdNotFoundError:
        return False
find_folder_name_of_exp_id 
find_folder_name_of_exp_id(exp_id)

Finds the folder name of the experiment id

Source code in niceml/experiments/localexperimentcache.py 
135
136
137
138
139
140
141
def find_folder_name_of_exp_id(self, exp_id) -> Optional[str]:
    """Finds the folder name of the experiment id"""

    for folder_name in get_exp_folder_list(self.store_folder):
        if exp_id in folder_name and isdir(join(self.store_folder, folder_name)):
            return folder_name
    raise ExpIdNotFoundError(f"Experiment with id: {exp_id} not in Cache")
get_exp_count_in_cache 
get_exp_count_in_cache()

Returns the amount of cached experiments

Source code in niceml/experiments/localexperimentcache.py 
131
132
133
def get_exp_count_in_cache(self):
    """Returns the amount of cached experiments"""
    return len(get_exp_folder_list(self.store_folder))
load_exp_info 
load_exp_info(exp_id)

Loads the experiment info from the cache

Source code in niceml/experiments/localexperimentcache.py 
143
144
145
146
147
148
149
150
151
def load_exp_info(self, exp_id: str) -> ExperimentInfo:
    """Loads the experiment info from the cache"""
    exp_folder = self.find_folder_name_of_exp_id(exp_id)
    exp_info_file = join(
        self.store_folder, exp_folder, ExperimentFilenames.EXP_INFO
    )
    exp_info = load_exp_info(exp_info_file)

    return exp_info
load_experiment 
load_experiment(exp_id, image_loader=None, df_loader=None)

Loads the experiment from the cache

Source code in niceml/experiments/localexperimentcache.py 
153
154
155
156
157
158
159
160
161
162
163
164
def load_experiment(
    self,
    exp_id: str,
    image_loader: Optional[ImageLoader] = None,
    df_loader: Optional[DfLoader] = None,
) -> ExperimentData:
    """Loads the experiment from the cache"""
    exp_folder = self.find_folder_name_of_exp_id(exp_id)
    storage = LocalStorage(self.store_folder)
    return create_expdata_from_storage(
        exp_folder, storage, image_loader=image_loader, df_loader=df_loader
    )
save_experiment 
save_experiment(exp_data)

Saves the experiment to the cache

Source code in niceml/experiments/localexperimentcache.py 
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
def save_experiment(self, exp_data: ExperimentData):
    """Saves the experiment to the cache"""
    if self.store_folder is None:
        return
    self._create_output_folders(exp_data=exp_data)

    exp_files_df = create_exp_file_df(exp_data=exp_data)

    for key, value in exp_data.exp_dict_data.items():
        with open(
            join(
                self.store_folder,
                exp_data.get_experiment_path(),
                key + ".yaml",
            ),
            "w",
            encoding="utf-8",
        ) as exp_data_path:
            yaml.dump(value, exp_data_path, indent=2)

    exp_data.log_data.to_csv(
        join(
            self.store_folder,
            exp_data.get_experiment_path(),
            ExperimentFilenames.TRAIN_LOGS,
        )
    )

    write_parquet(
        exp_files_df,
        join(
            self.store_folder,
            exp_data.get_experiment_path(),
            ExperimentFilenames.EXP_FILES_FILE,
        ),
    )

Functions

create_exp_file_df 

create_exp_file_df(exp_data)

Creates a dataframe with the experiment files

Source code in niceml/experiments/localexperimentcache.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def create_exp_file_df(exp_data: ExperimentData) -> pd.DataFrame:
    """Creates a dataframe with the experiment files"""
    exp_files = []
    for exp_file in exp_data.all_exp_files:
        ext = splitext(exp_file)[1]

        if len(ext) > 0:
            if ext == ".yml":
                ext = ".yaml"
            exp_files.append(
                {
                    ExperimentFilenames.EXP_FILES_COL: splitext(exp_file)[0] + ext,
                    "suffix": ext,
                }
            )
    return pd.DataFrame(exp_files)

get_exp_folder_list 

get_exp_folder_list(folder, root_folder=None)

Creates a list with names of experiment folders, which also includes parent
 directories if they exist

Args: folder: folder to search in root_folder: param for saving the first folder attribute due to a recursive function call

Returns:

  • List[str] –

    List of experiment folder names

Source code in niceml/experiments/localexperimentcache.py 
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def get_exp_folder_list(folder: str, root_folder: Optional[str] = None) -> List[str]:
    """
        Creates a list with names of experiment folders, which also includes parent
         directories if they exist
    Args:
        folder: folder to search in
        root_folder: param for saving the first `folder` attribute due to a recursive function call

    Returns:
        List of experiment folder names

    """
    if check_exp_name(folder):
        return [folder]

    folders = (
        list_dir(folder) if root_folder is None else list_dir(join(root_folder, folder))
    )
    if root_folder is None:
        root_folder = folder
    if not all(check_exp_name(split(path)[-1]) for path in folders):
        sub_folders = [
            get_exp_folder_list(path, root_folder)
            for path in folders
            if isdir(join(root_folder, path))
        ]
        sub_folders = sum(sub_folders, [])
        return sub_folders
    return (
        [join(folder, sub_folder) for sub_folder in folders]
        if folder != root_folder
        else folders
    )