Skip to content

remotettrainutils

remotettrainutils

Module for the loading the experiments for the dashboard

Classes

Functions

download_visu 

download_visu(experiment_downloader)

Download the visualization files for the experiment

Source code in niceml/dashboard/remotettrainutils.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def download_visu(experiment_downloader: ExperimentDownloader):
    """Download the visualization files for the experiment"""
    selected_extensions = []
    for ext in experiment_downloader.get_all_extensions():
        if st.checkbox(ext):
            selected_extensions.append(ext)
    if st.button("Download"):
        to_download_files = experiment_downloader.get_downloads(selected_extensions)
        download_count = len(to_download_files)
        status_text = st.empty()
        prog_bar = st.progress(0)
        for idx, file in enumerate(to_download_files):
            status_text.text(f"({idx + 1}/{download_count}) - File: {file.source_file}")
            prog_bar.progress((idx + 1) / download_count)
            file()

exp_manager_factory 

exp_manager_factory(*args)

Factory for the experiment manager cached with streamlit

Source code in niceml/dashboard/remotettrainutils.py 
19
20
21
22
@st.cache_resource()
def exp_manager_factory(*args):  # pylint: disable=unused-argument
    """Factory for the experiment manager cached with streamlit"""
    return ExperimentManager([])

load_experiments 

load_experiments(
    storage,
    exp_info_list,
    df_loader_factory,
    image_loader_factory,
    local_exp_cache=None,
)

Load the experiments from the cloud storage and stores them in the experiment manager. Additionally, they are saved in the local cache

Source code in niceml/dashboard/remotettrainutils.py 
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def load_experiments(
    storage,
    exp_info_list: List[ExperimentInfo],
    df_loader_factory: DfLoaderFactory,
    image_loader_factory: ImageLoaderFactory,
    local_exp_cache: Optional[ExperimentCache] = None,
):
    """Load the experiments from the cloud storage and
    stores them in the experiment manager. Additionally, they are saved in the local cache
    """
    experiments: List[ExperimentData]
    dir_info_list: List[str] = []
    load_exp_info_list: List[ExperimentInfo] = []

    # pylint: disable = unused-argument
    @st.cache_data()
    def _check_and_load_cache(
        *args,
    ) -> List[ExperimentData]:
        experiments_list = []
        for cur_exp_info in exp_info_list:
            if local_exp_cache is not None and not local_exp_cache.should_reload(
                cur_exp_info
            ):
                initialized_df_loader: DfLoader = df_loader_factory.create_df_loader(
                    storage, cur_exp_info.exp_filepath
                )
                initialized_image_loader: ImageLoader = (
                    image_loader_factory.create_image_loader(
                        storage, cur_exp_info.exp_filepath
                    )
                )
                exp_data = local_exp_cache.load_experiment(
                    cur_exp_info.short_id,
                    df_loader=initialized_df_loader,
                    image_loader=initialized_image_loader,
                )
                experiments_list.append(exp_data)
            else:
                dir_info_list.append(cur_exp_info.exp_filepath)
                load_exp_info_list.append(cur_exp_info)
        return experiments_list

    exp_cache_count = local_exp_cache.get_exp_count_in_cache()
    experiments = _check_and_load_cache(exp_info_list, exp_cache_count)
    if len(load_exp_info_list) > 0:
        load_exp_count = len(load_exp_info_list)
        prog_bar = st.progress(0)
        status_text = st.empty()
        status_text.text(f"Cached 0/{load_exp_count} experiments")

        for idx, dir_info in enumerate(dir_info_list):
            df_loader = df_loader_factory.create_df_loader(storage, dir_info)
            image_loader = image_loader_factory.create_image_loader(storage, dir_info)
            experiment = try_remote_exp_data_factory(
                dir_info, storage, df_loader=df_loader, image_loader=image_loader
            )
            if experiment is not None:
                experiments.append(experiment)
                if local_exp_cache is not None:
                    local_exp_cache.save_experiment(experiment)
            prog_bar.progress(idx / load_exp_count)
            status_text.text(f"Cached {idx}/{load_exp_count} experiments")
        status_text.text(f"Cached {load_exp_count}/{load_exp_count} experiments")
        prog_bar.progress(1.0)
        st.success("Done")
    return experiments

query_experiments 

query_experiments(storage)

Query the experiments from the cloud storage

Source code in niceml/dashboard/remotettrainutils.py 
25
26
27
28
29
30
31
32
33
def query_experiments(storage: StorageInterface) -> List[ExperimentInfo]:
    """Query the experiments from the cloud storage"""

    @st.cache_data(ttl=3600)
    def _local_query_exps(*args):  # pylint: disable=unused-argument
        exp_info_list: List[ExperimentInfo] = storage.list_experiments()
        return exp_info_list

    return _local_query_exps(id(storage))

select_to_load_exps 

select_to_load_exps(exp_info_list, exp_manager)

Select the experiments to load. That means which are not in the experiment manager

Source code in niceml/dashboard/remotettrainutils.py 
36
37
38
39
40
41
42
43
44
45
def select_to_load_exps(
    exp_info_list: List[ExperimentInfo], exp_manager: ExperimentManager
):
    """Select the experiments to load.
    That means which are not in the experiment manager"""
    experiments_to_load = []
    for exp_info in exp_info_list:
        if exp_manager.is_exp_modified(exp_info.short_id, exp_info.last_modified):
            experiments_to_load.append(exp_info)
    return experiments_to_load

try_remote_exp_data_factory 

try_remote_exp_data_factory(
    filepath, storage, df_loader, image_loader
)

Try to load experiment data from the filepath with the remote cloud storage

Source code in niceml/dashboard/remotettrainutils.py 
48
49
50
51
52
53
54
55
56
57
def try_remote_exp_data_factory(
    filepath, storage, df_loader, image_loader
) -> Optional[ExperimentData]:
    """Try to load experiment data from the filepath with the remote cloud storage"""
    try:
        return create_expdata_from_storage(
            filepath, storage, df_loader=df_loader, image_loader=image_loader
        )
    except FileNotFoundError:
        return None