Skip to content

hydra

hydra

Module containing all utils regarding hydra

Classes

HydraInitField 

HydraInitField(
    target_class,
    description=None,
    default_value=None,
    example_value=None,
    **kwargs
)

Bases: Field

Used to configure Dagster Ops

Used to configure Dagster Ops with a target class

Parameters:

  • target_class –

    class which is instantiated from the op

  • description (Optional[str], default: None ) –

    description of the class or field

  • default_value (Optional[dict], default: None ) –

    default value of the field when nothing is provided

  • example_value (Optional[dict], default: None ) –

    example value of the field shown in the documentation

  • **kwargs –

    additional kwargs passed to the Field class

Source code in niceml/config/hydra.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def __init__(
    self,
    target_class,
    description: Optional[str] = None,
    default_value: Optional[dict] = None,
    example_value: Optional[dict] = None,
    **kwargs,
):
    """
    Used to configure Dagster Ops with a target class

    Args:
        target_class: class which is instantiated from the op
        description: description of the class or field
        default_value: default value of the field when nothing is provided
        example_value: example value of the field shown in the documentation
        **kwargs: additional kwargs passed to the Field class
    """
    if description is None:
        description = target_class.__doc__
    if default_value is None:
        default_value = dict()
    if example_value is None:
        impl_str: str = "implementation_of_" if isabstract(target_class) else ""
        default_value = {"_target_": f"{impl_str}{target_class}"}
    super().__init__(
        dict, description=description, default_value=default_value, **kwargs
    )
    self.target_class = target_class
    self.example_value = example_value
Functions

HydraMapField 

HydraMapField(
    target_class,
    description=None,
    default_value=None,
    example_value=None,
    **kwargs
)

Bases: Field

Used to configure Dagster Ops

Used to configure Dagster Ops with a map

Parameters:

  • target_class –

    class which is instantiated from the op in the map

  • description (Optional[str], default: None ) –

    description of the class or field

  • default_value (Optional[dict], default: None ) –

    default value of the field when nothing is provided

  • example_value (Optional[dict], default: None ) –

    example value of the field shown in the documentation

  • **kwargs –

    additional kwargs passed to the Field class

Source code in niceml/config/hydra.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def __init__(
    self,
    target_class,
    description: Optional[str] = None,
    default_value: Optional[dict] = None,
    example_value: Optional[dict] = None,
    **kwargs,
):
    """
    Used to configure Dagster Ops with a map

    Args:
        target_class: class which is instantiated from the op in the map
        description: description of the class or field
        default_value: default value of the field when nothing is provided
        example_value: example value of the field shown in the documentation
        **kwargs: additional kwargs passed to the Field class
    """
    if description is None:
        description = target_class.__doc__
    if default_value is None:
        default_value = dict()
    if example_value is None:
        impl_str: str = "implementation_of_" if isabstract(target_class) else ""
        example_value = {"value": {"_target_": f"{impl_str}{target_class}"}}
    super().__init__(
        Map(str, dict),
        description=description,
        default_value=default_value,
        **kwargs,
    )
    self.target_class = target_class
    self.example_value = example_value
Functions

Functions

hydra_conf_mapping_factory 

hydra_conf_mapping_factory(drop=('globals'))

Load hydra configuration from config.

Parameters:

  • config –

    Configuration to be processed with hydra.

  • drop (Iterable[str], default: ('globals') ) –

    Keys to remove from the processed configuration after processing with hydra. Useful to define configuration variables that shall be used for interpolation during processing but not enter the processed configuration. Default: ("globals",).

Source code in niceml/config/hydra.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def hydra_conf_mapping_factory(drop: Iterable[str] = ("globals",)):
    """
    Load hydra configuration from ``config``.

    Args:
        config: Configuration to be processed with hydra.
        drop: Keys to remove from the processed configuration after processing
               with hydra. Useful to define configuration variables that shall be used
               for interpolation during processing but not enter the processed
               configuration. Default: ``("globals",)``.
    """

    @config_mapping
    def hydra_conf_mapping(config: Dict[str, Any]):
        register_niceml_resolvers()
        config = json.loads(json.dumps(config))
        config_dir = TemporaryDirectory()  # pylint: disable=consider-using-with

        _, config_file = mkstemp(suffix=".yaml", dir=config_dir.name, text=True)
        with open(config_file, "wt", encoding="utf-8") as file:
            yaml.dump(config, file, Dumper=yaml.SafeDumper)

        conf = load_hydra_conf(config_file)
        try:
            config_dir.cleanup()
        except (PermissionError, NotADirectoryError):
            pass

        conf = {key: value for key, value in conf.items() if key not in set(drop)}
        return conf

    return hydra_conf_mapping

instantiate_from_yaml 

instantiate_from_yaml(yaml_path, file_system=None)

uses hydra instantiate to a yaml config

Source code in niceml/config/hydra.py 
17
18
19
20
21
22
23
24
25
def instantiate_from_yaml(
    yaml_path: str, file_system: Optional[AbstractFileSystem] = None
) -> Any:
    """uses hydra instantiate to a yaml config"""
    file_system = file_system or LocalFileSystem()
    with file_system.open(yaml_path, "r", encoding="utf-8") as file:
        data = yaml.load(file, Loader=yaml.SafeLoader)

    return instantiate(data)

prepend_hydra_search_paths 

prepend_hydra_search_paths(config, searchpaths)

Add searchpaths to config under hydra/searchpath.

Source code in niceml/config/hydra.py 
28
29
30
31
32
33
34
def prepend_hydra_search_paths(
    config: Dict[str, Any], searchpaths: Iterable[str]
) -> Dict[str, Any]:
    """Add searchpaths to config under hydra/searchpath."""
    config_hydra = config.get("hydra", {})
    config_hydra_searchpaths = list(searchpaths) + config_hydra.get("searchpath", [])
    return {**config, "hydra": {**config_hydra, "searchpath": config_hydra_searchpaths}}