Source code for donfig.config_obj

#!/usr/bin/env python
#
# Copyright (c) 2018-2019 Donfig Developers
# Copyright (c) 2014-2018, Anaconda, Inc. and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import annotations

import ast
import base64
import contextlib
import json
import os
import pprint
import site
import sys
import warnings
from collections.abc import Mapping, Sequence
from contextlib import nullcontext
from copy import deepcopy
from typing import Any, Literal, MutableMapping

import yaml

from ._lock import SerializableLock

no_default = "__no_default__"


[docs] def canonical_name(k: str, config: Mapping[str, Any]) -> str: """Return the canonical name for a key. Handles user choice of '-' or '_' conventions by standardizing on whichever version was set first. If a key already exists in either hyphen or underscore form, the existing version is the canonical name. If neither version exists the original key is used as is. """ try: if k in config: return k except TypeError: # config is not a mapping, return the same name as provided return k altk = k.replace("_", "-") if "_" in k else k.replace("-", "_") if altk in config: return altk return k
[docs] def update( old: MutableMapping[str, Any], new: Mapping[str, Any], priority: Literal["old", "new", "new-defaults"] = "new", defaults: Mapping | None = None, ) -> Mapping[str, Any]: """Update a nested dictionary with values from another This is like dict.update except that it smoothly merges nested values This operates in-place and modifies old Parameters ---------- priority: string {'old', 'new', 'new-defaults'} If new (default) then the new dictionary has preference. Otherwise the old dictionary does. If 'new-defaults', a mapping should be given of the current defaults. Only if a value in ``old`` matches the current default, it will be updated with ``new``. Examples -------- >>> a = {'x': 1, 'y': {'a': 2}} >>> b = {'x': 2, 'y': {'b': 3}} >>> update(a, b) # doctest: +SKIP {'x': 2, 'y': {'a': 2, 'b': 3}} >>> a = {'x': 1, 'y': {'a': 2}} >>> b = {'x': 2, 'y': {'b': 3}} >>> update(a, b, priority='old') # doctest: +SKIP {'x': 1, 'y': {'a': 2, 'b': 3}} >>> d = {'x': 0, 'y': {'a': 2}} >>> a = {'x': 1, 'y': {'a': 2}} >>> b = {'x': 2, 'y': {'a': 3, 'b': 3}} >>> update(a, b, priority='new-defaults', defaults=d) # doctest: +SKIP {'x': 1, 'y': {'a': 3, 'b': 3}} See Also -------- donfig.config_obj.merge """ for k, v in new.items(): k = canonical_name(k, old) if isinstance(v, Mapping): if k not in old or old[k] is None: old[k] = {} update( old[k], v, priority=priority, defaults=defaults.get(k) if defaults else None, ) else: if ( priority == "new" or k not in old or (priority == "new-defaults" and defaults and k in defaults and defaults[k] == old[k]) ): old[k] = v return old
[docs] def merge(*dicts: Mapping) -> dict: """Update a sequence of nested dictionaries This prefers the values in the latter dictionaries to those in the former Examples -------- >>> a = {'x': 1, 'y': {'a': 2}} >>> b = {'y': {'b': 3}} >>> merge(a, b) # doctest: +SKIP {'x': 1, 'y': {'a': 2, 'b': 3}} See Also -------- donfig.config_obj.update """ result: dict = {} for d in dicts: update(result, d) return result
[docs] def collect_yaml(paths: Sequence[str]) -> list[dict]: """Collect configuration from yaml files This searches through a list of paths, expands to find all yaml or json files, and then parses each file. """ # Find all paths file_paths = [] for path in paths: if os.path.exists(path): if os.path.isdir(path): try: file_paths.extend( sorted( os.path.join(path, p) for p in os.listdir(path) if os.path.splitext(p)[1].lower() in (".json", ".yaml", ".yml") ) ) except OSError: # Ignore permission errors pass else: file_paths.append(path) configs = [] # Parse yaml files for path in file_paths: config = _load_config_file(path) if config is not None: configs.append(config) return configs
def _load_config_file(path: str) -> dict | None: try: with open(path) as f: config = yaml.safe_load(f.read()) except OSError: # Ignore permission errors return None except Exception as exc: raise ValueError(f"A config file at {path!r} is malformed, original error " f"message:\n\n{exc}") from None if config is not None and not isinstance(config, dict): raise ValueError( f"A config file at {path!r} is malformed - config files must have " f"a dict as the top level object, got a {type(config).__name__} instead" ) return config
[docs] def collect_env( prefix: str, env: Mapping[str, str] | None = None, deprecations: MutableMapping[str, str | None] | None = None ) -> dict: """Collect config from environment variables This grabs environment variables of the form "DASK_FOO__BAR_BAZ=123" and turns these into config variables of the form ``{"foo": {"bar-baz": 123}}`` It transforms the key and value in the following way: - Lower-cases the key text - Treats ``__`` (double-underscore) as nested access - Calls ``ast.literal_eval`` on the value """ if env is None: env = os.environ serial_env = f"{prefix}_INTERNAL_INHERIT_CONFIG" if serial_env in env: d = deserialize(env[serial_env]) else: d = {} prefix_len = len(prefix) for name, value in env.items(): if name.startswith(prefix): varname = name[prefix_len:].lower().replace("__", ".") try: d[varname] = ast.literal_eval(value) except (SyntaxError, ValueError): d[varname] = value result: dict = {} # fake thread lock to use set functionality lock = nullcontext() ConfigSet(result, lock, deprecations or {}, d) return result
[docs] class ConfigSet: """Temporarily set configuration values within a context manager Note, this class should be used directly from the `Config` object via the :meth:`donfig.Config.set` method. Examples -------- >>> from donfig.config_obj import ConfigSet >>> import mypkg >>> with ConfigSet(mypkg.config, {'foo': 123}): ... pass See Also -------- donfig.Config.set donfig.Config.get """ def __init__( self, config: MutableMapping, lock: SerializableLock | contextlib.AbstractContextManager, deprecations: MutableMapping[str, str | None], arg: Mapping | None = None, **kwargs, ): with lock: self.config = config self.deprecations = deprecations self._record: list[tuple[str, tuple, Any]] = [] if arg is not None: for key, value in arg.items(): key = self._check_deprecations(key) self._assign(key.split("."), value, config) if kwargs: for key, value in kwargs.items(): key = key.replace("__", ".") key = self._check_deprecations(key) self._assign(key.split("."), value, config) def _check_deprecations(self, key: str): """Check if the provided value has been renamed or removed. Parameters ---------- key : str The configuration key to check Returns ------- new: str The proper key, whether the original (if no deprecation) or the aliased value """ if key in self.deprecations: new = self.deprecations[key] if new: warnings.warn(f"Configuration key {key!r} has been deprecated. " f"Please use {new!r} instead") return new else: raise ValueError(f"Configuration value {key!r} has been removed") else: return key def __enter__(self): return self.config def __exit__(self, type, value, traceback): for op, path, value in reversed(self._record): d = self.config if op == "replace": for key in path[:-1]: d = d.setdefault(key, {}) d[path[-1]] = value else: # insert for key in path[:-1]: try: d = d[key] except KeyError: break else: d.pop(path[-1], None) def _assign( self, keys: Sequence[str], value: Any, d: MutableMapping, path: tuple[str, ...] = (), record: bool = True, ) -> None: """Assign value into a nested configuration dictionary Parameters ---------- keys : Sequence[str] The nested path of keys to assign the value, similar to toolz.put_in value: object d : dict The part of the nested dictionary into which we want to assign the value path : tuple[str], optional Used internally to hold the path of old values record : bool, optional Whether this operation needs to be recorded to allow for rollback. """ key = canonical_name(keys[0], d) path = path + (key,) if len(keys) == 1: if record: if key in d: self._record.append(("replace", path, d[key])) else: self._record.append(("insert", path, None)) d[key] = value else: if key not in d: if record: self._record.append(("insert", path, None)) d[key] = {} # No need to record subsequent operations after an insert record = False self._assign(keys[1:], value, d[key], path, record=record)
[docs] def expand_environment_variables(config): """Expand environment variables in a nested config dictionary This function will recursively search through any nested dictionaries and/or lists. Parameters ---------- config : dict, iterable, or str Input object to search for environment variables Returns ------- config : same type as input Examples -------- >>> expand_environment_variables({'x': [1, 2, '$USER']}) # doctest: +SKIP {'x': [1, 2, 'my-username']} """ if isinstance(config, Mapping): return {k: expand_environment_variables(v) for k, v in config.items()} elif isinstance(config, str): return os.path.expandvars(config) elif isinstance(config, (list, tuple, set)): return type(config)([expand_environment_variables(v) for v in config]) else: return config
[docs] class Config: def __init__( self, name: str, defaults: list[Mapping[str, Any]] | None = None, paths: list[str] | None = None, env: Mapping[str, str] | None = None, env_var: str | None = None, root_env_var: str | None = None, env_prefix: str | None = None, deprecations: Mapping[str, str | None] | None = None, ): if root_env_var is None: root_env_var = f"{name.upper()}_ROOT_CONFIG" if paths is None: paths = [ os.getenv(root_env_var, f"/etc/{name}"), os.path.join(sys.prefix, "etc", name), *[os.path.join(prefix, "etc", name) for prefix in site.PREFIXES], os.path.join(os.path.expanduser("~"), ".config", name), ] if env_prefix is None: env_prefix = f"{name.upper()}_" if env is None: env = os.environ if env_var is None: env_var = f"{name.upper()}_CONFIG" if env_var in os.environ: main_path = os.environ[env_var] paths.append(main_path) else: main_path = os.path.join(os.path.expanduser("~"), ".config", name) if deprecations is None: deprecations = {} # Remove duplicate paths while preserving ordering paths = list(reversed(list(dict.fromkeys(reversed(paths))))) self.name = name self.env_prefix = env_prefix self.env = env self.main_path = main_path self.paths = paths self.defaults = defaults or [] self.deprecations = deprecations self.config: MutableMapping[str, Any] = {} self.config_lock = SerializableLock() self.refresh() def __contains__(self, item): try: self[item] return True except (TypeError, IndexError, KeyError): return False def __getitem__(self, item): return self.get(item)
[docs] def pprint(self, **kwargs): return pprint.pprint(self.config, **kwargs)
[docs] def collect(self, paths: list[str] | None = None, env: Mapping[str, str] | None = None) -> dict: """Collect configuration from paths and environment variables Parameters ---------- paths : list[str] A list of paths to search for yaml config files. Defaults to the paths passed when creating this object. env : Mapping[str, str] The system environment variables to search through. Defaults to the environment dictionary passed when creating this object. Returns ------- config: dict See Also -------- donfig.Config.refresh: collect configuration and update into primary config """ if paths is None: paths = self.paths if env is None: env = self.env configs = [] if yaml: configs.extend(collect_yaml(paths=paths)) configs.append(collect_env(self.env_prefix, env=env)) return merge(*configs)
[docs] def refresh(self, **kwargs) -> None: """Update configuration by re-reading yaml files and env variables. This goes through the following stages: 1. Clearing out all old configuration 2. Updating from the stored defaults from downstream libraries (see update_defaults) 3. Updating from yaml files and environment variables Note that some functionality only checks configuration once at startup and may not change behavior, even if configuration changes. It is recommended to restart your python process if convenient to ensure that new configuration changes take place. See Also -------- donfig.Config.collect: for parameters donfig.Config.update_defaults """ self.clear() for d in self.defaults: update(self.config, d, priority="old") update(self.config, self.collect(**kwargs))
[docs] def get(self, key: str, default: Any = no_default) -> Any: """Get elements from global config Use '.' for nested access Examples -------- >>> from donfig import Config >>> config = Config('mypkg') >>> config.get('foo') # doctest: +SKIP {'x': 1, 'y': 2} >>> config.get('foo.x') # doctest: +SKIP 1 >>> config.get('foo.x.y', default=123) # doctest: +SKIP 123 See Also -------- donfig.Config.set """ keys = key.split(".") result = self.config for k in keys: k = canonical_name(k, result) try: result = result[k] except (TypeError, IndexError, KeyError): if default is not no_default: return default else: raise return result
[docs] def update_defaults(self, new: Mapping) -> None: """Add a new set of defaults to the configuration It does two things: 1. Add the defaults to a collection to be used by refresh() later 2. Updates the global config with the new configuration. Old values are prioritized over new ones, unless the current value is the old default, in which case it's updated to the new default. """ current_defaults = merge(*self.defaults) self.defaults.append(new) update(self.config, new, priority="new-defaults", defaults=current_defaults)
[docs] def to_dict(self): """Return dictionary copy of configuration. .. warning:: This will copy all keys and values. This includes values that may cause unwanted side effects depending on what values exist in the current configuration. """ return deepcopy(self.config)
[docs] def clear(self): """Clear all existing configuration.""" self.config.clear()
[docs] def merge(self, *dicts): """Merge this configuration with multiple dictionaries. See :func:`~donfig.config_obj.merge` for more information. """ self.config = merge(self.config, dicts)
[docs] def update(self, new, priority="new"): """Update the internal configuration dictionary with `new`. See :func:`~donfig.config_obj.update` for more information. """ self.config = update(self.config, new, priority=priority)
[docs] def expand_environment_variables(self) -> None: """Expand any environment variables in this configuration in-place. See :func:`~donfig.config_obj.expand_environment_variables` for more information. """ self.config = expand_environment_variables(self.config)
[docs] def rename(self, aliases: Mapping) -> None: """Rename old keys to new keys This helps migrate older configuration versions over time """ old = [] new = {} for o, n in aliases.items(): value = self.get(o, None) if value is not None: old.append(o) new[n] = value for k in old: del self.config[canonical_name(k, self.config)] # TODO: support nested keys self.set(new)
[docs] def set(self, arg=None, **kwargs): """Set configuration values within a context manager. Parameters ---------- arg : mapping or None, optional A mapping of configuration key-value pairs to set. **kwargs : Additional key-value pairs to set. If ``arg`` is provided, values set in ``arg`` will be applied before those in ``kwargs``. Double-underscores (``__``) in keyword arguments will be replaced with ``.``, allowing nested values to be easily set. Examples -------- >>> from donfig import Config >>> config = Config('mypkg') Set ``'foo.bar'`` in a context, by providing a mapping. >>> with config.set({'foo.bar': 123}): ... pass Set ``'foo.bar'`` in a context, by providing a keyword argument. >>> with config.set(foo__bar=123): ... pass Set ``'foo.bar'`` globally. >>> config.set(foo__bar=123) # doctest: +SKIP See Also -------- donfig.Config.get """ return ConfigSet(self.config, self.config_lock, self.deprecations, arg=arg, **kwargs)
[docs] def ensure_file(self, source: str, destination: str | None = None, comment: bool = True) -> None: """Copy file to default location if it does not already exist This tries to move a default configuration file to a default location if if does not already exist. It also comments out that file by default. This is to be used by downstream modules that may have default configuration files that they wish to include in the default configuration path. Parameters ---------- source : string, filename Source configuration file, typically within a source directory. destination : string, directory Destination directory. Configurable by ``<CONFIG NAME>_CONFIG`` environment variable, falling back to ~/.config/<config name>. comment : bool, True by default Whether or not to comment out the config file when copying. """ if destination is None: destination = self.main_path # destination is a file and already exists, never overwrite if os.path.isfile(destination): return # If destination is not an existing file, interpret as a directory, # use the source basename as the filename directory = destination destination = os.path.join(directory, os.path.basename(source)) try: if not os.path.exists(destination): os.makedirs(directory, exist_ok=True) # Atomically create destination. Parallel testing discovered # a race condition where a process can be busy creating the # destination while another process reads an empty config file. tmp = "%s.tmp.%d" % (destination, os.getpid()) with open(source) as f: lines = list(f) if comment: lines = ["# " + line if line.strip() and not line.startswith("#") else line for line in lines] with open(tmp, "w") as f: f.write("".join(lines)) try: os.rename(tmp, destination) except OSError: os.remove(tmp) except OSError: pass
[docs] def serialize(self) -> str: """Serialize conifg data into a string. See :func:`serialize` for more information. """ return serialize(self.config)
[docs] def serialize(data: Any) -> str: """Serialize config data into a string. Typically used to pass config via the ``MYPKG_INTERNAL_INHERIT_CONFIG`` environment variable. Parameters ---------- data: json-serializable object The data to serialize Returns ------- serialized_data: str The serialized data as a string """ return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
[docs] def deserialize(data: str) -> Any: """De-serialize config data into the original object. Typically used when receiving config via the ``MYPKG_INTERNAL_INHERIT_CONFIG`` environment variable. This is automatically called when a :class:`Config` is created and environment variables are loaded. Parameters ---------- data: str String serialized by :func:`donfig.serialize` Returns ------- deserialized_data: obj The de-serialized data """ return json.loads(base64.urlsafe_b64decode(data.encode()).decode())