Source code for minari.storage.local

import importlib.metadata
import os
import pathlib
import shutil
import tempfile
import warnings
from typing import Dict, Iterable, Optional, Tuple, Union

from minari.dataset.minari_dataset import (
    MinariDataset,
    gen_dataset_id,
    parse_dataset_id,
)
from minari.dataset.minari_storage import MinariStorage
from minari.storage import hosting
from minari.storage.datasets_root_dir import get_dataset_path


# Use importlib due to circular import when: "from minari import __version__"
__version__ = importlib.metadata.version("minari")


def list_non_hidden_dirs(path: pathlib.Path) -> Iterable[str]:
    """List all non-hidden subdirectories."""
    for d in path.iterdir():
        if d.is_dir() and (not d.name.startswith(".")):
            yield d.name


def dataset_id_sort_key(dataset_id: str) -> Tuple[str, str, int]:
    """Key for sorting dataset ids first by namespace, and then alphabetically."""
    namespace, dataset_name, version = parse_dataset_id(dataset_id)
    namespace = "" if namespace is None else namespace
    return (namespace, dataset_name, version)


[docs] def load_dataset(dataset_id: str, download: bool = False): """Retrieve Minari dataset from local database. Args: dataset_id (str): name id of Minari dataset download (bool): if `True` download the dataset if it is not found locally, or download it again if the local copy cannot be loaded, e.g. when files are missing or the metadata is unreadable after an interrupted download. Damaged data files are only detected when episodes are read. Default to `False`. Returns: MinariDataset """ file_path = get_dataset_path(dataset_id) data_path = os.path.join(file_path, "data") if not os.path.exists(data_path): if not download: raise FileNotFoundError( f"Dataset {dataset_id} not found locally at {file_path}. Use download=True to download the dataset." ) hosting.download_dataset(dataset_id) return MinariDataset(data_path) # Only storage-level errors mean the local copy is incomplete or corrupted, e.g. after an # interrupted download. Errors raised by MinariDataset (such as an unsupported dataset version) # are not fixed by downloading again, so they propagate unchanged. try: storage = MinariStorage.read(data_path) except ValueError as e: if not download: raise ValueError( f"Dataset {dataset_id} found locally at {file_path} could not be loaded, it may be incomplete or corrupted: {e}. " f"Use download=True or minari.download_dataset('{dataset_id}', force_download=True) to download it again, " f"or minari.delete_dataset('{dataset_id}') to remove it." ) from e warnings.warn( f"Dataset {dataset_id} found locally at {file_path} could not be loaded ({e}). Downloading it again." ) _redownload_dataset(dataset_id, file_path) storage = MinariStorage.read(data_path) return MinariDataset(storage)
def _redownload_dataset(dataset_id: str, file_path: pathlib.Path): """Replace a broken local dataset with a fresh download. The broken copy is moved aside, not forced over, so that the normal compatibility checks of `download_dataset` run. If the download fails, e.g. because the dataset only exists locally or the remote is unreachable, the broken copy is put back. """ backup_dir = pathlib.Path( tempfile.mkdtemp(prefix=f".{file_path.name}-backup-", dir=file_path.parent) ) backup_path = backup_dir / file_path.name shutil.move(file_path, backup_path) try: hosting.download_dataset(dataset_id) except BaseException: if file_path.exists(): shutil.rmtree(file_path) shutil.move(backup_path, file_path) shutil.rmtree(backup_dir) raise shutil.rmtree(backup_dir)
[docs] def list_local_datasets( latest_version: bool = False, compatible_minari_version: bool = False, prefix: Optional[str] = None, ) -> Dict[str, Dict[str, Union[str, int, bool]]]: """Get the ids and metadata of all the Minari datasets in the local database. Args: latest_version (bool): if `True` only the latest version of the datasets are returned i.e. from ['D4RL/door/human-v0', 'D4RL/door/human-v1`], only the metadata for v1 is returned. Default to `False`. compatible_minari_version (bool): if `True` only the datasets compatible with the current Minari version are returned. Default to `False`. Returns: Dict[str, Dict[str, str]]: keys the names of the Minari datasets and values the metadata """ from minari import supported_dataset_versions datasets_path = get_dataset_path() dataset_ids = [] def recurse_directories(base_path: pathlib.Path, namespace): parent_dir = base_path.joinpath(namespace) if not parent_dir.exists(): return # TODO: metadata.json of HDF5 should stay in dataset root directory not /data # Then we can use metadata.json to check if it is a dataset if parent_dir.joinpath("data").exists(): dataset_ids.append(namespace) return for dir_name in list_non_hidden_dirs(parent_dir): namespaced_dir_name = pathlib.Path(namespace, dir_name).as_posix() recurse_directories(base_path, namespaced_dir_name) recurse_directories(datasets_path, prefix or "") dataset_ids = sorted(dataset_ids, key=dataset_id_sort_key) local_datasets = {} for dst_id in dataset_ids: data_path = pathlib.Path(datasets_path, dst_id, "data").as_posix() try: metadata = MinariStorage.read_raw_metadata(data_path) metadata_id = metadata["dataset_id"] if dst_id != metadata_id: raise ValueError( f"Namespace location '{dst_id}' does not match id '{metadata_id}'." ) except Exception as e: warnings.warn(f"Misconfigured dataset named {dst_id}: {e}") continue if ( compatible_minari_version and metadata["minari_version"] not in supported_dataset_versions ): continue namespace, dataset_name, version = parse_dataset_id(dst_id) dataset = gen_dataset_id(namespace, dataset_name) if latest_version: if dataset not in local_datasets or version > local_datasets[dataset][0]: local_datasets[dataset] = (version, metadata) else: local_datasets[dst_id] = metadata if latest_version: # Return dict = {'dataset_id': metadata} return dict( map(lambda x: (f"{x[0]}-v{x[1][0]}", x[1][1]), local_datasets.items()) ) else: return local_datasets
[docs] def delete_dataset(dataset_id: str): """Delete a Minari dataset from the local Minari database. Args: dataset_id (str): name id of the Minari dataset """ dataset_path = get_dataset_path(dataset_id) shutil.rmtree(dataset_path) print(f"Dataset {dataset_id} deleted!")