icechunk.config#

class CachingConfig:
    """Configuration for how Icechunk caches its metadata files"""

    def __new__(
        cls,
        num_snapshot_nodes: int | None = None,
        num_chunk_refs: int | None = None,
        num_transaction_changes: int | None = None,
        num_bytes_attributes: int | None = None,
        num_bytes_chunks: int | None = None,
    ) -> Self:
        """
        Create a new `CachingConfig` object

        Parameters
        ----------
        num_snapshot_nodes: int | None
            The number of snapshot nodes to cache.
            Default: 500,000
        num_chunk_refs: int | None
            The number of chunk references to cache.
            Default: 15,000,000
        num_transaction_changes: int | None
            The number of transaction changes to cache.
            Default: 0
        num_bytes_attributes: int | None
            The number of bytes of attributes to cache.
            Default: 0
        num_bytes_chunks: int | None
            The number of bytes of chunks to cache.
            Default: 0
        """
    def _repr_html_(self, /) -> str: ...
    @property
    def num_snapshot_nodes(self) -> int | None:
        """
        The number of snapshot nodes to cache.

        Default: 500,000

        Returns
        -------
        int | None
            The number of snapshot nodes to cache.
        """
    @num_snapshot_nodes.setter
    def num_snapshot_nodes(self, value: int | None) -> None:
        """
        Set the number of snapshot nodes to cache.

        Parameters
        ----------
        value: int | None
            The number of snapshot nodes to cache.
        """
    @property
    def num_chunk_refs(self) -> int | None:
        """
        The number of chunk references to cache.

        Default: 15,000,000

        Returns
        -------
        int | None
            The number of chunk references to cache.
        """
    @num_chunk_refs.setter
    def num_chunk_refs(self, value: int | None) -> None:
        """
        Set the number of chunk references to cache.

        Parameters
        ----------
        value: int | None
            The number of chunk references to cache.
        """
    @property
    def num_transaction_changes(self) -> int | None:
        """
        The number of transaction changes to cache.

        Default: 0

        Returns
        -------
        int | None
            The number of transaction changes to cache.
        """
    @num_transaction_changes.setter
    def num_transaction_changes(self, value: int | None) -> None:
        """
        Set the number of transaction changes to cache.

        Parameters
        ----------
        value: int | None
            The number of transaction changes to cache.
        """
    @property
    def num_bytes_attributes(self) -> int | None:
        """
        The number of bytes of attributes to cache.

        Default: 0

        Returns
        -------
        int | None
            The number of bytes of attributes to cache.
        """
    @num_bytes_attributes.setter
    def num_bytes_attributes(self, value: int | None) -> None:
        """
        Set the number of bytes of attributes to cache.

        Parameters
        ----------
        value: int | None
            The number of bytes of attributes to cache.
        """
    @property
    def num_bytes_chunks(self) -> int | None:
        """
        The number of bytes of chunks to cache.

        Default: 0

        Returns
        -------
        int | None
            The number of bytes of chunks to cache.
        """
    @num_bytes_chunks.setter
    def num_bytes_chunks(self, value: int | None) -> None:
        """
        Set the number of bytes of chunks to cache.

        Parameters
        ----------
        value: int | None
            The number of bytes of chunks to cache.
        """

class CompressionAlgorithm(Enum):
    """Enum for selecting the compression algorithm used by Icechunk to write its metadata files

    Attributes
    ----------
    Zstd: int
        The Zstd compression algorithm.
    """

    Zstd = 0

    def __new__(cls) -> Self: ...
    @staticmethod
    def default() -> CompressionAlgorithm:
        """
        The default compression algorithm used by Icechunk to write its metadata files.

        Returns
        -------
        CompressionAlgorithm
            The default compression algorithm.
        """

class CompressionConfig:
    """Configuration for how Icechunk compresses its metadata files"""

    def __new__(
        cls, algorithm: CompressionAlgorithm | None = None, level: int | None = None
    ) -> Self:
        """
        Create a new `CompressionConfig` object

        Parameters
        ----------
        algorithm: CompressionAlgorithm | None
            The compression algorithm to use.
            Default: Zstd
        level: int | None
            The compression level to use.
            Default: 3
        """
    @property
    def algorithm(self) -> CompressionAlgorithm | None:
        """
        The compression algorithm used by Icechunk to write its metadata files.

        Default: Zstd

        Returns
        -------
        CompressionAlgorithm | None
            The compression algorithm used by Icechunk to write its metadata files.
        """
    @algorithm.setter
    def algorithm(self, value: CompressionAlgorithm | None) -> None:
        """
        Set the compression algorithm used by Icechunk to write its metadata files.

        Parameters
        ----------
        value: CompressionAlgorithm | None
            The compression algorithm to use.
        """
    @property
    def level(self) -> int | None:
        """
        The compression level used by Icechunk to write its metadata files.

        Default: 3

        Returns
        -------
        int | None
            The compression level used by Icechunk to write its metadata files.
        """
    @level.setter
    def level(self, value: int | None) -> None:
        """
        Set the compression level used by Icechunk to write its metadata files.

        Parameters
        ----------
        value: int | None
            The compression level to use.
        """
    @staticmethod
    def default() -> CompressionConfig:
        """
        The default compression configuration used by Icechunk to write its metadata files.

        Returns
        -------
        CompressionConfig
        """

class ManifestConfig:
    """Configuration for how Icechunk manifests"""

    def __new__(
        cls,
        preload: ManifestPreloadConfig | None = None,
        splitting: ManifestSplittingConfig | None = None,
        virtual_chunk_location_compression: ManifestVirtualChunkLocationCompressionConfig
        | None = None,
        max_concurrent_manifest_fetches_during_commit: int | None = None,
    ) -> Self:
        """
        Create a new `ManifestConfig` object

        Parameters
        ----------
        preload: ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded. When
            None, the default `ManifestPreloadConfig` is used.
            Default: None
        splitting: ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split. When
            None, the default `ManifestSplittingConfig` is used.
            Default: None
        virtual_chunk_location_compression: ManifestVirtualChunkLocationCompressionConfig | None
            The configuration for zstd compression of virtual chunk location URLs.
            When None, the default `ManifestVirtualChunkLocationCompressionConfig` is used.
            Default: None
        max_concurrent_manifest_fetches_during_commit: int | None
            How many manifests are fetched and updated concurrently during a
            commit, amend, flush, or rewrite_manifests.
            Default: 1
        """
    @property
    def preload(self) -> ManifestPreloadConfig | None:
        """
        The configuration for how Icechunk manifests will be preloaded.

        Default: None (uses the default `ManifestPreloadConfig`)

        Returns
        -------
        ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded.
        """
    @preload.setter
    def preload(self, value: ManifestPreloadConfig | None) -> None:
        """
        Set the configuration for how Icechunk manifests will be preloaded.

        Parameters
        ----------
        value: ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded.
        """

    @property
    def splitting(self) -> ManifestSplittingConfig | None:
        """
        The configuration for how Icechunk manifests will be split.

        Default: None (uses the default `ManifestSplittingConfig`)

        Returns
        -------
        ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split.
        """

    @splitting.setter
    def splitting(self, value: ManifestSplittingConfig | None) -> None:
        """
        Set the configuration for how Icechunk manifests will be split.

        Parameters
        ----------
        value: ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split.
        """

    @property
    def virtual_chunk_location_compression(
        self,
    ) -> ManifestVirtualChunkLocationCompressionConfig | None:
        """
        The configuration for zstd compression of virtual chunk location URLs.

        Default: None (uses the default `ManifestVirtualChunkLocationCompressionConfig`)

        Returns
        -------
        ManifestVirtualChunkLocationCompressionConfig | None
            The compression configuration.
        """

    @virtual_chunk_location_compression.setter
    def virtual_chunk_location_compression(
        self, value: ManifestVirtualChunkLocationCompressionConfig | None
    ) -> None:
        """
        Set the configuration for zstd compression of virtual chunk location URLs.

        Parameters
        ----------
        value: ManifestVirtualChunkLocationCompressionConfig | None
            The compression configuration.
        """

    @property
    def max_concurrent_manifest_fetches_during_commit(self) -> int | None:
        """
        How many manifests are fetched and updated concurrently during a
        commit, amend, flush, or rewrite_manifests.

        Default: 1

        Returns
        -------
        int | None
            The number of manifests fetched and updated concurrently during a commit.
        """

    @max_concurrent_manifest_fetches_during_commit.setter
    def max_concurrent_manifest_fetches_during_commit(self, value: int | None) -> None:
        """
        Set how many manifests are fetched and updated concurrently during a
        commit, amend, flush, or rewrite_manifests.

        Parameters
        ----------
        value: int | None
            The number of manifests to fetch and update concurrently.
        """

class ManifestPreloadCondition:
    """Configuration for conditions under which manifests will preload on session creation"""

    @staticmethod
    def or_conditions(
        conditions: list[ManifestPreloadCondition],
    ) -> ManifestPreloadCondition:
        """Create a preload condition that matches if any of `conditions` matches"""
    @staticmethod
    def and_conditions(
        conditions: list[ManifestPreloadCondition],
    ) -> ManifestPreloadCondition:
        """Create a preload condition that matches only if all passed `conditions` match"""
    @staticmethod
    def path_matches(regex: str) -> ManifestPreloadCondition:
        """Create a preload condition that matches if the full path to the array matches the passed regex.

        Array paths are absolute, as in `/path/to/my/array`
        """
    @staticmethod
    def name_matches(regex: str) -> ManifestPreloadCondition:
        """Create a preload condition that matches if the array's name matches the passed regex.

        Example, for an array  `/model/outputs/temperature`, the following will match:
        ```
        name_matches(".*temp.*")
        ```
        """
    @staticmethod
    def num_refs(from_refs: int | None, to_refs: int | None) -> ManifestPreloadCondition:
        """Create a preload condition that matches only if the number of chunk references in the manifest is within the given range.

        from_refs is inclusive, to_refs is exclusive.
        """
    @staticmethod
    def true() -> ManifestPreloadCondition:
        """Create a preload condition that always matches any manifest"""
    @staticmethod
    def false() -> ManifestPreloadCondition:
        """Create a preload condition that never matches any manifests"""
    def __and__(self, other: ManifestPreloadCondition, /) -> ManifestPreloadCondition:
        """Create a preload condition that matches if both this condition and `other` match."""
    def __or__(self, other: ManifestPreloadCondition, /) -> ManifestPreloadCondition:
        """Create a preload condition that matches if either this condition or `other` match."""

class ManifestPreloadConfig:
    """Configuration for how Icechunk manifest preload on session creation"""

    def __new__(
        cls,
        max_total_refs: int | None = None,
        preload_if: ManifestPreloadCondition | None = None,
        max_arrays_to_scan: int | None = None,
    ) -> Self:
        """
        Create a new `ManifestPreloadConfig` object

        Parameters
        ----------
        max_total_refs: int | None
            The maximum number of references to preload.
            Default: 10,000
        preload_if: ManifestPreloadCondition | None
            The condition under which manifests will be preloaded. When None,
            preloads arrays whose name matches CF-like coordinate names
            (e.g. ``time``, ``lat``, ``lon``, ``x``, ``y``, ``z``) and whose
            manifest has at most 1,000 chunk references. The name-matching
            regexes are lifted from cf-xarray's coordinate-axis heuristics;
            see https://cf-xarray.readthedocs.io/en/latest/generated/cf_xarray.accessor.CFAccessor.html#cf_xarray.accessor.CFAccessor.guess_coord_axis.
            Default: None
        max_arrays_to_scan: int | None
            The maximum number of arrays to scan when looking for manifests to preload.
            Increase for repositories with many nested groups.
            Default: 50
        """
    @property
    def max_total_refs(self) -> int | None:
        """
        The maximum number of references to preload.

        Default: 10,000

        Returns
        -------
        int | None
            The maximum number of references to preload.
        """
    @max_total_refs.setter
    def max_total_refs(self, value: int | None) -> None:
        """
        Set the maximum number of references to preload.

        Parameters
        ----------
        value: int | None
            The maximum number of references to preload.
        """
    @property
    def preload_if(self) -> ManifestPreloadCondition | None:
        """
        The condition under which manifests will be preloaded.

        Default: None (preload arrays with CF-like coordinate names and at most 1,000 chunk references)

        Returns
        -------
        ManifestPreloadCondition | None
            The condition under which manifests will be preloaded.
        """
    @preload_if.setter
    def preload_if(self, value: ManifestPreloadCondition | None) -> None:
        """
        Set the condition under which manifests will be preloaded.

        Parameters
        ----------
        value: ManifestPreloadCondition | None
            The condition under which manifests will be preloaded.
        """
    @property
    def max_arrays_to_scan(self) -> int | None:
        """
        The maximum number of arrays to scan when looking for manifests to preload.

        Default: 50

        Returns
        -------
        int | None
            The maximum number of arrays to scan.
        """
    @max_arrays_to_scan.setter
    def max_arrays_to_scan(self, value: int | None) -> None:
        """
        Set the maximum number of arrays to scan when looking for manifests to preload.

        Parameters
        ----------
        value: int | None
            The maximum number of arrays to scan.
        """

class ManifestSplitCondition:
    """Configuration for conditions under which manifests will be split into splits"""

    @staticmethod
    def or_conditions(
        conditions: list[ManifestSplitCondition],
    ) -> ManifestSplitCondition:
        """Create a splitting condition that matches if any of `conditions` matches"""
    @staticmethod
    def and_conditions(
        conditions: list[ManifestSplitCondition],
    ) -> ManifestSplitCondition:
        """Create a splitting condition that matches only if all passed `conditions` match"""
    @staticmethod
    def path_matches(regex: str) -> ManifestSplitCondition:
        """Create a splitting condition that matches if the full path to the array matches the passed regex.

        Array paths are absolute, as in `/path/to/my/array`
        """
    @staticmethod
    def name_matches(regex: str) -> ManifestSplitCondition:
        """Create a splitting condition that matches if the array's name matches the passed regex.

        Example, for an array  `/model/outputs/temperature`, the following will match:
        ```
        name_matches(".*temp.*")
        ```
        """

    @staticmethod
    def AnyArray() -> ManifestSplitCondition:
        """Create a splitting condition that matches any array."""

    def __or__(self, other: ManifestSplitCondition, /) -> ManifestSplitCondition:
        """Create a splitting condition that matches if either this condition or `other` matches"""

    def __and__(self, other: ManifestSplitCondition, /) -> ManifestSplitCondition:
        """Create a splitting condition that matches if both this condition and `other` match"""

class ManifestSplitDimCondition:
    """Conditions for specifying dimensions along which to shard manifests."""
    class Axis:
        """Split along specified integer axis."""
        def __new__(cls, axis: int) -> ManifestSplitDimCondition.Axis: ...

    class DimensionName:
        """Split along specified named dimension."""
        def __new__(cls, regex: str) -> ManifestSplitDimCondition.DimensionName: ...

    class Any:
        """Split along any other unspecified dimension."""
        def __new__(cls) -> ManifestSplitDimCondition.Any: ...

class ManifestSplittingConfig:
    """Configuration for manifest splitting."""

    @staticmethod
    def from_dict(
        split_sizes: dict[
            ManifestSplitCondition,
            dict[
                ManifestSplitDimCondition.Axis
                | ManifestSplitDimCondition.DimensionName
                | ManifestSplitDimCondition.Any,
                int,
            ],
        ],
    ) -> ManifestSplittingConfig: ...
    def to_dict(
        self,
    ) -> dict[
        ManifestSplitCondition,
        dict[
            ManifestSplitDimCondition.Axis
            | ManifestSplitDimCondition.DimensionName
            | ManifestSplitDimCondition.Any,
            int,
        ],
    ]: ...
    def __new__(cls, split_sizes: _SplitSizes | None = None) -> Self:
        """Configuration for how Icechunk manifests will be split.

        Parameters
        ----------
        split_sizes: tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be split.
            Default: None (a single rule matching every array with no splitting, i.e. one manifest per array)

        Examples
        --------

        Split manifests for the `temperature` array, with 3 chunks per shard along the `longitude` dimension.
        >>> ManifestSplittingConfig.from_dict(
        ...     {
        ...         ManifestSplitCondition.name_matches("temperature"): {
        ...             ManifestSplitDimCondition.DimensionName("longitude"): 3
        ...         }
        ...     }
        ... )
        """

    @property
    def split_sizes(self) -> _SplitSizes:
        """
        Configuration for how Icechunk manifests will be split.

        Default: None (a single rule matching every array with no splitting, i.e. one manifest per array)

        Returns
        -------
        tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be split.
        """

    @split_sizes.setter
    def split_sizes(self, value: _SplitSizes) -> None:
        """
        Set the sizes for how Icechunk manifests will be split.

        Parameters
        ----------
        value: tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be preloaded.
        """