icechunk.config#

class CachingConfig:
    """Configuration for how Icechunk caches its metadata files"""

    def __new__(
        cls,
        num_snapshot_nodes: int | None = None,
        num_chunk_refs: int | None = None,
        num_transaction_changes: int | None = None,
        num_bytes_attributes: int | None = None,
        num_bytes_chunks: int | None = None,
    ) -> CachingConfig:
        """
        Create a new `CachingConfig` object

        Parameters
        ----------
        num_snapshot_nodes: int | None
            The number of snapshot nodes to cache.
            Default: 500,000
        num_chunk_refs: int | None
            The number of chunk references to cache.
            Default: 15,000,000
        num_transaction_changes: int | None
            The number of transaction changes to cache.
            Default: 0
        num_bytes_attributes: int | None
            The number of bytes of attributes to cache.
            Default: 0
        num_bytes_chunks: int | None
            The number of bytes of chunks to cache.
            Default: 0
        """
    def __repr__(self, /) -> str: ...
    def __str__(self, /) -> str: ...
    def _repr_html_(self, /) -> str: ...
    @property
    def num_snapshot_nodes(self) -> int | None:
        """
        The number of snapshot nodes to cache.

        Returns
        -------
        int | None
            The number of snapshot nodes to cache.
        """
        ...
    @num_snapshot_nodes.setter
    def num_snapshot_nodes(self, value: int | None) -> None:
        """
        Set the number of snapshot nodes to cache.

        Parameters
        ----------
        value: int | None
            The number of snapshot nodes to cache.
        """
        ...
    @property
    def num_chunk_refs(self) -> int | None:
        """
        The number of chunk references to cache.

        Returns
        -------
        int | None
            The number of chunk references to cache.
        """
        ...
    @num_chunk_refs.setter
    def num_chunk_refs(self, value: int | None) -> None:
        """
        Set the number of chunk references to cache.

        Parameters
        ----------
        value: int | None
            The number of chunk references to cache.
        """
        ...
    @property
    def num_transaction_changes(self) -> int | None:
        """
        The number of transaction changes to cache.

        Returns
        -------
        int | None
            The number of transaction changes to cache.
        """
        ...
    @num_transaction_changes.setter
    def num_transaction_changes(self, value: int | None) -> None:
        """
        Set the number of transaction changes to cache.

        Parameters
        ----------
        value: int | None
            The number of transaction changes to cache.
        """
        ...
    @property
    def num_bytes_attributes(self) -> int | None:
        """
        The number of bytes of attributes to cache.

        Returns
        -------
        int | None
            The number of bytes of attributes to cache.
        """
        ...
    @num_bytes_attributes.setter
    def num_bytes_attributes(self, value: int | None) -> None:
        """
        Set the number of bytes of attributes to cache.

        Parameters
        ----------
        value: int | None
            The number of bytes of attributes to cache.
        """
        ...
    @property
    def num_bytes_chunks(self) -> int | None:
        """
        The number of bytes of chunks to cache.

        Returns
        -------
        int | None
            The number of bytes of chunks to cache.
        """
        ...
    @num_bytes_chunks.setter
    def num_bytes_chunks(self, value: int | None) -> None:
        """
        Set the number of bytes of chunks to cache.

        Parameters
        ----------
        value: int | None
            The number of bytes of chunks to cache.
        """
        ...

class CompressionAlgorithm(Enum):
    """Enum for selecting the compression algorithm used by Icechunk to write its metadata files

    Attributes
    ----------
    Zstd: int
        The Zstd compression algorithm.
    """

    Zstd = 0

    def __new__(cls) -> CompressionAlgorithm: ...
    @staticmethod
    def default() -> CompressionAlgorithm:
        """
        The default compression algorithm used by Icechunk to write its metadata files.

        Returns
        -------
        CompressionAlgorithm
            The default compression algorithm.
        """
        ...

class CompressionConfig:
    """Configuration for how Icechunk compresses its metadata files"""

    def __new__(
        cls, algorithm: CompressionAlgorithm | None = None, level: int | None = None
    ) -> CompressionConfig:
        """
        Create a new `CompressionConfig` object

        Parameters
        ----------
        algorithm: CompressionAlgorithm | None
            The compression algorithm to use.
            Default: Zstd
        level: int | None
            The compression level to use.
            Default: 3
        """
        ...
    @property
    def algorithm(self) -> CompressionAlgorithm | None:
        """
        The compression algorithm used by Icechunk to write its metadata files.

        Returns
        -------
        CompressionAlgorithm | None
            The compression algorithm used by Icechunk to write its metadata files.
        """
        ...
    @algorithm.setter
    def algorithm(self, value: CompressionAlgorithm | None) -> None:
        """
        Set the compression algorithm used by Icechunk to write its metadata files.

        Parameters
        ----------
        value: CompressionAlgorithm | None
            The compression algorithm to use.
        """
        ...
    @property
    def level(self) -> int | None:
        """
        The compression level used by Icechunk to write its metadata files.

        Returns
        -------
        int | None
            The compression level used by Icechunk to write its metadata files.
        """
        ...
    @level.setter
    def level(self, value: int | None) -> None:
        """
        Set the compression level used by Icechunk to write its metadata files.

        Parameters
        ----------
        value: int | None
            The compression level to use.
        """
        ...
    @staticmethod
    def default() -> CompressionConfig:
        """
        The default compression configuration used by Icechunk to write its metadata files.

        Returns
        -------
        CompressionConfig
        """

class ManifestConfig:
    """Configuration for how Icechunk manifests"""

    def __new__(
        cls,
        preload: ManifestPreloadConfig | None = None,
        splitting: ManifestSplittingConfig | None = None,
        virtual_chunk_location_compression: ManifestVirtualChunkLocationCompressionConfig
        | None = None,
    ) -> ManifestConfig:
        """
        Create a new `ManifestConfig` object

        Parameters
        ----------
        preload: ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded.
        splitting: ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split.
        virtual_chunk_location_compression: ManifestVirtualChunkLocationCompressionConfig | None
            The configuration for zstd compression of virtual chunk location URLs.
        """
        ...
    @property
    def preload(self) -> ManifestPreloadConfig | None:
        """
        The configuration for how Icechunk manifests will be preloaded.

        Returns
        -------
        ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded.
        """
        ...
    @preload.setter
    def preload(self, value: ManifestPreloadConfig | None) -> None:
        """
        Set the configuration for how Icechunk manifests will be preloaded.

        Parameters
        ----------
        value: ManifestPreloadConfig | None
            The configuration for how Icechunk manifests will be preloaded.
        """
        ...

    @property
    def splitting(self) -> ManifestSplittingConfig | None:
        """
        The configuration for how Icechunk manifests will be split.

        Returns
        -------
        ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split.
        """
        ...

    @splitting.setter
    def splitting(self, value: ManifestSplittingConfig | None) -> None:
        """
        Set the configuration for how Icechunk manifests will be split.

        Parameters
        ----------
        value: ManifestSplittingConfig | None
            The configuration for how Icechunk manifests will be split.
        """
        ...

    @property
    def virtual_chunk_location_compression(
        self,
    ) -> ManifestVirtualChunkLocationCompressionConfig | None:
        """
        The configuration for zstd compression of virtual chunk location URLs.

        Returns
        -------
        ManifestVirtualChunkLocationCompressionConfig | None
            The compression configuration.
        """
        ...

    @virtual_chunk_location_compression.setter
    def virtual_chunk_location_compression(
        self, value: ManifestVirtualChunkLocationCompressionConfig | None
    ) -> None:
        """
        Set the configuration for zstd compression of virtual chunk location URLs.

        Parameters
        ----------
        value: ManifestVirtualChunkLocationCompressionConfig | None
            The compression configuration.
        """
        ...

class ManifestPreloadCondition:
    """Configuration for conditions under which manifests will preload on session creation"""

    @staticmethod
    def or_conditions(
        conditions: list[ManifestPreloadCondition],
    ) -> ManifestPreloadCondition:
        """Create a preload condition that matches if any of `conditions` matches"""
        ...
    @staticmethod
    def and_conditions(
        conditions: list[ManifestPreloadCondition],
    ) -> ManifestPreloadCondition:
        """Create a preload condition that matches only if all passed `conditions` match"""
        ...
    @staticmethod
    def path_matches(regex: str) -> ManifestPreloadCondition:
        """Create a preload condition that matches if the full path to the array matches the passed regex.

        Array paths are absolute, as in `/path/to/my/array`
        """
        ...
    @staticmethod
    def name_matches(regex: str) -> ManifestPreloadCondition:
        """Create a preload condition that matches if the array's name matches the passed regex.

        Example, for an array  `/model/outputs/temperature`, the following will match:
        ```
        name_matches(".*temp.*")
        ```
        """
        ...
    @staticmethod
    def num_refs(from_refs: int | None, to_refs: int | None) -> ManifestPreloadCondition:
        """Create a preload condition that matches only if the number of chunk references in the manifest is within the given range.

        from_refs is inclusive, to_refs is exclusive.
        """
        ...
    @staticmethod
    def true() -> ManifestPreloadCondition:
        """Create a preload condition that always matches any manifest"""
        ...
    @staticmethod
    def false() -> ManifestPreloadCondition:
        """Create a preload condition that never matches any manifests"""
        ...
    def __and__(self, other: ManifestPreloadCondition, /) -> ManifestPreloadCondition:
        """Create a preload condition that matches if both this condition and `other` match."""
        ...
    def __or__(self, other: ManifestPreloadCondition, /) -> ManifestPreloadCondition:
        """Create a preload condition that matches if either this condition or `other` match."""
        ...

class ManifestPreloadConfig:
    """Configuration for how Icechunk manifest preload on session creation"""

    def __new__(
        cls,
        max_total_refs: int | None = None,
        preload_if: ManifestPreloadCondition | None = None,
        max_arrays_to_scan: int | None = None,
    ) -> ManifestPreloadConfig:
        """
        Create a new `ManifestPreloadConfig` object

        Parameters
        ----------
        max_total_refs: int | None
            The maximum number of references to preload.
        preload_if: ManifestPreloadCondition | None
            The condition under which manifests will be preloaded.
        max_arrays_to_scan: int | None
            The maximum number of arrays to scan when looking for manifests to preload.
            Default is 50. Increase for repositories with many nested groups.
        """
        ...
    @property
    def max_total_refs(self) -> int | None:
        """
        The maximum number of references to preload.

        Returns
        -------
        int | None
            The maximum number of references to preload.
        """
        ...
    @max_total_refs.setter
    def max_total_refs(self, value: int | None) -> None:
        """
        Set the maximum number of references to preload.

        Parameters
        ----------
        value: int | None
            The maximum number of references to preload.
        """
        ...
    @property
    def preload_if(self) -> ManifestPreloadCondition | None:
        """
        The condition under which manifests will be preloaded.

        Returns
        -------
        ManifestPreloadCondition | None
            The condition under which manifests will be preloaded.
        """
        ...
    @preload_if.setter
    def preload_if(self, value: ManifestPreloadCondition | None) -> None:
        """
        Set the condition under which manifests will be preloaded.

        Parameters
        ----------
        value: ManifestPreloadCondition | None
            The condition under which manifests will be preloaded.
        """
        ...
    @property
    def max_arrays_to_scan(self) -> int | None:
        """
        The maximum number of arrays to scan when looking for manifests to preload.

        Returns
        -------
        int | None
            The maximum number of arrays to scan. Default is 50.
        """
        ...
    @max_arrays_to_scan.setter
    def max_arrays_to_scan(self, value: int | None) -> None:
        """
        Set the maximum number of arrays to scan when looking for manifests to preload.

        Parameters
        ----------
        value: int | None
            The maximum number of arrays to scan.
        """
        ...

class ManifestSplitCondition:
    """Configuration for conditions under which manifests will be split into splits"""

    @staticmethod
    def or_conditions(
        conditions: list[ManifestSplitCondition],
    ) -> ManifestSplitCondition:
        """Create a splitting condition that matches if any of `conditions` matches"""
        ...
    @staticmethod
    def and_conditions(
        conditions: list[ManifestSplitCondition],
    ) -> ManifestSplitCondition:
        """Create a splitting condition that matches only if all passed `conditions` match"""
        ...
    @staticmethod
    def path_matches(regex: str) -> ManifestSplitCondition:
        """Create a splitting condition that matches if the full path to the array matches the passed regex.

        Array paths are absolute, as in `/path/to/my/array`
        """
        ...
    @staticmethod
    def name_matches(regex: str) -> ManifestSplitCondition:
        """Create a splitting condition that matches if the array's name matches the passed regex.

        Example, for an array  `/model/outputs/temperature`, the following will match:
        ```
        name_matches(".*temp.*")
        ```
        """
        ...

    @staticmethod
    def AnyArray() -> ManifestSplitCondition:
        """Create a splitting condition that matches any array."""
        ...

    def __or__(self, other: ManifestSplitCondition, /) -> ManifestSplitCondition:
        """Create a splitting condition that matches if either this condition or `other` matches"""
        ...

    def __and__(self, other: ManifestSplitCondition, /) -> ManifestSplitCondition:
        """Create a splitting condition that matches if both this condition and `other` match"""
        ...

class ManifestSplitDimCondition:
    """Conditions for specifying dimensions along which to shard manifests."""
    class Axis:
        """Split along specified integer axis."""
        def __new__(cls, axis: int) -> ManifestSplitDimCondition.Axis: ...

    class DimensionName:
        """Split along specified named dimension."""
        def __new__(cls, regex: str) -> ManifestSplitDimCondition.DimensionName: ...

    class Any:
        """Split along any other unspecified dimension."""
        def __new__(cls) -> ManifestSplitDimCondition.Any: ...

class ManifestSplittingConfig:
    """Configuration for manifest splitting."""

    @staticmethod
    def from_dict(
        split_sizes: dict[
            ManifestSplitCondition,
            dict[
                ManifestSplitDimCondition.Axis
                | ManifestSplitDimCondition.DimensionName
                | ManifestSplitDimCondition.Any,
                int,
            ],
        ],
    ) -> ManifestSplittingConfig: ...
    def to_dict(
        self,
    ) -> dict[
        ManifestSplitCondition,
        dict[
            ManifestSplitDimCondition.Axis
            | ManifestSplitDimCondition.DimensionName
            | ManifestSplitDimCondition.Any,
            int,
        ],
    ]: ...
    def __new__(cls, split_sizes: _SplitSizes | None = None) -> ManifestSplittingConfig:
        """Configuration for how Icechunk manifests will be split.

        Parameters
        ----------
        split_sizes: tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be preloaded.

        Examples
        --------

        Split manifests for the `temperature` array, with 3 chunks per shard along the `longitude` dimension.
        >>> ManifestSplittingConfig.from_dict(
        ...     {
        ...         ManifestSplitCondition.name_matches("temperature"): {
        ...             ManifestSplitDimCondition.DimensionName("longitude"): 3
        ...         }
        ...     }
        ... )
        """
        pass

    @property
    def split_sizes(self) -> _SplitSizes:
        """
        Configuration for how Icechunk manifests will be split.

        Returns
        -------
        tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be preloaded.
        """
        ...

    @split_sizes.setter
    def split_sizes(self, value: _SplitSizes) -> None:
        """
        Set the sizes for how Icechunk manifests will be split.

        Parameters
        ----------
        value: tuple[tuple[ManifestSplitCondition, tuple[tuple[ManifestSplitDimCondition, int], ...]], ...]
            The configuration for how Icechunk manifests will be preloaded.
        """
        ...