icechunk.storage#

class ChunkType(Enum):
    """Enum for Zarr chunk types

    Attributes
    ----------
    Uninitialized: int
        Chunk doesn't have a materialized type yet
    Native: int
        Regular Zarr chunks
    Virtual: int
        Chunk conforming to the VirtualiZarr spec
    Inline: int
        Chunk is store inline in the manifest
    """

    uninitialized = 0
    native = 1
    virtual = 2
    inline = 3

class S3Options:
    """Options for accessing an S3-compatible storage backend"""
    def __new__(
        cls,
        region: str | None = None,
        endpoint_url: str | None = None,
        allow_http: bool = False,
        anonymous: bool = False,
        force_path_style: bool = False,
        network_stream_timeout_seconds: int | None = None,
        requester_pays: bool = False,
    ) -> S3Options:
        """
        Create a new `S3Options` object

        Parameters
        ----------
        region: str | None
            Optional, the region to use for the storage backend.
        endpoint_url: str | None
            Optional, the endpoint URL to use for the storage backend.
        allow_http: bool
            Whether to allow HTTP requests to the storage backend.
        anonymous: bool
            Whether to use anonymous credentials to the storage backend. When `True`, the s3 requests will not be signed.
        force_path_style: bool
            Whether to force use of path-style addressing for buckets.
        network_stream_timeout_seconds: int | None
            Timeout requests if no bytes can be transmitted during this period of time.
            If set to 0, timeout is disabled. Default: 60.
        requester_pays: bool
            Enable requester pays for S3 buckets
        """

    @property
    def region(self) -> str | None:
        """
        Optional region to use for the storage backend.

        Returns
        -------
        str | None
            The region configured for the storage backend.
        """
        ...

    @region.setter
    def region(self, value: str | None) -> None:
        """
        Set the region to use for the storage backend.

        Parameters
        ----------
        value: str | None
            The region to use for the storage backend.
        """
        ...

    @property
    def endpoint_url(self) -> str | None:
        """
        Optional endpoint URL for the storage backend.

        Returns
        -------
        str | None
            The endpoint URL configured for the storage backend.
        """
        ...

    @endpoint_url.setter
    def endpoint_url(self, value: str | None) -> None:
        """
        Set the endpoint URL for the storage backend.

        Parameters
        ----------
        value: str | None
            The endpoint URL to use for the storage backend.
        """
        ...

    @property
    def allow_http(self) -> bool:
        """
        Whether HTTP requests are allowed for the storage backend.

        Returns
        -------
        bool
            ``True`` when HTTP requests to the storage backend are permitted.
        """
        ...

    @allow_http.setter
    def allow_http(self, value: bool) -> None:
        """
        Set whether HTTP requests are allowed for the storage backend.

        Parameters
        ----------
        value: bool
            ``True`` to allow HTTP requests to the storage backend, ``False`` otherwise.
        """
        ...

    @property
    def anonymous(self) -> bool:
        """
        Whether to use anonymous credentials (unsigned requests).

        Returns
        -------
        bool
            ``True`` when anonymous access is configured.
        """
        ...

    @anonymous.setter
    def anonymous(self, value: bool) -> None:
        """
        Set whether to use anonymous credentials.

        Parameters
        ----------
        value: bool
            ``True`` to perform unsigned requests, ``False`` to sign requests.
        """
        ...

    @property
    def force_path_style(self) -> bool:
        """
        Whether to force path-style bucket addressing.

        Returns
        -------
        bool
            ``True`` when path-style addressing is forced.
        """
        ...

    @force_path_style.setter
    def force_path_style(self, value: bool) -> None:
        """
        Set whether to force path-style bucket addressing.

        Parameters
        ----------
        value: bool
            ``True`` to always use path-style addressing, ``False`` to allow virtual-host style.
        """
        ...

    @property
    def network_stream_timeout_seconds(self) -> int | None:
        """
        Timeout in seconds for idle network streams.

        Returns
        -------
        int | None
            The timeout duration; ``0`` disables the timeout and ``None`` uses the default.
        """
        ...

    @network_stream_timeout_seconds.setter
    def network_stream_timeout_seconds(self, value: int | None) -> None:
        """
        Set the timeout for idle network streams.

        Parameters
        ----------
        value: int | None
            Timeout duration in seconds. Use ``0`` to disable or ``None`` for the default.
        """
        ...

class Storage:
    """Storage configuration for an IcechunkStore

    Currently supports memory, filesystem S3, azure blob, and google cloud storage backends.
    Use the following methods to create a Storage object with the desired backend.

    Ex:
    ```
    storage = icechunk.in_memory_storage()
    storage = icechunk.local_filesystem_storage("/path/to/root")
    storage = icechunk.s3_storage("bucket", "prefix", ...)
    storage = icechunk.gcs_storage("bucket", "prefix", ...)
    storage = icechunk.azure_storage("container", "prefix", ...)
    ```
    """

    @classmethod
    def new_s3(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_s3_object_store(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_tigris(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        use_weak_consistency: bool,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_in_memory(cls) -> Storage: ...
    @classmethod
    def new_local_filesystem(cls, path: str) -> Storage: ...
    @classmethod
    def new_gcs(
        cls,
        bucket: str,
        prefix: str | None,
        credentials: _AnyGcsCredential | None = None,
        *,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_r2(
        cls,
        config: S3Options,
        bucket: str | None = None,
        prefix: str | None = None,
        account_id: str | None = None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_azure_blob(
        cls,
        account: str,
        container: str,
        prefix: str,
        credentials: _AnyAzureCredential | None = None,
        *,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_http(
        cls,
        base_url: str,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_redirect(
        cls,
        base_url: str,
    ) -> Storage: ...
    def __repr__(self) -> str: ...
    def __str__(self) -> str: ...
    def _repr_html_(self) -> str: ...
    def default_settings(self) -> StorageSettings: ...
    def list_objects(
        self, settings: StorageSettings | None = None, prefix: str | None = None
    ) -> list[tuple[str, int]]:
        """List objects in the storage backend, optionally filtered by a key prefix.

        Deprecated: use ``list_objects_metadata`` instead, which also returns
        the ``created_at`` timestamp.

        Parameters
        ----------
        settings : StorageSettings | None
            Optional storage settings to override the defaults (retries, concurrency, etc.).
        prefix : str | None
            If provided, only objects whose keys start with this prefix are returned.
            When ``None`` or empty, all objects under the repository root are listed.

        Returns
        -------
        list[tuple[str, int]]
            A list of ``(key, size_in_bytes)`` tuples for each object found.
        """
        ...
    def list_objects_metadata(
        self, settings: StorageSettings | None = None, prefix: str | None = None
    ) -> list[StorageObjectInfo]:
        """List objects with full metadata, optionally filtered by a key prefix.

        Parameters
        ----------
        settings : StorageSettings | None
            Optional storage settings to override the defaults.
        prefix : str | None
            If provided, only objects whose keys start with this prefix are returned.

        Returns
        -------
        list[StorageObjectInfo]
            A list of :class:`StorageObjectInfo` objects.
        """
        ...

class StorageConcurrencySettings:
    """Configuration for how Icechunk uses its Storage instance"""

    def __new__(
        cls,
        max_concurrent_requests_for_object: int | None = None,
        ideal_concurrent_request_size: int | None = None,
    ) -> StorageConcurrencySettings:
        """
        Create a new `StorageConcurrencySettings` object

        Parameters
        ----------
        max_concurrent_requests_for_object: int | None
            The maximum number of concurrent requests for an object.
            Default: 18
        ideal_concurrent_request_size: int | None
            The ideal concurrent request size in bytes.
            Default: 12,582,912 (12 MB)
        """
        ...
    @property
    def max_concurrent_requests_for_object(self) -> int | None:
        """
        The maximum number of concurrent requests for an object.

        Returns
        -------
        int | None
            The maximum number of concurrent requests for an object.
        """
        ...
    @max_concurrent_requests_for_object.setter
    def max_concurrent_requests_for_object(self, value: int | None) -> None:
        """
        Set the maximum number of concurrent requests for an object.

        Parameters
        ----------
        value: int | None
            The maximum number of concurrent requests for an object.
        """
        ...
    @property
    def ideal_concurrent_request_size(self) -> int | None:
        """
        The ideal concurrent request size.

        Returns
        -------
        int | None
            The ideal concurrent request size.
        """
        ...
    @ideal_concurrent_request_size.setter
    def ideal_concurrent_request_size(self, value: int | None) -> None:
        """
        Set the ideal concurrent request size.

        Parameters
        ----------
        value: int | None
            The ideal concurrent request size.
        """
        ...

class StorageRetriesSettings:
    """Configuration for how Icechunk retries requests.

    Icechunk retries failed requests with an exponential backoff algorithm."""

    def __new__(
        cls,
        max_tries: int | None = None,
        initial_backoff_ms: int | None = None,
        max_backoff_ms: int | None = None,
    ) -> StorageRetriesSettings:
        """
        Create a new `StorageRetriesSettings` object

        Parameters
        ----------
        max_tries: int | None
            The maximum number of tries, including the initial one. Set to 1 to disable retries.
            Default: 10
        initial_backoff_ms: int | None
            The initial backoff duration in milliseconds.
            Default: 100
        max_backoff_ms: int | None
            The limit to backoff duration in milliseconds.
            Default: 180,000 (3 minutes)
        """
        ...
    @property
    def max_tries(self) -> int | None:
        """
        The maximum number of tries, including the initial one.

        Returns
        -------
        int | None
            The maximum number of tries.
        """
        ...
    @max_tries.setter
    def max_tries(self, value: int | None) -> None:
        """
        Set the maximum number of tries. Set to 1 to disable retries.

        Parameters
        ----------
        value: int | None
            The maximum number of tries
        """
        ...
    @property
    def initial_backoff_ms(self) -> int | None:
        """
        The initial backoff duration in milliseconds.

        Returns
        -------
        int | None
            The initial backoff duration in milliseconds.
        """
        ...
    @initial_backoff_ms.setter
    def initial_backoff_ms(self, value: int | None) -> None:
        """
        Set the initial backoff duration in milliseconds.

        Parameters
        ----------
        value: int | None
            The initial backoff duration in milliseconds.
        """
        ...
    @property
    def max_backoff_ms(self) -> int | None:
        """
        The maximum backoff duration in milliseconds.

        Returns
        -------
        int | None
            The maximum backoff duration in milliseconds.
        """
        ...
    @max_backoff_ms.setter
    def max_backoff_ms(self, value: int | None) -> None:
        """
        Set the maximum backoff duration in milliseconds.

        Parameters
        ----------
        value: int | None
            The maximum backoff duration in milliseconds.
        """
        ...

class StorageSettings:
    """Configuration for how Icechunk uses its Storage instance"""

    def __new__(
        cls,
        concurrency: StorageConcurrencySettings | None = None,
        retries: StorageRetriesSettings | None = None,
        unsafe_use_conditional_create: bool | None = None,
        unsafe_use_conditional_update: bool | None = None,
        unsafe_use_metadata: bool | None = None,
        storage_class: str | None = None,
        metadata_storage_class: str | None = None,
        chunks_storage_class: str | None = None,
        minimum_size_for_multipart_upload: int | None = None,
        timeouts: StorageTimeoutSettings | None = None,
    ) -> StorageSettings:
        """
        Create a new `StorageSettings` object

        Parameters
        ----------
        concurrency: StorageConcurrencySettings | None
            The configuration for how Icechunk uses its Storage instance.

        retries: StorageRetriesSettings | None
            The configuration for how Icechunk retries failed requests.

        unsafe_use_conditional_update: bool | None
            If set to False, Icechunk loses some of its consistency guarantees.
            This is only useful in object stores that don't support the feature.
            Use it at your own risk.
            Default: True

        unsafe_use_conditional_create: bool | None
            If set to False, Icechunk loses some of its consistency guarantees.
            This is only useful in object stores that don't support the feature.
            Use at your own risk.
            Default: True

        unsafe_use_metadata: bool | None
            Don't write metadata fields in Icechunk files.
            This is only useful in object stores that don't support the feature.
            Use at your own risk.
            Default: True

        storage_class: str | None
            Store all objects using this object store storage class
            If None the object store default will be used.
            Currently not supported in GCS.
            Example: STANDARD_IA

        metadata_storage_class: str | None
            Store metadata objects using this object store storage class.
            Currently not supported in GCS.
            Defaults to storage_class.

        chunks_storage_class: str | None
            Store chunk objects using this object store storage class.
            Currently not supported in GCS.
            Defaults to storage_class.

        minimum_size_for_multipart_upload: int | None
            Use object store's multipart upload for objects larger than this size in bytes.
            Default: 100 MB if None is passed.

        timeouts: StorageTimeoutSettings | None
            The configuration for AWS SDK timeout settings.
        """
        ...
    def __repr__(self, /) -> str: ...
    def __str__(self, /) -> str: ...
    def _repr_html_(self, /) -> str: ...
    @property
    def concurrency(self) -> StorageConcurrencySettings | None:
        """
        The configuration for how much concurrency Icechunk store uses

        Returns
        -------
        StorageConcurrencySettings | None
            The configuration for how Icechunk uses its Storage instance.
        """

    @concurrency.setter
    def concurrency(self, value: StorageConcurrencySettings | None) -> None: ...
    @property
    def retries(self) -> StorageRetriesSettings | None:
        """
        The configuration for how Icechunk retries failed requests.

        Returns
        -------
        StorageRetriesSettings | None
            The configuration for how Icechunk retries failed requests.
        """

    @retries.setter
    def retries(self, value: StorageRetriesSettings | None) -> None: ...
    @property
    def timeouts(self) -> StorageTimeoutSettings | None:
        """
        The configuration for AWS SDK timeout settings.

        Returns
        -------
        StorageTimeoutSettings | None
            The timeout configuration.
        """

    @timeouts.setter
    def timeouts(self, value: StorageTimeoutSettings | None) -> None: ...
    @property
    def unsafe_use_conditional_update(self) -> bool | None:
        """True if Icechunk will use conditional PUT operations for updates in the object store"""
        ...

    @unsafe_use_conditional_update.setter
    def unsafe_use_conditional_update(self, value: bool) -> None: ...
    @property
    def unsafe_use_conditional_create(self) -> bool | None:
        """True if Icechunk will use conditional PUT operations for creation in the object store"""
        ...

    @unsafe_use_conditional_create.setter
    def unsafe_use_conditional_create(self, value: bool) -> None: ...
    @property
    def unsafe_use_metadata(self) -> bool | None:
        """True if Icechunk will write object metadata in the object store"""
        ...

    @unsafe_use_metadata.setter
    def unsafe_use_metadata(self, value: bool) -> None: ...
    @property
    def storage_class(self) -> str | None:
        """All objects in object store will use this storage class or the default if None"""
        ...

    @storage_class.setter
    def storage_class(self, value: str) -> None: ...
    @property
    def metadata_storage_class(self) -> str | None:
        """Metadata objects in object store will use this storage class or self.storage_class if None"""
        ...

    @metadata_storage_class.setter
    def metadata_storage_class(self, value: str) -> None: ...
    @property
    def chunks_storage_class(self) -> str | None:
        """Chunk objects in object store will use this storage class or self.storage_class if None"""
        ...

    @chunks_storage_class.setter
    def chunks_storage_class(self, value: str) -> None: ...
    @property
    def minimum_size_for_multipart_upload(self) -> int | None:
        """Use object store's multipart upload for objects larger than this size in bytes"""
        ...

    @minimum_size_for_multipart_upload.setter
    def minimum_size_for_multipart_upload(self, value: int) -> None: ...

@final
class StorageTimeoutSettings:
    """Configuration for AWS SDK timeout settings.

    Controls connect, read, and operation timeouts for the underlying S3 client."""

    def __new__(
        cls,
        connect_timeout_ms: int | None = None,
        read_timeout_ms: int | None = None,
        operation_timeout_ms: int | None = None,
        operation_attempt_timeout_ms: int | None = None,
    ) -> StorageTimeoutSettings:
        """
        Create a new `StorageTimeoutSettings` object

        All timeouts default to None, meaning the underlying
        `AWS SDK default <https://docs.aws.amazon.com/sdk-for-rust/latest/dg/timeouts.html>`_
        is used.

        Parameters
        ----------
        connect_timeout_ms: int | None
            The timeout for establishing a connection in milliseconds.
        read_timeout_ms: int | None
            The timeout for reading a response in milliseconds.
        operation_timeout_ms: int | None
            The timeout for the entire operation (including retries) in milliseconds.
        operation_attempt_timeout_ms: int | None
            The timeout for a single attempt of an operation in milliseconds.
        """
        ...
    @property
    def connect_timeout_ms(self) -> int | None: ...
    @connect_timeout_ms.setter
    def connect_timeout_ms(self, value: int | None) -> None: ...
    @property
    def read_timeout_ms(self) -> int | None: ...
    @read_timeout_ms.setter
    def read_timeout_ms(self, value: int | None) -> None: ...
    @property
    def operation_timeout_ms(self) -> int | None: ...
    @operation_timeout_ms.setter
    def operation_timeout_ms(self, value: int | None) -> None: ...
    @property
    def operation_attempt_timeout_ms(self) -> int | None: ...
    @operation_attempt_timeout_ms.setter
    def operation_attempt_timeout_ms(self, value: int | None) -> None: ...