icechunk.storage#

@final
class ChecksumAlgorithm(Enum):
    """Checksum algorithm to use on S3 write requests.

    When :attr:`S3Options.checksum_algorithm` is unset, the AWS SDK picks its
    own default for the ``x-amz-checksum-*`` header on ``PutObject``,
    ``UploadPart``, and ``DeleteObjects``. Some S3-compatible services only
    accept a subset of algorithms; set this option to override the SDK's
    choice.

    Attributes
    ----------
    Crc32: int
    Crc32c: int
    Crc64Nvme: int
    Sha1: int
    Sha256: int
    """

    Crc32 = 0
    Crc32c = 1
    Crc64Nvme = 2
    Sha1 = 3
    Sha256 = 4

class ChunkType(Enum):
    """Enum for Zarr chunk types

    Attributes
    ----------
    Uninitialized: int
        Chunk doesn't have a materialized type yet
    Native: int
        Regular Zarr chunks
    Virtual: int
        Chunk conforming to the VirtualiZarr spec
    Inline: int
        Chunk is store inline in the manifest
    """

    uninitialized = 0
    native = 1
    virtual = 2
    inline = 3

class S3Options:
    """Options for accessing an S3-compatible storage backend"""
    def __new__(
        cls,
        region: str | None = None,
        endpoint_url: str | None = None,
        allow_http: bool = False,
        anonymous: bool = False,
        force_path_style: bool = False,
        network_stream_timeout_seconds: int | None = None,
        requester_pays: bool = False,
        checksum_algorithm: ChecksumAlgorithm | None = None,
    ) -> S3Options:
        """
        Create a new `S3Options` object

        Parameters
        ----------
        region: str | None
            Optional, the region to use for the storage backend.
            Default: None
        endpoint_url: str | None
            Optional, the endpoint URL to use for the storage backend.
            Default: None
        allow_http: bool
            Whether to allow HTTP requests to the storage backend.
            Default: False
        anonymous: bool
            Whether to use anonymous credentials to the storage backend. When `True`, the s3 requests will not be signed.
            Default: False
        force_path_style: bool
            Whether to force use of path-style addressing for buckets.
            Default: False
        network_stream_timeout_seconds: int | None
            Timeout requests if no bytes can be transmitted during this period of time.
            If set to 0, timeout is disabled. When None, the AWS SDK default applies.
            Default: None
        requester_pays: bool
            Enable requester pays for S3 buckets.
            Default: False
        checksum_algorithm: ChecksumAlgorithm | None
            Override the checksum algorithm used for write requests. When None,
            the AWS SDK picks its own default. Set explicitly when targeting an
            S3-compatible service that rejects the SDK's default.
            Default: None
        """

    @property
    def region(self) -> str | None:
        """
        Optional region to use for the storage backend.

        Default: None

        Returns
        -------
        str | None
            The region configured for the storage backend.
        """
        ...

    @region.setter
    def region(self, value: str | None) -> None:
        """
        Set the region to use for the storage backend.

        Parameters
        ----------
        value: str | None
            The region to use for the storage backend.
        """
        ...

    @property
    def endpoint_url(self) -> str | None:
        """
        Optional endpoint URL for the storage backend.

        Default: None

        Returns
        -------
        str | None
            The endpoint URL configured for the storage backend.
        """
        ...

    @endpoint_url.setter
    def endpoint_url(self, value: str | None) -> None:
        """
        Set the endpoint URL for the storage backend.

        Parameters
        ----------
        value: str | None
            The endpoint URL to use for the storage backend.
        """
        ...

    @property
    def allow_http(self) -> bool:
        """
        Whether HTTP requests are allowed for the storage backend.

        Default: False

        Returns
        -------
        bool
            ``True`` when HTTP requests to the storage backend are permitted.
        """
        ...

    @allow_http.setter
    def allow_http(self, value: bool) -> None:
        """
        Set whether HTTP requests are allowed for the storage backend.

        Parameters
        ----------
        value: bool
            ``True`` to allow HTTP requests to the storage backend, ``False`` otherwise.
        """
        ...

    @property
    def anonymous(self) -> bool:
        """
        Whether to use anonymous credentials (unsigned requests).

        Default: False

        Returns
        -------
        bool
            ``True`` when anonymous access is configured.
        """
        ...

    @anonymous.setter
    def anonymous(self, value: bool) -> None:
        """
        Set whether to use anonymous credentials.

        Parameters
        ----------
        value: bool
            ``True`` to perform unsigned requests, ``False`` to sign requests.
        """
        ...

    @property
    def force_path_style(self) -> bool:
        """
        Whether to force path-style bucket addressing.

        Default: False

        Returns
        -------
        bool
            ``True`` when path-style addressing is forced.
        """
        ...

    @force_path_style.setter
    def force_path_style(self, value: bool) -> None:
        """
        Set whether to force path-style bucket addressing.

        Parameters
        ----------
        value: bool
            ``True`` to always use path-style addressing, ``False`` to allow virtual-host style.
        """
        ...

    @property
    def network_stream_timeout_seconds(self) -> int | None:
        """
        Timeout in seconds for idle network streams.

        Default: None (AWS SDK default)

        Returns
        -------
        int | None
            The timeout duration; ``0`` disables the timeout and ``None`` uses the default.
        """
        ...

    @network_stream_timeout_seconds.setter
    def network_stream_timeout_seconds(self, value: int | None) -> None:
        """
        Set the timeout for idle network streams.

        Parameters
        ----------
        value: int | None
            Timeout duration in seconds. Use ``0`` to disable or ``None`` for the default.
        """
        ...

class Storage:
    """Storage configuration for an IcechunkStore

    Currently supports memory, filesystem S3, azure blob, and google cloud storage backends.
    Use the following methods to create a Storage object with the desired backend.

    Ex:
    ```
    storage = icechunk.in_memory_storage()
    storage = icechunk.local_filesystem_storage("/path/to/root")
    storage = icechunk.s3_storage("bucket", "prefix", ...)
    storage = icechunk.gcs_storage("bucket", "prefix", ...)
    storage = icechunk.azure_storage("container", "prefix", ...)
    ```
    """

    @classmethod
    def new_s3(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_s3_object_store(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_tigris(
        cls,
        config: S3Options,
        bucket: str,
        prefix: str | None,
        use_weak_consistency: bool,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_in_memory(cls) -> Storage: ...
    @classmethod
    def new_local_filesystem(cls, path: str) -> Storage: ...
    @classmethod
    def new_gcs(
        cls,
        bucket: str,
        prefix: str | None,
        credentials: _AnyGcsCredential | None = None,
        *,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_r2(
        cls,
        config: S3Options,
        bucket: str | None = None,
        prefix: str | None = None,
        account_id: str | None = None,
        credentials: _AnyS3Credential | None = None,
    ) -> Storage: ...
    @classmethod
    def new_azure_blob(
        cls,
        account: str,
        container: str,
        prefix: str,
        credentials: _AnyAzureCredential | None = None,
        *,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_http(
        cls,
        base_url: str,
        config: dict[str, str] | None = None,
    ) -> Storage: ...
    @classmethod
    def new_redirect(
        cls,
        base_url: str,
    ) -> Storage: ...
    def __repr__(self) -> str: ...
    def __str__(self) -> str: ...
    def _repr_html_(self) -> str: ...
    def default_settings(self) -> StorageSettings: ...
    def list_objects(
        self, settings: StorageSettings | None = None, prefix: str | None = None
    ) -> list[tuple[str, int]]:
        """List objects in the storage backend, optionally filtered by a key prefix.

        Deprecated: use ``list_objects_metadata`` instead, which also returns
        the ``created_at`` timestamp.

        Parameters
        ----------
        settings : StorageSettings | None
            Optional storage settings to override the defaults (retries, concurrency, etc.).
        prefix : str | None
            If provided, only objects whose keys start with this prefix are returned.
            When ``None`` or empty, all objects under the repository root are listed.

        Returns
        -------
        list[tuple[str, int]]
            A list of ``(key, size_in_bytes)`` tuples for each object found.
        """
        ...
    def list_objects_metadata(
        self, settings: StorageSettings | None = None, prefix: str | None = None
    ) -> list[StorageObjectInfo]:
        """List objects with full metadata, optionally filtered by a key prefix.

        Parameters
        ----------
        settings : StorageSettings | None
            Optional storage settings to override the defaults.
        prefix : str | None
            If provided, only objects whose keys start with this prefix are returned.

        Returns
        -------
        list[StorageObjectInfo]
            A list of :class:`StorageObjectInfo` objects.
        """
        ...

class StorageConcurrencySettings:
    """Configuration for how Icechunk uses its Storage instance"""

    def __new__(
        cls,
        max_concurrent_requests_for_object: int | None = None,
        ideal_concurrent_request_size: int | None = None,
    ) -> StorageConcurrencySettings:
        """
        Create a new `StorageConcurrencySettings` object

        Parameters
        ----------
        max_concurrent_requests_for_object: int | None
            The maximum number of concurrent requests for an object.
            Default: 18
        ideal_concurrent_request_size: int | None
            The ideal concurrent request size in bytes.
            Default: 12,582,912 (12 MB)
        """
        ...
    @property
    def max_concurrent_requests_for_object(self) -> int | None:
        """
        The maximum number of concurrent requests for an object.

        Default: 18

        Returns
        -------
        int | None
            The maximum number of concurrent requests for an object.
        """
        ...
    @max_concurrent_requests_for_object.setter
    def max_concurrent_requests_for_object(self, value: int | None) -> None:
        """
        Set the maximum number of concurrent requests for an object.

        Parameters
        ----------
        value: int | None
            The maximum number of concurrent requests for an object.
        """
        ...
    @property
    def ideal_concurrent_request_size(self) -> int | None:
        """
        The ideal concurrent request size in bytes.

        Default: 12,582,912 (12 MB)

        Returns
        -------
        int | None
            The ideal concurrent request size in bytes.
        """
        ...
    @ideal_concurrent_request_size.setter
    def ideal_concurrent_request_size(self, value: int | None) -> None:
        """
        Set the ideal concurrent request size.

        Parameters
        ----------
        value: int | None
            The ideal concurrent request size.
        """
        ...

class StorageRetriesSettings:
    """Configuration for how Icechunk retries requests.

    Icechunk retries failed requests with an exponential backoff algorithm."""

    def __new__(
        cls,
        max_tries: int | None = None,
        initial_backoff_ms: int | None = None,
        max_backoff_ms: int | None = None,
    ) -> StorageRetriesSettings:
        """
        Create a new `StorageRetriesSettings` object

        Parameters
        ----------
        max_tries: int | None
            The maximum number of tries, including the initial one. Set to 1 to disable retries.
            Default: 10
        initial_backoff_ms: int | None
            The initial backoff duration in milliseconds.
            Default: 100
        max_backoff_ms: int | None
            The limit to backoff duration in milliseconds.
            Default: 180,000 (3 minutes)
        """
        ...
    @property
    def max_tries(self) -> int | None:
        """
        The maximum number of tries, including the initial one.

        Default: 10

        Returns
        -------
        int | None
            The maximum number of tries.
        """
        ...
    @max_tries.setter
    def max_tries(self, value: int | None) -> None:
        """
        Set the maximum number of tries. Set to 1 to disable retries.

        Parameters
        ----------
        value: int | None
            The maximum number of tries
        """
        ...
    @property
    def initial_backoff_ms(self) -> int | None:
        """
        The initial backoff duration in milliseconds.

        Default: 100

        Returns
        -------
        int | None
            The initial backoff duration in milliseconds.
        """
        ...
    @initial_backoff_ms.setter
    def initial_backoff_ms(self, value: int | None) -> None:
        """
        Set the initial backoff duration in milliseconds.

        Parameters
        ----------
        value: int | None
            The initial backoff duration in milliseconds.
        """
        ...
    @property
    def max_backoff_ms(self) -> int | None:
        """
        The maximum backoff duration in milliseconds.

        Default: 180,000 (3 minutes)

        Returns
        -------
        int | None
            The maximum backoff duration in milliseconds.
        """
        ...
    @max_backoff_ms.setter
    def max_backoff_ms(self, value: int | None) -> None:
        """
        Set the maximum backoff duration in milliseconds.

        Parameters
        ----------
        value: int | None
            The maximum backoff duration in milliseconds.
        """
        ...

class StorageSettings:
    """Configuration for how Icechunk uses its Storage instance"""

    def __new__(
        cls,
        concurrency: StorageConcurrencySettings | None = None,
        retries: StorageRetriesSettings | None = None,
        unsafe_use_conditional_create: bool | None = None,
        unsafe_use_conditional_update: bool | None = None,
        unsafe_use_metadata: bool | None = None,
        storage_class: str | None = None,
        metadata_storage_class: str | None = None,
        chunks_storage_class: str | None = None,
        minimum_size_for_multipart_upload: int | None = None,
        timeouts: StorageTimeoutSettings | None = None,
    ) -> StorageSettings:
        """
        Create a new `StorageSettings` object

        Parameters
        ----------
        concurrency: StorageConcurrencySettings | None
            The configuration for how Icechunk uses its Storage instance. When
            None, the default `StorageConcurrencySettings` is used.
            Default: None

        retries: StorageRetriesSettings | None
            The configuration for how Icechunk retries failed requests. When
            None, the default `StorageRetriesSettings` is used.
            Default: None

        unsafe_use_conditional_update: bool | None
            If set to False, Icechunk loses some of its consistency guarantees.
            This is only useful in object stores that don't support the feature.
            Use it at your own risk.
            Default: True

        unsafe_use_conditional_create: bool | None
            If set to False, Icechunk loses some of its consistency guarantees.
            This is only useful in object stores that don't support the feature.
            Use at your own risk.
            Default: True

        unsafe_use_metadata: bool | None
            Don't write metadata fields in Icechunk files.
            This is only useful in object stores that don't support the feature.
            Use at your own risk.
            Default: True

        storage_class: str | None
            Store all objects using this object store storage class.
            If None the object store default will be used.
            Currently not supported in GCS.
            Example: STANDARD_IA
            Default: None

        metadata_storage_class: str | None
            Store metadata objects using this object store storage class.
            Currently not supported in GCS.
            Default: None (falls back to `storage_class`)

        chunks_storage_class: str | None
            Store chunk objects using this object store storage class.
            Currently not supported in GCS.
            Default: None (falls back to `storage_class`)

        minimum_size_for_multipart_upload: int | None
            Use object store's multipart upload for objects larger than this size in bytes.
            Default: 104,857,600 (100 MB)

        timeouts: StorageTimeoutSettings | None
            The configuration for AWS SDK timeout settings. When None, all
            timeouts use the AWS SDK defaults.
            Default: None
        """
        ...
    def __repr__(self, /) -> str: ...
    def __str__(self, /) -> str: ...
    def _repr_html_(self, /) -> str: ...
    @property
    def concurrency(self) -> StorageConcurrencySettings | None:
        """
        The configuration for how much concurrency Icechunk store uses.

        Default: None (uses the default `StorageConcurrencySettings`)

        Returns
        -------
        StorageConcurrencySettings | None
            The configuration for how Icechunk uses its Storage instance.
        """

    @concurrency.setter
    def concurrency(self, value: StorageConcurrencySettings | None) -> None: ...
    @property
    def retries(self) -> StorageRetriesSettings | None:
        """
        The configuration for how Icechunk retries failed requests.

        Default: None (uses the default `StorageRetriesSettings`)

        Returns
        -------
        StorageRetriesSettings | None
            The configuration for how Icechunk retries failed requests.
        """

    @retries.setter
    def retries(self, value: StorageRetriesSettings | None) -> None: ...
    @property
    def timeouts(self) -> StorageTimeoutSettings | None:
        """
        The configuration for AWS SDK timeout settings.

        Default: None (all timeouts use the AWS SDK defaults)

        Returns
        -------
        StorageTimeoutSettings | None
            The timeout configuration.
        """

    @timeouts.setter
    def timeouts(self, value: StorageTimeoutSettings | None) -> None: ...
    @property
    def unsafe_use_conditional_update(self) -> bool | None:
        """True if Icechunk will use conditional PUT operations for updates in the object store.

        Default: True
        """
        ...

    @unsafe_use_conditional_update.setter
    def unsafe_use_conditional_update(self, value: bool) -> None: ...
    @property
    def unsafe_use_conditional_create(self) -> bool | None:
        """True if Icechunk will use conditional PUT operations for creation in the object store.

        Default: True
        """
        ...

    @unsafe_use_conditional_create.setter
    def unsafe_use_conditional_create(self, value: bool) -> None: ...
    @property
    def unsafe_use_metadata(self) -> bool | None:
        """True if Icechunk will write object metadata in the object store.

        Default: True
        """
        ...

    @unsafe_use_metadata.setter
    def unsafe_use_metadata(self, value: bool) -> None: ...
    @property
    def storage_class(self) -> str | None:
        """All objects in object store will use this storage class or the default if None.

        Default: None
        """
        ...

    @storage_class.setter
    def storage_class(self, value: str) -> None: ...
    @property
    def metadata_storage_class(self) -> str | None:
        """Metadata objects in object store will use this storage class or self.storage_class if None.

        Default: None (falls back to `storage_class`)
        """
        ...

    @metadata_storage_class.setter
    def metadata_storage_class(self, value: str) -> None: ...
    @property
    def chunks_storage_class(self) -> str | None:
        """Chunk objects in object store will use this storage class or self.storage_class if None.

        Default: None (falls back to `storage_class`)
        """
        ...

    @chunks_storage_class.setter
    def chunks_storage_class(self, value: str) -> None: ...
    @property
    def minimum_size_for_multipart_upload(self) -> int | None:
        """Use object store's multipart upload for objects larger than this size in bytes.

        Default: 104,857,600 (100 MB)
        """
        ...

    @minimum_size_for_multipart_upload.setter
    def minimum_size_for_multipart_upload(self, value: int) -> None: ...