Skip to content

feat: make TableConfig accessible #3518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion python/deltalake/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ from typing import (
TYPE_CHECKING,
Any,
Literal,
Optional,
Union,
)

Expand Down Expand Up @@ -93,7 +94,10 @@ class RawDeltaTable:
def get_add_file_sizes(self) -> dict[str, int]: ...
def get_latest_version(self) -> int: ...
def metadata(self) -> RawDeltaTableMetaData: ...
def protocol_versions(self) -> list[Any]: ...
def protocol_versions(
self,
) -> tuple[int, int, Optional[list[str]], Optional[list[str]]]: ...
def table_config(self) -> tuple[bool, int]: ...
def load_version(self, version: int) -> None: ...
def load_with_datetime(self, ds: str) -> None: ...
def files(self, partition_filters: PartitionFilterType | None) -> list[str]: ...
Expand Down
9 changes: 9 additions & 0 deletions python/deltalake/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,11 @@ def __str__(self) -> str:
)


class DeltaTableConfig(NamedTuple):
without_files: bool
log_buffer_size: int


class ProtocolVersions(NamedTuple):
min_reader_version: int
min_writer_version: int
Expand Down Expand Up @@ -166,6 +171,10 @@ def __init__(
log_buffer_size=log_buffer_size,
)

@property
def table_config(self) -> DeltaTableConfig:
return DeltaTableConfig(*self._table.table_config())

@staticmethod
def is_deltatable(
table_uri: str, storage_options: dict[str, str] | None = None
Expand Down
8 changes: 8 additions & 0 deletions python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,14 @@ impl RawDeltaTable {
self.with_table(|t| Ok(t.config.require_files))
}

pub fn table_config(&self) -> PyResult<(bool, usize)> {
self.with_table(|t| {
let config = t.config.clone();
// Require_files inverted to reflect without_files
Ok((!config.require_files, config.log_buffer_size))
})
}

pub fn metadata(&self) -> PyResult<RawDeltaTableMetaData> {
let metadata = self.with_table(|t| {
t.metadata()
Expand Down
28 changes: 28 additions & 0 deletions python/tests/test_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
from arro3.core import Table

from deltalake import write_deltalake
from deltalake.table import DeltaTable, DeltaTableConfig


def test_config_roundtrip(tmp_path, sample_table: Table):
write_deltalake(tmp_path, sample_table)

config = DeltaTableConfig(without_files=True, log_buffer_size=100)

dt = DeltaTable(
tmp_path,
without_files=config.without_files,
log_buffer_size=config.log_buffer_size,
)

assert config == dt.table_config

config = DeltaTableConfig(without_files=False, log_buffer_size=1)

dt = DeltaTable(
tmp_path,
without_files=config.without_files,
log_buffer_size=config.log_buffer_size,
)

assert config == dt.table_config
Loading