diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index bd902cd1ff..adec150d84 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -46,8 +46,6 @@ PREVIOUS_METADATA_LOCATION, TABLE_TYPE, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -67,7 +65,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, update_table_metadata from pyiceberg.table.metadata import TableMetadata, new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index c24355f6fb..18bbcfe084 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -59,8 +59,6 @@ METADATA_LOCATION, TABLE_TYPE, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -79,7 +77,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, TableProperties, update_table_metadata from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/pyiceberg/catalog/noop.py b/pyiceberg/catalog/noop.py index a8b7154621..e294390e61 100644 --- a/pyiceberg/catalog/noop.py +++ b/pyiceberg/catalog/noop.py @@ -28,10 +28,9 @@ from pyiceberg.table import ( CommitTableRequest, CommitTableResponse, - SortOrder, Table, ) -from pyiceberg.table.sorting import UNSORTED_SORT_ORDER +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties if TYPE_CHECKING: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 79fc37a398..c401339e18 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -65,8 +65,8 @@ CommitTableResponse, Table, TableIdentifier, - TableMetadata, ) +from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder, assign_fresh_sort_order_ids from pyiceberg.typedef import EMPTY_DICT, UTF8, IcebergBaseModel from pyiceberg.types import transform_dict_value_to_str diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index b6b2feeeb0..d44d4996b6 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -44,8 +44,6 @@ from pyiceberg.catalog import ( METADATA_LOCATION, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -64,7 +62,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, update_table_metadata from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties if TYPE_CHECKING: import pyarrow as pa diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index a185164cd1..79bc995198 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -54,7 +54,6 @@ BoundStartsWith, BoundTerm, BoundUnaryPredicate, - L, Not, Or, UnboundPredicate, @@ -63,7 +62,7 @@ from pyiceberg.manifest import DataFile, ManifestFile, PartitionFieldSummary from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.typedef import EMPTY_DICT, StructProtocol +from pyiceberg.typedef import EMPTY_DICT, L, StructProtocol from pyiceberg.types import ( DoubleType, FloatType, diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index be944ffb36..7192513a2d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -75,8 +75,8 @@ AlwaysTrue, BooleanExpression, BoundTerm, - Literal, ) +from pyiceberg.expressions.literals import Literal from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, bind, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 1a4183c914..674dcb0322 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -46,6 +46,8 @@ from sortedcontainers import SortedList from typing_extensions import Annotated +import pyiceberg.expressions.parser as parser +import pyiceberg.expressions.visitors as visitors from pyiceberg.exceptions import CommitFailedException, ResolveError, ValidationError from pyiceberg.expressions import ( AlwaysTrue, @@ -53,10 +55,7 @@ BooleanExpression, EqualTo, Reference, - parser, - visitors, ) -from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator, inclusive_projection from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ( POSITIONAL_DELETE_SCHEMA, @@ -1375,7 +1374,9 @@ def _match_deletes_to_data_file(data_entry: ManifestEntry, positional_delete_ent relevant_entries = positional_delete_entries[positional_delete_entries.bisect_right(data_entry) :] if len(relevant_entries) > 0: - evaluator = _InclusiveMetricsEvaluator(POSITIONAL_DELETE_SCHEMA, EqualTo("file_path", data_entry.data_file.file_path)) + evaluator = visitors._InclusiveMetricsEvaluator( + POSITIONAL_DELETE_SCHEMA, EqualTo("file_path", data_entry.data_file.file_path) + ) return { positional_delete_entry.data_file for positional_delete_entry in relevant_entries @@ -1399,7 +1400,7 @@ def __init__( super().__init__(table, row_filter, selected_fields, case_sensitive, snapshot_id, options, limit) def _build_partition_projection(self, spec_id: int) -> BooleanExpression: - project = inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) + project = visitors.inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) return project(self.row_filter) @cached_property @@ -1466,7 +1467,7 @@ def plan_files(self) -> Iterable[FileScanTask]: # this filter depends on the partition spec used to write the manifest file partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator) - metrics_evaluator = _InclusiveMetricsEvaluator( + metrics_evaluator = visitors._InclusiveMetricsEvaluator( self.table.schema(), self.row_filter, self.case_sensitive, self.options.get("include_empty_files") == "true" ).eval diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 1defb0996f..1f0060780e 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -31,8 +31,6 @@ from pyiceberg.catalog import ( Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -58,7 +56,7 @@ from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import IntegerType, LongType, NestedField diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 51bc286267..850e5f0180 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -23,7 +23,7 @@ from requests_mock import Mocker import pyiceberg -from pyiceberg.catalog import PropertiesUpdateSummary, Table, load_catalog +from pyiceberg.catalog import PropertiesUpdateSummary, load_catalog from pyiceberg.catalog.rest import AUTH_URL, RestCatalog from pyiceberg.exceptions import ( AuthorizationExpiredError, @@ -36,6 +36,7 @@ from pyiceberg.io import load_file_io from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import IdentityTransform, TruncateTransform diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index 0b869d6826..3a77f8678a 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -25,7 +25,6 @@ from pytest_lazyfixture import lazy_fixture from sqlalchemy.exc import ArgumentError, IntegrityError -from pyiceberg.catalog import Identifier from pyiceberg.catalog.sql import SqlCatalog from pyiceberg.exceptions import ( CommitFailedException, @@ -49,6 +48,7 @@ SortOrder, ) from pyiceberg.transforms import IdentityTransform +from pyiceberg.typedef import Identifier from pyiceberg.types import IntegerType diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 3ce2f2226c..0bccc9b80f 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -17,6 +17,7 @@ import pytest from pyparsing import ParseException +import pyiceberg.expressions.parser as parser from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, @@ -37,7 +38,6 @@ NotStartsWith, Or, StartsWith, - parser, ) diff --git a/tests/integration/test_writes.py b/tests/integration/test_writes.py index 3b6d476b74..82c4ace711 100644 --- a/tests/integration/test_writes.py +++ b/tests/integration/test_writes.py @@ -32,11 +32,12 @@ from pyspark.sql import SparkSession from pytest_mock.plugin import MockerFixture -from pyiceberg.catalog import Catalog, Properties, Table +from pyiceberg.catalog import Catalog from pyiceberg.catalog.sql import SqlCatalog from pyiceberg.exceptions import NoSuchTableError from pyiceberg.schema import Schema -from pyiceberg.table import _dataframe_to_data_files +from pyiceberg.table import Table, _dataframe_to_data_files +from pyiceberg.typedef import Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a3dd56db7f..2acffdfdf9 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -53,8 +53,8 @@ GreaterThan, Not, Or, - literal, ) +from pyiceberg.expressions.literals import literal from pyiceberg.io import InputStream, OutputStream, load_file_io from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, diff --git a/tests/table/test_init.py b/tests/table/test_init.py index b8097f5fcf..f734211510 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -57,7 +57,6 @@ SetDefaultSortOrderUpdate, SetPropertiesUpdate, SetSnapshotRefUpdate, - SnapshotRef, StaticTable, Table, UpdateSchema, @@ -68,6 +67,7 @@ update_table_metadata, ) from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadataUtil, TableMetadataV2, _generate_snapshot_id +from pyiceberg.table.refs import SnapshotRef from pyiceberg.table.snapshots import ( Operation, Snapshot, diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index c05700ecbb..0cf17b11a2 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -29,7 +29,6 @@ from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import FromByteStream -from pyiceberg.table import SortOrder from pyiceberg.table.metadata import ( TableMetadataUtil, TableMetadataV1, @@ -37,7 +36,7 @@ new_table_metadata, ) from pyiceberg.table.refs import SnapshotRef, SnapshotRefType -from pyiceberg.table.sorting import NullOrder, SortDirection, SortField +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import UTF8 from pyiceberg.types import ( diff --git a/tests/test_schema.py b/tests/test_schema.py index 6394b72ba6..7e10dd5b0d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,8 +23,8 @@ from pyiceberg import schema from pyiceberg.exceptions import ResolveError, ValidationError -from pyiceberg.expressions import Accessor from pyiceberg.schema import ( + Accessor, Schema, build_position_accessors, promote, diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 6ef11a47ea..3e789cb854 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -37,8 +37,7 @@ ) from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Snapshot -from pyiceberg.table.snapshots import Operation, Summary +from pyiceberg.table.snapshots import Operation, Snapshot, Summary from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import Record from pyiceberg.types import IntegerType, NestedField