Deprecate for 0.8.0 release (apache#1269)

* remove deprecated functions

* remove deprecated properties

* fix tests based on deprecated properties

* update readme

* get_first_property_value

* deprecate properties

Author: kevinjqliu

mkdocs/docs/configuration.md

@@ -340,8 +340,8 @@ catalog:
 
 <!-- prettier-ignore-start -->
 
-!!! warning "Deprecated Properties"
-    `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0:
+!!! warning "Removed Properties"
+    The properties `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were deprecated and removed in 0.8.0
 
 <!-- prettier-ignore-end -->
 
@@ -396,8 +396,8 @@ catalog:
 
 <!-- prettier-ignore-start -->
 
-!!! warning "Deprecated Properties"
-    `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0:
+!!! warning "Removed Properties"
+    The properties `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were deprecated and removed in 0.8.0
 
 <!-- prettier-ignore-end -->
 

pyiceberg/catalog/__init__.py

@@ -106,21 +106,6 @@
     re.X,
 )
 
-DEPRECATED_PROFILE_NAME = "profile_name"
-DEPRECATED_REGION = "region_name"
-DEPRECATED_BOTOCORE_SESSION = "botocore_session"
-DEPRECATED_ACCESS_KEY_ID = "aws_access_key_id"
-DEPRECATED_SECRET_ACCESS_KEY = "aws_secret_access_key"
-DEPRECATED_SESSION_TOKEN = "aws_session_token"
-DEPRECATED_PROPERTY_NAMES = {
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_ACCESS_KEY_ID,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
-}
-
 
 class CatalogType(Enum):
     REST = "rest"
@@ -794,14 +779,6 @@ class MetastoreCatalog(Catalog, ABC):
     def __init__(self, name: str, **properties: str):
         super().__init__(name, **properties)
 
-        for property_name in DEPRECATED_PROPERTY_NAMES:
-            if self.properties.get(property_name):
-                deprecation_message(
-                    deprecated_in="0.7.0",
-                    removed_in="0.8.0",
-                    help_message=f"The property {property_name} is deprecated. Please use properties that start with client., glue., and dynamo. instead",
-                )
-
     def create_table_transaction(
         self,
         identifier: Union[str, Identifier],

pyiceberg/catalog/dynamodb.py

@@ -30,12 +30,6 @@
 import boto3
 
 from pyiceberg.catalog import (
-    DEPRECATED_ACCESS_KEY_ID,
-    DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
     ICEBERG,
     METADATA_LOCATION,
     PREVIOUS_METADATA_LOCATION,
@@ -102,18 +96,11 @@ def __init__(self, name: str, **properties: str):
         super().__init__(name, **properties)
 
         session = boto3.Session(
-            profile_name=get_first_property_value(properties, DYNAMODB_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
-            region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION, DEPRECATED_REGION),
-            botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
-            aws_access_key_id=get_first_property_value(
-                properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
-            ),
-            aws_secret_access_key=get_first_property_value(
-                properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
-            ),
-            aws_session_token=get_first_property_value(
-                properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
-            ),
+            profile_name=properties.get(DYNAMODB_PROFILE_NAME),
+            region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION),
+            aws_access_key_id=get_first_property_value(properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=get_first_property_value(properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+            aws_session_token=get_first_property_value(properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN),
         )
         self.dynamodb = session.client(DYNAMODB_CLIENT)
         self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT)

pyiceberg/catalog/glue.py

@@ -40,12 +40,6 @@
 )
 
 from pyiceberg.catalog import (
-    DEPRECATED_ACCESS_KEY_ID,
-    DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
     EXTERNAL_TABLE,
     ICEBERG,
     LOCATION,
@@ -303,18 +297,11 @@ def __init__(self, name: str, **properties: Any):
         super().__init__(name, **properties)
 
         session = boto3.Session(
-            profile_name=get_first_property_value(properties, GLUE_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
-            region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION, DEPRECATED_REGION),
-            botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
-            aws_access_key_id=get_first_property_value(
-                properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
-            ),
-            aws_secret_access_key=get_first_property_value(
-                properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
-            ),
-            aws_session_token=get_first_property_value(
-                properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
-            ),
+            profile_name=properties.get(GLUE_PROFILE_NAME),
+            region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION),
+            aws_access_key_id=get_first_property_value(properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=get_first_property_value(properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+            aws_session_token=get_first_property_value(properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN),
         )
         self.glue: GlueClient = session.client("glue", endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT))
 

pyiceberg/io/pyarrow.py

@@ -1688,23 +1688,6 @@ def project_batches(
             total_row_count += len(batch)
 
 
-@deprecated(
-    deprecated_in="0.7.0",
-    removed_in="0.8.0",
-    help_message="The public API for 'to_requested_schema' is deprecated and is replaced by '_to_requested_schema'",
-)
-def to_requested_schema(requested_schema: Schema, file_schema: Schema, table: pa.Table) -> pa.Table:
-    struct_array = visit_with_partner(requested_schema, table, ArrowProjectionVisitor(file_schema), ArrowAccessor(file_schema))
-
-    arrays = []
-    fields = []
-    for pos, field in enumerate(requested_schema.fields):
-        array = struct_array.field(pos)
-        arrays.append(array)
-        fields.append(pa.field(field.name, array.type, field.optional))
-    return pa.Table.from_arrays(arrays, schema=pa.schema(fields))
-
-
 def _to_requested_schema(
     requested_schema: Schema,
     file_schema: Schema,

pyiceberg/table/__init__.py

@@ -89,7 +89,6 @@
 from pyiceberg.table.update import (
     AddPartitionSpecUpdate,
     AddSchemaUpdate,
-    AddSnapshotUpdate,
     AddSortOrderUpdate,
     AssertCreate,
     AssertRefSnapshotId,
@@ -314,55 +313,6 @@ def set_properties(self, properties: Properties = EMPTY_DICT, **kwargs: Any) ->
         updates = properties or kwargs
         return self._apply((SetPropertiesUpdate(updates=updates),))
 
-    @deprecated(
-        deprecated_in="0.7.0",
-        removed_in="0.8.0",
-        help_message="Please use one of the functions in ManageSnapshots instead",
-    )
-    def add_snapshot(self, snapshot: Snapshot) -> Transaction:
-        """Add a new snapshot to the table.
-
-        Returns:
-            The transaction with the add-snapshot staged.
-        """
-        updates = (AddSnapshotUpdate(snapshot=snapshot),)
-
-        return self._apply(updates, ())
-
-    @deprecated(
-        deprecated_in="0.7.0",
-        removed_in="0.8.0",
-        help_message="Please use one of the functions in ManageSnapshots instead",
-    )
-    def set_ref_snapshot(
-        self,
-        snapshot_id: int,
-        parent_snapshot_id: Optional[int],
-        ref_name: str,
-        type: str,
-        max_ref_age_ms: Optional[int] = None,
-        max_snapshot_age_ms: Optional[int] = None,
-        min_snapshots_to_keep: Optional[int] = None,
-    ) -> Transaction:
-        """Update a ref to a snapshot.
-
-        Returns:
-            The transaction with the set-snapshot-ref staged
-        """
-        updates = (
-            SetSnapshotRefUpdate(
-                snapshot_id=snapshot_id,
-                ref_name=ref_name,
-                type=type,
-                max_ref_age_ms=max_ref_age_ms,
-                max_snapshot_age_ms=max_snapshot_age_ms,
-                min_snapshots_to_keep=min_snapshots_to_keep,
-            ),
-        )
-
-        requirements = (AssertRefSnapshotId(snapshot_id=parent_snapshot_id, ref="main"),)
-        return self._apply(updates, requirements)
-
     def _set_ref_snapshot(
         self,
         snapshot_id: int,

tests/catalog/test_dynamodb.py

@@ -45,7 +45,6 @@
 from pyiceberg.typedef import Properties
 from tests.conftest import (
     BUCKET_NAME,
-    DEPRECATED_AWS_SESSION_PROPERTIES,
     TABLE_METADATA_LOCATION_REGEX,
     UNIFIED_AWS_SESSION_PROPERTIES,
 )
@@ -563,28 +562,6 @@ def test_update_namespace_properties_overlap_update_removal(_bucket_initialize:
     assert test_catalog.load_namespace_properties(database_name) == test_properties
 
 
-def test_passing_provided_profile() -> None:
-    catalog_name = "test_ddb_catalog"
-    session_props = {
-        "aws_access_key_id": "abc",
-        "aws_secret_access_key": "def",
-        "aws_session_token": "ghi",
-        "region_name": "eu-central-1",
-        "botocore_session": None,
-        "profile_name": None,
-    }
-    props = {"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}
-    props.update(session_props)  # type: ignore
-    with mock.patch("boto3.Session", return_value=mock.Mock()) as mock_session:
-        mock_client = mock.Mock()
-        mock_session.return_value.client.return_value = mock_client
-        mock_client.describe_table.return_value = {"Table": {"TableStatus": "ACTIVE"}}
-        test_catalog = DynamoDbCatalog(catalog_name, **props)
-        assert test_catalog.dynamodb is mock_client
-        mock_session.assert_called_with(**session_props)
-        assert test_catalog.dynamodb is mock_session().client()
-
-
 @mock_aws
 def test_passing_glue_session_properties() -> None:
     session_properties: Properties = {
@@ -594,7 +571,6 @@ def test_passing_glue_session_properties() -> None:
         "dynamodb.region": "dynamodb.region",
         "dynamodb.session-token": "dynamodb.session-token",
         **UNIFIED_AWS_SESSION_PROPERTIES,
-        **DEPRECATED_AWS_SESSION_PROPERTIES,
     }
 
     with mock.patch("boto3.Session") as mock_session:
@@ -609,7 +585,6 @@ def test_passing_glue_session_properties() -> None:
             aws_session_token="dynamodb.session-token",
             region_name="dynamodb.region",
             profile_name="dynamodb.profile-name",
-            botocore_session=None,
         )
         assert test_catalog.dynamodb is mock_session().client()
 
@@ -619,7 +594,6 @@ def test_passing_unified_session_properties_to_dynamodb() -> None:
     session_properties: Properties = {
         "dynamodb.profile-name": "dynamodb.profile-name",
         **UNIFIED_AWS_SESSION_PROPERTIES,
-        **DEPRECATED_AWS_SESSION_PROPERTIES,
     }
 
     with mock.patch("boto3.Session") as mock_session:
@@ -634,7 +608,6 @@ def test_passing_unified_session_properties_to_dynamodb() -> None:
             aws_session_token="client.session-token",
             region_name="client.region",
             profile_name="dynamodb.profile-name",
-            botocore_session=None,
         )
         assert test_catalog.dynamodb is mock_session().client()
 

tests/catalog/test_glue.py

@@ -14,7 +14,7 @@
 #  KIND, either express or implied.  See the License for the
 #  specific language governing permissions and limitations
 #  under the License.
-from typing import Any, Dict, List
+from typing import List
 from unittest import mock
 
 import boto3
@@ -40,7 +40,6 @@
 from pyiceberg.types import IntegerType
 from tests.conftest import (
     BUCKET_NAME,
-    DEPRECATED_AWS_SESSION_PROPERTIES,
     TABLE_METADATA_LOCATION_REGEX,
     UNIFIED_AWS_SESSION_PROPERTIES,
 )
@@ -635,26 +634,6 @@ def test_update_namespace_properties_overlap_update_removal(
     assert test_catalog.load_namespace_properties(database_name) == test_properties
 
 
-@mock_aws
-def test_passing_profile_name() -> None:
-    session_properties: Dict[str, Any] = {
-        "aws_access_key_id": "abc",
-        "aws_secret_access_key": "def",
-        "aws_session_token": "ghi",
-        "region_name": "eu-central-1",
-        "profile_name": "sandbox",
-        "botocore_session": None,
-    }
-    test_properties = {"type": "glue"}
-    test_properties.update(session_properties)
-
-    with mock.patch("boto3.Session") as mock_session:
-        test_catalog = GlueCatalog("glue", **test_properties)
-
-    mock_session.assert_called_with(**session_properties)
-    assert test_catalog.glue is mock_session().client()
-
-
 @mock_aws
 def test_passing_glue_session_properties() -> None:
     session_properties: Properties = {
@@ -664,7 +643,6 @@ def test_passing_glue_session_properties() -> None:
         "glue.region": "glue.region",
         "glue.session-token": "glue.session-token",
         **UNIFIED_AWS_SESSION_PROPERTIES,
-        **DEPRECATED_AWS_SESSION_PROPERTIES,
     }
 
     with mock.patch("boto3.Session") as mock_session:
@@ -676,7 +654,6 @@ def test_passing_glue_session_properties() -> None:
         aws_session_token="glue.session-token",
         region_name="glue.region",
         profile_name="glue.profile-name",
-        botocore_session=None,
     )
     assert test_catalog.glue is mock_session().client()
 
@@ -686,7 +663,6 @@ def test_passing_unified_session_properties_to_glue() -> None:
     session_properties: Properties = {
         "glue.profile-name": "glue.profile-name",
         **UNIFIED_AWS_SESSION_PROPERTIES,
-        **DEPRECATED_AWS_SESSION_PROPERTIES,
     }
 
     with mock.patch("boto3.Session") as mock_session:
@@ -698,7 +674,6 @@ def test_passing_unified_session_properties_to_glue() -> None:
         aws_session_token="client.session-token",
         region_name="client.region",
         profile_name="glue.profile-name",
-        botocore_session=None,
     )
     assert test_catalog.glue is mock_session().client()
 

tests/conftest.py

@@ -2029,14 +2029,6 @@ def hierarchical_namespace_list(hierarchical_namespace_name: str) -> List[str]:
     re.X,
 )
 
-DEPRECATED_AWS_SESSION_PROPERTIES = {
-    "aws_access_key_id": "aws_access_key_id",
-    "aws_secret_access_key": "aws_secret_access_key",
-    "aws_session_token": "aws_session_token",
-    "region_name": "region_name",
-    "profile_name": "profile_name",
-}
-
 UNIFIED_AWS_SESSION_PROPERTIES = {
     "client.access-key-id": "client.access-key-id",
     "client.secret-access-key": "client.secret-access-key",