We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent e9745d5 commit e5d1bb8Copy full SHA for e5d1bb8
pyiceberg/io/pyarrow.py
@@ -1421,11 +1421,15 @@ def _task_to_record_batches(
1421
1422
# Apply the user filter
1423
if pyarrow_filter is not None:
1424
- current_batch = current_batch.filter(pyarrow_filter)
+ # Temporary fix until PyArrow 21 is released ( https://github.com/apache/arrow/pull/46057 )
1425
+ table = pa.Table.from_batches([current_batch])
1426
+ table = table.filter(pyarrow_filter)
1427
# skip empty batches
- if current_batch.num_rows == 0:
1428
+ if table.num_rows == 0:
1429
continue
1430
1431
+ current_batch = table.combine_chunks().to_batches()[0]
1432
+
1433
result_batch = _to_requested_schema(
1434
projected_schema,
1435
file_project_schema,
0 commit comments