File tree Expand file tree Collapse file tree 1 file changed +3
-4
lines changed
Expand file tree Collapse file tree 1 file changed +3
-4
lines changed Original file line number Diff line number Diff line change 1212import psutil
1313import pyarrow as pa
1414import pystac
15+ from pyarrow .dataset import dataset
1516
1617from stac_geoparquet .arrow ._batch import StacArrowBatch , StacJsonBatch
1718from stac_geoparquet .arrow ._constants import (
@@ -136,11 +137,9 @@ def parse_stac_items_to_arrow(
136137 output_path = fname ,
137138 )
138139 memlog (f"Batch { cnt } " )
139- ds = pa .dataset .dataset (
140- tmpdir , schema = schema , format = "parquet" , batch_size = chunk_size
141- )
140+ ds = dataset (tmpdir , schema = schema , format = "parquet" )
142141 memlog ("Created Dataset" )
143- batches = ds .to_batches ()
142+ batches = ds .to_batches (batch_size = chunk_size )
144143 memlog ("Created Batches" )
145144 return pa .RecordBatchReader .from_batches (schema , batches )
146145
You can’t perform that action at this time.
0 commit comments