use bytes instead of BytesIO for xet

lhoestq · lhoestq · commit fc6916434ee1 · 2025-08-12T18:27:48.000+02:00
diff --git a/pyspark_huggingface/huggingface_sink.py b/pyspark_huggingface/huggingface_sink.py
@@ -213,10 +213,9 @@ def flush(writer: pq.ParquetWriter):
                     f"{self.prefix}-{self.uuid}-part-{partition_id}-{num_files}.parquet"
                 )
                 num_files += 1
-                parquet.seek(0)
 
                 addition = CommitOperationAdd(
-                    path_in_repo=name, path_or_fileobj=parquet
+                    path_in_repo=name, path_or_fileobj=parquet.getvalue()
                 )
                 api.preupload_lfs_files(
                     repo_id=self.repo_id,

Original file line number	Diff line number	Diff line change
`@@ -213,10 +213,9 @@ def flush(writer: pq.ParquetWriter):`
`213`	`213`	`f"{self.prefix}-{self.uuid}-part-{partition_id}-{num_files}.parquet"`
`214`	`214`	`)`
`215`	`215`	`num_files += 1`
`216`		`- parquet.seek(0)`
`217`	`216`
`218`	`217`	`addition = CommitOperationAdd(`
`219`		`- path_in_repo=name, path_or_fileobj=parquet`
	`218`	`+ path_in_repo=name, path_or_fileobj=parquet.getvalue()`
`220`	`219`	`)`
`221`	`220`	`api.preupload_lfs_files(`
`222`	`221`	`repo_id=self.repo_id,`