Update app.py

app.py

@@ -14,20 +14,16 @@ def convert_parquet_to_jsonl(parquet_file=None, parquet_url=None):
     else:
         raise ValueError("Either parquet_file or parquet_url must be provided")
 
-    # Function to clean any invalid UTF-8 sequences by replacing them
-    def clean_string(val):
-        if isinstance(val, str):
-            # Re-encode string to UTF-8, replacing invalid characters
-            return val.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
-        return val
+    # Clean string columns to replace invalid UTF-8 sequences
+    for col in df.select_dtypes(include=["object"]).columns:
+        df[col] = df[col].apply(
+            lambda x: x.encode("utf-8", errors="replace").decode("utf-8", errors="replace") if isinstance(x, str) else x
+        )
 
-    # Apply cleaning to every cell in the DataFrame
-    df = df.applymap(clean_string)
-
-    # Now safely convert to JSON Lines
+    # Convert to JSON Lines
     jsonl_data = df.to_json(orient="records", lines=True)
-
-    # Write out to a file
+    
+    # Write the output to a file using UTF-8 encoding explicitly
     output_file_path = "output.jsonl"
     with open(output_file_path, "w", encoding="utf-8") as f:
         f.write(jsonl_data)