Update app.py

app.py

@@ -5,7 +5,8 @@ from langchain.text_splitter import (
 )
 
 
-def chunk(text, length, splitter_selection):
+def chunk(text, length, splitter_selection, separators_str):
+    separators = separators_str[1:-1].split(", ")
     if splitter_selection == "LangChain's CharacterTextSplitter":
         text_splitter = CharacterTextSplitter(
             separator="",
@@ -16,22 +17,13 @@ def chunk(text, length, splitter_selection):
         )
         splits = text_splitter.create_documents([text])
         text_splits = [split.page_content for split in splits]
-    elif splitter_selection == "Langchain's RecursiveCharacterTextSplitter - vanilla":
+    elif splitter_selection == "Langchain's RecursiveCharacterTextSplitter":
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=length,
             chunk_overlap=0,
             length_function=len,
             add_start_index=True,
-        )
-        splits = text_splitter.create_documents([text])
-        text_splits = [split.page_content for split in splits]
-    elif splitter_selection == "Langchain's RecursiveCharacterTextSplitter - with '.'":
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=length,
-            chunk_overlap=0,
-            length_function=len,
-            add_start_index=True,
-            separators=["\n\n", "\n", ".", " ", ""],
+            separators=separators,
         )
         splits = text_splitter.create_documents([text])
         text_splits = [split.page_content for split in splits]
@@ -76,7 +68,10 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         label="Chunking method ",
         info="How should we split our chunks?",
     )
-    separator_selection = gr.Textbox(value=["\n\n", "\n", ".", " ", ""], label="Separators used in RecursiveCharacterTextSplitter")
+    separator_selection = gr.Textbox(
+        value=["\n\n", "\n", ".", " ", ""],
+        label="Separators used in RecursiveCharacterTextSplitter",
+    )
     length_unit_selection = gr.Dropdown(
         choices=[
             "Character count",
@@ -96,22 +91,22 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     )
     text.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection],
+        inputs=[text, slider_count, split_selection, separator_selection],
         outputs=out,
     )
     length_unit_selection.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection],
+        inputs=[text, slider_count, split_selection, separator_selection],
         outputs=out,
     )
     split_selection.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection],
+        inputs=[text, slider_count, split_selection, separator_selection],
         outputs=out,
     )
     slider_count.change(
         fn=chunk,
-        inputs=[text, slider_count, split_selection],
+        inputs=[text, slider_count, split_selection, separator_selection],
         outputs=out,
     )
 demo.launch()