Merge branch 'main' of https://huggingface.co/spaces/CarperAI/pile-v2-eda

app.py

@@ -14,7 +14,7 @@ else:
 contribution_json = "contributors.json"
 
 contribution_dict = json.load(open(contribution_json,"r"))
-IGNORE_LIST = ["Bible","Tanzil",""]
+IGNORE_LIST = ["Bible","Tanzil","GNOME"]
 
 splits = [split for split in os.listdir(CACHE_DIR) if split not in IGNORE_LIST]
 
@@ -44,17 +44,18 @@ def load_page(split):
             meta = data["meta"]
             with st.expander("Render Content"):
                 st.write(content)
-            st.write("### Content:")
-            st.text(content)
-            st.write("### Meta:")
-            st.write(ast.literal_eval(meta))
-            #Tokenizer related count
-            tokenized =  tokenizer(content, return_length=True)['length'][0]
-            token_count_metric = st.metric("Token Count(compared to 2048)",value=tokenized,delta=4096-tokenized)
+            with st.expander("Raw Content"):
+                st.text(content)
+            with st.expander("Metadata and Metrics"):    
+                st.write("### Meta:")
+                st.write(ast.literal_eval(meta))
+            # Tokenizer-related count
+                tokenized =  tokenizer(content, return_length=True)['length'][0]
+                token_count_metric = st.metric("Token Count(compared to 2048)",value=tokenized,delta=4096-tokenized)
             #Word related count
-            split_words = re.findall(r'\w+', content)
-            word_count_metric = st.metric("Word Count",value=len(split_words))
-            
+                split_words = re.findall(r'\w+', content)
+                word_count_metric = st.metric("Word Count",value=len(split_words))
+                
 
 
 demo_name = st.sidebar.selectbox("Choose a demo", splits)