adjust caches, update spaces card

README.md

@@ -1,19 +1,37 @@
 ---
-title: Dashboard
-emoji: 🌐
-colorFrom: blue
-colorTo: red
+title: NeurIPS Demo
+emoji: ⚡
+colorFrom: green
+colorTo: blue
 sdk: streamlit
 app_file: app.py
 pinned: false
 ---
 
-# Training transformers together dashboard
+# Configuration
 
-[![Generic badge](https://img.shields.io/badge/🤗-Open%20In%20Spaces-blue.svg)](https://huggingface.co/spaces/training-transformers-together/training-transformers-together-dashboard)
+`title`: _string_  
+Display title for the Space
 
-A dashboard app for Hugging Face Spaces
+`emoji`: _string_  
+Space emoji (emoji-only character allowed)
 
----
+`colorFrom`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_
+Can be either `gradio` or `streamlit`
+
+`sdk_version` : _string_  
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_  
+Path to your main application file (which contains either `gradio` or `streamlit` Python code).  
+Path is relative to the root of the repository.
 
-Autogenerated using [this template](https://github.com/nateraw/spaces-template)
+`pinned`: _boolean_  
+Whether the Space stays on top of your list.

app.py

@@ -1,109 +1,30 @@
 """
-This part of the website was bodged together by ham-handed hedgehogs. If something looks wrong, it's because it is.
+This specific file was bodged together by ham-handed hedgehogs. If something looks wrong, it's because it is.
 If you're not a hedgehog, you shouldn't reuse this code. Use this instead: https://docs.streamlit.io/library/get-started
 """
-import os
 
 import streamlit as st
-import streamlit.components.v1 as components
-import wandb
 
-from dashboard_utils.bubbles import get_new_bubble_data
-from dashboard_utils.main_metrics import get_main_metrics
-from streamlit_observable import observable
 
+from st_helpers import make_header, content_text, cite
+from charts import draw_current_progress
 
 st.set_page_config(page_title="Training Transformers Together", layout="centered")
-with open("static/header.html", 'r', encoding='utf-8') as f:
-    header_html = f.read()
-with open("static/header_style.css", 'r', encoding='utf-8') as f:
-    header_style_css = f.read()
-with open("static/header_animate.js") as f:
-    header_animate_js = f.read()
-with open("static/content_style.css", 'r', encoding='utf-8') as f:
-    content_style_css = f.read()
-with open("static/meta.html", 'r', encoding='utf-8') as f:
-    meta_html = f.read()
 
-st.markdown("## Full demo content will be posted here on December 7th!")
-
-components.html(f"<style>{header_style_css}</style>{header_html}<script>{header_animate_js}</script>", height=260)
 
-st.markdown(meta_html, unsafe_allow_html=True)
-st.markdown(f"<style>{content_style_css}</style>", unsafe_allow_html=True)  # apply css to the rest of the document
+st.markdown("## Full demo content will be posted here on December 7th!")
 
-def content_text(text: str, vspace_before: int = 0, vspace_after: int = 0):
-    st.markdown(f'<center><div class="padded faded main_text" '
-                f'style="padding-top: {vspace_before}px;padding-bottom: {vspace_after}px;">'
-                f'{text}</div><center>',
-                unsafe_allow_html=True)
-CITATIONS = {}
-def cite(tag):
-    CITATIONS[tag] = len(CITATIONS) + 1
-    return f"[{CITATIONS[tag]}]"
+make_header()
 
 content_text(f"""
 There was a time when you could comfortably train SoTA vision and language models at home on your workstation.
-The first ConvNet to beat ImageNet took in 5-6 days on two gamer-grade GPUs {cite("alexnet")}. Today's top-1 imagenet model 
-took 20,000 TPU-v3 days {cite("coatnet")}. And things are even worse in the NLP world: training GPT-3 on a top-tier server
- with 8 A100 would still take decades {cite("gpt-3")} .""")
+The first ConvNet to beat ImageNet took in 5-6 days on two gamer-grade GPUs{cite("alexnet")}. Today's top-1 imagenet model 
+took 20,000 TPU-v3 days{cite("coatnet")}. And things are even worse in the NLP world: training GPT-3 on a top-tier server
+ with 8 A100 would still take decades{cite("gpt-3")}.""")
 
 content_text(f"""
 So, can individual researchers and small labs still train state-of-the-art? Yes we can!
 All it takes is for a bunch of us to come together. In fact, we're doing it right now and <b>you're invited to join!</b>
 """, vspace_before=12, vspace_after=16)
 
-source = get_main_metrics()
-st.vega_lite_chart(
-    source, {
-        "height": 200,
-        "title": "Training DALLE with volunteers. Updated every few minutes during NeurIPS.",
-        "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-        "description": "Current training progress",
-        "encoding": {"x": {"field": "date", "type": "temporal"}},
-        "config": {"axisX": {"labelAngle": -40}},
-        "resolve": {"scale": {"y": "independent"}},
-        "layer": [
-            {
-                "mark": {"type": "line", "point": {"tooltip": True, "filled": False, "strokeOpacity": 0},
-                         "color": "#85A9C5"},
-                "encoding": {"y": {"field": "training loss", "type": "quantitative", "axis": {"titleColor": "#85A9C5"}}},
-            },
-            {
-                "mark": {"type": "line", "point": {"tooltip": True, "filled": False, "strokeOpacity": 0.0},
-                         "color": "#85C5A6", "opacity": 0.5},
-                "encoding": {
-                    "y": {"field": "active participants", "type": "quantitative", "axis": {"titleColor": "#85C5A6"}}},
-            },
-        ],
-    },
-    use_container_width=True,
-)
-
-#
-# st.caption("Number of alive runs over time")
-# st.vega_lite_chart(
-#     source,
-#     use_container_width=True,
-# )
-# st.caption("Number of steps")
-# st.vega_lite_chart(
-#     source,
-#     {
-#         "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-#         "description": "Training Loss",
-#         "mark": {"type": "line", "point": {"tooltip": True, "filled": False, "strokeOpacity": 0}},
-#         "encoding": {"x": {"field": "date", "type": "temporal"}, "y": {"field": "steps", "type": "quantitative"}},
-#         "config": {"axisX": {"labelAngle": -40}},
-#     },
-#     use_container_width=True,
-# )
-#
-# st.header("Collaborative training participants")
-# serialized_data, profiles = get_new_bubble_data()
-# observable(
-#     "Participants",
-#     notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
-#     targets=["c_noaws"],
-#     redefine={"serializedData": serialized_data, "profileSimple": profiles},
-# )
+draw_current_progress()

charts.py

@@ -0,0 +1,47 @@
+import streamlit as st
+
+from dashboard_utils.bubbles import get_new_bubble_data
+from dashboard_utils.main_metrics import get_main_metrics
+from streamlit_observable import observable
+
+
+def draw_current_progress():
+    source = get_main_metrics()
+    st.vega_lite_chart(
+        source, {
+            "height": 200,
+            "title": "Training DALLE with volunteers. Updated every few minutes during NeurIPS.",
+            "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+            "description": "Current training progress",
+            "encoding": {"x": {"field": "wall time", "type": "temporal"}},
+            "config": {"axisX": {"labelAngle": -40}},
+            "resolve": {"scale": {"y": "independent"}},
+            "layer": [
+                {
+                    "mark": {"type": "line", "point": {"tooltip": True, "filled": False, "strokeOpacity": 0},
+                             "color": "#85A9C5"},
+                    "encoding": {
+                        "y": {"field": "training loss", "type": "quantitative", "axis": {"titleColor": "#85A9C5"}}},
+                },
+                {
+                    "mark": {"type": "line", "point": {"tooltip": True, "filled": False, "strokeOpacity": 0.0},
+                             "color": "#85C5A6", "opacity": 0.5},
+                    "encoding": {
+                        "y": {"field": "active participants", "type": "quantitative",
+                              "axis": {"titleColor": "#85C5A6"}}},
+                },
+            ],
+        },
+        use_container_width=True,
+    )
+
+    with st.expander("Who's training?", expanded=False):
+        st.markdown("### Collaborative training participants\n(may take a few seconds to load)")
+
+        serialized_data, profiles = get_new_bubble_data()
+        observable(
+            "Participants",
+            notebook="d/9ae236a507f54046",  # "@huggingface/participants-bubbles-chart",
+            targets=["c_noaws"],
+            redefine={"serializedData": serialized_data, "profileSimple": profiles},
+        )

dashboard_utils/bubbles.py

@@ -10,7 +10,7 @@ from dashboard_utils.time_tracker import _log, simple_time_tracker
 
 URL_QUICKSEARCH = "https://huggingface.co/api/quicksearch?"
 WANDB_REPO = "learning-at-home/Worker_logs"
-CACHE_TTL = 100
+CACHE_TTL = 600
 
 
 @st.cache(ttl=CACHE_TTL)

dashboard_utils/main_metrics.py

@@ -30,4 +30,4 @@ def get_main_metrics():
         alive_peers.append(row["alive peers"])
         dates.append(datetime.datetime.utcfromtimestamp(row["_timestamp"]))
 
-    return pd.DataFrame({"steps": steps, "training loss": losses, "active participants": alive_peers, "date": dates})
+    return pd.DataFrame({"steps": steps, "training loss": losses, "active participants": alive_peers, "wall time": dates})

st_helpers.py

@@ -0,0 +1,34 @@
+import streamlit as st
+import streamlit.components.v1 as components
+
+with open("static/header.html", 'r', encoding='utf-8') as f:
+    header_html = f.read()
+with open("static/header_style.css", 'r', encoding='utf-8') as f:
+    header_style_css = f.read()
+with open("static/header_animate.js") as f:
+    header_animate_js = f.read()
+with open("static/content_style.css", 'r', encoding='utf-8') as f:
+    content_style_css = f.read()
+with open("static/meta.html", 'r', encoding='utf-8') as f:
+    meta_html = f.read()
+
+
+def make_header():
+    components.html(f"<style>{header_style_css}</style>{header_html}<script>{header_animate_js}</script>", height=260)
+    st.markdown(meta_html, unsafe_allow_html=True)
+    st.markdown(f"<style>{content_style_css}</style>", unsafe_allow_html=True)  # apply css to the rest of the document
+
+
+def content_text(text: str, vspace_before: int = 0, vspace_after: int = 0):
+    st.markdown(f'<center><div class="padded faded main_text" '
+                f'style="padding-top: {vspace_before}px; padding-bottom: {vspace_after}px; text-align: justify;">'
+                f'{text}</div><center>',
+                unsafe_allow_html=True)
+
+
+CITATIONS = {}
+
+
+def cite(tag):
+    CITATIONS[tag] = len(CITATIONS) + 1
+    return f"&nbsp;[{CITATIONS[tag]}]"