Upload app.py

app.py

@@ -0,0 +1,135 @@
+import gradio as gr
+import sys
+import requests
+import zipfile
+import io
+import ast
+
+
+def is_file_type(file_path, file_extension):
+    """Check if the file has the specified file extension."""
+    return file_path.endswith(file_extension)
+
+
+def is_likely_useful_file(file_path, lang="python"):
+    """Determine if the file is likely to be useful by excluding certain directories and specific file types."""
+    excluded_dirs = ["docs", "examples", "tests", "test", "scripts", "utils", "benchmarks"]
+    utility_or_config_files = []
+    github_workflow_or_docs = [".github", ".gitignore", "LICENSE"]
+
+    if lang == "python":
+        excluded_dirs.append("__pycache__")
+        utility_or_config_files.extend(["hubconf.py", "setup.py"])
+        github_workflow_or_docs.extend(["stale.py", "gen-card-", "write_model_card"])
+    elif lang == "go":
+        excluded_dirs.append("vendor")
+        utility_or_config_files.extend(["go.mod", "go.sum", "Makefile"])
+
+    if any(part.startswith(".") for part in file_path.split("/")):
+        return False
+    if "test" in file_path.lower():
+        return False
+    for excluded_dir in excluded_dirs:
+        if f"/{excluded_dir}/" in file_path or file_path.startswith(excluded_dir + "/"):
+            return False
+    for file_name in utility_or_config_files:
+        if file_name in file_path:
+            return False
+    for doc_file in github_workflow_or_docs:
+        if doc_file in file_path:
+            return False
+    return True
+
+
+def is_test_file(file_content, lang):
+    """Determine if the file content suggests it is a test file."""
+    test_indicators = {"python": ["unittest", "pytest"], "go": ["testing"]}.get(lang, [])
+
+    if lang == "python":
+        try:
+            module = ast.parse(file_content)
+            for node in ast.walk(module):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        if alias.name in test_indicators:
+                            return True
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module in test_indicators:
+                        return True
+        except SyntaxError:
+            pass
+
+    return False
+
+
+def has_sufficient_content(file_content, min_line_count=10):
+    """Check if the file has a minimum number of substantive lines."""
+    lines = [line for line in file_content.split("\n") if line.strip() and not line.strip().startswith(("#", "//"))]
+    return len(lines) >= min_line_count
+
+
+def remove_comments_and_docstrings(source):
+    """Remove comments and docstrings from the Python source code."""
+    tree = ast.parse(source)
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)) and ast.get_docstring(node):
+            node.body = node.body[1:]  # Remove docstring
+        elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):
+            node.value.s = ""  # Remove comments
+    return ast.unparse(tree)
+
+
+def download_repo(repo_url, branch_or_tag="master"):
+    """Download and process files from a GitHub repository."""
+    download_url = f"{repo_url}/archive/refs/heads/{branch_or_tag}.zip"
+    lang = "python"
+
+    print(download_url)
+    response = requests.get(download_url)
+
+    if response.status_code == 200:
+        zip_file = zipfile.ZipFile(io.BytesIO(response.content))
+        file_contents = ""
+        print(zip_file.namelist())
+        for file_path in zip_file.namelist():
+            # Skip directories, non-language files, less likely useful files, hidden directories, and test files
+            if file_path.endswith("/") or not is_file_type(file_path, ".py") or not is_likely_useful_file(file_path):
+                print("Dir or non-lang or useless:", file_path)
+                continue
+            file_content = zip_file.read(file_path).decode("utf-8")
+
+            # Skip test files based on content
+            if is_test_file(file_content, lang):
+                print("Test file:", file_path)
+                continue
+            print("Appending", file_path)
+
+            file_contents += f"// File: {file_path}\n" if lang == "go" else f"# File: {file_path}\n"
+            file_contents += file_content
+            file_contents += "\n\n"
+        return file_contents
+    else:
+        print(f"Failed to download the repository. Status code: {response.status_code}")
+        sys.exit(1)
+
+
+def download_and_process(repo_url, branch_or_tag="master"):
+    file_contents = download_repo(repo_url, branch_or_tag)
+    return file_contents
+
+
+iface = gr.Interface(
+    fn=download_and_process,
+    inputs=[
+        gr.components.Textbox(label="GitHub Repository URL", value="https://github.com/cognitivecomputations/github2file"),
+        gr.components.Textbox(label="Branch or Tag", value="master"),
+    ],
+    outputs=gr.components.Code(
+        label="Output File",
+        language="python",
+        interactive=True,
+    ),
+)
+
+if __name__ == "__main__":
+    iface.launch()