CERBERUS/awesome-python
1
0
Fork 0
mirror of https://github.com/vinta/awesome-python.git synced 2026-08-03 22:40:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat: use explicit Projects section in README

Browse source
This commit is contained in:
Vinta Chen 2026-05-04 21:24:57 +08:00
parent 921d47b455
commit 6c18b6447e
No known key found for this signature in database
GPG key ID: B93DE4F003C33630
5 changed files with 190 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

9
website/build.py
View file

@ -243,13 +243,14 @@ def write_sitemap_xml(path: Path, urls: Sequence[tuple[str, str]]) -> None:
def top_level_heading_text(line: str) -> str | None:
stripped = line.strip()
if not stripped.startswith("# "):
match = re.match(r"^(#{1,2})\s+(.+)$", stripped)
if match is None:
return None
return stripped.removeprefix("#").strip().strip("#").strip().strip("*").strip()
return match.group(2).strip().strip("#").strip().strip("*").strip()
def extract_categories_body(markdown: str) -> str:
"""Return content under the `# Categories` heading, excluding the heading line itself."""
"""Return content from `Categories` through `Projects`, excluding later sections."""
lines = markdown.splitlines(keepends=True)
start_idx = None
end_idx = len(lines)
@ -261,7 +262,7 @@ def extract_categories_body(markdown: str) -> str:
start_idx = i + 1
while start_idx < len(lines) and lines[start_idx].strip() == "":
start_idx += 1
elif start_idx is not None and i >= start_idx:
elif start_idx is not None and heading.lower() in ("resources", "contributing"):
end_idx = i
break
if start_idx is None:

51
website/readme_parser.py
View file

@ -114,6 +114,13 @@ def _heading_text(node: SyntaxTreeNode) -> str:
return ""
def _heading_level(node: SyntaxTreeNode) -> int | None:
"""Return the numeric level for a heading node."""
if node.type != "heading" or not node.tag.startswith("h"):
return None
return int(node.tag[1:])
def _extract_description_children(nodes: list[SyntaxTreeNode]) -> list[SyntaxTreeNode]:
"""Extract description children from the first paragraph if it's a single <em> block.
@ -303,7 +310,7 @@ def _parse_grouped_sections(
) -> list[ParsedGroup]:
"""Parse nodes into groups of categories using bold markers as group boundaries.
Bold-only paragraphs (**Group Name**) delimit groups. H2 headings under each
Bold-only paragraphs (**Group Name**) delimit groups. H3 headings under each
bold marker become categories within that group. Categories appearing before
any bold marker go into an "Other" group.
"""
@ -341,7 +348,7 @@ def _parse_grouped_sections(
flush_group()
current_group_name = bold_name
current_cat_body = []
elif node.type == "heading" and node.tag == "h2":
elif node.type == "heading" and node.tag in ("h2", "h3"):
flush_cat()
current_cat_name = _heading_text(node)
current_cat_body = []
@ -383,7 +390,7 @@ def _parse_sponsor_item(inline: SyntaxTreeNode) -> ParsedSponsor | None:
def parse_sponsors(text: str) -> list[ParsedSponsor]:
"""Parse the `# Sponsors` section of README.md into a list of sponsors.
"""Parse the `Sponsors` section of README.md into a list of sponsors.
Expects bullets in the form `**[name](url)**: description`.
Returns [] if no Sponsors section exists.
@ -395,14 +402,18 @@ def parse_sponsors(text: str) -> list[ParsedSponsor]:
start_idx = None
end_idx = len(children)
start_level = None
for i, node in enumerate(children):
if node.type == "heading" and node.tag == "h1":
title = _heading_text(node).strip().lower()
if start_idx is None and title == "sponsors":
start_idx = i + 1
elif start_idx is not None:
end_idx = i
break
level = _heading_level(node)
if level is None:
continue
title = _heading_text(node).strip().lower()
if start_idx is None and title == "sponsors":
start_idx = i + 1
start_level = level
elif start_idx is not None and start_level is not None and level <= start_level:
end_idx = i
break
if start_idx is None:
return []
@ -426,26 +437,26 @@ def parse_readme(text: str) -> list[ParsedGroup]:
"""Parse README.md text into grouped categories.
Returns a list of ParsedGroup dicts containing nested categories.
Content between the thematic break (---) and # Resources or # Contributing
is parsed as categories grouped by bold markers (**Group Name**).
Content between the Projects heading and Resources or Contributing is parsed
as categories grouped by bold markers (**Group Name**).
"""
md = MarkdownIt("commonmark")
tokens = md.parse(text)
root = SyntaxTreeNode(tokens)
children = root.children
# Find thematic break (---) and section boundaries in one pass
hr_idx = None
# Find Projects and section boundaries in one pass.
projects_idx = None
cat_end_idx = None
for i, node in enumerate(children):
if hr_idx is None and node.type == "hr":
hr_idx = i
elif node.type == "heading" and node.tag == "h1":
if _heading_level(node) in (1, 2):
text_content = _heading_text(node)
if cat_end_idx is None and text_content in ("Resources", "Contributing"):
if projects_idx is None and text_content == "Projects":
projects_idx = i
elif cat_end_idx is None and text_content in ("Resources", "Contributing"):
cat_end_idx = i
if hr_idx is None:
if projects_idx is None:
return []
cat_nodes = children[hr_idx + 1 : cat_end_idx or len(children)]
cat_nodes = children[projects_idx + 1 : cat_end_idx or len(children)]
return _parse_grouped_sections(cat_nodes)

62
website/tests/test_build.py
View file

@ -137,31 +137,31 @@ class TestBuild:
Intro.
---
## Projects
**Tools**
## Widgets
### Widgets
_Widget libraries. Also see [awesome-widgets](https://example.com/widgets)._
- [w1](https://example.com) - A widget.
## Gadgets
### Gadgets
_Gadget tools._
- [g1](https://example.com) - A gadget.
# Resources
## Resources
Info.
## Newsletters
### Newsletters
- [NL](https://example.com)
# Contributing
## Contributing
Help!
""")
@ -179,17 +179,17 @@ class TestBuild:
Intro.
---
## Projects
**Tools**
## Widgets
### Widgets
- Sync
- [w1](https://example.com) - A widget.
# Contributing
## Contributing
Help!
""")
@ -232,7 +232,7 @@ class TestBuild:
Intro.
---
## Projects
**Tools**
@ -298,28 +298,28 @@ class TestBuild:
Intro.
# **Sponsors**
## **Sponsors**
- **[Sponsor](https://sponsor.example.com)**: Sponsored tool.
> Become a sponsor: [Sponsor us](SPONSORSHIP.md).
# Categories
## Categories
**Tools**
- [Widgets](#widgets)
---
## Projects
**Tools**
## Widgets
### Widgets
- [w1](https://example.com) - A widget.
- [w2](https://github.com/owner/w2) - A starred widget.
# Contributing
## Contributing
Help!
""")
@ -353,7 +353,7 @@ class TestBuild:
assert "## Categories" in llms_txt
assert "**Tools**" in llms_txt
assert "- [Widgets](#widgets)" in llms_txt
assert "## Widgets" in llms_txt
assert "### Widgets" in llms_txt
assert "- [w1](https://example.com) - A widget." in llms_txt
assert "- [w2](https://github.com/owner/w2) - A starred widget. (GitHub stars: 42)" in llms_txt
assert llms_txt != readme
@ -363,7 +363,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
## Only
@ -387,7 +387,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
## Stuff
@ -431,7 +431,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**Widgets**
@ -538,7 +538,7 @@ class TestBuild:
Intro.
---
## Projects
**Tools**
@ -591,7 +591,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**AI & ML**
@ -624,7 +624,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**Web**
@ -659,7 +659,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**Web**
@ -691,7 +691,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**AI & ML**
@ -731,7 +731,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
## Sneaky </script><script>x=1</script>
@ -760,7 +760,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**AI & ML**
@ -800,7 +800,7 @@ class TestBuild:
readme = textwrap.dedent("""\
# T
---
## Projects
**AI & ML**
@ -980,7 +980,7 @@ class TestExtractEntries:
readme = textwrap.dedent("""\
# T
---
## Projects
**Tools**
@ -1004,7 +1004,7 @@ class TestExtractEntries:
readme = textwrap.dedent("""\
# T
---
## Projects
**Tools**
@ -1031,7 +1031,7 @@ class TestExtractEntries:
readme = textwrap.dedent("""\
# T
---
## Projects
## Stdlib
@ -1050,7 +1050,7 @@ class TestExtractEntries:
readme = textwrap.dedent("""\
# T
---
## Projects
**Tools**

88
website/tests/test_readme_parser.py
View file

@ -81,35 +81,35 @@ MINIMAL_README = textwrap.dedent("""\
Some intro text.
---
## Projects
## Alpha
### Alpha
_Libraries for alpha stuff._
- [lib-a](https://example.com/a) - Does A.
- [lib-b](https://example.com/b) - Does B.
## Beta
### Beta
_Tools for beta._
- [lib-c](https://example.com/c) - Does C.
# Resources
## Resources
Where to discover resources.
## Newsletters
### Newsletters
- [News One](https://example.com/n1)
- [News Two](https://example.com/n2)
## Podcasts
### Podcasts
- [Pod One](https://example.com/p1)
# Contributing
## Contributing
Please contribute!
""")
@ -120,11 +120,11 @@ GROUPED_README = textwrap.dedent("""\
Some intro text.
---
## Projects
**Group One**
## Alpha
### Alpha
_Libraries for alpha stuff._
@ -133,25 +133,25 @@ GROUPED_README = textwrap.dedent("""\
**Group Two**
## Beta
### Beta
_Tools for beta._
- [lib-c](https://example.com/c) - Does C.
## Gamma
### Gamma
- [lib-d](https://example.com/d) - Does D.
# Resources
## Resources
Where to discover resources.
## Newsletters
### Newsletters
- [News One](https://example.com/n1)
# Contributing
## Contributing
Please contribute!
""")
@ -191,7 +191,7 @@ class TestParseReadmeSections:
all_names.extend(c["name"] for c in g["categories"])
assert "Contributing" not in all_names
def test_no_separator(self):
def test_no_projects_heading(self):
groups = parse_readme("# Just a heading\n\nSome text.\n")
assert groups == []
@ -199,19 +199,19 @@ class TestParseReadmeSections:
readme = textwrap.dedent("""\
# Title
---
## Projects
## NullDesc
### NullDesc
- [item](https://x.com) - Thing.
# Resources
## Resources
## Tips
### Tips
- [tip](https://x.com)
# Contributing
## Contributing
Done.
""")
@ -225,15 +225,15 @@ class TestParseReadmeSections:
readme = textwrap.dedent("""\
# T
---
## Projects
## Algos
### Algos
_Algorithms. Also see [awesome-algos](https://example.com)._
- [lib](https://x.com) - Lib.
# Contributing
## Contributing
Done.
""")
@ -273,17 +273,17 @@ class TestParseGroupedReadme:
readme = textwrap.dedent("""\
# T
---
## Projects
**Empty**
**HasCats**
## Cat
### Cat
- [x](https://x.com) - X.
# Contributing
## Contributing
Done.
""")
@ -295,15 +295,15 @@ class TestParseGroupedReadme:
readme = textwrap.dedent("""\
# T
---
## Projects
**Note:** This is not a group marker.
## Cat
### Cat
- [x](https://x.com) - X.
# Contributing
## Contributing
Done.
""")
@ -317,19 +317,19 @@ class TestParseGroupedReadme:
readme = textwrap.dedent("""\
# T
---
## Projects
## Orphan
### Orphan
- [x](https://x.com) - X.
**A Group**
## Grouped
### Grouped
- [y](https://x.com) - Y.
# Contributing
## Contributing
Done.
""")
@ -405,15 +405,15 @@ class TestParseSectionEntries:
readme = textwrap.dedent("""\
# T
---
## Projects
## Async
### Async
- [asyncio](https://x.com) - Async I/O.
- [awesome-asyncio](https://y.com)
- [trio](https://z.com) - Friendly async.
# Contributing
## Contributing
Done.
""")
@ -480,21 +480,21 @@ class TestParseRealReadme:
md = MarkdownIt("commonmark")
root = SyntaxTreeNode(md.parse(self.readme_text))
# Find category section boundaries (between --- and # Resources/Contributing)
hr_idx = None
# Find category section boundaries (between Projects and Resources/Contributing)
projects_idx = None
end_idx = None
for i, node in enumerate(root.children):
if hr_idx is None and node.type == "hr":
hr_idx = i
elif node.type == "heading" and node.tag == "h1":
if node.type == "heading" and node.tag in ("h1", "h2"):
text = render_inline_text(node.children[0].children) if node.children else ""
if end_idx is None and text in ("Resources", "Contributing"):
if projects_idx is None and text == "Projects":
projects_idx = i
elif end_idx is None and text in ("Resources", "Contributing"):
end_idx = i
if hr_idx is None:
if projects_idx is None:
return
bad = []
cat_nodes = root.children[hr_idx + 1 : end_idx or len(root.children)]
cat_nodes = root.children[projects_idx + 1 : end_idx or len(root.children)]
for node in cat_nodes:
if node.type != "bullet_list":
continue