From 81edf08d8ef145db63d715051351838e576acd41 Mon Sep 17 00:00:00 2001 From: Carl Vargklint Date: Sat, 10 Oct 2026 15:33:37 +0200 Subject: [PATCH] Add noindex pages, robots.txt and operator footer - Page.noindex: marked pages get a robots meta tag and a Disallow line in the generated robots.txt (single source of truth in src/pages.py) - templates/robots.txt: public pages indexable, noindex pages and data files not - Footer shows an optional operator line from contents/snippets/footer_operator.md - Tests for robots.txt, noindex meta and footer Co-Authored-By: Claude Sonnet 5.5 --- src/build.py | 24 ++++++++++++++++---- src/pages.py | 6 +++++ templates/html/page.html | 6 +++-- templates/robots.txt | 11 +++++++++ test/test_robots.py | 48 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 templates/robots.txt create mode 100644 test/test_robots.py diff --git a/src/build.py b/src/build.py index c24ec87..5c3d0a1 100644 --- a/src/build.py +++ b/src/build.py @@ -17,7 +17,7 @@ from renderer.index import Index from .config import Config from .edition import Edition -from .pages import EXAM_RESULT, EXAM_SIMULATOR, HOMEWORK, PAGES, page_urls +from .pages import EXAM_RESULT, EXAM_SIMULATOR, HOMEWORK, NOINDEX_DIRECTIVES, PAGES, page_urls class Build: @@ -149,6 +149,7 @@ class Build: page_stylesheets=None, page_scripts=None, social_image=None, + robots=None, ): page_template = self.env.get_template("html/page.html") return page_template.render( @@ -158,6 +159,7 @@ class Build: page_stylesheets=page_stylesheets or [], page_scripts=page_scripts or [], social_image=social_image, + robots=robots, ) def __build_exam_simulator(self): @@ -586,11 +588,15 @@ class Build: result = self.__build_page(result) file.write(result) - def __build_html_page(self, contents, page): + def __build_html_page(self, contents, page, noindex=False): for content in contents: if content["url_part"] == page: with (self.config.p_build / f"{page}.html").open("w", encoding="utf-8") as file: - result = self.__build_page(content=content["content"], sidebar=content["sidebar"]) + result = self.__build_page( + content=content["content"], + sidebar=content["sidebar"], + robots=NOINDEX_DIRECTIVES if noindex else None, + ) file.write(result) return True return False @@ -600,14 +606,24 @@ class Build: snippets = self.__parse_snippets() contents = self.__parse_contents() + # Operator line in the footer of every page (contents/snippets/footer_operator.md, optional). + self.env.globals["footer_operator"] = snippets.get("footer_operator", "") self.__build_index(snippets) for page in PAGES: # Prefer the static page of the content repository, fall back to the generic template. - if not self.__build_html_page(contents, page.name) and page.template is not None: + if not self.__build_html_page(contents, page.name, page.noindex) and page.template is not None: self.__build_course_page(snippets, page.template, page.name) + self.__build_robots() self.__build_exam_simulator() self.__build_homework_tool() + def __build_robots(self): + """robots.txt: the public site may be indexed, pages marked noindex (contact details) may not be crawled.""" + template = self.env.get_template("robots.txt") + noindex_pages = [f"{page.name}.html" for page in PAGES if page.noindex] + with (self.config.p_build / "robots.txt").open("w", encoding="utf-8") as file: + file.write(template.render(noindex_pages=noindex_pages)) + def build_solutions(self): if not self.has_question_pool: return diff --git a/src/pages.py b/src/pages.py index 16d50d3..8aa34a1 100644 --- a/src/pages.py +++ b/src/pages.py @@ -14,6 +14,8 @@ class Page: # Template in templates/html/ used as fallback if the content repository # has no static page with this name. template: str | None = None + # Never index this page: robots meta tag + Disallow in robots.txt (e.g. contact details). + noindex: bool = False # Pages whose body comes from contents/static/.html. @@ -24,8 +26,12 @@ PAGES = ( Page("kurser_karta", template="kurser-karta"), Page("kurser_lista", template="kurser-lista"), Page("mentorer", template="mentorer"), + Page("kontakt", noindex=True), ) +# Robots directives for pages marked noindex (meta tag; the nginx configs send the same as X-Robots-Tag). +NOINDEX_DIRECTIVES = "noindex, nofollow, noarchive, nosnippet" + # Pages generated by the generator itself. The last two only exist if the # content repository contains a question pool. INDEX = "index" diff --git a/templates/html/page.html b/templates/html/page.html index 6290d1f..3e12157 100644 --- a/templates/html/page.html +++ b/templates/html/page.html @@ -4,6 +4,9 @@ 50Ω – lär dig amatörradio + {% if robots %} + + {% endif %} {% if social_image %} @@ -153,8 +156,7 @@ Inte godkänd av DARC eller SSA.
- {# TODO: The operator must add their own imprint / contact details and privacy information here. #} - [Utgivare, kontakt och integritetspolicy – ska fyllas i av driftansvarig] + {% if footer_operator %}{{ footer_operator }}{% endif %}
diff --git a/templates/robots.txt b/templates/robots.txt new file mode 100644 index 0000000..427ddc7 --- /dev/null +++ b/templates/robots.txt @@ -0,0 +1,11 @@ +# robots.txt – generated by the site generator, do not edit by hand. +# The course pages may be crawled and indexed. Pages with contact details may not. + +User-agent: * +Allow: / +{% for page in noindex_pages -%} +Disallow: /{{ page }} +{% endfor -%} +# Data files (search indexes etc.) are not pages +Disallow: /*.json$ +Disallow: /*.zip$ diff --git a/test/test_robots.py b/test/test_robots.py new file mode 100644 index 0000000..9803fb5 --- /dev/null +++ b/test/test_robots.py @@ -0,0 +1,48 @@ +from src.build import Build +from src.config import Config +from src.pages import NOINDEX_DIRECTIVES + + +def write_site(tmp_path): + (tmp_path / "toc").mkdir() + (tmp_path / "contents" / "static").mkdir(parents=True) + (tmp_path / "contents" / "snippets").mkdir(parents=True) + (tmp_path / "contents" / "static" / "kontakt.html").write_text( + "

Kontakt

hemlig@example.se", encoding="utf-8" + ) + (tmp_path / "contents" / "static" / "info.html").write_text("

Info

", encoding="utf-8") + (tmp_path / "contents" / "snippets" / "footer_operator.md").write_text( + 'Ansvarig: Test (SA0TST) · Kontakt\n', encoding="utf-8" + ) + + +def test_contact_page_is_noindex_and_listed_in_robots_txt(tmp_path): + write_site(tmp_path) + Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website() + out = tmp_path / "build" + + robots = (out / "robots.txt").read_text(encoding="utf-8") + assert "User-agent: *" in robots + assert "Allow: /\n" in robots + assert "Disallow: /kontakt.html" in robots + assert "Disallow: /info.html" not in robots # normal pages stay indexable + + assert f'' in (out / "kontakt.html").read_text(encoding="utf-8") + assert 'name="robots"' not in (out / "info.html").read_text(encoding="utf-8") + + +def test_footer_names_operator_without_contact_details(tmp_path): + write_site(tmp_path) + Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website() + info = (tmp_path / "build" / "info.html").read_text(encoding="utf-8") + + assert "Ansvarig: Test (SA0TST)" in info + assert 'href="kontakt.html"' in info + assert "hemlig@example.se" not in info + + +def test_no_footer_operator_without_snippet(tmp_path): + write_site(tmp_path) + (tmp_path / "contents" / "snippets" / "footer_operator.md").unlink() + Build(Config(content_path=tmp_path, build_path=tmp_path / "build")).build_website() + assert "Ansvarig" not in (tmp_path / "build" / "info.html").read_text(encoding="utf-8")