refactor: removed question-directus download

build.py

@@ -4,33 +4,23 @@ import src.config as config
 import src.download as download
 
 conf = config.Config()
-api = directus.DirectusAPI(conf.question_base_url, conf.question_access_token)
 content_api = directus.DirectusAPI(conf.content_base_url, conf.content_access_token)
-dl = download.Download(api, content_api, conf)
+dl = download.Download(content_api, conf)
-bd = build.Build(conf)
 
-# Download the text files:
+# Download the content files:
-dl.download_edition("N")
+dl.download_git_content()
-dl.download_edition("E")
-dl.download_edition("A")
-dl.download_edition("NE")
-dl.download_edition("EA")
-dl.download_edition("NEA")
 
 # Download additional files:
-# dl.download_question_metadata() # (now constant file metadata3b.json)
-dl.download_photos()
-dl.download_pictures()
-dl.download_includes()
 dl.download_snippets()
 dl.download_content()
 
-# Build Everything:
+## Build Everything:
-bd.build_website()
+bd = build.Build(conf)
-bd.build_edition("N")
+# bd.build_website()
-bd.build_edition("E")
+# bd.build_edition("N")
-bd.build_edition("A")
+# bd.build_edition("E")
-bd.build_edition("NE")
+# bd.build_edition("A")
-bd.build_edition("EA")
+# bd.build_edition("NE")
-bd.build_edition("NEA")
+# bd.build_edition("EA")
-bd.build_assets()
+# bd.build_edition("NEA")
+# bd.build_assets()

config/config.json.example

@@ -1,7 +1,5 @@
 {
-  "directus_question_base_url" : "https://fragenkatalog.darc.de/",
-  "directus_question_api_key" : "",
   "directus_content_base_url" : "https://redaktion.50ohm.de/",
   "directus_content_api_key" : "",
-  "no_latex": true
+  "git_url" : "[email protected]:DARC-e-V/50ohm-contents-dl.git",
 }

src/config.py

@@ -13,17 +13,17 @@ class Config:
         else:
             self.config = {}
 
-        self.question_access_token = self.get_config_value("directus_question_api_key")
-        self.question_base_url = self.get_config_value("directus_question_base_url", "https://fragenkatalog.darc.de/")
         self.content_access_token = self.get_config_value("directus_content_api_key")
         self.content_base_url = self.get_config_value("directus_content_base_url", "https://redaktion.50ohm.de/")
-        self.no_latex = self.get_config_value("no_latex", False)
+        self.git_url = self.get_config_value("git_url", "[email protected]:DARC-e-V/50ohm-contents-dl.git")
 
-        self.p_fragenkatalog = Path(self.get_config_value("path_fragenkatalog", "./data/fragenkatalog3b.json"))
+        self.p_fragenkatalog = Path(
+            self.get_config_value("path_fragenkatalog", "./data/git_content/contents/questions/fragenkatalog3b.json")
+        )
 
         self.p_data = Path("./data")
-        self.p_data_photos = self.p_data / "photos"
+        self.p_data_photos = self.p_data / "git_content" / "photos"
-        self.p_data_pictures = self.p_data / "pictures"
+        self.p_data_pictures = self.p_data / "git_content" / "pictures"
 
         self.p_build = Path("./build")
         self.p_build_photos = self.p_build / "photos"

src/download.py

@@ -1,5 +1,6 @@
 import json
 
+from git import Repo
 from tqdm import tqdm
 
 from api.directus import DirectusAPI
@@ -8,136 +9,10 @@ from .config import Config
 
 
 class Download:
-    def __init__(self, api: DirectusAPI, content_api: DirectusAPI, config: Config):
+    def __init__(self, content_api: DirectusAPI, config: Config):
-        self.api = api
         self.config = config
         self.content_api = content_api
 
-    def download_edition(self, edition):
-        in_edition = self.api.get_one("items/edition", {"filter": {"ident": {"_eq": edition}}})
-
-        in_chapters = self.api.get(
-            "items/chapter",
-            {"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
-        )
-
-        out_chapters = []
-
-        for in_chapter in tqdm(in_chapters, desc="Downloading edition"):
-            in_sections = self.api.get(
-                "items/section",
-                {"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
-            )
-
-            out_chapters.append(
-                {
-                    "title": in_chapter["title"],
-                    "abstract": in_chapter["abstract"],
-                    "ident": in_chapter["ident"],
-                    "video_url": in_chapter["video_url"],
-                    "sections": [],
-                }
-            )
-
-            for in_section in in_sections:
-                out_chapters[-1]["sections"].append(
-                    {
-                        "title": in_section["title"],
-                        "ident": in_section["ident"],
-                        "status": in_section["status"],
-                        "class": in_section["class"],
-                        "content": in_section["content"],
-                        "slide": in_section["slide"],
-                        "video_url": in_section["video_url"],
-                        "questions": [],
-                    }
-                )
-
-            out_edition = {
-                "title": in_edition["title"],
-                "abstract": in_edition["abstract"],
-                "chapters": out_chapters,
-                "edition": edition,
-            }
-
-            with (self.config.p_data / f"book_{edition}.json").open("w", encoding="utf-8") as file:
-                json.dump(out_edition, file, ensure_ascii=False, indent=4)
-
-    def download_question_metadata(self):
-        questions = self.api.get(
-            "items/questions", params={"filter": {"class_3": {"_in": [1, 2, 3]}}, "sort": "position", "limit": -1}
-        )
-
-        result = {}
-
-        for question in tqdm(questions, desc="Downloading metadata"):
-            result[question["id"]] = {
-                "number": question["number"],
-                "picture_question": question["picture_question"] if question["picture_question"] is not None else "",
-                "picture_a": question["picture_a"] if question["picture_a"] is not None else "",
-                "picture_b": question["picture_b"] if question["picture_b"] is not None else "",
-                "picture_c": question["picture_c"] if question["picture_c"] is not None else "",
-                "picture_d": question["picture_d"] if question["picture_d"] is not None else "",
-                "layout": question["layout"] if question["layout"] is not None else "",
-            }
-
-        # Serializing json:
-        with (self.config.p_data / "metadata.json").open("w", encoding="utf-8") as f:
-            json.dump(result, f, indent=4)
-        f.close()
-
-    def download_photos(self):
-        self.config.p_data_photos.mkdir(parents=True, exist_ok=True)
-
-        photos = self.api.get("items/Fotos", params={"limit": -1})
-
-        for photo in tqdm(photos, desc="Downloading photos"):
-            # Save Photo to file
-            data = self.api.get_file("assets/" + photo["photo"])
-            with (self.config.p_data_photos / f"{photo['id']}.jpg").open("wb") as file:
-                file.write(data)
-
-            # Save Alt-Text to file
-            prefix = ""
-            if photo["alt_text"] is not None:
-                if photo["alt_text_reviewed"] is None:
-                    prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
-                with (self.config.p_data_photos / f"{photo['id']}.txt").open("w") as file:
-                    file.write(prefix + photo["alt_text"])
-
-    def download_pictures(self):
-        self.config.p_data_pictures.mkdir(parents=True, exist_ok=True)
-
-        pictures = self.api.get("items/pictures", {"limit": -1})
-
-        for picture in tqdm(pictures, desc="Downloading pictures"):
-            # Save LaTeX sources to file
-            with (self.config.p_data_pictures / f"{picture['id']}include.tex").open("w") as file:
-                file.write(picture["latex"])
-
-            # Save Alt-Text to file
-            prefix = ""
-            if picture["alt_text"] is not None:
-                if picture["alt_text_reviewed"] is None:
-                    prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
-                with (self.config.p_data_pictures / f"{picture['id']}.txt").open("w") as file:
-                    file.write(prefix + picture["alt_text"])
-
-            # Download SVG
-            data = self.api.get_file(f"assets/{picture['picture']}")
-            with (self.config.p_data_pictures / f"{picture['id']}.svg").open("wb") as file:
-                file.write(data)
-
-    def download_includes(self):
-        includes = self.api.get("items/include", params={"limit": -1})
-
-        result = {}
-        for include in tqdm(includes, desc="Downloading includes"):
-            result[include["ident"]] = include["content"]
-
-            with (self.config.p_data / "includes.json").open("w", encoding="utf-8") as file:
-                json.dump(result, file, indent=4)
-
     def download_snippets(self):
         snippets = {}
         for snippet in tqdm(self.content_api.get("items/snippet"), desc="Downloading snippets"):
@@ -160,3 +35,15 @@ class Download:
 
         with (self.config.p_data / "content.json").open("w", encoding="utf-8") as file:
             json.dump(contents, file, ensure_ascii=False, indent=4)
+
+    def download_git_content(self):
+        repo_path = self.config.p_data / "git_content"
+        if not repo_path.exists():
+            print("Cloning git repository, this may take a while ...")
+            Repo.clone_from(self.config.git_url, repo_path)
+            print("... done.")
+        else:
+            repo = Repo(repo_path)
+            origin = repo.remotes.origin
+            origin.pull()
+            print("Repository updated.")