refactor: removed question-directus download
This commit is contained in:
Matthias Jung
@@ -4,33 +4,23 @@ import src.config as config
|
||||
import src.download as download
|
||||
|
||||
conf = config.Config()
|
||||
api = directus.DirectusAPI(conf.question_base_url, conf.question_access_token)
|
||||
content_api = directus.DirectusAPI(conf.content_base_url, conf.content_access_token)
|
||||
dl = download.Download(api, content_api, conf)
|
||||
bd = build.Build(conf)
|
||||
dl = download.Download(content_api, conf)
|
||||
|
||||
# Download the text files:
|
||||
dl.download_edition("N")
|
||||
dl.download_edition("E")
|
||||
dl.download_edition("A")
|
||||
dl.download_edition("NE")
|
||||
dl.download_edition("EA")
|
||||
dl.download_edition("NEA")
|
||||
# Download the content files:
|
||||
dl.download_git_content()
|
||||
|
||||
# Download additional files:
|
||||
# dl.download_question_metadata() # (now constant file metadata3b.json)
|
||||
dl.download_photos()
|
||||
dl.download_pictures()
|
||||
dl.download_includes()
|
||||
dl.download_snippets()
|
||||
dl.download_content()
|
||||
|
||||
# Build Everything:
|
||||
bd.build_website()
|
||||
bd.build_edition("N")
|
||||
bd.build_edition("E")
|
||||
bd.build_edition("A")
|
||||
bd.build_edition("NE")
|
||||
bd.build_edition("EA")
|
||||
bd.build_edition("NEA")
|
||||
bd.build_assets()
|
||||
## Build Everything:
|
||||
bd = build.Build(conf)
|
||||
# bd.build_website()
|
||||
# bd.build_edition("N")
|
||||
# bd.build_edition("E")
|
||||
# bd.build_edition("A")
|
||||
# bd.build_edition("NE")
|
||||
# bd.build_edition("EA")
|
||||
# bd.build_edition("NEA")
|
||||
# bd.build_assets()
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
{
|
||||
"directus_question_base_url" : "https://fragenkatalog.darc.de/",
|
||||
"directus_question_api_key" : "",
|
||||
"directus_content_base_url" : "https://redaktion.50ohm.de/",
|
||||
"directus_content_api_key" : "",
|
||||
"no_latex": true
|
||||
"git_url" : "[email protected]:DARC-e-V/50ohm-contents-dl.git",
|
||||
}
|
||||
|
||||
+6
-6
@@ -13,17 +13,17 @@ class Config:
|
||||
else:
|
||||
self.config = {}
|
||||
|
||||
self.question_access_token = self.get_config_value("directus_question_api_key")
|
||||
self.question_base_url = self.get_config_value("directus_question_base_url", "https://fragenkatalog.darc.de/")
|
||||
self.content_access_token = self.get_config_value("directus_content_api_key")
|
||||
self.content_base_url = self.get_config_value("directus_content_base_url", "https://redaktion.50ohm.de/")
|
||||
self.no_latex = self.get_config_value("no_latex", False)
|
||||
self.git_url = self.get_config_value("git_url", "[email protected]:DARC-e-V/50ohm-contents-dl.git")
|
||||
|
||||
self.p_fragenkatalog = Path(self.get_config_value("path_fragenkatalog", "./data/fragenkatalog3b.json"))
|
||||
self.p_fragenkatalog = Path(
|
||||
self.get_config_value("path_fragenkatalog", "./data/git_content/contents/questions/fragenkatalog3b.json")
|
||||
)
|
||||
|
||||
self.p_data = Path("./data")
|
||||
self.p_data_photos = self.p_data / "photos"
|
||||
self.p_data_pictures = self.p_data / "pictures"
|
||||
self.p_data_photos = self.p_data / "git_content" / "photos"
|
||||
self.p_data_pictures = self.p_data / "git_content" / "pictures"
|
||||
|
||||
self.p_build = Path("./build")
|
||||
self.p_build_photos = self.p_build / "photos"
|
||||
|
||||
+14
-127
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
|
||||
from git import Repo
|
||||
from tqdm import tqdm
|
||||
|
||||
from api.directus import DirectusAPI
|
||||
@@ -8,136 +9,10 @@ from .config import Config
|
||||
|
||||
|
||||
class Download:
|
||||
def __init__(self, api: DirectusAPI, content_api: DirectusAPI, config: Config):
|
||||
self.api = api
|
||||
def __init__(self, content_api: DirectusAPI, config: Config):
|
||||
self.config = config
|
||||
self.content_api = content_api
|
||||
|
||||
def download_edition(self, edition):
|
||||
in_edition = self.api.get_one("items/edition", {"filter": {"ident": {"_eq": edition}}})
|
||||
|
||||
in_chapters = self.api.get(
|
||||
"items/chapter",
|
||||
{"filter": {"edition": {"edition_id": {"_eq": in_edition["id"]}}}},
|
||||
)
|
||||
|
||||
out_chapters = []
|
||||
|
||||
for in_chapter in tqdm(in_chapters, desc="Downloading edition"):
|
||||
in_sections = self.api.get(
|
||||
"items/section",
|
||||
{"filter": {"chapter": {"chapter_id": {"_eq": in_chapter["id"]}}}},
|
||||
)
|
||||
|
||||
out_chapters.append(
|
||||
{
|
||||
"title": in_chapter["title"],
|
||||
"abstract": in_chapter["abstract"],
|
||||
"ident": in_chapter["ident"],
|
||||
"video_url": in_chapter["video_url"],
|
||||
"sections": [],
|
||||
}
|
||||
)
|
||||
|
||||
for in_section in in_sections:
|
||||
out_chapters[-1]["sections"].append(
|
||||
{
|
||||
"title": in_section["title"],
|
||||
"ident": in_section["ident"],
|
||||
"status": in_section["status"],
|
||||
"class": in_section["class"],
|
||||
"content": in_section["content"],
|
||||
"slide": in_section["slide"],
|
||||
"video_url": in_section["video_url"],
|
||||
"questions": [],
|
||||
}
|
||||
)
|
||||
|
||||
out_edition = {
|
||||
"title": in_edition["title"],
|
||||
"abstract": in_edition["abstract"],
|
||||
"chapters": out_chapters,
|
||||
"edition": edition,
|
||||
}
|
||||
|
||||
with (self.config.p_data / f"book_{edition}.json").open("w", encoding="utf-8") as file:
|
||||
json.dump(out_edition, file, ensure_ascii=False, indent=4)
|
||||
|
||||
def download_question_metadata(self):
|
||||
questions = self.api.get(
|
||||
"items/questions", params={"filter": {"class_3": {"_in": [1, 2, 3]}}, "sort": "position", "limit": -1}
|
||||
)
|
||||
|
||||
result = {}
|
||||
|
||||
for question in tqdm(questions, desc="Downloading metadata"):
|
||||
result[question["id"]] = {
|
||||
"number": question["number"],
|
||||
"picture_question": question["picture_question"] if question["picture_question"] is not None else "",
|
||||
"picture_a": question["picture_a"] if question["picture_a"] is not None else "",
|
||||
"picture_b": question["picture_b"] if question["picture_b"] is not None else "",
|
||||
"picture_c": question["picture_c"] if question["picture_c"] is not None else "",
|
||||
"picture_d": question["picture_d"] if question["picture_d"] is not None else "",
|
||||
"layout": question["layout"] if question["layout"] is not None else "",
|
||||
}
|
||||
|
||||
# Serializing json:
|
||||
with (self.config.p_data / "metadata.json").open("w", encoding="utf-8") as f:
|
||||
json.dump(result, f, indent=4)
|
||||
f.close()
|
||||
|
||||
def download_photos(self):
|
||||
self.config.p_data_photos.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
photos = self.api.get("items/Fotos", params={"limit": -1})
|
||||
|
||||
for photo in tqdm(photos, desc="Downloading photos"):
|
||||
# Save Photo to file
|
||||
data = self.api.get_file("assets/" + photo["photo"])
|
||||
with (self.config.p_data_photos / f"{photo['id']}.jpg").open("wb") as file:
|
||||
file.write(data)
|
||||
|
||||
# Save Alt-Text to file
|
||||
prefix = ""
|
||||
if photo["alt_text"] is not None:
|
||||
if photo["alt_text_reviewed"] is None:
|
||||
prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
|
||||
with (self.config.p_data_photos / f"{photo['id']}.txt").open("w") as file:
|
||||
file.write(prefix + photo["alt_text"])
|
||||
|
||||
def download_pictures(self):
|
||||
self.config.p_data_pictures.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pictures = self.api.get("items/pictures", {"limit": -1})
|
||||
|
||||
for picture in tqdm(pictures, desc="Downloading pictures"):
|
||||
# Save LaTeX sources to file
|
||||
with (self.config.p_data_pictures / f"{picture['id']}include.tex").open("w") as file:
|
||||
file.write(picture["latex"])
|
||||
|
||||
# Save Alt-Text to file
|
||||
prefix = ""
|
||||
if picture["alt_text"] is not None:
|
||||
if picture["alt_text_reviewed"] is None:
|
||||
prefix = "Der folgende Alt-Text wurde noch nicht geprüft: "
|
||||
with (self.config.p_data_pictures / f"{picture['id']}.txt").open("w") as file:
|
||||
file.write(prefix + picture["alt_text"])
|
||||
|
||||
# Download SVG
|
||||
data = self.api.get_file(f"assets/{picture['picture']}")
|
||||
with (self.config.p_data_pictures / f"{picture['id']}.svg").open("wb") as file:
|
||||
file.write(data)
|
||||
|
||||
def download_includes(self):
|
||||
includes = self.api.get("items/include", params={"limit": -1})
|
||||
|
||||
result = {}
|
||||
for include in tqdm(includes, desc="Downloading includes"):
|
||||
result[include["ident"]] = include["content"]
|
||||
|
||||
with (self.config.p_data / "includes.json").open("w", encoding="utf-8") as file:
|
||||
json.dump(result, file, indent=4)
|
||||
|
||||
def download_snippets(self):
|
||||
snippets = {}
|
||||
for snippet in tqdm(self.content_api.get("items/snippet"), desc="Downloading snippets"):
|
||||
@@ -160,3 +35,15 @@ class Download:
|
||||
|
||||
with (self.config.p_data / "content.json").open("w", encoding="utf-8") as file:
|
||||
json.dump(contents, file, ensure_ascii=False, indent=4)
|
||||
|
||||
def download_git_content(self):
|
||||
repo_path = self.config.p_data / "git_content"
|
||||
if not repo_path.exists():
|
||||
print("Cloning git repository, this may take a while ...")
|
||||
Repo.clone_from(self.config.git_url, repo_path)
|
||||
print("... done.")
|
||||
else:
|
||||
repo = Repo(repo_path)
|
||||
origin = repo.remotes.origin
|
||||
origin.pull()
|
||||
print("Repository updated.")
|
||||
|
||||
Reference in New Issue
Block a user