generate category

README.md

@@ -1,4 +1,4 @@
-# gawa
+# Gawa
 
 Gawa is my personal website. I've personally written it using the Django framework.
 
@@ -7,21 +7,28 @@ Gawa is my personal website. I've personally written it using the Django framewo
 These are the Credentials for logging into the admin panel:
 | Username | Password |
 |----------|----------|
-| root     | root     |
+| `root`   | `root`   |
 
 ### Blog
 
-| Username           | Password     |
-|--------------------|--------------|
-| contact@cscherr.de | hrCcDa0jBspG |
+| Username                                         | Password       |
+|--------------------------------------------------|----------------|
+| [`contact@cscherr.de`](mailto:contact@cscherr.de) | `hrCcDa0jBspG` |
 
 ## License
 
-Bootstrap:  MIT Licensed  
-Django:     BSD 3-Clause "New" or "Revised" License  
+Bootstrap:  MIT Licensed
+Django:     BSD 3-Clause "New" or "Revised" License
 
 __Gawa: MIT Licensed, see LICENSE__
 
+## Dependencies
+
+| Description    | Package (fedora) |
+|----------------|------------------|
+| Database stuff | `libpq-devel`    |
+| Database stuff | `mariadb`        |
+
 ## Security
 
 - [ ] Do something about the files in the blog dir

gawa/blog/models.py

@@ -1,3 +1,8 @@
+import ast
+import re
+import os
+import pathlib
+import markdown
 from django.db import models
 from django.utils.translation import gettext as _
 from start.models import Searchable
@@ -5,7 +10,6 @@ from start.models import Searchable
 import logging
 logger = logging.getLogger(__name__)
 
-import markdown
 EXTENSIONS = [
     "extra",
     "admonition",
@@ -20,11 +24,9 @@ EXTENSION_CONFIGS = {
     },
 }
 
-MD = markdown.Markdown(extensions=EXTENSIONS, extension_configs=EXTENSION_CONFIGS)
+MD = markdown.Markdown(extensions=EXTENSIONS,
+                       extension_configs=EXTENSION_CONFIGS)
 
-import pathlib
-import os
-import re
 
 class Category(models.Model):
     """
@@ -33,8 +35,8 @@ class Category(models.Model):
     Name not translated because it would make i18n in urls and Searchables specifically a pain.
     Maybe some day it would be cool if these were Searchable
     """
-    name= models.CharField(max_length=50)
-    slug = models.SlugField()
+    name = models.CharField(max_length=50)
+    slug = models.SlugField(unique=True)
 
     class Meta:
         verbose_name = _("Category")
@@ -43,16 +45,19 @@ class Category(models.Model):
     def __str__(self):
         return f"{{<{self.__class__.__name__}>\"{self.name}\"}}"
 
+
 class BlogPost(Searchable):
     """
     Should contain a blogpost
     """
     body_en = models.TextField(blank=True, default="")
     body_de = models.TextField(blank=True, default="")
-    category = models.ForeignKey(Category, on_delete=models.SET_NULL, null=True)
+    category = models.ForeignKey(
+        Category, on_delete=models.SET_NULL, null=True)
     thumbnail = models.ImageField(blank=True, upload_to="img/thumbnails")
     featured = models.BooleanField(default=False)
-    langs = models.CharField(default="['en': False, 'de': False]", max_length=64)
+    langs = models.CharField(
+        default="['en': False, 'de': False]", max_length=64)
     slug = models.SlugField()
 
     # TODO autodiscover new blog posts based on markdown files?
@@ -88,21 +93,38 @@ class BlogPost(Searchable):
 
                 html_en: str = MD.convert(body_en)
                 try:
+                    # NOTE: MD.Meta is generated after MD.convert() by the meta
+                    # extension.
+                    if not hasattr(MD, 'Meta'):
+                        logger.error("Metadata extension for markdown\
+                        not loaded")
+                        raise ValueError("Metadata extension for markdown\
+                        not loaded")
                     meta_en = MD.Meta
                     self.title_en = meta_en["title"][0]
                     self.subtitle_en = meta_en["subtitle"][0]
                     self.desc_en = meta_en["desc"][0]
-                    # TODO: parse date from markdown
+                    self.date = meta_en["date"][0]
                     self.featured = meta_en["featured"][0] == "True"
                     self.public = meta_en["public"][0] == "True"
                     # self.thumbnail = meta_en["thumbnail"]
                     # TODO: parse keywords from markdown
                     # TODO: parse category from markdown
 
+                    try:
+                        category: Category = Category.objects.get(
+                            slug=meta_en['category'][0])
+                    except Category.DoesNotExist:
+                        category = Category.objects.create(
+                            name=meta_en['category'], slug=meta_en['category'])
+                    logger.debug(f"category of {self}: {category}")
+                    self.category = category
+
                     # if keyword or category do not exist, create them
                     # I suppose
                 except Exception as e:
-                    logger.warning(f"could not generate metadata {self.slug} from markdown: {e}")
+                    logger.warning(
+                        f"could not generate metadata {self.slug} from markdown: {e}")
 
                 self.body_en = ""
                 self.body_en = html_en
@@ -110,7 +132,8 @@ class BlogPost(Searchable):
             # TODO: mark as untranslated
             pass
         except Exception as e:
-            logger.warning(f"could not generate article {self.slug} from markdown: {e}")
+            logger.warning(
+                f"could not generate article {self.slug} from markdown: {e}")
         try:
             MD.reset()
             with open(f"{self.DATA_DIR}/de-{self.slug}.md") as f_de:
@@ -119,6 +142,13 @@ class BlogPost(Searchable):
 
                 html_de: str = MD.convert(body_de)
                 try:
+                    # NOTE: MD.Meta is generated after MD.convert() by the meta
+                    # extension.
+                    if not hasattr(MD, 'Meta'):
+                        logger.error("Metadata extension for markdown\
+                        not loaded")
+                        raise ValueError("Metadata extension for markdown\
+                        not loaded")
                     meta_de = MD.Meta
                     self.title_de = meta_de["title"][0]
                     self.subtitle_de = meta_de["subtitle"][0]
@@ -133,7 +163,8 @@ class BlogPost(Searchable):
                     # if keyword or category do not exist, create them
                     # I suppose
                 except Exception as e:
-                    logger.warning(f"could not generate metadata {self.slug} from markdown: {e}")
+                    logger.warning(
+                        f"could not generate metadata {self.slug} from markdown: {e}")
 
                 self.body_de = ""
                 self.body_de = html_de
@@ -141,9 +172,10 @@ class BlogPost(Searchable):
             # TODO: mark as untranslated
             pass
         except Exception as e:
-            logger.warning(f"could not generate article {self.slug} from markdown: {e}")
+            logger.warning(
+                f"could not generate article {self.slug} from markdown: {e}")
 
-    def get_langs(self) -> dict[str, bool]:
+    def get_langs(self) -> dict[str, bool] | None:
         """
         get available languages
         """
@@ -152,7 +184,13 @@ class BlogPost(Searchable):
         # SECURITY:
         # If someone could inject the langs field, arbitrary python code might
         # run, Potentially ending in a critical RCE vulnerability
-        return eval(str(self.langs))
+        try:
+            langs = ast.literal_eval(str(self.langs))
+            return langs
+        except ValueError as e:
+            logger.error(
+                f"could not safely evaluate 'langs' for '{self}': {e}")
+            return None
 
     def set_langs(self, langs: dict[str, bool]):
         """
@@ -179,7 +217,8 @@ class BlogPost(Searchable):
         if not data_dir.is_dir():
             logger.error(f"'{cls.DATA_DIR} is not a directory'")
 
-        files = [f for f in os.listdir(data_dir) if (data_dir.joinpath(f)).is_file()]
+        files = [f for f in os.listdir(data_dir) if (
+            data_dir.joinpath(f)).is_file()]
         logger.debug(f"discovered files: {files}")
 
         # finding lang and title
@@ -191,9 +230,15 @@ class BlogPost(Searchable):
             # parse file name
             try:
                 matches = re.match(regex, file[0])
-                current_lang = matches.group(1)
-                file[1][current_lang] = True
-                file[2] = matches.group(2)
+                if matches is None:
+                    logger.warning(
+                        f"Data file '{file[0]}' does not fit to the filename\
+                        regex")
+                    files.remove(file)
+                else:
+                    current_lang = matches.group(1)
+                    file[1][current_lang] = True
+                    file[2] = matches.group(2)
             except Exception as e:
                 logger.error(e)
                 files.remove(file)
@@ -215,12 +260,14 @@ class BlogPost(Searchable):
                     # only a single version of this file
                     continue
             except Exception as e:
-                logger.error(f"Could not combine BlogPosts for '{file[0]}': {e}")
+                logger.error(
+                    f"Could not combine BlogPosts for '{file[0]}': {e}")
 
         try:
             # deduplicate
             _files = []
-            for f in [[_f[1],_f[2]] for _f in files]:   # dont care about fname
+            for f in [[_f[1], _f[2]]
+                      for _f in files]:   # dont care about fname
                 if f not in _files:
                     _files.append(f)
             files = _files
@@ -236,8 +283,6 @@ class BlogPost(Searchable):
             except Exception as e:
                 logger.error(f"Could not create BlogPost for '{file[1]}': {e}")
 
-
-
     class Meta:
         verbose_name = _("blog post")
         verbose_name_plural = _("blog posts")

gawa/blog/templates/admin/blogpost.html

@@ -1,9 +1,6 @@
-{% extends 'admin/change_list.html' %}
-
-{% block object-tools %}
-		<form action="sync" method="POST">
-				{% csrf_token %}
-				<button type="submit">Sync with FS</button>
-		</form>
-    {{ block.super }}
-{% endblock %}
+{% extends 'admin/change_list.html' %} {% block object-tools %}
+<form action="sync" method="POST">
+  {% csrf_token %}
+  <button type="submit" class="button" style="padding: 4px">Sync with FS</button>
+</form>
+{{ block.super }} {% endblock %}

gawa/gawa/settings.py

@@ -11,6 +11,8 @@ https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
 # for getting envvars
+import logging
+from django.utils.translation import gettext_lazy as _
 import os
 
 # default django
@@ -35,7 +37,7 @@ ALLOWED_HOSTS = ["*"]
 
 # Allow inclusion of stuff from these origins
 CORS_ALLOWED_ORIGINS = [
-        "https://static.cscherr.de",
+    "https://static.cscherr.de",
 ]
 
 # Application definition
@@ -100,7 +102,6 @@ DATABASES = {
 }
 
 
-
 # Password validation
 # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
 
@@ -123,7 +124,6 @@ AUTH_PASSWORD_VALIDATORS = [
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
 
-from django.utils.translation import gettext_lazy as _
 
 LANGUAGES = [
     ("de", _("German")),
@@ -155,10 +155,10 @@ STATIC_URL = '/static/'
 COMPRESS_ENABLED = True
 
 STATICFILES_FINDERS = [
-        'django.contrib.staticfiles.finders.FileSystemFinder',
-        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
-        'compressor.finders.CompressorFinder',
-        ]
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'compressor.finders.CompressorFinder',
+]
 
 COMPRESS_PRECOMPILERS = (
     ('text/x-scss', 'django_libsass.SassCompiler'),
@@ -173,7 +173,6 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 # Logging configs
 
-import logging
 
 myServerFormatter = ServerFormatter
 myServerFormatter.default_time_format = "%Y-%M-%d %H:%M:%S"
@@ -294,7 +293,7 @@ LOGGING = {
 # Media stuff
 # this is where user uploaded files will go.
 # TODO change this for prod
-#MEDIA_ROOT = "/home/plex/Documents/code/python/gawa/media"
+# MEDIA_ROOT = "/home/plex/Documents/code/python/gawa/media"
 MEDIA_ROOT = "/app/media"
 MEDIA_URL = "/media/"
 # FILE_UPLOAD_TEMP_DIR = "/tmp/gawa/upload"

gawa/gawa/urls.py

@@ -21,7 +21,7 @@ from django.conf import settings
 from django.conf.urls.static import static
 
 urlpatterns = [
-    re_path(r'^i18n/', include('django.conf.urls.i18n')), 
+    re_path(r'^i18n/', include('django.conf.urls.i18n')),
 ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
 urlpatterns += i18n_patterns(

gawa/start/admin.py

@@ -10,6 +10,7 @@ def regenerate(modeladmin, request, queryset):
     for obj in queryset:
         obj.regenerate()
 
+
 @admin.register(Keyword)
 class KeywordAdmin(admin.ModelAdmin):
     """
@@ -17,21 +18,25 @@ class KeywordAdmin(admin.ModelAdmin):
     """
     list_display = ["text_en", "text_de"]
 
+
 @admin.register(StaticSite)
 class StaticSiteAdmin(admin.ModelAdmin):
     """
     Admin Interface for StaticSite
     """
-    list_display = ["title_en", "subtitle_en", "title_de", "subtitle_de", "suburl"]
+    list_display = ["title_en", "subtitle_en",
+                    "title_de", "subtitle_de", "suburl"]
     ordering = ['title_de', 'title_en']
     actions = [regenerate]
 
+
 @admin.register(Searchable)
 class SearchableAdmin(admin.ModelAdmin):
     """
     Abstract Admin Interface for all Searchables
     """
-    list_display = ["title_en", "subtitle_en", "title_de", "subtitle_de", "suburl"]
+    list_display = ["title_en", "subtitle_en",
+                    "title_de", "subtitle_de", "suburl"]
     ordering = ['title_de', 'title_en']
     actions = [regenerate]
 
@@ -41,6 +46,7 @@ class LinkAdmin(admin.ModelAdmin):
     """
     Admin Interface for Links
     """
-    list_display = ["title_en", "title_de", "url", "suburl", "favicon", "status", "personal"]
+    list_display = ["title_en", "title_de", "url",
+                    "suburl", "favicon", "status", "personal"]
     ordering = ['status', 'title_de', 'title_en']
     actions = [regenerate]

gawa/start/apps.py

@@ -1,5 +1,6 @@
 from django.apps import AppConfig
 
+
 class StartConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'start'

gawa/start/forms.py

@@ -1,10 +1,11 @@
 from django import forms
 from django.utils.translation import gettext as _
 
+
 class MainSearchForm(forms.Form):
     search = forms.CharField(
-    max_length=100,
-    label=''
+        max_length=100,
+        label=''
     )
     search.widget = forms.TextInput(
         attrs={

gawa/start/middleware.py

@@ -15,7 +15,8 @@ class LangBasedOnUrlMiddleware(MiddlewareMixin):
     def process_request(request):
 
         if hasattr(request, 'session'):
-            active_session_lang = request.session.get(translation.LANGUAGE_SESSION_KEY)
+            active_session_lang = request.session.get(
+                translation.LANGUAGE_SESSION_KEY)
 
             if active_session_lang == request.LANGUAGE_CODE:
                 return

gawa/start/models.py

@@ -1,3 +1,6 @@
+import random
+import favicon
+import requests
 from django.db import models
 from django.db.models.options import override
 from django.utils.translation import gettext as _
@@ -8,9 +11,6 @@ from django.conf import settings
 import logging
 logger = logging.getLogger(__name__)
 
-import requests
-import favicon
-import random
 
 class Keyword(models.Model):
     """
@@ -26,6 +26,7 @@ class Keyword(models.Model):
         verbose_name = _("Keyword")
         verbose_name_plural = _("keywords")
 
+
 class Searchable(models.Model):
     """
     Abstract class for any model that should be searchable.
@@ -37,8 +38,10 @@ class Searchable(models.Model):
     title_en = models.CharField(max_length=50, default="title EN")
     subtitle_de = models.CharField(max_length=50, blank=True)
     subtitle_en = models.CharField(max_length=50, blank=True)
-    desc_de = models.TextField(blank=True, max_length=250, unique=False, default="Beschreibung DE")
-    desc_en = models.TextField(blank=True, max_length=250, unique=False, default="Description EN")
+    desc_de = models.TextField(
+        blank=True, max_length=250, unique=False, default="Beschreibung DE")
+    desc_en = models.TextField(
+        blank=True, max_length=250, unique=False, default="Description EN")
     # may be empty/blank for some entries
     date = models.DateField(blank=True, null=True)
     keywords = models.ManyToManyField(Keyword)
@@ -61,12 +64,14 @@ class Searchable(models.Model):
         """
         regenerate a object
         """
-        raise NotImplementedError(f"{self.__class__.__name__} does not implement regenerate")
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement regenerate")
 
     class Meta:
         verbose_name = _("Searchable")
         verbose_name_plural = _("Searchables")
 
+
 class StaticSite(Searchable):
     """
     This model represents any static site, such as start:index, 
@@ -75,20 +80,21 @@ class StaticSite(Searchable):
     Every searchable view should inherit from start.views.SearchableView.
     # TODO automate scanning for SearchableView classes
     """
-    
+
     def regenerate(self):
         """
         regenerate a object
         """
         logger.info(f"regenerating {self.__class__.__name__} object: {self}")
         logger.warning(f"{self.__class__.__name__} cannot regenerate.")
-        #self.save()
+        # self.save()
         pass
 
     class Meta:
         verbose_name = _("static site")
         verbose_name_plural = _("static sites")
 
+
 class Link(Searchable):
     """
     contains all my interesting links
@@ -121,11 +127,13 @@ class Link(Searchable):
             icons = favicon.get(self.url, timeout=2)
         except (ConnectionError) as ce:
             # just keep whatever was stored if we cant get a new favicon
-            logger.warn(f"unable to download favicon for {self}: {ce.with_traceback(None)}")
+            logger.warn(
+                f"unable to download favicon for {self}: {ce.with_traceback(None)}")
             self.status = False
 
         except Exception as e:
-            logger.warn(f"Unexpected Exception while downloading {self}: {e.with_traceback(None)}")
+            logger.warn(
+                f"Unexpected Exception while downloading {self}: {e.with_traceback(None)}")
             self.status = False
 
         else:
@@ -144,9 +152,10 @@ class Link(Searchable):
             except FileNotFoundError as fe:
                 logger.error(f"cant write favicon to file for {self}: {fe}")
                 self.favicon = None
-            
+
             except Exception as e:
-                logger.warn(f"Unexpected Exception while downloading {self}: {e.with_traceback(None)}")
+                logger.warn(
+                    f"Unexpected Exception while downloading {self}: {e.with_traceback(None)}")
                 self.favicon = None
 
         self.save()

gawa/start/templatetags/helper_tags.py

@@ -36,4 +36,3 @@ def change_lang(context, lang="de", *args, **kwargs):
     finally:
         activate(lang)
         return "%s" % url
-

gawa/start/urls.py

@@ -9,5 +9,6 @@ urlpatterns = [
     path("legal/", views.LegalInfo.as_view(), name="legal"),
     path("links/", views.Links.as_view(), name="links"),
     path("professional/", views.LegalInfo.as_view(), name="professional"),
-    path('language/activate/<language_code>/', views.ActivateLanguage.as_view(), name='activate_language'),
+    path('language/activate/<language_code>/',
+         views.ActivateLanguage.as_view(), name='activate_language'),
 ]

gawa/start/views.py

@@ -18,6 +18,7 @@ from abc import ABC
 import logging
 logger = logging.getLogger(__name__)
 
+
 class SearchableView(View, ABC):
     """
     This abstract view implements some traits of views that should show up
@@ -40,6 +41,7 @@ class Index(TemplateView, SearchableView):
 
     template_name: str = "start/index.html"
 
+
 class Professional(TemplateView, SearchableView):
     """
     Professional informations that might interest a professional employer
@@ -47,6 +49,7 @@ class Professional(TemplateView, SearchableView):
     # TODO
     template_name: str = "start/legalinfo.html"
 
+
 class LegalInfo(TemplateView, SearchableView):
     """
     Legal info that the german authorities want.
@@ -54,20 +57,22 @@ class LegalInfo(TemplateView, SearchableView):
     # TODO
     template_name: str = "start/legalinfo.html"
 
+
 class ActivateLanguage(View):
     """
     Set the language to whatever
     """
     language_code = ''
-    redirect_to   = ''
+    redirect_to = ''
 
     def get(self, request, *args, **kwargs):
-        self.redirect_to   = request.META.get('HTTP_REFERER')
+        self.redirect_to = request.META.get('HTTP_REFERER')
         self.language_code = kwargs.get('language_code')
         translation.activate(self.language_code)
         request.session[translation.LANGUAGE_SESSION_KEY] = self.language_code
         return redirect(self.redirect_to)
 
+
 class MainSearch(ListView):
     """
     Search for anything.
@@ -94,6 +99,7 @@ class MainSearch(ListView):
             return render(request, "errors/bad_request.html")
         return super().get(request, *args, **kwargs)
 
+
 class Links(ListView):
     """
     This View contains links to various interesting sites.
@@ -115,6 +121,6 @@ class Links(ListView):
         return object_list
 
     def get_context_data(self, *, object_list=None, **kwargs):
-        context =  super().get_context_data(object_list=object_list, **kwargs)
+        context = super().get_context_data(object_list=object_list, **kwargs)
         context['personal_links'] = self.get_queryset_personal_links()
         return context