revamp-file-discover #33

Merged
PlexSheep merged 2 commits from revamp-file-discover into devel 2023-10-02 21:09:58 +02:00
3 changed files with 39 additions and 55 deletions

View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.5 on 2023-10-02 19:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('blog', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='blogpost',
name='slug',
field=models.SlugField(unique=True),
),
]

View File

@ -76,7 +76,7 @@ class BlogPost(Searchable):
body_de = models.TextField(blank=True, body_de = models.TextField(blank=True,
default="""This aritcle is not available in english.""") default="""This aritcle is not available in english.""")
category = models.ForeignKey( category = models.ForeignKey(
Category, on_delete=models.SET_DEFAULT, null=False, Category, on_delete=models.SET_DEFAULT, blank=False,
default=Category.get_or_create_uncategorized) default=Category.get_or_create_uncategorized)
thumbnail = models.ImageField( thumbnail = models.ImageField(
blank=True, blank=True,
@ -85,7 +85,15 @@ class BlogPost(Searchable):
featured = models.BooleanField(default=False) featured = models.BooleanField(default=False)
langs = models.CharField( langs = models.CharField(
default=DEFAULT_LANGS.__repr__(), max_length=64) default=DEFAULT_LANGS.__repr__(), max_length=64)
slug = models.SlugField() slug = models.SlugField(unique=True, blank=False)
def save(self):
# check if the slug is empty if we remove whitespaces
if len(self.slug.strip()) == 0:
logger.error(
f"trying to save '{self.__class__}' with empty slug: {self}")
raise ValueError(f"trying to save '{self.__class__}' with empty slug: {self}")
super().save()
def regenerate(self): def regenerate(self):
""" """
@ -228,72 +236,30 @@ class BlogPost(Searchable):
files = [f for f in os.listdir(data_dir) if ( files = [f for f in os.listdir(data_dir) if (
data_dir.joinpath(f)).is_file()] data_dir.joinpath(f)).is_file()]
logger.debug(f"discovered files: {files}")
# finding lang and title # find the meta file
regex = r"^(en|de)-(.*)\.md" regex = r"^(.*)\.toml"
# TODO: only find toml files
# filepath, language codes, slug # filepath, slug
files = [[f, cls.DEFAULT_LANGS, ""] for f in files] files = [[f, ""] for f in files]
for file in files: for file in files:
# parse file name # parse file name
try: try:
matches = re.match(regex, file[0]) matches = re.match(regex, file[0])
if matches is None: if matches is None:
logger.warning( # file is not a toml / meta file
f"Data file '{file[0]}' does not fit to the filename\
regex")
files.remove(file) files.remove(file)
continue
else: else:
current_lang = matches.group(1) current_lang = matches.group(1)
file[1][current_lang] = True file[1] = matches.group(1)
file[2] = matches.group(2)
except Exception as e:
logger.error(e)
files.remove(file)
# PERF: obj = BlogPost(slug=file[1])
# Optimize for single loop, should be doable and better design
# collapse diffrent versions
for file in files:
try:
if [_f[2] for _f in files].count(file[2]) >= 2:
logger.debug(f"multiple versions of '{file[2]}'")
versions = [_f for _f in files if _f[2] == file[2]]
lang: dict[str, bool] = file[1]
for version in versions:
for key in version[1]:
lang[key] |= version[1][key]
else:
# only a single version of this file
continue
except Exception as e:
logger.error(
f"Could not combine BlogPosts for '{file[0]}': {e}")
# TODO: not needed when relying on toml files
try:
# deduplicate
_files = []
for f in [[_f[1], _f[2]]
for _f in files]: # dont care about fname
if f not in _files:
_files.append(f)
files = _files
logger.debug(f"to save: {files}")
except Exception as e:
logger.error(f"Could not dedup BlogPosts: {e}")
for file in files:
try:
obj = BlogPost(langs=file[0], slug=file[1])
obj.sync_file() obj.sync_file()
obj.regenerate() obj.regenerate()
obj.save()
except Exception as e: except Exception as e:
logger.error(f"Could not create BlogPost for '{file[1]}': {e}") logger.error(f"Could not create BlogPost for '{file[1]}': {e}")
files.remove(file)
class Meta: class Meta:
verbose_name = _("blog post") verbose_name = _("blog post")

View File

@ -59,7 +59,7 @@ class Searchable(models.Model):
obj.regenerate() obj.regenerate()
def __str__(self): def __str__(self):
return f"{{<{self.__class__.__name__}>\"{self.slug}\"}}" return f"{{<{self.__class__.__name__}>\"{self.title_en}\"}}"
def regenerate(self): def regenerate(self):
""" """
@ -75,7 +75,7 @@ class Searchable(models.Model):
class StaticSite(Searchable): class StaticSite(Searchable):
""" """
This model represents any static site, such as start:index, This model represents any static site, such as start:index,
that should show up in search. that should show up in search.
Every searchable view should inherit from start.views.SearchableView. Every searchable view should inherit from start.views.SearchableView.