Merge pull request 'revamp-file-discover' (#33) from revamp-file-discover into devel
Reviewed-on: #33
This commit is contained in:
commit
9b7c73a977
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 4.2.5 on 2023-10-02 19:03
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('blog', '0001_initial'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='blogpost',
|
||||||
|
name='slug',
|
||||||
|
field=models.SlugField(unique=True),
|
||||||
|
),
|
||||||
|
]
|
|
@ -76,7 +76,7 @@ class BlogPost(Searchable):
|
||||||
body_de = models.TextField(blank=True,
|
body_de = models.TextField(blank=True,
|
||||||
default="""This aritcle is not available in english.""")
|
default="""This aritcle is not available in english.""")
|
||||||
category = models.ForeignKey(
|
category = models.ForeignKey(
|
||||||
Category, on_delete=models.SET_DEFAULT, null=False,
|
Category, on_delete=models.SET_DEFAULT, blank=False,
|
||||||
default=Category.get_or_create_uncategorized)
|
default=Category.get_or_create_uncategorized)
|
||||||
thumbnail = models.ImageField(
|
thumbnail = models.ImageField(
|
||||||
blank=True,
|
blank=True,
|
||||||
|
@ -85,7 +85,15 @@ class BlogPost(Searchable):
|
||||||
featured = models.BooleanField(default=False)
|
featured = models.BooleanField(default=False)
|
||||||
langs = models.CharField(
|
langs = models.CharField(
|
||||||
default=DEFAULT_LANGS.__repr__(), max_length=64)
|
default=DEFAULT_LANGS.__repr__(), max_length=64)
|
||||||
slug = models.SlugField()
|
slug = models.SlugField(unique=True, blank=False)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
# check if the slug is empty if we remove whitespaces
|
||||||
|
if len(self.slug.strip()) == 0:
|
||||||
|
logger.error(
|
||||||
|
f"trying to save '{self.__class__}' with empty slug: {self}")
|
||||||
|
raise ValueError(f"trying to save '{self.__class__}' with empty slug: {self}")
|
||||||
|
super().save()
|
||||||
|
|
||||||
def regenerate(self):
|
def regenerate(self):
|
||||||
"""
|
"""
|
||||||
|
@ -228,72 +236,30 @@ class BlogPost(Searchable):
|
||||||
|
|
||||||
files = [f for f in os.listdir(data_dir) if (
|
files = [f for f in os.listdir(data_dir) if (
|
||||||
data_dir.joinpath(f)).is_file()]
|
data_dir.joinpath(f)).is_file()]
|
||||||
logger.debug(f"discovered files: {files}")
|
|
||||||
|
|
||||||
# finding lang and title
|
# find the meta file
|
||||||
regex = r"^(en|de)-(.*)\.md"
|
regex = r"^(.*)\.toml"
|
||||||
# TODO: only find toml files
|
|
||||||
|
|
||||||
# filepath, language codes, slug
|
# filepath, slug
|
||||||
files = [[f, cls.DEFAULT_LANGS, ""] for f in files]
|
files = [[f, ""] for f in files]
|
||||||
for file in files:
|
for file in files:
|
||||||
# parse file name
|
# parse file name
|
||||||
try:
|
try:
|
||||||
matches = re.match(regex, file[0])
|
matches = re.match(regex, file[0])
|
||||||
if matches is None:
|
if matches is None:
|
||||||
logger.warning(
|
# file is not a toml / meta file
|
||||||
f"Data file '{file[0]}' does not fit to the filename\
|
|
||||||
regex")
|
|
||||||
files.remove(file)
|
files.remove(file)
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
current_lang = matches.group(1)
|
current_lang = matches.group(1)
|
||||||
file[1][current_lang] = True
|
file[1] = matches.group(1)
|
||||||
file[2] = matches.group(2)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e)
|
|
||||||
files.remove(file)
|
|
||||||
|
|
||||||
# PERF:
|
obj = BlogPost(slug=file[1])
|
||||||
# Optimize for single loop, should be doable and better design
|
|
||||||
|
|
||||||
# collapse diffrent versions
|
|
||||||
for file in files:
|
|
||||||
try:
|
|
||||||
if [_f[2] for _f in files].count(file[2]) >= 2:
|
|
||||||
logger.debug(f"multiple versions of '{file[2]}'")
|
|
||||||
versions = [_f for _f in files if _f[2] == file[2]]
|
|
||||||
lang: dict[str, bool] = file[1]
|
|
||||||
for version in versions:
|
|
||||||
for key in version[1]:
|
|
||||||
lang[key] |= version[1][key]
|
|
||||||
else:
|
|
||||||
# only a single version of this file
|
|
||||||
continue
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(
|
|
||||||
f"Could not combine BlogPosts for '{file[0]}': {e}")
|
|
||||||
|
|
||||||
# TODO: not needed when relying on toml files
|
|
||||||
try:
|
|
||||||
# deduplicate
|
|
||||||
_files = []
|
|
||||||
for f in [[_f[1], _f[2]]
|
|
||||||
for _f in files]: # dont care about fname
|
|
||||||
if f not in _files:
|
|
||||||
_files.append(f)
|
|
||||||
files = _files
|
|
||||||
logger.debug(f"to save: {files}")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Could not dedup BlogPosts: {e}")
|
|
||||||
|
|
||||||
for file in files:
|
|
||||||
try:
|
|
||||||
obj = BlogPost(langs=file[0], slug=file[1])
|
|
||||||
obj.sync_file()
|
obj.sync_file()
|
||||||
obj.regenerate()
|
obj.regenerate()
|
||||||
obj.save()
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Could not create BlogPost for '{file[1]}': {e}")
|
logger.error(f"Could not create BlogPost for '{file[1]}': {e}")
|
||||||
|
files.remove(file)
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
verbose_name = _("blog post")
|
verbose_name = _("blog post")
|
||||||
|
|
|
@ -59,7 +59,7 @@ class Searchable(models.Model):
|
||||||
obj.regenerate()
|
obj.regenerate()
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"{{<{self.__class__.__name__}>\"{self.slug}\"}}"
|
return f"{{<{self.__class__.__name__}>\"{self.title_en}\"}}"
|
||||||
|
|
||||||
def regenerate(self):
|
def regenerate(self):
|
||||||
"""
|
"""
|
||||||
|
@ -75,7 +75,7 @@ class Searchable(models.Model):
|
||||||
|
|
||||||
class StaticSite(Searchable):
|
class StaticSite(Searchable):
|
||||||
"""
|
"""
|
||||||
This model represents any static site, such as start:index,
|
This model represents any static site, such as start:index,
|
||||||
that should show up in search.
|
that should show up in search.
|
||||||
|
|
||||||
Every searchable view should inherit from start.views.SearchableView.
|
Every searchable view should inherit from start.views.SearchableView.
|
||||||
|
|
Loading…
Reference in New Issue