linus/Anime-Movies-Upcomming-Realese-Bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finalising the Querries and the rest

Browse Source
This commit is contained in:
ProgrammGamer
2026-04-21 22:41:43 +02:00
parent 2b3dffb62f
commit e5784bc1b8
3 changed files with 372 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

356
movie_pipeline.py
View File

@@ -2,147 +2,213 @@
from __future__ import annotations
from datetime import date, timedelta
from difflib import SequenceMatcher
from app_config import get_settings
from data_sources.anilist_source import fetch_anilist_movie_by_search
from data_sources.anilist_source import fetch_anilist_movie_candidates
from data_sources.animeschedule_source import fetch_animeschedule_anime_by_title
from data_sources.tmdb_source import fetch_tmdb_anime_movies
from data_sources.tmdb_source import (
extract_de_theatrical_dates,
fetch_tmdb_release_dates,
search_tmdb_movies,
select_release_in_range,
)
FUZZY_MATCH_THRESHOLD = 0.65
SCHEDULE_MATCH_THRESHOLD = 2
def add_months(d: date, months: int) -> date:
year = d.year + ((d.month - 1 + months) // 12)
month = ((d.month - 1 + months) % 12) + 1
def add_months(value: date, months: int) -> date:
year = value.year + ((value.month - 1 + months) // 12)
month = ((value.month - 1 + months) % 12) + 1
return date(year, month, 1)
def format_date(d: date, locale: str) -> str:
def normalize_title(text: str) -> str:
cleaned = "".join(ch.lower() if ch.isalnum() else " " for ch in (text or ""))
return " ".join(cleaned.split())
def fuzzy_ratio(left: str, right: str) -> float:
a = normalize_title(left)
b = normalize_title(right)
if not a or not b:
return 0.0
return SequenceMatcher(None, a, b).ratio()
def format_date(value: date, locale: str) -> str:
if locale == "de-DE":
return f"{d.day:02d}.{d.month:02d}.{d.year}"
return d.isoformat()
return f"{value.day:02d}.{value.month:02d}.{value.year}"
return value.isoformat()
def select_title(anilist: dict, schedule: dict, tmdb: dict) -> str:
return (
(schedule.get("english_title") or "").strip()
or (anilist.get("title_english") or "").strip()
or (anilist.get("title_best") or "").strip()
or (tmdb.get("title") or "").strip()
or (tmdb.get("original_title") or "").strip()
)
def parse_record_release_date(record: dict) -> date:
text = str(record.get("releaseDate") or "").strip()
try:
return date.fromisoformat(text)
except ValueError:
return date.max
def structure_movie_record(tmdb_movie: dict, anilist: dict, schedule: dict, locale: str) -> dict:
release_date = tmdb_movie.get("release_date")
release = format_date(release_date, locale) if isinstance(release_date, date) else "n/a"
def is_year_match(anilist_year: int, tmdb_year: int) -> bool:
if anilist_year <= 0 or tmdb_year <= 0:
return False
return abs(anilist_year - tmdb_year) <= 1
has_anilist = bool(anilist)
has_schedule = bool(schedule and (schedule.get("title") or schedule.get("english_title") or schedule.get("url")))
final_description = (
(tmdb_movie.get("overview") or "").strip()
or (anilist.get("description") or "").strip()
or (schedule.get("description") or "").strip()
or "n/a"
)
def title_score(anilist_english: str, anilist_romaji: str, anilist_native: str, tmdb_title: str, tmdb_original: str) -> tuple[int, int, int, float]:
english_norm = normalize_title(anilist_english)
romaji_norm = normalize_title(anilist_romaji)
native_norm = normalize_title(anilist_native)
tmdb_options = [normalize_title(tmdb_title), normalize_title(tmdb_original)]
# User requirement: cover images should come from AniList.
cover_image = (anilist.get("cover_image") or "").strip()
exact_english = 1 if english_norm and english_norm in tmdb_options else 0
exact_romaji = 1 if romaji_norm and romaji_norm in tmdb_options else 0
exact_native = 1 if native_norm and native_norm in tmdb_options else 0
genres_text = (
(tmdb_movie.get("genres_text") or "").strip()
or (anilist.get("genres_text") or "").strip()
or (schedule.get("genres") or "").strip()
or "n/a"
)
best_ratio = 0.0
for option in (tmdb_title, tmdb_original):
best_ratio = max(
best_ratio,
fuzzy_ratio(anilist_english, option),
fuzzy_ratio(anilist_romaji, option),
fuzzy_ratio(anilist_native, option),
)
record = {
"title": select_title(anilist, schedule, tmdb_movie),
"title_english_anilist": (anilist.get("title_english") or "").strip(),
"title_anilist": (anilist.get("title_best") or "").strip(),
"title_schedule_english": (schedule.get("english_title") or "").strip(),
"title_romaji": (anilist.get("title_romaji") or "").strip() or "n/a",
"title_native": (anilist.get("title_native") or "").strip() or "n/a",
"studio": (anilist.get("studio_text") or "").strip() or (schedule.get("studios") or "").strip() or "n/a",
"genres": genres_text,
"tags": (anilist.get("tags_text") or "").strip() or "n/a",
"release": release,
"anilist_url": (anilist.get("anilist_url") or "").strip() or "n/a",
"format": (anilist.get("format") or "").strip() or (schedule.get("format") or "").strip() or "MOVIE",
"episodes": anilist.get("episodes") or "n/a",
"duration": anilist.get("duration") or "n/a",
"source": (anilist.get("source") or "").strip() or "n/a",
"cover_image": cover_image,
"description": final_description,
"schedule_url": (schedule.get("url") or "").strip(),
"schedule_title": (schedule.get("title") or "").strip(),
"release_source": "TMDb DE",
"source_presence": {
"tmdb": True,
"anilist": has_anilist,
"animeschedule": has_schedule,
return exact_english, exact_romaji, exact_native, best_ratio
def collect_tmdb_candidates_for_anilist(anilist_entry: dict, tmdb_token: str, search_cache: dict[str, list[dict]]) -> list[dict]:
english = (anilist_entry.get("title_english") or "").strip()
romaji = (anilist_entry.get("title_romaji") or "").strip()
native = (anilist_entry.get("title_native") or "").strip()
queries = []
if english:
queries.append(english)
if romaji and normalize_title(romaji) != normalize_title(english):
queries.append(romaji)
if native and normalize_title(native) not in {normalize_title(english), normalize_title(romaji)}:
queries.append(native)
if not queries:
return []
candidates_by_id = {}
languages = ["de-DE", "en-US", "ja-JP"]
for query in queries:
normalized_query = normalize_title(query)
for language in languages:
cache_key = f"{language}:{normalized_query}"
if cache_key not in search_cache:
search_cache[cache_key] = search_tmdb_movies(query, tmdb_token, language=language)
for item in search_cache[cache_key]:
tmdb_id = int(item.get("tmdb_id") or 0)
if tmdb_id <= 0:
continue
existing = candidates_by_id.get(tmdb_id)
if not existing:
candidates_by_id[tmdb_id] = item
continue
# Prefer candidate carrying a release date if duplicate appears from different language queries.
if not existing.get("release_date") and item.get("release_date"):
candidates_by_id[tmdb_id] = item
return list(candidates_by_id.values())
def pick_best_tmdb_match(anilist_entry: dict, tmdb_candidates: list[dict]) -> dict | None:
english = (anilist_entry.get("title_english") or "").strip()
romaji = (anilist_entry.get("title_romaji") or "").strip()
native = (anilist_entry.get("title_native") or "").strip()
anilist_year = int(anilist_entry.get("start_year") or 0)
best = None
best_tuple = (-1, -1, -1, 0.0)
for candidate in tmdb_candidates:
exact_english, exact_romaji, exact_native, ratio = title_score(
english,
romaji,
native,
str(candidate.get("title") or ""),
str(candidate.get("original_title") or ""),
)
tmdb_release = candidate.get("release_date")
tmdb_year = tmdb_release.year if tmdb_release else 0
has_exact_match = exact_english == 1 or exact_romaji == 1 or exact_native == 1
if not has_exact_match and not is_year_match(anilist_year, tmdb_year):
continue
if ratio < FUZZY_MATCH_THRESHOLD and exact_english == 0 and exact_romaji == 0 and exact_native == 0:
continue
score_tuple = (exact_english, exact_romaji, exact_native, ratio)
if score_tuple > best_tuple:
best_tuple = score_tuple
best = candidate
return best
def resolve_titles(anilist_entry: dict, schedule_token: str, schedule_cache: dict[str, dict]) -> tuple[str, str]:
english = (anilist_entry.get("title_english") or "").strip()
romaji = (anilist_entry.get("title_romaji") or "").strip()
native = (anilist_entry.get("title_native") or "").strip()
schedule_english = ""
if not english and romaji:
schedule = fetch_animeschedule_anime_by_title(romaji, schedule_token, schedule_cache)
if int(schedule.get("match_score") or 0) >= SCHEDULE_MATCH_THRESHOLD:
schedule_english = (schedule.get("english_title") or "").strip()
preferred_title = schedule_english or english or romaji or native
return preferred_title, schedule_english
def build_record(anilist_entry: dict, tmdb_entry: dict, release_date: date, locale: str, title: str, schedule_english: str) -> dict:
return {
"title": title,
"title_english_anilist": (anilist_entry.get("title_english") or "").strip(),
"title_anilist": (anilist_entry.get("title_english") or "").strip() or (anilist_entry.get("title_romaji") or "").strip(),
"title_schedule_english": schedule_english,
"title_romaji": (anilist_entry.get("title_romaji") or "").strip(),
"title_native": (anilist_entry.get("title_native") or "").strip(),
"studio": (anilist_entry.get("studio_text") or "").strip(),
"genres": (anilist_entry.get("genres_text") or "").strip() or "n/a",
"tags": (anilist_entry.get("tags_text") or "").strip(),
"release": format_date(release_date, locale),
"releaseDate": release_date.isoformat(),
"anilist_url": (anilist_entry.get("anilist_url") or "").strip() or "n/a",
"format": (anilist_entry.get("format") or "").strip() or "MOVIE",
"cover_image": (anilist_entry.get("cover_image") or "").strip(),
"description": (anilist_entry.get("description") or "").strip(),
"tmdb_title": (tmdb_entry.get("title") or "").strip(),
"tmdb_id": int(tmdb_entry.get("tmdb_id") or 0),
"ids": {
"anilist": int(anilist_entry.get("anilist_id") or 0),
"tmdb": int(tmdb_entry.get("tmdb_id") or 0),
},
"tmdb": tmdb_movie,
"anilist": anilist,
"animeschedule": schedule,
}
return record
def sort_and_structure_movies(tmdb_movies: list[dict], locale: str, schedule_token: str) -> list[dict]:
records = []
anilist_cache: dict[str, dict | None] = {}
schedule_cache: dict[str, dict] = {}
def sort_dedup_records(records: list[dict]) -> list[dict]:
unique = {}
for record in records:
ids = record.get("ids") or {}
key = (int(ids.get("anilist") or 0), int(ids.get("tmdb") or 0), str(record.get("releaseDate") or ""))
if key not in unique:
unique[key] = record
sorted_tmdb = sorted(
tmdb_movies,
key=lambda item: item.get("release_date") or date.min,
reverse=True,
)
for tmdb_movie in sorted_tmdb:
anilist = None
for candidate in ((tmdb_movie.get("title") or "").strip(), (tmdb_movie.get("original_title") or "").strip()):
if not candidate:
continue
anilist = fetch_anilist_movie_by_search(candidate, anilist_cache)
if anilist:
break
anilist = anilist or {}
schedule_candidates = [
(anilist.get("title_english") or "").strip(),
(anilist.get("title_best") or "").strip(),
(tmdb_movie.get("title") or "").strip(),
(tmdb_movie.get("original_title") or "").strip(),
]
best_schedule = {}
best_schedule_score = 0
seen_schedule_candidates = set()
for schedule_candidate in schedule_candidates:
if not schedule_candidate:
continue
norm_key = schedule_candidate.lower()
if norm_key in seen_schedule_candidates:
continue
seen_schedule_candidates.add(norm_key)
candidate_result = fetch_animeschedule_anime_by_title(schedule_candidate, schedule_token, schedule_cache)
score = int(candidate_result.get("match_score") or 0)
if score > best_schedule_score:
best_schedule = candidate_result
best_schedule_score = score
# Prefer results that have explicit English title when scores are tied.
if score == best_schedule_score and score > 0:
if not (best_schedule.get("english_title") or "").strip() and (candidate_result.get("english_title") or "").strip():
best_schedule = candidate_result
schedule = best_schedule if best_schedule_score > 0 else {}
records.append(structure_movie_record(tmdb_movie, anilist, schedule, locale))
return records
result = list(unique.values())
result.sort(key=lambda item: (parse_record_release_date(item), str(item.get("title") or "")))
return result
def get_upcoming_movie_records(
@@ -155,9 +221,59 @@ def get_upcoming_movie_records(
schedule_token = (anime_schedule_token or settings.animeschedule_api_token).strip()
tmdb_token = (tmdb_read_access_token or settings.tmdb_read_access_token).strip()
if not tmdb_token:
return []
current_day = today or date.today()
month_start = date(current_day.year, current_day.month, 1)
end_date = add_months(month_start, 2) - timedelta(days=1)
tmdb_movies = fetch_tmdb_anime_movies(current_day, end_date, tmdb_token, language=locale)
return sort_and_structure_movies(tmdb_movies, locale, schedule_token)
anilist_candidates = fetch_anilist_movie_candidates(current_day)
schedule_cache: dict[str, dict] = {}
tmdb_search_cache: dict[str, list[dict]] = {}
tmdb_release_cache: dict[int, dict] = {}
output_records = []
for anilist_entry in anilist_candidates:
tmdb_candidates = collect_tmdb_candidates_for_anilist(anilist_entry, tmdb_token, tmdb_search_cache)
if not tmdb_candidates:
continue
matched_tmdb = pick_best_tmdb_match(anilist_entry, tmdb_candidates)
if not matched_tmdb:
continue
tmdb_id = int(matched_tmdb.get("tmdb_id") or 0)
if tmdb_id <= 0:
continue
if tmdb_id not in tmdb_release_cache:
tmdb_release_cache[tmdb_id] = fetch_tmdb_release_dates(tmdb_id, tmdb_token)
release_payload = tmdb_release_cache[tmdb_id]
de_theatrical_dates = extract_de_theatrical_dates(release_payload)
release_date = select_release_in_range(de_theatrical_dates, current_day, end_date)
if not release_date:
candidate_release = matched_tmdb.get("release_date")
if isinstance(candidate_release, date) and current_day <= candidate_release <= end_date:
release_date = candidate_release
else:
continue
preferred_title, schedule_english = resolve_titles(anilist_entry, schedule_token, schedule_cache)
if not preferred_title:
continue
output_records.append(
build_record(
anilist_entry=anilist_entry,
tmdb_entry=matched_tmdb,
release_date=release_date,
locale=locale,
title=preferred_title,
schedule_english=schedule_english,
)
)
return sort_dedup_records(output_records)