using System.Globalization; using System.Text.Json; using System.Text.RegularExpressions; using AngleSharp.Dom; using AngleSharp.Html.Parser; using FozFc.Api.Infrastructure.Zerozero; namespace FozFc.Api.Application.Zerozero; /// /// HTML parsers ported from the ZeroZero worker (cheerio → AngleSharp). Produce /// plain dictionaries matching the frontend DTOs; image rewriting is applied by /// the feed service afterwards. /// public static class ZerozeroParsers { private const int ExcerptMaxLength = 180; private static readonly HtmlParser Parser = new(); private static string Txt(IElement? el) => el?.TextContent.Trim() ?? ""; // ---------------------------------------------------------------- dates / status private static TimeZoneInfo LisbonTz() { try { return TimeZoneInfo.FindSystemTimeZoneById("Europe/Lisbon"); } catch { try { return TimeZoneInfo.FindSystemTimeZoneById("GMT Standard Time"); } catch { return TimeZoneInfo.Utc; } } } private static string TodayInLisbon() => TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, LisbonTz()).ToString("yyyy-MM-dd"); private static int? ParseMatchday(string value) { var m = Regex.Match(value ?? "", @"J\s*(\d+)", RegexOptions.IgnoreCase); return m.Success ? int.Parse(m.Groups[1].Value) : null; } private static (int? home, int? away) ParseScore(string value) { var cleaned = Regex.Replace(value ?? "", @"\s+", "").Trim(); var m = Regex.Match(cleaned, @"^(\d+)-(\d+)$"); if (!m.Success) return (null, null); return (int.Parse(m.Groups[1].Value), int.Parse(m.Groups[2].Value)); } private static string ParseStatusFromRow(string resultCell, string date, string time, string? rowClasses = null) { if (!string.IsNullOrEmpty(rowClasses) && Regex.IsMatch(rowClasses, @"\b(live|ao-?vivo|em-?jogo)\b", RegexOptions.IgnoreCase)) return "live"; var (home, away) = ParseScore(resultCell); var hasScore = home.HasValue && away.HasValue; var today = TodayInLisbon(); if (hasScore && date == today && !string.IsNullOrEmpty(time)) { var parts = time.Split(':'); if (parts.Length == 2 && int.TryParse(parts[0], out var h) && int.TryParse(parts[1], out var min)) { try { var local = new DateTime(int.Parse(date[..4]), int.Parse(date.Substring(5, 2)), int.Parse(date.Substring(8, 2)), h, min, 0, DateTimeKind.Unspecified); var kickoffUtc = TimeZoneInfo.ConvertTimeToUtc(local, LisbonTz()); var elapsed = (DateTime.UtcNow - kickoffUtc).TotalMinutes; if (elapsed >= -15 && elapsed <= 135) return "live"; } catch { /* ignore */ } } } if (hasScore) return "finished"; if (string.IsNullOrEmpty(date)) return "scheduled"; return string.CompareOrdinal(date, today) < 0 ? "postponed" : "scheduled"; } private static string ParsePtDateTime(string? value) { var raw = (value ?? "").Trim(); var m = Regex.Match(raw, @"^(\d{2})-(\d{2})-(\d{4})\s+(\d{2}):(\d{2})$"); if (!m.Success) return DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); return $"{m.Groups[3].Value}-{m.Groups[2].Value}-{m.Groups[1].Value}T{m.Groups[4].Value}:{m.Groups[5].Value}:00Z"; } private static string? ParsePublishedAtFromArticle(string? value) { var m = Regex.Match((value ?? "").Trim(), @"(\d{4})/(\d{2})/(\d{2})\s+(\d{2}):(\d{2})"); if (!m.Success) return null; return $"{m.Groups[1].Value}-{m.Groups[2].Value}-{m.Groups[3].Value}T{m.Groups[4].Value}:{m.Groups[5].Value}:00Z"; } private static string NormalizeNewsCategory(string? value) { var n = (value ?? "").ToLowerInvariant(); if (n.Contains("opini")) return "opinião"; if (n.Contains("comunicad")) return "comunicado"; if (n.Contains("transfer")) return "transferências"; if (n.Contains("match") || n.Contains("jogo")) return "match-report"; return "notícias"; } private static string TruncateText(string value, int maxLength) { var text = Regex.Replace((value ?? "").Trim(), @"\s+", " "); if (text.Length == 0) return ""; if (text.Length <= maxLength) return text; return text[..Math.Max(0, maxLength - 1)].TrimEnd() + "…"; } private static bool IsOpinionContent(Dictionary item, string? html = null) { var url = (item.GetValueOrDefault("externalUrl")?.ToString() ?? "").ToLowerInvariant(); var title = (item.GetValueOrDefault("title")?.ToString() ?? "").ToLowerInvariant(); var category = (item.GetValueOrDefault("category")?.ToString() ?? "").ToLowerInvariant(); if (url.Contains("/opiniao/") || url.Contains("/opiniao-")) return true; if (title.Contains("colunas de opini") || title.Contains("opinião")) return true; if (category.Contains("opini")) return true; if (html != null) { var lower = html.ToLowerInvariant(); if (lower.Contains("hp_widget_opinions")) return true; if (lower.Contains("colunas de opinião") || lower.Contains("colunas de opini")) return true; } return false; } // ---------------------------------------------------------------- matches private static string SliceAroundAllMatchesTable(string html) { var idx = html.ToLowerInvariant().IndexOf("todos os jogos", StringComparison.Ordinal); if (idx == -1) return html; var start = Math.Max(0, idx - 5000); const int maxSpan = 1_800_000; return html.Substring(start, Math.Min(html.Length - start, maxSpan)); } public static List> ParseMatches(string html) { var doc = Parser.ParseDocument(SliceAroundAllMatchesTable(html)); var teamLogo = ZerozeroImages.ToAbsoluteUrl(doc.QuerySelector(".zz-entity-header-team img[alt='FC Foz']")?.GetAttribute("src")); var headerEl = doc.QuerySelectorAll("h2.header").FirstOrDefault(el => el.TextContent.ToLowerInvariant().Contains("todos os jogos")); var rows = headerEl?.Closest(".box")?.QuerySelectorAll("table.zztable.stats tbody tr.parent") ?? Array.Empty().AsEnumerable().Cast(); var result = new List>(); foreach (var row in rows) { var tds = row.QuerySelectorAll("td"); if (tds.Length < 9) continue; var date = Txt(tds[1]); var time = Txt(tds[2]); var side = Txt(tds[3]); var opponentLogo = ZerozeroImages.ToAbsoluteUrl(tds[4].QuerySelector("img")?.GetAttribute("src")); var opponentName = Txt(tds[5]); var resultCell = Txt(tds[6]); var competition = Txt(tds[7]); var round = Txt(tds[8]); if (string.IsNullOrEmpty(date) || string.IsNullOrEmpty(opponentName)) continue; var isHome = side.Contains("(C)"); var (home, away) = ParseScore(resultCell); var matchId = row.GetAttribute("id") ?? $"ext-{Guid.NewGuid():N}"; var item = new Dictionary { ["id"] = matchId, ["homeTeam"] = isHome ? ZerozeroSite.FcFozName : opponentName, ["awayTeam"] = isHome ? opponentName : ZerozeroSite.FcFozName, ["homeLogo"] = isHome ? teamLogo : opponentLogo, ["awayLogo"] = isHome ? opponentLogo : teamLogo, ["date"] = date, ["time"] = time, ["venue"] = isHome ? "Casa" : "Fora", ["competition"] = string.IsNullOrEmpty(competition) ? "Competição" : competition, ["status"] = ParseStatusFromRow(resultCell, date, time, row.GetAttribute("class")), ["matchday"] = ParseMatchday(round), ["source"] = "external", }; if (home.HasValue) item["homeScore"] = home; if (away.HasValue) item["awayScore"] = away; result.Add(item); } return result.OrderBy(m => $"{m["date"]} {m["time"]}", StringComparer.Ordinal).ToList(); } public static List> ParseCupProgress(string html) { var doc = Parser.ParseDocument(html); var matches = new List>(); foreach (var table in doc.QuerySelectorAll("table.zztable, table.zztable.stats")) { foreach (var row in table.QuerySelectorAll("tbody tr")) { if (!row.TextContent.Contains(ZerozeroSite.FcFozName)) continue; var cells = row.QuerySelectorAll("td"); if (cells.Length < 4) continue; var href = row.QuerySelector("a[href*='/jogo/'], a[href*='/match/']")?.GetAttribute("href"); var matchId = !string.IsNullOrEmpty(href) ? href.Split('/', StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? $"ext-{Guid.NewGuid():N}" : $"ext-{Guid.NewGuid():N}"; var date = Txt(cells[0]); var homeTeam = Txt(cells[1]); var awayTeam = Txt(cells[3]); var result = Txt(cells[2]); var (home, away) = ParseScore(result); if (string.IsNullOrEmpty(date) || string.IsNullOrEmpty(homeTeam) || string.IsNullOrEmpty(awayTeam)) continue; var item = new Dictionary { ["id"] = matchId, ["homeTeam"] = homeTeam, ["awayTeam"] = awayTeam, ["date"] = date, ["time"] = "", ["venue"] = homeTeam == ZerozeroSite.FcFozName ? "Casa" : awayTeam == ZerozeroSite.FcFozName ? "Fora" : "-", ["competition"] = "AF Porto Taça", ["status"] = ParseStatusFromRow(result, date, ""), ["source"] = "external", }; if (home.HasValue) item["homeScore"] = home; if (away.HasValue) item["awayScore"] = away; matches.Add(item); } } return matches.OrderByDescending(m => m["date"]?.ToString(), StringComparer.Ordinal).ToList(); } // ---------------------------------------------------------------- news list public static List> ParseNewsList(string html) { var doc = Parser.ParseDocument(html); var seen = new HashSet(); var news = new List>(); void Push(Dictionary? item) { var slug = item?.GetValueOrDefault("slug")?.ToString() ?? ""; if (string.IsNullOrEmpty(slug) || !seen.Add(slug)) return; news.Add(item!); } foreach (var a in doc.QuerySelectorAll("#highlights > a")) { var href = a.GetAttribute("href") ?? ""; var slug = href.Split('/', StringSplitOptions.RemoveEmptyEntries).ElementAtOrDefault(1) ?? ""; var title = Txt(a.QuerySelector("h2")); if (string.IsNullOrEmpty(slug) || string.IsNullOrEmpty(title)) continue; Push(BuildNewsListItem(slug, title, Txt(a.QuerySelector("span.text")), Txt(a.QuerySelector(".dateauthor")), a.QuerySelector("img")?.GetAttribute("src"), Txt(a.QuerySelector(".keywords div")), href)); } foreach (var box in doc.QuerySelectorAll(".news_box_list")) { var titleLink = box.QuerySelector(".newstitle a"); var href = titleLink?.GetAttribute("href") ?? ""; var slug = href.Split('/', StringSplitOptions.RemoveEmptyEntries).ElementAtOrDefault(1) ?? ""; var title = Txt(titleLink); if (string.IsNullOrEmpty(slug) || string.IsNullOrEmpty(title)) continue; Push(BuildNewsListItem(slug, title, Txt(box.QuerySelector(".newstext")), Txt(box.QuerySelector(".dateauthor")), box.QuerySelector(".newsphoto img")?.GetAttribute("src"), Txt(box.QuerySelector(".keywords div")), href)); } return news .Where(item => !IsOpinionContent(item)) .OrderByDescending(item => item.GetValueOrDefault("publishedAt")?.ToString(), StringComparer.Ordinal) .ToList(); } private static Dictionary BuildNewsListItem( string slug, string title, string excerpt, string dateAuthor, string? imageSrc, string categoryText, string href) => new() { ["id"] = $"ext-{slug}", ["title"] = title, ["slug"] = slug, ["excerpt"] = string.IsNullOrEmpty(excerpt) ? title : excerpt, ["content"] = "", ["image"] = ZerozeroImages.ToAbsoluteUrl(imageSrc), ["category"] = NormalizeNewsCategory(categoryText), ["publishedAt"] = ParsePtDateTime(dateAuthor), ["author"] = "Redação", ["source"] = "external", ["externalUrl"] = ZerozeroImages.ToAbsoluteUrl(href), }; // ---------------------------------------------------------------- news article public static Dictionary? ParseNewsArticle(string html, Dictionary baseItem) { if (IsOpinionContent(baseItem, html)) return null; var doc = Parser.ParseDocument(html); var article = doc.QuerySelector("#news_body"); string? Meta(string sel, string attr) => doc.QuerySelector(sel)?.GetAttribute(attr); var title = Txt(article?.QuerySelector("header.title h1")); if (string.IsNullOrEmpty(title)) title = (Meta("meta[property='og:title']", "content") ?? "").Trim(); if (string.IsNullOrEmpty(title)) title = baseItem.GetValueOrDefault("title")?.ToString() ?? ""; if (title.ToLowerInvariant().Contains("colunas de opini")) return null; var heroCandidates = new List { ExtractHeroImageFromArticleDom(doc), FindNewsArticleImageFromJsonLd(doc), Meta("meta[property='og:image']", "content"), Meta("meta[property='og:image:secure_url']", "content"), Meta("meta[name='twitter:image']", "content") ?? Meta("meta[name='twitter:image:src']", "content"), Meta("link[rel='image_src']", "href"), FirstAttr(article, new[] { ".photo_frame img", ".photo_frame a img", ".article-photo img", ".news-photo img", ".news_photo img", ".photo img", "figure.photo img" }, "src"), }; var imageRaw = ZerozeroImages.PickNewsHeroImage(heroCandidates) ?? ZerozeroImages.PickNewsHeroImage(new[] { baseItem.GetValueOrDefault("image")?.ToString() }); var imageUpscaled = ZerozeroImages.TryUpscaleNewsMediaUrl(imageRaw) ?? imageRaw; var image = imageUpscaled is null ? null : ZerozeroImages.NormalizeNoticiasHeroImageUrl(ZerozeroImages.UpgradeNewsImageToStaticzzCdn(imageUpscaled)); var authorRaw = Txt(article?.QuerySelector(".author-name")); var authorResolved = !string.IsNullOrEmpty(authorRaw) ? Regex.Replace(authorRaw, @"^Autor:\s*", "", RegexOptions.IgnoreCase).Trim() : baseItem.GetValueOrDefault("author")?.ToString(); var author = authorResolved == "zerozero.pt" ? "Redação" : authorResolved; var publishedAt = ParsePublishedAtFromArticle(Txt(article?.QuerySelector(".info_bar"))) ?? baseItem.GetValueOrDefault("publishedAt")?.ToString(); var contentRoot = new[] { article?.QuerySelector(".text.article-content"), article?.QuerySelector(".article-content"), article?.QuerySelector(".text"), article, }.Where(c => c != null).OrderByDescending(c => c!.TextContent.Trim().Length).FirstOrDefault() ?? article; if (contentRoot != null) foreach (var junk in contentRoot.QuerySelectorAll(".mid_article, script, style, noscript")) junk.Remove(); var paragraphs = contentRoot?.QuerySelectorAll("p").Select(p => p.TextContent.Trim()).Where(t => t.Length > 0).ToList() ?? new(); var content = paragraphs.Count > 0 ? string.Join("\n\n", paragraphs) : contentRoot?.TextContent.Trim() ?? ""; var result = new Dictionary(baseItem) { ["title"] = title, ["image"] = image, ["author"] = author, ["publishedAt"] = publishedAt, ["content"] = content, }; var baseExcerpt = baseItem.GetValueOrDefault("excerpt")?.ToString(); result["excerpt"] = !string.IsNullOrEmpty(baseExcerpt) ? baseExcerpt : !string.IsNullOrEmpty(TruncateText(content, ExcerptMaxLength)) ? TruncateText(content, ExcerptMaxLength) : title; return result; } private static string? FirstAttr(IElement? scope, string[] selectors, string attr) { if (scope == null) return null; foreach (var sel in selectors) { var v = scope.QuerySelector(sel)?.GetAttribute(attr); if (!string.IsNullOrWhiteSpace(v)) return v; } return null; } private static IEnumerable GatherUrlsFromImgElement(IElement img) { var attrs = new[] { "data-src", "data-lazy-src", "data-original", "data-full", "data-large", "data-zoom-image", "data-image", "data-srcset" }; foreach (var attr in attrs) { var v = img.GetAttribute(attr); if (string.IsNullOrWhiteSpace(v)) continue; if (attr == "data-srcset") { var best = ZerozeroImages.PickLargestFromSrcset(v); if (best != null) yield return best; } else yield return v.Trim(); } var fromSs = ZerozeroImages.PickLargestFromSrcset(img.GetAttribute("srcset")); if (fromSs != null) yield return fromSs; var src = img.GetAttribute("src"); if (!string.IsNullOrWhiteSpace(src)) yield return src.Trim(); } private static string? ExtractHeroImageFromArticleDom(IDocument doc) { var urls = new List(); foreach (var source in doc.QuerySelectorAll("#news_body picture source, #news_body .photo_frame picture source")) { var best = ZerozeroImages.PickLargestFromSrcset(source.GetAttribute("srcset")); if (best != null) urls.Add(best); var s = source.GetAttribute("src"); if (!string.IsNullOrWhiteSpace(s)) urls.Add(s.Trim()); } var imgSelectors = new[] { "#news_body .photo_frame img", "#news_body .photo_frame a img", "#news_body .article-photo img", "#news_body .main_photo img", "#news_body .headline-photo img", "#news_body figure.photo img", "#news_body .news_photo img", "#news_body .news-photo img", }; foreach (var sel in imgSelectors) { var el = doc.QuerySelector(sel); if (el != null) urls.AddRange(GatherUrlsFromImgElement(el)); } if (urls.Count == 0) return null; var resolved = urls.Select(u => ZerozeroImages.ToAbsoluteUrl(u) ?? u) .Where(u => Regex.IsMatch(u, @"^https?://", RegexOptions.IgnoreCase)).Cast(); return ZerozeroImages.PickNewsHeroImage(resolved); } private static string? FindNewsArticleImageFromJsonLd(IDocument doc) { var urls = new List(); foreach (var script in doc.QuerySelectorAll("script[type='application/ld+json']")) { var raw = script.TextContent.Trim(); if (raw.Length == 0) continue; JsonElement root; try { using var d = JsonDocument.Parse(raw); root = d.RootElement.Clone(); } catch { continue; } void Collect(JsonElement node) { if (node.ValueKind == JsonValueKind.Array) { foreach (var n in node.EnumerateArray()) Collect(n); return; } if (node.ValueKind != JsonValueKind.Object) return; if (node.TryGetProperty("@graph", out var graph) && graph.ValueKind == JsonValueKind.Array) { foreach (var g in graph.EnumerateArray()) Collect(g); return; } if (node.TryGetProperty("mainEntity", out var main)) Collect(main); var typeStr = ""; if (node.TryGetProperty("@type", out var t)) typeStr = t.ValueKind == JsonValueKind.Array ? string.Join(" ", t.EnumerateArray().Select(x => x.ToString().ToLowerInvariant())) : t.ToString().ToLowerInvariant(); if (!typeStr.Contains("newsarticle") && !typeStr.Contains("article")) return; if (node.TryGetProperty("image", out var img)) { void PushVal(JsonElement v) { if (v.ValueKind == JsonValueKind.String && Regex.IsMatch(v.GetString() ?? "", @"^https?://", RegexOptions.IgnoreCase)) urls.Add(v.GetString()!.Trim()); else if (v.ValueKind == JsonValueKind.Object && v.TryGetProperty("url", out var urlProp) && urlProp.ValueKind == JsonValueKind.String && Regex.IsMatch(urlProp.GetString() ?? "", @"^https?://", RegexOptions.IgnoreCase)) urls.Add(urlProp.GetString()!.Trim()); } if (img.ValueKind == JsonValueKind.Array) foreach (var v in img.EnumerateArray()) PushVal(v); else PushVal(img); } } Collect(root); } if (urls.Count == 0) return null; return ZerozeroImages.PickNewsHeroImage(urls.Distinct().Cast()); } // ---------------------------------------------------------------- standings public static List> ParseStandings(string html) { var doc = Parser.ParseDocument(html); var header = doc.QuerySelectorAll("h2.header").FirstOrDefault(el => el.TextContent.Trim().ToLowerInvariant().Contains("classificação")); IElement? table = null; if (header != null) table = header.Closest(".box, .card-data")?.QuerySelector("table"); if (table == null) { foreach (var t in doc.QuerySelectorAll("table")) { var headerText = string.Join("|", t.QuerySelectorAll("thead th").Select(th => th.TextContent.Trim().ToLowerInvariant())); if (headerText.Contains("pts") && headerText.Contains("j") && (headerText.Contains("v") || headerText.Contains("e"))) { table = t; break; } } } if (table == null) return new(); var headers = table.QuerySelectorAll("thead th").Select(th => th.TextContent.Trim().ToLowerInvariant()).ToList(); int IndexOf(string label) => headers.FindIndex(h => h == label || h.Contains(label)); var idxPos = headers.FindIndex(h => h is "#" or "pos" or "º" || h.Contains("pos")); var idxTeam = headers.FindIndex(h => h.Contains("equipa") || h.Contains("team") || h.Contains("clube")); var idxPts = headers.FindIndex(h => h is "pts" or "p" || h.Contains("pontos")); var idxJ = IndexOf("j"); var idxV = IndexOf("v"); var idxE = IndexOf("e"); var idxD = IndexOf("d"); var idxGM = headers.FindIndex(h => h.Contains("gm") || h.Contains("gp")); var idxGS = headers.FindIndex(h => h.Contains("gs") || h.Contains("gc")); if (idxPos == -1) idxPos = 0; if (idxTeam == -1) idxTeam = 2; if (idxPts == -1) idxPts = 3; if (idxJ == -1) idxJ = 4; if (idxV == -1) idxV = 5; if (idxE == -1) idxE = 6; if (idxD == -1) idxD = 7; if (idxGM == -1) idxGM = 8; if (idxGS == -1) idxGS = 9; var result = new List>(); foreach (var row in table.QuerySelectorAll("tbody tr")) { var cells = row.QuerySelectorAll("td"); if (cells.Length < 5) continue; if (!int.TryParse(Txt(cells[idxPos]), out var position)) continue; var teamCell = cells[idxTeam]; var team = Regex.Replace(Txt(teamCell), @"\s+", " "); var logo = ZerozeroImages.ToAbsoluteUrl(teamCell.QuerySelector("img")?.GetAttribute("src")) ?? (idxTeam > 0 ? ZerozeroImages.ToAbsoluteUrl(cells[idxTeam - 1].QuerySelector("img")?.GetAttribute("src")) : null); int NumAt(int idx) { if (idx < 0 || idx >= cells.Length) return 0; var cleaned = Regex.Replace(Txt(cells[idx]), @"[^\d]", ""); return int.TryParse(cleaned, out var n) ? n : 0; } result.Add(new Dictionary { ["position"] = position, ["team"] = team, ["played"] = NumAt(idxJ), ["won"] = NumAt(idxV), ["drawn"] = NumAt(idxE), ["lost"] = NumAt(idxD), ["goalsFor"] = NumAt(idxGM), ["goalsAgainst"] = NumAt(idxGS), ["points"] = NumAt(idxPts), ["logo"] = logo, }); } return result; } }