using System.Text; using System.Text.RegularExpressions; using FozFc.Api.Configuration; namespace FozFc.Api.Infrastructure.Zerozero; /// Shared constants for the zerozero.pt scraper. public static class ZerozeroSite { public const string Origin = "https://www.zerozero.pt"; public const string TeamGamesPath = "/equipa/fc-foz/8493/jogos"; public const string TeamNewsPath = "/equipa/fc-foz/8493/noticias"; public const string LigaProPath = "/competicao/af-porto-divisao-liga-pro-2383"; public const string TacaPath = "/competicao/af-porto-taca"; public const string FcFozName = "FC Foz"; } /// /// Fetches HTML from zerozero.pt with browser-like headers, retries and correct /// charset decoding (the site sometimes serves ISO-8859-1 / Windows-1252). /// Optionally routes through a render proxy (anti-bot) when configured. /// public sealed class ZerozeroHttpClient { private const int RetryAttempts = 3; private const int RetryBaseDelayMs = 600; private static readonly string[] UserAgents = { "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", }; private readonly IHttpClientFactory _httpClientFactory; private readonly AppOptions _options; private readonly ILogger _logger; static ZerozeroHttpClient() { // Enable Windows-1252 / ISO-8859-1 decoders (not in the default .NET set). Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); } public ZerozeroHttpClient(IHttpClientFactory httpClientFactory, AppOptions options, ILogger logger) { _httpClientFactory = httpClientFactory; _options = options; _logger = logger; } private static string RandomUserAgent() => UserAgents[Random.Shared.Next(UserAgents.Length)]; private void ApplyBrowserHeaders(HttpRequestMessage req, string referer) { req.Headers.TryAddWithoutValidation("User-Agent", RandomUserAgent()); req.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"); req.Headers.TryAddWithoutValidation("Accept-Language", "pt-PT,pt;q=0.9,en-US;q=0.8,en;q=0.7"); req.Headers.TryAddWithoutValidation("Cache-Control", "no-cache"); req.Headers.TryAddWithoutValidation("Pragma", "no-cache"); req.Headers.TryAddWithoutValidation("Sec-Fetch-Dest", "document"); req.Headers.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate"); req.Headers.TryAddWithoutValidation("Sec-Fetch-Site", "none"); req.Headers.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1"); req.Headers.TryAddWithoutValidation("Referer", string.IsNullOrEmpty(referer) ? $"{ZerozeroSite.Origin}/" : referer); } /// Fetches the raw bytes for a zerozero path (or absolute URL for media). public async Task<(byte[] Bytes, string ContentType)> FetchBytesAsync(string url, string? referer = null, CancellationToken ct = default) { var client = _httpClientFactory.CreateClient("zerozero"); Exception? last = null; for (var attempt = 1; attempt <= RetryAttempts; attempt++) { try { using var req = new HttpRequestMessage(HttpMethod.Get, ResolveTarget(url, out var proxied)); if (!proxied) ApplyBrowserHeaders(req, referer ?? $"{ZerozeroSite.Origin}/"); else if (!string.IsNullOrEmpty(_options.RenderProxySecret)) req.Headers.TryAddWithoutValidation("x-proxy-secret", _options.RenderProxySecret); using var res = await client.SendAsync(req, ct); if (!res.IsSuccessStatusCode) throw new HttpRequestException($"HTTP {(int)res.StatusCode} from {url}"); var bytes = await res.Content.ReadAsByteArrayAsync(ct); var contentType = res.Content.Headers.ContentType?.ToString() ?? ""; return (bytes, contentType); } catch (Exception ex) { last = ex; _logger.LogWarning("[zerozero] attempt {Attempt}/{Total} for {Url}: {Message}", attempt, RetryAttempts, url, ex.Message); if (attempt < RetryAttempts) await Task.Delay(RetryBaseDelayMs * (int)Math.Pow(2, attempt - 1), ct); } } throw last ?? new Exception("Fetch failed"); } /// Fetches a zerozero path and decodes it to HTML with the right charset. public async Task FetchHtmlAsync(string path, string? referer = null, CancellationToken ct = default) { var target = path.StartsWith("http", StringComparison.OrdinalIgnoreCase) ? path : $"{ZerozeroSite.Origin}{path}"; var (bytes, contentType) = await FetchBytesAsync(target, referer, ct); return DecodeHtml(bytes, contentType); } private string ResolveTarget(string url, out bool proxied) { if (!string.IsNullOrEmpty(_options.RenderProxyUrl) && url.StartsWith(ZerozeroSite.Origin, StringComparison.OrdinalIgnoreCase)) { proxied = true; var proxyBase = _options.RenderProxyUrl.TrimEnd('/'); return $"{proxyBase}/fetch?url={Uri.EscapeDataString(url)}"; } proxied = false; return url; } private static string DecodeHtml(byte[] bytes, string contentType) { var headerCharset = Regex.Match(contentType, @"charset\s*=\s*([^\s;]+)", RegexOptions.IgnoreCase).Groups[1].Value; var probe = Encoding.Latin1.GetString(bytes, 0, Math.Min(bytes.Length, 2048)); var metaCharset = Regex.Match(probe, @"charset\s*=\s*[""']?\s*([a-z0-9\-_]+)", RegexOptions.IgnoreCase).Groups[1].Value; var charset = (string.IsNullOrEmpty(headerCharset) ? metaCharset : headerCharset).ToLowerInvariant(); Encoding encoding; if (charset.Contains("iso-8859-1") || charset.Contains("latin1") || charset.Contains("windows-1252")) { try { encoding = Encoding.GetEncoding("windows-1252"); } catch { encoding = Encoding.Latin1; } } else { encoding = Encoding.UTF8; } return encoding.GetString(bytes); } }