using System.Text;
using System.Text.RegularExpressions;
using FozFc.Api.Configuration;
namespace FozFc.Api.Infrastructure.Zerozero;
/// Shared constants for the zerozero.pt scraper.
public static class ZerozeroSite
{
public const string Origin = "https://www.zerozero.pt";
public const string TeamGamesPath = "/equipa/fc-foz/8493/jogos";
public const string TeamNewsPath = "/equipa/fc-foz/8493/noticias";
public const string LigaProPath = "/competicao/af-porto-divisao-liga-pro-2383";
public const string TacaPath = "/competicao/af-porto-taca";
public const string FcFozName = "FC Foz";
}
///
/// Fetches HTML from zerozero.pt with browser-like headers, retries and correct
/// charset decoding (the site sometimes serves ISO-8859-1 / Windows-1252).
/// Optionally routes through a render proxy (anti-bot) when configured.
///
public sealed class ZerozeroHttpClient
{
private const int RetryAttempts = 3;
private const int RetryBaseDelayMs = 600;
private static readonly string[] UserAgents =
{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly AppOptions _options;
private readonly ILogger _logger;
static ZerozeroHttpClient()
{
// Enable Windows-1252 / ISO-8859-1 decoders (not in the default .NET set).
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
}
public ZerozeroHttpClient(IHttpClientFactory httpClientFactory, AppOptions options, ILogger logger)
{
_httpClientFactory = httpClientFactory;
_options = options;
_logger = logger;
}
private static string RandomUserAgent() => UserAgents[Random.Shared.Next(UserAgents.Length)];
private void ApplyBrowserHeaders(HttpRequestMessage req, string referer)
{
req.Headers.TryAddWithoutValidation("User-Agent", RandomUserAgent());
req.Headers.TryAddWithoutValidation("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8");
req.Headers.TryAddWithoutValidation("Accept-Language", "pt-PT,pt;q=0.9,en-US;q=0.8,en;q=0.7");
req.Headers.TryAddWithoutValidation("Cache-Control", "no-cache");
req.Headers.TryAddWithoutValidation("Pragma", "no-cache");
req.Headers.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
req.Headers.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
req.Headers.TryAddWithoutValidation("Sec-Fetch-Site", "none");
req.Headers.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1");
req.Headers.TryAddWithoutValidation("Referer", string.IsNullOrEmpty(referer) ? $"{ZerozeroSite.Origin}/" : referer);
}
/// Fetches the raw bytes for a zerozero path (or absolute URL for media).
public async Task<(byte[] Bytes, string ContentType)> FetchBytesAsync(string url, string? referer = null, CancellationToken ct = default)
{
var client = _httpClientFactory.CreateClient("zerozero");
Exception? last = null;
for (var attempt = 1; attempt <= RetryAttempts; attempt++)
{
try
{
using var req = new HttpRequestMessage(HttpMethod.Get, ResolveTarget(url, out var proxied));
if (!proxied) ApplyBrowserHeaders(req, referer ?? $"{ZerozeroSite.Origin}/");
else if (!string.IsNullOrEmpty(_options.RenderProxySecret))
req.Headers.TryAddWithoutValidation("x-proxy-secret", _options.RenderProxySecret);
using var res = await client.SendAsync(req, ct);
if (!res.IsSuccessStatusCode)
throw new HttpRequestException($"HTTP {(int)res.StatusCode} from {url}");
var bytes = await res.Content.ReadAsByteArrayAsync(ct);
var contentType = res.Content.Headers.ContentType?.ToString() ?? "";
return (bytes, contentType);
}
catch (Exception ex)
{
last = ex;
_logger.LogWarning("[zerozero] attempt {Attempt}/{Total} for {Url}: {Message}", attempt, RetryAttempts, url, ex.Message);
if (attempt < RetryAttempts)
await Task.Delay(RetryBaseDelayMs * (int)Math.Pow(2, attempt - 1), ct);
}
}
throw last ?? new Exception("Fetch failed");
}
/// Fetches a zerozero path and decodes it to HTML with the right charset.
public async Task FetchHtmlAsync(string path, string? referer = null, CancellationToken ct = default)
{
var target = path.StartsWith("http", StringComparison.OrdinalIgnoreCase) ? path : $"{ZerozeroSite.Origin}{path}";
var (bytes, contentType) = await FetchBytesAsync(target, referer, ct);
return DecodeHtml(bytes, contentType);
}
private string ResolveTarget(string url, out bool proxied)
{
if (!string.IsNullOrEmpty(_options.RenderProxyUrl) && url.StartsWith(ZerozeroSite.Origin, StringComparison.OrdinalIgnoreCase))
{
proxied = true;
var proxyBase = _options.RenderProxyUrl.TrimEnd('/');
return $"{proxyBase}/fetch?url={Uri.EscapeDataString(url)}";
}
proxied = false;
return url;
}
private static string DecodeHtml(byte[] bytes, string contentType)
{
var headerCharset = Regex.Match(contentType, @"charset\s*=\s*([^\s;]+)", RegexOptions.IgnoreCase).Groups[1].Value;
var probe = Encoding.Latin1.GetString(bytes, 0, Math.Min(bytes.Length, 2048));
var metaCharset = Regex.Match(probe, @"charset\s*=\s*[""']?\s*([a-z0-9\-_]+)", RegexOptions.IgnoreCase).Groups[1].Value;
var charset = (string.IsNullOrEmpty(headerCharset) ? metaCharset : headerCharset).ToLowerInvariant();
Encoding encoding;
if (charset.Contains("iso-8859-1") || charset.Contains("latin1") || charset.Contains("windows-1252"))
{
try { encoding = Encoding.GetEncoding("windows-1252"); }
catch { encoding = Encoding.Latin1; }
}
else
{
encoding = Encoding.UTF8;
}
return encoding.GetString(bytes);
}
}