using Backend.DatabaseHandler; using Backend.Interface; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; namespace Backend.ServiceHandler { public sealed class EdgarFilingService : IDisposable { private static readonly HashSet RelevantForms = ["8-K", "10-Q", "10-K"]; private static readonly Dictionary EightKItems = new() { ["1.01"] = "Material Definitive Agreement", ["1.02"] = "Termination of Material Agreement", ["1.03"] = "Bankruptcy or Receivership", ["2.01"] = "Acquisition or Disposition of Assets", ["2.02"] = "Results of Operations and Financial Condition", ["2.03"] = "Creation of Direct Financial Obligation", ["2.05"] = "Exit or Disposal Activities", ["2.06"] = "Material Impairments", ["3.01"] = "Notice of Delisting", ["3.02"] = "Unregistered Sales of Equity Securities", ["3.03"] = "Material Modification to Rights of Security Holders", ["4.01"] = "Changes in Certifying Accountant", ["4.02"] = "Non-Reliance on Prior Financial Statements", ["5.01"] = "Changes in Control of Registrant", ["5.02"] = "Departure/Election of Directors or Officers", ["5.03"] = "Amendments to Articles or Bylaws", ["5.07"] = "Submission of Matters to Vote", ["7.01"] = "Regulation FD Disclosure", ["8.01"] = "Other Events", }; private readonly SemaphoreSlim _throttle = new(1, 1); private readonly TimeSpan _requestDelay; private readonly HttpClient _http; private readonly INewsSentimentStore _store; private readonly ILogger _logger; private static readonly SemaphoreSlim _cikLock = new(1, 1); private static Dictionary? _tickerToCik; private static DateTime _cikCacheExpiry = DateTime.MinValue; public EdgarFilingService( INewsSentimentStore store, IConfiguration config, ILogger logger) { _store = store; _logger = logger; var userAgent = config["Edgar:UserAgent"] ?? "SoftTrader contact@example.com"; var delayMs = config.GetValue("Edgar:RequestDelayMs", 600); _requestDelay = TimeSpan.FromMilliseconds(delayMs); _http = new HttpClient(); _http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", userAgent); _http.DefaultRequestHeaders.Accept.ParseAdd("application/json"); } public async Task> FetchAndStoreAsync( string ticker, CancellationToken cancellationToken = default) { ticker = ticker.Trim().ToUpperInvariant(); var cik = await ResolveCikAsync(ticker, cancellationToken); if (cik is null) { _logger.LogDebug("No CIK found for '{Ticker}', skipping EDGAR.", ticker); return []; } var existingHashes = await _store.GetExistingHashesAsync(ticker, cancellationToken); var filings = await FetchSubmissionsAsync(cik, cancellationToken); var newRecords = new List(); foreach (var filing in filings) { var hash = ComputeHash(filing.AccessionNumber); if (existingHashes.Contains(hash)) continue; string title; if (filing.Form == "8-K") { var items = await ParseEightKItemsAsync(cik, filing, cancellationToken); title = items.Count > 0 ? $"8-K: {string.Join("; ", items)}" : "8-K Filing"; } else { title = filing.Form == "10-Q" ? "Quarterly Report (10-Q)" : "Annual Report (10-K)"; } var accessionNoDashes = filing.AccessionNumber.Replace("-", ""); var url = $"https://www.sec.gov/Archives/edgar/data/{cik}/{accessionNoDashes}/{filing.PrimaryDocument}"; newRecords.Add(new NewsSentimentRecord { Source = "sec_edgar", Keyword = ticker, PublishedAt = filing.FilingDate, Title = title, Score = 50, SentimentLabel = "filing", TitleHash = hash, Publisher = "SEC EDGAR", Url = url, }); } if (newRecords.Count > 0) { await _store.InsertAsync(newRecords, cancellationToken); _logger.LogInformation("Stored {Count} new EDGAR filings for {Ticker}.", newRecords.Count, ticker); } return newRecords; } private async Task ResolveCikAsync(string ticker, CancellationToken ct) { await _cikLock.WaitAsync(ct); try { if (_tickerToCik is null || DateTime.UtcNow > _cikCacheExpiry) { var json = await ThrottledGetStringAsync( "https://www.sec.gov/files/company_tickers.json", ct); using var doc = JsonDocument.Parse(json); var map = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var prop in doc.RootElement.EnumerateObject()) { var t = prop.Value.GetProperty("ticker").GetString(); var c = prop.Value.GetProperty("cik_str").GetInt64(); if (t is not null) map[t] = c.ToString().PadLeft(10, '0'); } _tickerToCik = map; _cikCacheExpiry = DateTime.UtcNow.AddHours(24); } } finally { _cikLock.Release(); } return _tickerToCik.GetValueOrDefault(ticker); } private async Task> FetchSubmissionsAsync(string cik, CancellationToken ct) { var json = await ThrottledGetStringAsync( $"https://data.sec.gov/submissions/CIK{cik}.json", ct); using var doc = JsonDocument.Parse(json); var recent = doc.RootElement.GetProperty("recent"); return ParseFilingArrays(recent); } private static List ParseFilingArrays(JsonElement recent) { var accessions = recent.GetProperty("accessionNumber"); var dates = recent.GetProperty("filingDate"); var forms = recent.GetProperty("form"); var primaryDocs = recent.GetProperty("primaryDocument"); var filings = new List(); int count = accessions.GetArrayLength(); for (int i = 0; i < count; i++) { var form = forms[i].GetString() ?? ""; if (!RelevantForms.Contains(form)) continue; var dateStr = dates[i].GetString(); if (!DateTimeOffset.TryParse(dateStr, out var filingDate)) continue; filings.Add(new EdgarFiling { AccessionNumber = accessions[i].GetString() ?? "", FilingDate = filingDate, Form = form, PrimaryDocument = primaryDocs[i].GetString() ?? "", }); } return filings; } private async Task> ParseEightKItemsAsync( string cik, EdgarFiling filing, CancellationToken ct) { try { var accessionNoDashes = filing.AccessionNumber.Replace("-", ""); var url = $"https://www.sec.gov/Archives/edgar/data/{cik}/{accessionNoDashes}/{filing.PrimaryDocument}"; var html = await ThrottledGetStringAsync(url, ct); var matches = Regex.Matches(html, @"Item\s+(\d+\.\d+)", RegexOptions.IgnoreCase); var items = new List(); var seen = new HashSet(); foreach (Match match in matches) { var itemNum = match.Groups[1].Value; // Skip 9.01 (Financial Statements and Exhibits) — boilerplate in every 8-K if (itemNum == "9.01") continue; if (EightKItems.TryGetValue(itemNum, out var desc) && seen.Add(itemNum)) items.Add($"{desc} (Item {itemNum})"); } return items; } catch (Exception ex) { _logger.LogWarning(ex, "Failed to parse 8-K items for {Accession}.", filing.AccessionNumber); return []; } } private async Task ThrottledGetStringAsync(string url, CancellationToken ct) { await _throttle.WaitAsync(ct); try { await Task.Delay(_requestDelay, ct); return await _http.GetStringAsync(url, ct); } finally { _throttle.Release(); } } private static string ComputeHash(string text) { byte[] bytes = SHA256.HashData(Encoding.UTF8.GetBytes(text)); return Convert.ToHexStringLower(bytes); } public void Dispose() { _http.Dispose(); _throttle.Dispose(); } private sealed class EdgarFiling { public string AccessionNumber { get; init; } = ""; public DateTimeOffset FilingDate { get; init; } public string Form { get; init; } = ""; public string PrimaryDocument { get; init; } = ""; } } }