SoftTraderBackend/ServiceHandler/EdgarFilingService.cs

using Backend.DatabaseHandler;
using Backend.Interface;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;

namespace Backend.ServiceHandler
{
	public sealed class EdgarFilingService : IDisposable
	{
		private static readonly HashSet<string> RelevantForms = ["8-K", "10-Q", "10-K"];

		private static readonly Dictionary<string, string> EightKItems = new()
		{
			["1.01"] = "Material Definitive Agreement",
			["1.02"] = "Termination of Material Agreement",
			["1.03"] = "Bankruptcy or Receivership",
			["2.01"] = "Acquisition or Disposition of Assets",
			["2.02"] = "Results of Operations and Financial Condition",
			["2.03"] = "Creation of Direct Financial Obligation",
			["2.05"] = "Exit or Disposal Activities",
			["2.06"] = "Material Impairments",
			["3.01"] = "Notice of Delisting",
			["3.02"] = "Unregistered Sales of Equity Securities",
			["3.03"] = "Material Modification to Rights of Security Holders",
			["4.01"] = "Changes in Certifying Accountant",
			["4.02"] = "Non-Reliance on Prior Financial Statements",
			["5.01"] = "Changes in Control of Registrant",
			["5.02"] = "Departure/Election of Directors or Officers",
			["5.03"] = "Amendments to Articles or Bylaws",
			["5.07"] = "Submission of Matters to Vote",
			["7.01"] = "Regulation FD Disclosure",
			["8.01"] = "Other Events",
		};

		private readonly SemaphoreSlim _throttle = new(1, 1);
		private readonly TimeSpan _requestDelay;
		private readonly HttpClient _http;
		private readonly INewsSentimentStore _store;
		private readonly ILogger<EdgarFilingService> _logger;

		private static readonly SemaphoreSlim _cikLock = new(1, 1);
		private static Dictionary<string, string>? _tickerToCik;
		private static DateTime _cikCacheExpiry = DateTime.MinValue;

		public EdgarFilingService(
			INewsSentimentStore store,
			IConfiguration config,
			ILogger<EdgarFilingService> logger)
		{
			_store = store;
			_logger = logger;

			var userAgent = config["Edgar:UserAgent"]
				?? "SoftTrader contact@example.com";
			var delayMs = config.GetValue("Edgar:RequestDelayMs", 600);
			_requestDelay = TimeSpan.FromMilliseconds(delayMs);

			_http = new HttpClient();
			_http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", userAgent);
			_http.DefaultRequestHeaders.Accept.ParseAdd("application/json");
		}

		public async Task<List<NewsSentimentRecord>> FetchAndStoreAsync(
			string ticker,
			CancellationToken cancellationToken = default)
		{
			ticker = ticker.Trim().ToUpperInvariant();

			var cik = await ResolveCikAsync(ticker, cancellationToken);
			if (cik is null)
			{
				_logger.LogDebug("No CIK found for '{Ticker}', skipping EDGAR.", ticker);
				return [];
			}

			var existingHashes = await _store.GetExistingHashesAsync(ticker, cancellationToken);
			var filings = await FetchSubmissionsAsync(cik, cancellationToken);

			var newRecords = new List<NewsSentimentRecord>();

			foreach (var filing in filings)
			{
				var hash = ComputeHash(filing.AccessionNumber);
				if (existingHashes.Contains(hash))
					continue;

				string title;
				if (filing.Form == "8-K")
				{
					var items = await ParseEightKItemsAsync(cik, filing, cancellationToken);
					title = items.Count > 0
						? $"8-K: {string.Join("; ", items)}"
						: "8-K Filing";
				}
				else
				{
					title = filing.Form == "10-Q"
						? "Quarterly Report (10-Q)"
						: "Annual Report (10-K)";
				}

				var accessionNoDashes = filing.AccessionNumber.Replace("-", "");
				var url = $"https://www.sec.gov/Archives/edgar/data/{cik}/{accessionNoDashes}/{filing.PrimaryDocument}";

				newRecords.Add(new NewsSentimentRecord
				{
					Source = "sec_edgar",
					Keyword = ticker,
					PublishedAt = filing.FilingDate,
					Title = title,
					Score = 50,
					SentimentLabel = "filing",
					TitleHash = hash,
					Publisher = "SEC EDGAR",
					Url = url,
				});
			}

			if (newRecords.Count > 0)
			{
				await _store.InsertAsync(newRecords, cancellationToken);
				_logger.LogInformation("Stored {Count} new EDGAR filings for {Ticker}.", newRecords.Count, ticker);
			}

			return newRecords;
		}

		private async Task<string?> ResolveCikAsync(string ticker, CancellationToken ct)
		{
			await _cikLock.WaitAsync(ct);
			try
			{
				if (_tickerToCik is null || DateTime.UtcNow > _cikCacheExpiry)
				{
					var json = await ThrottledGetStringAsync(
						"https://www.sec.gov/files/company_tickers.json", ct);

					using var doc = JsonDocument.Parse(json);
					var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

					foreach (var prop in doc.RootElement.EnumerateObject())
					{
						var t = prop.Value.GetProperty("ticker").GetString();
						var c = prop.Value.GetProperty("cik_str").GetInt64();
						if (t is not null)
							map[t] = c.ToString().PadLeft(10, '0');
					}

					_tickerToCik = map;
					_cikCacheExpiry = DateTime.UtcNow.AddHours(24);
				}
			}
			finally
			{
				_cikLock.Release();
			}

			return _tickerToCik.GetValueOrDefault(ticker);
		}

		private async Task<List<EdgarFiling>> FetchSubmissionsAsync(string cik, CancellationToken ct)
		{
			var json = await ThrottledGetStringAsync(
				$"https://data.sec.gov/submissions/CIK{cik}.json", ct);

			using var doc = JsonDocument.Parse(json);
			var recent = doc.RootElement.GetProperty("recent");
			return ParseFilingArrays(recent);
		}

		private static List<EdgarFiling> ParseFilingArrays(JsonElement recent)
		{
			var accessions = recent.GetProperty("accessionNumber");
			var dates = recent.GetProperty("filingDate");
			var forms = recent.GetProperty("form");
			var primaryDocs = recent.GetProperty("primaryDocument");

			var filings = new List<EdgarFiling>();
			int count = accessions.GetArrayLength();

			for (int i = 0; i < count; i++)
			{
				var form = forms[i].GetString() ?? "";
				if (!RelevantForms.Contains(form))
					continue;

				var dateStr = dates[i].GetString();
				if (!DateTimeOffset.TryParse(dateStr, out var filingDate))
					continue;

				filings.Add(new EdgarFiling
				{
					AccessionNumber = accessions[i].GetString() ?? "",
					FilingDate = filingDate,
					Form = form,
					PrimaryDocument = primaryDocs[i].GetString() ?? "",
				});
			}

			return filings;
		}

		private async Task<List<string>> ParseEightKItemsAsync(
			string cik, EdgarFiling filing, CancellationToken ct)
		{
			try
			{
				var accessionNoDashes = filing.AccessionNumber.Replace("-", "");
				var url = $"https://www.sec.gov/Archives/edgar/data/{cik}/{accessionNoDashes}/{filing.PrimaryDocument}";

				var html = await ThrottledGetStringAsync(url, ct);

				var matches = Regex.Matches(html, @"Item\s+(\d+\.\d+)", RegexOptions.IgnoreCase);
				var items = new List<string>();
				var seen = new HashSet<string>();

				foreach (Match match in matches)
				{
					var itemNum = match.Groups[1].Value;
					// Skip 9.01 (Financial Statements and Exhibits) — boilerplate in every 8-K
					if (itemNum == "9.01")
						continue;
					if (EightKItems.TryGetValue(itemNum, out var desc) && seen.Add(itemNum))
						items.Add($"{desc} (Item {itemNum})");
				}

				return items;
			}
			catch (Exception ex)
			{
				_logger.LogWarning(ex, "Failed to parse 8-K items for {Accession}.", filing.AccessionNumber);
				return [];
			}
		}

		private async Task<string> ThrottledGetStringAsync(string url, CancellationToken ct)
		{
			await _throttle.WaitAsync(ct);
			try
			{
				await Task.Delay(_requestDelay, ct);
				return await _http.GetStringAsync(url, ct);
			}
			finally
			{
				_throttle.Release();
			}
		}

		private static string ComputeHash(string text)
		{
			byte[] bytes = SHA256.HashData(Encoding.UTF8.GetBytes(text));
			return Convert.ToHexStringLower(bytes);
		}

		public void Dispose()
		{
			_http.Dispose();
			_throttle.Dispose();
		}

		private sealed class EdgarFiling
		{
			public string AccessionNumber { get; init; } = "";
			public DateTimeOffset FilingDate { get; init; }
			public string Form { get; init; } = "";
			public string PrimaryDocument { get; init; } = "";
		}
	}
}