From f85779fde83faea1ac665d4984e4207e4cd86bed Mon Sep 17 00:00:00 2001 From: caspar Date: Thu, 19 Mar 2026 15:55:48 -0400 Subject: [PATCH] Integrated news headline functions. --- Backend.cs | 1 + Controller/NewsSentimentController.cs | 99 ++++++++++++++++++- Controller/UserInterestController.cs | 16 +-- DatabaseHandler/HistoricalRequestParser.cs | 7 +- DatabaseHandler/NewsSentimentRecord.cs | 1 + .../PostgresHistoricalMarketDataStore.cs | 58 +++++------ DatabaseHandler/PostgresNewsSentimentStore.cs | 87 ++++++++++++---- DatabaseHandler/PostgresUserInterestStore.cs | 71 ++++++++----- Interface/INewsSentimentStore.cs | 5 + Interface/IUserInterestCache.cs | 1 + ServiceHandler/NewsSentimentRefreshService.cs | 78 +++++++++++++++ ServiceHandler/NewsSentimentService.cs | 14 ++- 12 files changed, 355 insertions(+), 83 deletions(-) create mode 100644 ServiceHandler/NewsSentimentRefreshService.cs diff --git a/Backend.cs b/Backend.cs index 68002eb..8e007d8 100644 --- a/Backend.cs +++ b/Backend.cs @@ -15,6 +15,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddHostedService(); builder.Services.AddCors(options => { diff --git a/Controller/NewsSentimentController.cs b/Controller/NewsSentimentController.cs index 9abd135..6ef7d4c 100644 --- a/Controller/NewsSentimentController.cs +++ b/Controller/NewsSentimentController.cs @@ -1,3 +1,4 @@ +using Backend.Interface; using Backend.ServiceHander; using Microsoft.AspNetCore.Mvc; @@ -7,11 +8,20 @@ namespace Backend.Controller [Route("api/news-sentiment")] public sealed class NewsSentimentController : ControllerBase { + private static DateTime _lastRefresh = DateTime.MinValue; + private static readonly Lock _refreshLock = new(); private readonly NewsSentimentService _service; + private readonly IUserInterestStore _interests; + private readonly INewsSentimentStore _store; - public NewsSentimentController(NewsSentimentService service) + public NewsSentimentController( + NewsSentimentService service, + IUserInterestStore interests, + INewsSentimentStore store) { _service = service; + _interests = interests; + _store = store; } [HttpGet] @@ -30,6 +40,7 @@ namespace Backend.Controller title = r.Title, score = r.Score, sentimentLabel = r.SentimentLabel, + publisher = r.Publisher, url = r.Url })); } @@ -42,5 +53,91 @@ namespace Backend.Controller return StatusCode(500, new { error = ex.ToString() }); } } + + [HttpGet("feed")] + public async Task Feed( + [FromQuery] string user, + [FromQuery] string? keyword = null, + [FromQuery(Name = "request-amount")] int requestAmount = 20, + CancellationToken cancellationToken = default) + { + try + { + if (string.IsNullOrWhiteSpace(user)) + return BadRequest("User cannot be empty."); + + if (requestAmount < 1) + requestAmount = 1; + + List keywords; + + if (!string.IsNullOrWhiteSpace(keyword)) + { + keywords = [keyword.Trim()]; + } + else + { + var symbols = await _interests.FetchAllAsync(user, "symbol"); + var topics = await _interests.FetchAllAsync(user, "topic"); + keywords = symbols.Concat(topics).Distinct().ToList(); + } + + if (keywords.Count == 0) + return Ok(Array.Empty()); + + var results = await _store.GetFeedAsync(keywords, requestAmount, cancellationToken); + + return Ok(results.Select(r => new + { + title = r.Title, + publisher = r.Publisher, + score = r.Score, + publishedAt = r.PublishedAt.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ssZ"), + url = r.Url + })); + } + catch (ArgumentException ex) + { + return BadRequest(ex.Message); + } + catch (Exception ex) + { + return StatusCode(500, new { error = ex.ToString() }); + } + } + [HttpGet("refresh")] + public async Task> Refresh( + [FromQuery] string user, + CancellationToken cancellationToken = default) + { + try + { + if (string.IsNullOrWhiteSpace(user)) + return BadRequest("User cannot be empty."); + + // 5 second refresh cooldown - abuse protection + lock (_refreshLock) + { + if ((DateTime.UtcNow - _lastRefresh).TotalSeconds < 5) + return Ok(new BoolResponse { Success = false }); + _lastRefresh = DateTime.UtcNow; + } + + var symbols = await _interests.FetchAllAsync(user, "symbol"); + var topics = await _interests.FetchAllAsync(user, "topic"); + var keywords = symbols.Concat(topics).Distinct().ToList(); + + foreach (var keyword in keywords) + { + await _service.AnalyzeAsync(keyword, cancellationToken); + } + + return Ok(new BoolResponse { Success = true }); + } + catch (Exception) + { + return Ok(new BoolResponse { Success = false }); + } + } } } diff --git a/Controller/UserInterestController.cs b/Controller/UserInterestController.cs index a95c9a0..f46ba06 100644 --- a/Controller/UserInterestController.cs +++ b/Controller/UserInterestController.cs @@ -35,13 +35,15 @@ namespace Backend.Controllers } } - [HttpPost("add")] + [HttpGet("add")] public async Task> Add( - [FromBody] UserInterestModifyRequest request) + [FromQuery] string user, + [FromQuery] string type, + [FromQuery] string value) { try { - bool success = await _store.AddAsync(request.User, request.Type, request.Value); + bool success = await _store.AddAsync(user, type, value); return Ok(new BoolResponse { Success = success }); } catch (ArgumentException ex) @@ -50,13 +52,15 @@ namespace Backend.Controllers } } - [HttpDelete("remove")] + [HttpGet("remove")] public async Task> Remove( - [FromBody] UserInterestModifyRequest request) + [FromQuery] string user, + [FromQuery] string type, + [FromQuery] string value) { try { - bool success = await _store.RemoveAsync(request.User, request.Type, request.Value); + bool success = await _store.RemoveAsync(user, type, value); return Ok(new BoolResponse { Success = success }); } catch (ArgumentException ex) diff --git a/DatabaseHandler/HistoricalRequestParser.cs b/DatabaseHandler/HistoricalRequestParser.cs index 6b656cf..1f1d06c 100644 --- a/DatabaseHandler/HistoricalRequestParser.cs +++ b/DatabaseHandler/HistoricalRequestParser.cs @@ -94,11 +94,14 @@ Console.WriteLine(raw); // Intraday fraction: (6.5h × 5 days) / (24h × 7 days) ≈ 0.194 // Daily/weekly fraction (weekdays only): 5 / 7 ≈ 0.714 // Monthly bars already map 1:1 to calendar months, no adjustment needed. + // NYSE has ~252 trading days per year (weekdays minus ~10 holidays). + // Using 252/365 instead of 5/7 avoids overestimating for 3+ month windows + // where accumulated holidays cause the cache check to fail. double marketFraction = barSize switch { HistoricalBarSize.OneMonth => 1.0, - HistoricalBarSize.OneDay or HistoricalBarSize.FiveDays => 5.0 / 7.0, - _ => (6.5 * 5.0) / (24.0 * 7.0) + HistoricalBarSize.OneDay or HistoricalBarSize.FiveDays => 252.0 / 365.0, + _ => (6.5 * 252.0) / (24.0 * 365.0) }; return Math.Max(1, (int)Math.Floor(duration.TotalSeconds * marketFraction / secondsPerBar)); diff --git a/DatabaseHandler/NewsSentimentRecord.cs b/DatabaseHandler/NewsSentimentRecord.cs index 5e272ad..96dd5be 100644 --- a/DatabaseHandler/NewsSentimentRecord.cs +++ b/DatabaseHandler/NewsSentimentRecord.cs @@ -9,6 +9,7 @@ namespace Backend.DatabaseHandler public double Score { get; init; } public string SentimentLabel { get; init; } = string.Empty; public string TitleHash { get; init; } = string.Empty; + public string Publisher { get; init; } = string.Empty; public string Url { get; init; } = string.Empty; } } diff --git a/DatabaseHandler/PostgresHistoricalMarketDataStore.cs b/DatabaseHandler/PostgresHistoricalMarketDataStore.cs index 50a4cd9..e1aa5ec 100644 --- a/DatabaseHandler/PostgresHistoricalMarketDataStore.cs +++ b/DatabaseHandler/PostgresHistoricalMarketDataStore.cs @@ -29,17 +29,17 @@ namespace Backend.DatabaseHandler await conn.OpenAsync(cancellationToken); const string sql = """ - SELECT symbol, bar_size, what_to_show, use_rth, timestamp_utc, - open, high, low, close, volume - FROM historical_equity_bars - WHERE symbol = @symbol - AND bar_size = @bar_size - AND what_to_show = @what_to_show - AND use_rth = @use_rth - AND timestamp_utc >= @start_utc - AND timestamp_utc <= @end_utc - ORDER BY timestamp_utc ASC; - """; + SELECT symbol, bar_size, what_to_show, use_rth, timestamp_utc, + open, high, low, close, volume + FROM historical_equity_bars + WHERE symbol = @symbol + AND bar_size = @bar_size + AND what_to_show = @what_to_show + AND use_rth = @use_rth + AND timestamp_utc >= @start_utc + AND timestamp_utc <= @end_utc + ORDER BY timestamp_utc ASC; + """; await using var cmd = new NpgsqlCommand(sql, conn); cmd.Parameters.AddWithValue("symbol", symbol); @@ -84,24 +84,24 @@ namespace Backend.DatabaseHandler await using var tx = await conn.BeginTransactionAsync(cancellationToken); const string sql = """ - INSERT INTO historical_equity_bars - ( - symbol, bar_size, what_to_show, use_rth, timestamp_utc, - open, high, low, close, volume - ) - VALUES - ( - @symbol, @bar_size, @what_to_show, @use_rth, @timestamp_utc, - @open, @high, @low, @close, @volume - ) - ON CONFLICT (symbol, bar_size, what_to_show, use_rth, timestamp_utc) - DO UPDATE SET - open = EXCLUDED.open, - high = EXCLUDED.high, - low = EXCLUDED.low, - close = EXCLUDED.close, - volume = EXCLUDED.volume; - """; + INSERT INTO historical_equity_bars + ( + symbol, bar_size, what_to_show, use_rth, timestamp_utc, + open, high, low, close, volume + ) + VALUES + ( + @symbol, @bar_size, @what_to_show, @use_rth, @timestamp_utc, + @open, @high, @low, @close, @volume + ) + ON CONFLICT (symbol, bar_size, what_to_show, use_rth, timestamp_utc) + DO UPDATE SET + open = EXCLUDED.open, + high = EXCLUDED.high, + low = EXCLUDED.low, + close = EXCLUDED.close, + volume = EXCLUDED.volume; + """; foreach (var bar in barList) { diff --git a/DatabaseHandler/PostgresNewsSentimentStore.cs b/DatabaseHandler/PostgresNewsSentimentStore.cs index 5f38d68..524150b 100644 --- a/DatabaseHandler/PostgresNewsSentimentStore.cs +++ b/DatabaseHandler/PostgresNewsSentimentStore.cs @@ -23,12 +23,12 @@ namespace Backend.DatabaseHandler await conn.OpenAsync(cancellationToken); const string sql = """ - SELECT source, keyword, published_at, title, score, - sentiment_label, title_hash, url - FROM news_sentiment_cache - WHERE keyword = @keyword - ORDER BY published_at DESC; - """; + SELECT source, keyword, published_at, title, score, + sentiment_label, title_hash, publisher, url + FROM news_sentiment_cache + WHERE keyword = @keyword + ORDER BY published_at DESC; + """; await using var cmd = new NpgsqlCommand(sql, conn); cmd.Parameters.AddWithValue("keyword", keyword); @@ -45,7 +45,55 @@ namespace Backend.DatabaseHandler Score = reader.GetDouble(4), SentimentLabel = reader.GetString(5), TitleHash = reader.GetString(6).TrimEnd(), - Url = reader.GetString(7) + Publisher = reader.GetString(7), + Url = reader.GetString(8) + }); + } + + return results; + } + + public async Task> GetFeedAsync( + IEnumerable keywords, + int limit, + CancellationToken cancellationToken = default) + { + var keywordList = keywords.ToList(); + if (keywordList.Count == 0) + return []; + + var results = new List(); + + await using var conn = new NpgsqlConnection(_connectionString); + await conn.OpenAsync(cancellationToken); + + const string sql = """ + SELECT source, keyword, published_at, title, score, + sentiment_label, title_hash, publisher, url + FROM news_sentiment_cache + WHERE keyword = ANY(@keywords) + ORDER BY published_at DESC + LIMIT @limit; + """; + + await using var cmd = new NpgsqlCommand(sql, conn); + cmd.Parameters.AddWithValue("keywords", keywordList.ToArray()); + cmd.Parameters.AddWithValue("limit", limit); + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + results.Add(new NewsSentimentRecord + { + Source = reader.GetString(0), + Keyword = reader.GetString(1), + PublishedAt = reader.GetFieldValue(2), + Title = reader.GetString(3), + Score = reader.GetDouble(4), + SentimentLabel = reader.GetString(5), + TitleHash = reader.GetString(6).TrimEnd(), + Publisher = reader.GetString(7), + Url = reader.GetString(8) }); } @@ -66,14 +114,14 @@ namespace Backend.DatabaseHandler await using var tx = await conn.BeginTransactionAsync(cancellationToken); const string sql = """ - INSERT INTO news_sentiment_cache - (source, keyword, published_at, title, score, - sentiment_label, title_hash, url) - VALUES - (@source, @keyword, @published_at, @title, @score, - @sentiment_label, @title_hash, @url) - ON CONFLICT (title_hash) DO NOTHING; - """; + INSERT INTO news_sentiment_cache + (source, keyword, published_at, title, score, + sentiment_label, title_hash, publisher, url) + VALUES + (@source, @keyword, @published_at, @title, @score, + @sentiment_label, @title_hash, @publisher, @url) + ON CONFLICT (title_hash) DO NOTHING; + """; foreach (var record in recordList) { @@ -85,6 +133,7 @@ namespace Backend.DatabaseHandler cmd.Parameters.AddWithValue("score", record.Score); cmd.Parameters.AddWithValue("sentiment_label", record.SentimentLabel); cmd.Parameters.AddWithValue("title_hash", record.TitleHash); + cmd.Parameters.AddWithValue("publisher", record.Publisher); cmd.Parameters.AddWithValue("url", record.Url); await cmd.ExecuteNonQueryAsync(cancellationToken); @@ -103,10 +152,10 @@ namespace Backend.DatabaseHandler await conn.OpenAsync(cancellationToken); const string sql = """ - SELECT title_hash - FROM news_sentiment_cache - WHERE keyword = @keyword; - """; + SELECT title_hash + FROM news_sentiment_cache + WHERE keyword = @keyword; + """; await using var cmd = new NpgsqlCommand(sql, conn); cmd.Parameters.AddWithValue("keyword", keyword); diff --git a/DatabaseHandler/PostgresUserInterestStore.cs b/DatabaseHandler/PostgresUserInterestStore.cs index d9f39bb..c09e969 100644 --- a/DatabaseHandler/PostgresUserInterestStore.cs +++ b/DatabaseHandler/PostgresUserInterestStore.cs @@ -20,17 +20,17 @@ public sealed class PostgresUserInterestStore : IUserInterestStore string normalizedType = NormalizeType(type); string sql = normalizedType == "symbol" ? """ - SELECT symbol - FROM user_interest_symbols - WHERE username = @user - ORDER BY symbol ASC; - """ + SELECT symbol + FROM user_interest_symbols + WHERE username = @user + ORDER BY symbol ASC; + """ : """ - SELECT topic - FROM user_interest_topics - WHERE username = @user - ORDER BY topic ASC; - """; + SELECT topic + FROM user_interest_topics + WHERE username = @user + ORDER BY topic ASC; + """; var results = new List(); @@ -49,6 +49,29 @@ public sealed class PostgresUserInterestStore : IUserInterestStore return results; } + public async Task> FetchAllDistinctValuesAsync(CancellationToken cancellationToken = default) + { + const string sql = """ + SELECT symbol AS value FROM user_interest_symbols + UNION + SELECT topic AS value FROM user_interest_topics; + """; + + var results = new List(); + + await using var conn = new NpgsqlConnection(_connectionString); + await conn.OpenAsync(cancellationToken); + + await using var cmd = new NpgsqlCommand(sql, conn); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + results.Add(reader.GetString(0)); + } + + return results; + } + public async Task AddAsync(string user, string type, string value) { ValidateUser(user); @@ -61,15 +84,15 @@ public sealed class PostgresUserInterestStore : IUserInterestStore string sql = normalizedType == "symbol" ? """ - INSERT INTO user_interest_symbols (username, symbol) - VALUES (@user, @value) - ON CONFLICT (username, symbol) DO NOTHING; - """ + INSERT INTO user_interest_symbols (username, symbol) + VALUES (@user, @value) + ON CONFLICT (username, symbol) DO NOTHING; + """ : """ - INSERT INTO user_interest_topics (username, topic) - VALUES (@user, @value) - ON CONFLICT (username, topic) DO NOTHING; - """; + INSERT INTO user_interest_topics (username, topic) + VALUES (@user, @value) + ON CONFLICT (username, topic) DO NOTHING; + """; await using var conn = new NpgsqlConnection(_connectionString); await conn.OpenAsync(); @@ -94,13 +117,13 @@ public sealed class PostgresUserInterestStore : IUserInterestStore string sql = normalizedType == "symbol" ? """ - DELETE FROM user_interest_symbols - WHERE username = @user AND symbol = @value; - """ + DELETE FROM user_interest_symbols + WHERE username = @user AND symbol = @value; + """ : """ - DELETE FROM user_interest_topics - WHERE username = @user AND topic = @value; - """; + DELETE FROM user_interest_topics + WHERE username = @user AND topic = @value; + """; await using var conn = new NpgsqlConnection(_connectionString); await conn.OpenAsync(); diff --git a/Interface/INewsSentimentStore.cs b/Interface/INewsSentimentStore.cs index f11c9fe..277112c 100644 --- a/Interface/INewsSentimentStore.cs +++ b/Interface/INewsSentimentStore.cs @@ -8,6 +8,11 @@ namespace Backend.Interface string keyword, CancellationToken cancellationToken = default); + Task> GetFeedAsync( + IEnumerable keywords, + int limit, + CancellationToken cancellationToken = default); + Task InsertAsync( IEnumerable records, CancellationToken cancellationToken = default); diff --git a/Interface/IUserInterestCache.cs b/Interface/IUserInterestCache.cs index 01233e1..3b2dfff 100644 --- a/Interface/IUserInterestCache.cs +++ b/Interface/IUserInterestCache.cs @@ -3,6 +3,7 @@ namespace Backend.Interface public interface IUserInterestStore { Task> FetchAllAsync(string user, string type); + Task> FetchAllDistinctValuesAsync(CancellationToken cancellationToken = default); Task AddAsync(string user, string type, string value); Task RemoveAsync(string user, string type, string value); } diff --git a/ServiceHandler/NewsSentimentRefreshService.cs b/ServiceHandler/NewsSentimentRefreshService.cs new file mode 100644 index 0000000..3447eb4 --- /dev/null +++ b/ServiceHandler/NewsSentimentRefreshService.cs @@ -0,0 +1,78 @@ +using Backend.Interface; + +namespace Backend.ServiceHander +{ + public sealed class NewsSentimentRefreshService : BackgroundService + { + private static readonly TimeSpan PollInterval = TimeSpan.FromMinutes(15); + private static readonly TimeZoneInfo Pacific = TimeZoneInfo.FindSystemTimeZoneById("America/Los_Angeles"); + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public NewsSentimentRefreshService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + if (IsWithinActiveWindow()) + { + await RefreshAsync(stoppingToken); + } + else + { + _logger.LogDebug("Outside active window (weekdays 07:00-20:00 PST), skipping refresh."); + } + + await Task.Delay(PollInterval, stoppingToken); + } + } + + private static bool IsWithinActiveWindow() + { + var pacificNow = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, Pacific); + + if (pacificNow.DayOfWeek is DayOfWeek.Saturday or DayOfWeek.Sunday) + return false; + + var hour = pacificNow.Hour; + return hour >= 7 && hour < 20; + } + + private async Task RefreshAsync(CancellationToken cancellationToken) + { + try + { + var interests = _services.GetRequiredService(); + var sentiment = _services.GetRequiredService(); + + var keywords = await interests.FetchAllDistinctValuesAsync(cancellationToken); + + _logger.LogInformation("Refreshing news for {Count} keywords.", keywords.Count); + + foreach (var keyword in keywords) + { + try + { + await sentiment.AnalyzeAsync(keyword, cancellationToken); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to refresh news for keyword '{Keyword}'.", keyword); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, "News sentiment refresh cycle failed."); + } + } + } +} diff --git a/ServiceHandler/NewsSentimentService.cs b/ServiceHandler/NewsSentimentService.cs index b071417..3da86c4 100644 --- a/ServiceHandler/NewsSentimentService.cs +++ b/ServiceHandler/NewsSentimentService.cs @@ -40,17 +40,19 @@ namespace Backend.ServiceHander if (existingHashes.Contains(titleHash)) continue; - var (score, label) = _scorer.Score(title); + var (headline, publisher) = ParsePublisher(title); + var (score, label) = _scorer.Score(headline); newRecords.Add(new NewsSentimentRecord { Source = "google_rss", Keyword = keyword, PublishedAt = publishedAt, - Title = title, + Title = headline, Score = score, SentimentLabel = label, TitleHash = titleHash, + Publisher = publisher, Url = url }); } @@ -92,6 +94,14 @@ namespace Backend.ServiceHander return items; } + private static (string Headline, string Publisher) ParsePublisher(string title) + { + int idx = title.LastIndexOf(" - "); + if (idx < 0) + return (title, "Unknown"); + return (title[..idx].Trim(), title[(idx + 3)..].Trim()); + } + private static string ComputeHash(string text) { byte[] bytes = SHA256.HashData(Encoding.UTF8.GetBytes(text));