rust/forkmessager
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Очистка удаленных

Browse Source
This commit is contained in:
Халимов Рустам
2026-03-17 15:55:41 +03:00
parent 67bf8319aa
commit 408cb446c3
13 changed files with 494 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server-net/src/Host/Controllers/AdminController.cs
+141
View File
@@ -157,9 +157,150 @@ public class AdminController : ControllerBase
}
});
}
[HttpGet("clean/dry-run")]
public async Task<IActionResult> CleanDryRun(
[FromServices] Knot.Shared.Kernel.Storage.IFileStorageService fileStorage,
[FromServices] Knot.Modules.Identity.Infrastructure.Persistence.IdentityDbContext identityDb,
CancellationToken ct)
{
var orphanedMessages = await _chatsDbContext.Messages
.Include(m => m.Media)
.Where(m => m.IsDeleted || !_chatsDbContext.Chats.Any(c => c.Id == m.ChatId))
.ToListAsync(ct);
var orphanedMessagesCount = orphanedMessages.Count;
// Fetch all current physical files from MinIO
var allMinioFiles = (await fileStorage.ListFilesAsync()).ToList();
// Collect ALL active URLs that we must preserve
var keptMessages = await _chatsDbContext.Messages
.AsNoTracking()
.Include(m => m.Media)
.Where(m => !m.IsDeleted && _chatsDbContext.Chats.Any(c => c.Id == m.ChatId))
.ToListAsync(ct);
var allChats = await _chatsDbContext.Chats.AsNoTracking().ToListAsync(ct);
var allUsers = await identityDb.Users.AsNoTracking().ToListAsync(ct);
var validUrls = new HashSet<string>();
var activeMessageUrls = keptMessages
.Where(m => m.Media != null)
.SelectMany(m => m.Media)
.Select(me => me.Url)
.Where(u => !string.IsNullOrEmpty(u));
var activeChatUrls = allChats
.Where(c => !string.IsNullOrEmpty(c.Avatar))
.Select(c => c.Avatar!);
var activeUserUrls = allUsers
.Where(u => !string.IsNullOrEmpty(u.Avatar))
.Select(u => u.Avatar!);
foreach(var u in activeMessageUrls) validUrls.Add(u!);
foreach(var u in activeChatUrls) validUrls.Add(u);
foreach(var u in activeUserUrls) validUrls.Add(u);
var validFileIds = validUrls
.Where(u => u.Contains("/api/files/"))
.Select(u => u.Split('/').Last())
.ToHashSet();
long safeBytes = 0;
foreach(var file in allMinioFiles)
{
if (!validFileIds.Contains(file.FileId))
{
safeBytes += file.Size;
}
}
return Ok(new CleanupDryRunResultDto
{
OrphanedMessagesCount = orphanedMessagesCount,
OrphanedMediaBytes = safeBytes
});
}
[HttpPost("clean/run")]
public async Task<IActionResult> CleanRun(
[FromServices] Knot.Shared.Kernel.Storage.IFileStorageService fileStorage,
[FromServices] Knot.Modules.Identity.Infrastructure.Persistence.IdentityDbContext identityDb,
CancellationToken ct)
{
var orphanMessages = await _chatsDbContext.Messages
.Include(m => m.Media)
.Where(m => m.IsDeleted || !_chatsDbContext.Chats.Any(c => c.Id == m.ChatId))
.ToListAsync(ct);
// Fetch all current physical files from MinIO
var allMinioFiles = (await fileStorage.ListFilesAsync()).ToList();
// Collect ALL active URLs that we must preserve
var keptMessages = await _chatsDbContext.Messages
.AsNoTracking()
.Include(m => m.Media)
.Where(m => !m.IsDeleted && _chatsDbContext.Chats.Any(c => c.Id == m.ChatId))
.ToListAsync(ct);
var allChats = await _chatsDbContext.Chats.AsNoTracking().ToListAsync(ct);
var allUsers = await identityDb.Users.AsNoTracking().ToListAsync(ct);
var validUrls = new HashSet<string>();
var activeMessageUrls = keptMessages
.Where(m => m.Media != null)
.SelectMany(m => m.Media)
.Select(me => me.Url)
.Where(u => !string.IsNullOrEmpty(u));
var activeChatUrls = allChats
.Where(c => !string.IsNullOrEmpty(c.Avatar))
.Select(c => c.Avatar!);
var activeUserUrls = allUsers
.Where(u => !string.IsNullOrEmpty(u.Avatar))
.Select(u => u.Avatar!);
foreach(var u in activeMessageUrls) validUrls.Add(u!);
foreach(var u in activeChatUrls) validUrls.Add(u);
foreach(var u in activeUserUrls) validUrls.Add(u);
var validFileIds = validUrls
.Where(u => u.Contains("/api/files/"))
.Select(u => u.Split('/').Last())
.ToHashSet();
// Physically delete from Storage
foreach (var file in allMinioFiles)
{
if (!validFileIds.Contains(file.FileId))
{
await fileStorage.DeleteFileAsync(file.FileId);
}
}
if(orphanMessages.Any()) {
_chatsDbContext.Messages.RemoveRange(orphanMessages);
await _chatsDbContext.SaveChangesAsync(ct);
}
return Ok(new { message = "Cleanup completed successfully" });
}
}
public class ResetPasswordDto
{
public string NewPassword { get; set; } = string.Empty;
}
public class CleanupDryRunResultDto
{
public int OrphanedMessagesCount { get; set; }
public long OrphanedMediaBytes { get; set; }
}
apps/server-net/src/Host/Controllers/KlipyController.cs
+44 -7
View File
@@ -29,18 +29,38 @@ public class KlipyController : ControllerBase
if (!conf.EnableKlipy || string.IsNullOrEmpty(conf.KlipyApiKey))
return BadRequest(new { error = "Klipy is disabled or not configured." });
if (_cache.TryGetValue("klipy_trending", out JsonElement cachedResult))
var cacheKeyTrending = $"klipy_trending_{conf.KlipyApiKey}";
if (_cache.TryGetValue(cacheKeyTrending, out JsonElement cachedResult))
{
return Ok(cachedResult);
}
try
{
var customerId = string.IsNullOrWhiteSpace(conf.KlipyCustomerId) ? "anonymous" : conf.KlipyCustomerId;
var client = _httpClientFactory.CreateClient();
var url = $"https://api.klipy.com/api/v1/{conf.KlipyApiKey}/gifs/trending?page=1&per_page=30&customer_id={conf.KlipyCustomerId ?? "anonymous"}";
var result = await client.GetFromJsonAsync<JsonElement>(url);
var url = $"https://api.klipy.co/api/v1/{conf.KlipyApiKey}/gifs/trending?page=1&per_page=30&customer_id={customerId}";
_cache.Set("klipy_trending", result, TimeSpan.FromMinutes(60)); // Cache trending for 60 minutes
var response = await client.GetAsync(url);
if (!response.IsSuccessStatusCode)
{
var errorContent = await response.Content.ReadAsStringAsync();
// fallback to api.klipy.com if .co failed with 404 or something, though maybe not needed
if (response.StatusCode == System.Net.HttpStatusCode.NotFound) {
url = $"https://api.klipy.com/api/v1/{conf.KlipyApiKey}/gifs/trending?page=1&per_page=30&customer_id={customerId}";
response = await client.GetAsync(url);
errorContent = await response.Content.ReadAsStringAsync();
}
if (!response.IsSuccessStatusCode)
return StatusCode(500, new { error = "Failed to fetch from Klipy", details = errorContent, url });
}
var result = await response.Content.ReadFromJsonAsync<JsonElement>();
_cache.Set(cacheKeyTrending, result, TimeSpan.FromMinutes(60)); // Cache trending for 60 minutes
return Ok(result);
}
catch (Exception ex)
@@ -59,7 +79,7 @@ public class KlipyController : ControllerBase
if (string.IsNullOrWhiteSpace(q))
return BadRequest(new { error = "Query is empty." });
var cacheKey = $"klipy_search_{q.ToLowerInvariant()}";
var cacheKey = $"klipy_search_{conf.KlipyApiKey}_{q.ToLowerInvariant()}";
if (_cache.TryGetValue(cacheKey, out JsonElement cachedResult))
{
return Ok(cachedResult);
@@ -67,9 +87,26 @@ public class KlipyController : ControllerBase
try
{
var customerId = string.IsNullOrWhiteSpace(conf.KlipyCustomerId) ? "anonymous" : conf.KlipyCustomerId;
var client = _httpClientFactory.CreateClient();
var url = $"https://api.klipy.com/api/v1/{conf.KlipyApiKey}/gifs/search?page=1&per_page=30&q={Uri.EscapeDataString(q)}&customer_id={conf.KlipyCustomerId ?? "anonymous"}";
var result = await client.GetFromJsonAsync<JsonElement>(url);
var url = $"https://api.klipy.co/api/v1/{conf.KlipyApiKey}/gifs/search?page=1&per_page=30&q={Uri.EscapeDataString(q)}&customer_id={customerId}";
var response = await client.GetAsync(url);
if (!response.IsSuccessStatusCode)
{
var errorContent = await response.Content.ReadAsStringAsync();
if (response.StatusCode == System.Net.HttpStatusCode.NotFound) {
url = $"https://api.klipy.com/api/v1/{conf.KlipyApiKey}/gifs/search?page=1&per_page=30&q={Uri.EscapeDataString(q)}&customer_id={customerId}";
response = await client.GetAsync(url);
errorContent = await response.Content.ReadAsStringAsync();
}
if (!response.IsSuccessStatusCode)
return StatusCode(500, new { error = "Failed to fetch from Klipy", details = errorContent, url });
}
var result = await response.Content.ReadFromJsonAsync<JsonElement>();
_cache.Set(cacheKey, result, TimeSpan.FromMinutes(15)); // Cache searches for 15 minutes
return Ok(result);
apps/server-net/src/Host/Program.cs
+1 -1
View File
@@ -25,7 +25,7 @@ var builder = WebApplication.CreateBuilder(args);
// Маппинг стандартных переменных окружения в иерархию .NET
var envMappings = new Dictionary<string, string?>
{
["ConnectionStrings:Database"] = builder.Configuration["DATABASE_URL"],
["ConnectionStrings:DefaultConnection"] = builder.Configuration["DATABASE_URL"],
["Jwt:Secret"] = builder.Configuration["JWT_SECRET"],
["Jwt:Issuer"] = builder.Configuration["JWT_ISSUER"],
["Jwt:Audience"] = builder.Configuration["JWT_AUDIENCE"],
apps/server-net/src/Host/appsettings.json
+3 -2
View File
@@ -7,12 +7,13 @@
},
"AllowedHosts": "*",
"ConnectionStrings": {
"Database": "Host=localhost;Port=5432;Database=knot_db;Username=knot;Password=knot_pass"
"DefaultConnection": "Host=localhost;Port=5432;Database=knot_db;Username=knot;Password=knot_pass"
},
"Jwt": {
"Secret": "knot_super_secret_key_1234567890_knot",
"Issuer": "Knot",
"Audience": "KnotUsers",
"ExpiryInMinutes": 1440
}
},
"KNOT_MASTER_ENCRYPTION_KEY": "knot_super_secret_key_1234567890_knot"
}
apps/server-net/src/Modules/Chats/DependencyInjection.cs
+1 -1
View File
@@ -19,7 +19,7 @@ public static class DependencyInjection
IConfiguration configuration)
{
// Настройка базы данных
string connectionString = configuration.GetConnectionString("Database")!;
string connectionString = configuration.GetConnectionString("DefaultConnection")!;
services.AddDbContext<ChatsDbContext>(options =>
options.UseNpgsql(connectionString));
apps/server-net/src/Modules/Chats/Domain/Message.cs
+15 -8
View File
@@ -63,8 +63,15 @@ public sealed class Message : AggregateRoot<Guid>
{
return new Message(Guid.NewGuid(), chatId, senderId, content, type, replyToId, quote, forwardedFromId, storyId, storyMediaUrl, storyMediaType, DateTime.UtcNow, false);
}
public static Message Import(Guid chatId, Guid senderId, string? content, string type, DateTime createdAt, Guid? replyToId = null, Guid? forwardedFromId = null)
public static Message Import(
Guid chatId,
Guid senderId,
string? content,
string type,
DateTime createdAt,
Guid? replyToId = null,
Guid? forwardedFromId = null)
{
return new Message(Guid.NewGuid(), chatId, senderId, content, type, replyToId, null, forwardedFromId, null, null, null, createdAt, true);
}
@@ -105,7 +112,7 @@ public sealed class Message : AggregateRoot<Guid>
{
Content = null;
IsDeleted = true;
_media.Clear();
// _media.Clear(); // DO NOT CLEAR! Data cleanup needs to know the URLs to delete from S3
_reactions.Clear();
}
@@ -133,9 +140,9 @@ public sealed class Reaction : Entity<Guid>
Emoji = emoji;
}
private Reaction() : base(Guid.Empty)
{
Emoji = string.Empty;
private Reaction() : base(Guid.Empty)
{
Emoji = string.Empty;
}
}
@@ -159,8 +166,8 @@ public sealed class Media : Entity<Guid>
Size = size;
}
private Media() : base(Guid.Empty)
{
private Media() : base(Guid.Empty)
{
Type = string.Empty;
Url = string.Empty;
}
apps/server-net/src/Modules/Identity/DependencyInjection.cs
+1 -1
View File
@@ -19,7 +19,7 @@ public static class DependencyInjection
IConfiguration configuration)
{
// Настройка базы данных
string connectionString = configuration.GetConnectionString("Database")!;
string connectionString = configuration.GetConnectionString("DefaultConnection")!;
services.AddDbContext<IdentityDbContext>(options =>
options.UseNpgsql(connectionString));
apps/server-net/src/Shared/Knot.Shared.Infrastructure/Storage/S3FileStorageService.cs
+34
View File
@@ -149,4 +149,38 @@ public class S3FileStorageService : IFileStorageService
return (msResult, contentType, fileName);
}
public async Task DeleteFileAsync(string fileId)
{
try
{
var removeArgs = new RemoveObjectArgs()
.WithBucket(_bucketName)
.WithObject(fileId);
await _minioClient.RemoveObjectAsync(removeArgs).ConfigureAwait(false);
}
catch (MinioException e)
{
Console.WriteLine($"[Bucket] Error deleting file {fileId}: {e.Message}");
}
}
public async Task<IEnumerable<(string FileId, long Size)>> ListFilesAsync()
{
var result = new List<(string FileId, long Size)>();
try
{
var listArgs = new ListObjectsArgs().WithBucket(_bucketName).WithRecursive(true);
await foreach (var item in _minioClient.ListObjectsEnumAsync(listArgs).ConfigureAwait(false))
{
result.Add((item.Key, (long)item.Size));
}
}
catch (Exception ex)
{
Console.WriteLine($"[Bucket] Error listing files: {ex.Message}");
}
return result;
}
}
apps/server-net/src/Shared/Knot.Shared.Kernel/Storage/IFileStorageService.cs
+6
View File
@@ -10,4 +10,10 @@ public interface IFileStorageService
// Скачивает файл и возвращает его расшифрованный поток и тип содержимого.
Task<(Stream Stream, string ContentType, string FileName)> DownloadFileAsync(string fileId);
// Удаляет файл из хранилища.
Task DeleteFileAsync(string fileId);
// Получает список всех файлов в хранилище с их размерами.
Task<System.Collections.Generic.IEnumerable<(string FileId, long Size)>> ListFilesAsync();
}