rust/forkmessager
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Импорт чатов с телеграмм, асинхронная загрузка сообщений

Browse Source
This commit is contained in:
Халимов Рустам
2026-03-16 23:13:20 +03:00
parent 5600067f8d
commit 97b70250de
10 changed files with 331 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server-net/src/Host/Controllers/TelegramImportController.cs
+215 -51
View File
@@ -77,24 +77,28 @@ public sealed class TelegramImportController : ControllerBase
// Open zip and quickly scan messages.html
using (var archive = ZipFile.OpenRead(tempPath))
{
var htmlEntries = archive.Entries.Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase));
var htmlEntries = archive.Entries
.Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase));
foreach (var entry in htmlEntries)
{
using var stream = entry.Open();
var doc = new HtmlDocument();
doc.Load(stream);
var parser = new AngleSharp.Html.Parser.HtmlParser();
var doc = parser.ParseDocument(stream);
var messageNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'message ')]");
var messageNodes = doc.QuerySelectorAll(".message");
if (messageNodes == null) continue;
foreach (var node in messageNodes)
{
var fromNameNode = node.SelectSingleNode(".//div[contains(@class, 'from_name')]");
var fromNameNode = node.QuerySelector(".from_name");
if (fromNameNode != null)
{
var name = fromNameNode.InnerText.Trim();
// Ignore standard system names if obvious (for now everything is recorded)
var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone();
var innerSpans = nameNodeText.QuerySelectorAll("span");
foreach (var span in innerSpans) span.Remove();
var name = nameNodeText.TextContent.Trim();
if (!string.IsNullOrWhiteSpace(name))
{
names.Add(name);
@@ -169,15 +173,25 @@ public sealed class TelegramImportController : ControllerBase
{
var htmlEntries = archive.Entries
.Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase))
.OrderBy(e => e.FullName); // Read chronologically
.OrderBy(e =>
{
var name = e.Name.ToLower().Replace("messages", "").Replace(".html", "");
return string.IsNullOrEmpty(name) ? 0 : int.TryParse(name, out var num) ? num : 999999;
})
.ToList();
Guid lastSenderGuid = myId;
DateTime lastCreatedAt = DateTime.UtcNow;
Dictionary<string, Guid> messageIdMap = new();
Message? lastSavedMessage = null;
foreach (var entry in htmlEntries)
{
using var stream = entry.Open();
var doc = new HtmlDocument();
doc.Load(stream);
var parser = new AngleSharp.Html.Parser.HtmlParser();
var doc = parser.ParseDocument(stream);
var messageNodes = doc.DocumentNode.SelectNodes("//div[contains(@class, 'message ')]");
var messageNodes = doc.QuerySelectorAll(".message");
if (messageNodes == null) continue;
var baseDir = Path.GetDirectoryName(entry.FullName)?.Replace("\\", "/") ?? "";
@@ -185,56 +199,193 @@ public sealed class TelegramImportController : ControllerBase
foreach (var node in messageNodes)
{
try
try
{
var fromNameNode = node.SelectSingleNode(".//div[contains(@class, 'from_name')]");
var textNode = node.SelectSingleNode(".//div[contains(@class, 'text')]");
var dateNode = node.SelectSingleNode(".//div[contains(@class, 'pull_right date')]");
var fromNameNode = node.QuerySelector(".from_name");
var textNode = node.QuerySelector(".text");
// Default values
var senderGuid = myId; // Fallback
var dateNode = node.QuerySelector(".date[title]")
?? node.QuerySelector(".pull_right[title]")
?? node.QuerySelector("[title]");
if (fromNameNode != null)
{
var name = fromNameNode.InnerText.Trim();
var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone();
var innerSpans = nameNodeText.QuerySelectorAll("span");
foreach (var span in innerSpans) span.Remove();
var name = nameNodeText.TextContent.Trim();
if (req.Mapping.TryGetValue(name, out var mappedId) && mappedId != Guid.Empty)
senderGuid = mappedId;
lastSenderGuid = mappedId;
else
lastSenderGuid = myId;
}
var content = textNode?.InnerText?.Trim() ?? "";
Guid senderGuid = lastSenderGuid;
string content = "";
if (textNode != null)
{
var html = textNode.InnerHtml
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase)
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase);
var tempParser = new AngleSharp.Html.Parser.HtmlParser();
var tempDoc = tempParser.ParseDocument("<div>" + html + "</div>");
content = tempDoc.Body.TextContent.Trim();
}
DateTime createdAt = DateTime.UtcNow;
if (dateNode != null && dateNode.Attributes["title"] != null)
{
var dateStr = dateNode.Attributes["title"].Value;
if (DateTime.TryParse(dateStr, out var d))
createdAt = d.ToUniversalTime();
}
DateTime createdAt = lastCreatedAt;
var titleNodes = node.QuerySelectorAll("[title]");
bool parsed = false;
var mediaNodes = node.SelectNodes(".//a[contains(@class, 'photo_wrap')] | .//video | .//a[contains(@class, 'document')] | .//a[contains(@class, 'media_voice_message')]");
var messageType = "text";
if (mediaNodes != null && mediaNodes.Count > 0)
if (titleNodes != null)
{
var firstHref = mediaNodes[0].Attributes["href"]?.Value ?? mediaNodes[0].Attributes["src"]?.Value;
if (firstHref != null)
foreach (var tnode in titleNodes)
{
if (firstHref.EndsWith(".jpg") || firstHref.EndsWith(".png")) messageType = "image";
else if (firstHref.EndsWith(".mp4")) messageType = "video";
else if (firstHref.EndsWith(".ogg")) messageType = "voice";
else messageType = "file";
var dateStr = tnode.GetAttribute("title")?.Trim() ?? "";
if (dateStr.Length >= 10 && char.IsDigit(dateStr[0]) && char.IsDigit(dateStr[1]))
{
// Remove "UTC" so we can parse timezone offset generically, e.g. "17.10.2025 12:56:25 +03:00"
var cleanStr = dateStr.Replace("UTC", "", StringComparison.OrdinalIgnoreCase).Trim();
if (DateTimeOffset.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss zzz", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out var dto))
{
createdAt = dto.UtcDateTime;
parsed = true;
break;
}
else if (DateTime.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, out var dt))
{
createdAt = dt;
parsed = true;
break;
}
else if (DateTime.TryParse(cleanStr, out var dFallback))
{
createdAt = dFallback.ToUniversalTime();
parsed = true;
break;
}
}
}
}
var newMessage = Message.Import(chatId, senderGuid, content, messageType, createdAt);
if (!parsed)
{
Console.WriteLine("Warning: Could not parse date in imported message! Using lastCreatedAt.");
}
else
{
lastCreatedAt = createdAt;
}
// Parse forwards
var forwardedNode = node.QuerySelector(".forwarded.body");
if (forwardedNode != null)
{
var fwdNameNode = forwardedNode.QuerySelector(".from_name");
var fwdNameText = fwdNameNode != null ? (AngleSharp.Dom.IElement)fwdNameNode.Clone() : null;
if (fwdNameText != null)
{
var innerSpans = fwdNameText.QuerySelectorAll("span");
foreach (var s in innerSpans) s.Remove();
}
var fwdName = fwdNameText != null ? fwdNameText.TextContent.Trim() : "Неизвестного";
var fwdTextNode = forwardedNode.QuerySelector(".text");
string fwdContent = "";
if (fwdTextNode != null)
{
var fHtml = fwdTextNode.InnerHtml
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase)
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase);
var tempParser = new AngleSharp.Html.Parser.HtmlParser();
var tempDoc = tempParser.ParseDocument("<div>" + fHtml + "</div>");
fwdContent = tempDoc.Body.TextContent.Trim();
}
content = string.IsNullOrEmpty(content)
? $"[Переслано от {fwdName}]:\n{fwdContent}"
: $"{content}\n\n[Переслано от {fwdName}]:\n{fwdContent}";
}
// Parse replies
Guid? replyToId = null;
var replyNode = node.QuerySelector(".reply_to a");
if (replyNode != null)
{
var href = replyNode.GetAttribute("href");
if (href != null && href.StartsWith("#go_to_"))
{
var tgId = href.Substring("#go_to_".Length);
if (messageIdMap.TryGetValue(tgId, out var mappedMsgId))
{
replyToId = mappedMsgId;
}
}
}
var mediaNodes = node.QuerySelectorAll("a.photo_wrap, a.animated_wrap, video, audio, a.document, a.media_voice_message, a.media_video, img.sticker").ToList();
if (mediaNodes.Count == 0)
{
var fallback = node.QuerySelectorAll(".media_wrap a[href]");
mediaNodes.AddRange(fallback);
}
var messageType = "text";
// Helper for categorizing files
(string mType, string cType) GetMediaTypes(string fileUrl)
{
var ext = Path.GetExtension(fileUrl)?.ToLower();
return ext switch
{
".jpg" or ".jpeg" or ".png" or ".webp" => ("image", "image/jpeg"),
".mp4" or ".mov" or ".avi" => ("video", "video/mp4"),
".ogg" or ".mp3" => ("voice", "audio/ogg"),
_ => ("file", "application/octet-stream")
};
}
if (mediaNodes != null && mediaNodes.Count > 0)
{
var firstHref = mediaNodes[0].GetAttribute("href") ?? mediaNodes[0].GetAttribute("src");
if (firstHref != null)
{
messageType = GetMediaTypes(firstHref).mType;
}
}
bool isJoined = fromNameNode == null;
bool isMediaOnly = string.IsNullOrEmpty(content) && forwardedNode == null && replyToId == null;
Message? targetMessage = null;
if (isJoined && isMediaOnly && lastSavedMessage != null && Math.Abs((createdAt - lastSavedMessage.CreatedAt).TotalSeconds) <= 60 && lastSavedMessage.SenderId == senderGuid)
{
targetMessage = lastSavedMessage;
}
else
{
string finalContent = content;
targetMessage = Message.Import(chatId, senderGuid, finalContent, messageType, createdAt, replyToId, null);
var idAttr = node.GetAttribute("id");
if (!string.IsNullOrEmpty(idAttr))
{
messageIdMap[idAttr] = targetMessage.Id;
}
}
if (mediaNodes != null)
{
foreach (var mediaNode in mediaNodes)
{
string? href = mediaNode.Attributes["href"]?.Value ?? mediaNode.Attributes["src"]?.Value;
string? href = mediaNode.GetAttribute("href") ?? mediaNode.GetAttribute("src");
if (!string.IsNullOrEmpty(href) && !href.StartsWith("http"))
{
// Local file in zip
var zipPath = baseDir + href.Replace("\\", "/");
var zipEntry = archive.GetEntry(zipPath);
if (zipEntry != null)
@@ -244,23 +395,36 @@ public sealed class TelegramImportController : ControllerBase
await zipfs.CopyToAsync(ms, ct);
ms.Position = 0;
string cType = "application/octet-stream";
var mType = "file";
if (href.EndsWith(".jpg") || href.EndsWith(".png")) { cType = "image/jpeg"; mType = "image"; messageType = "image"; }
else if (href.EndsWith(".mp4")) { cType = "video/mp4"; mType = "video"; messageType = "video"; }
else if (href.EndsWith(".ogg")) { cType = "audio/ogg"; mType = "voice"; messageType = "voice"; }
var fileId = await _fileStorage.UploadFileAsync(ms, Path.GetFileName(href), cType);
newMessage.AddMedia(mType, $"/api/files/{fileId}", Path.GetFileName(href), zipEntry.Length);
var types = GetMediaTypes(href);
var fileId = await _fileStorage.UploadFileAsync(ms, Path.GetFileName(href), types.cType);
targetMessage.AddMedia(types.mType, $"/api/files/{fileId}", Path.GetFileName(href), zipEntry.Length);
}
}
}
}
// Save text message without media if content is just empty, but wait, if it's empty and has no media, it was probably a system message (like pinned, joined).
if (!string.IsNullOrEmpty(content) || newMessage.Media.Any())
// Parse reactions
var reactionNodes = node.QuerySelectorAll(".reactions .reaction");
foreach (var reactionNode in reactionNodes)
{
_messageRepository.Add(newMessage);
var emojiNode = reactionNode.QuerySelector(".emoji");
if (emojiNode == null) continue;
var emoji = emojiNode.TextContent.Trim();
var userpicNodes = reactionNode.QuerySelectorAll(".userpics .userpic .initials[title]");
foreach (var userpicNode in userpicNodes)
{
var title = userpicNode.GetAttribute("title")?.Trim();
if (!string.IsNullOrEmpty(title) && req.Mapping.TryGetValue(title, out var rUserId) && rUserId != Guid.Empty)
{
targetMessage.AddReaction(rUserId, emoji);
}
}
}
if (targetMessage != lastSavedMessage && (!string.IsNullOrEmpty(content) || targetMessage.Media.Any()))
{
_messageRepository.Add(targetMessage);
lastSavedMessage = targetMessage;
importedCount++;
}
}