Добавить общий Storage-сервис данных

Новый сервис Deal.Storage (src/storage-service): gRPC Upload/
Download/Stat/Delete, определение типа контент-снифингом, MinIO-бэкенд,
токен-валидация через общий интерцептор. Контракт storage.proto в
Deal.Proto; sln сервиса; подключение в compose.dev и эндпоинт в core;
тесты (снифер + хост/токен).
This commit is contained in:
Rustam Khalimov
2026-09-11 14:33:43 +03:00
parent e1aebd1d78
commit 770dba7257
18 changed files with 1120 additions and 1 deletions
+1
View File
@@ -41,6 +41,7 @@
<Protobuf Include="telegram.proto" GrpcServices="Both" />
<Protobuf Include="ml.proto" GrpcServices="Both" />
<Protobuf Include="ai.proto" GrpcServices="Both" />
<Protobuf Include="storage.proto" GrpcServices="Both" />
</ItemGroup>
</Project>
+78
View File
@@ -0,0 +1,78 @@
//
// Контракт общего сервиса данных: загрузка/выгрузка/стат/удаление объектов вложений.
//
// tenant-id — id тенанта (metadata); объект хранится в пространстве тенанта;
// service-token — общий токен сервисов (env DEAL_SERVICE_TOKEN); неверный/пустой →
// UNAUTHENTICATED (интерцептор хоста).
//
// Тип объекта (image/video/audio/document/archive/other) определяет сервис —
// контракт типы вложений не задаёт.
syntax = "proto3";
package deal.storage.v1;
option csharp_namespace = "Deal.Grpc.Storage";
service StorageService {
// Поток: первое сообщение несёт meta, далее — data.
rpc Upload(stream UploadRequest) returns (UploadReply);
// Поток содержимого объекта частями.
rpc Download(DownloadRequest) returns (stream DownloadChunk);
// Дескриптор объекта (без содержимого).
rpc Stat(StatRequest) returns (StatReply);
// Удаление объекта (отсутствующий — успех).
rpc Delete(DeleteRequest) returns (DeleteReply);
}
message UploadRequest {
UploadMeta meta = 1;
bytes data = 2;
}
message UploadMeta {
string tenant_id = 1;
string file_name = 2;
string content_type = 3;
}
message UploadReply {
// Идентификатор объекта — ключ в хранилище.
string id = 1;
// Ссылка на скачивание/отображение.
string ref = 2;
// Тип, определённый сервисом (image/video/audio/document/archive/other).
string kind = 3;
string mime_type = 4;
string file_name = 5;
int64 size = 6;
optional int32 width = 7;
optional int32 height = 8;
optional double duration_sec = 9;
optional string preview_ref = 10;
}
message DownloadRequest {
string id = 1;
}
message DownloadChunk {
bytes data = 1;
}
message StatRequest {
string id = 1;
}
message StatReply {
bool found = 1;
UploadReply info = 2;
}
message DeleteRequest {
string id = 1;
}
message DeleteReply {}
@@ -0,0 +1,3 @@
// Тесты storage-service поднимают Kestrel-хост и меняют env-переменные процесса на время сценария —
// параллельный прогон классов дал бы гонки на env, поэтому тесты сериализованы.
[assembly: CollectionBehavior(DisableTestParallelization = true)]
@@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Grpc.Net.Client" Version="2.83.0" />
<PackageReference Include="Grpc.HealthCheck" Version="2.83.0" />
</ItemGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Deal.Storage\Deal.Storage.csproj" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>
@@ -0,0 +1,128 @@
using System.Net;
using System.Net.Sockets;
using Deal.Grpc.Storage;
using Grpc.Core;
using Grpc.Health.V1;
using Grpc.Net.Client;
using Microsoft.AspNetCore.Builder;
namespace Deal.Storage.Tests.Grpc;
/// <summary>
/// Тесты хоста storage-service: health и проверка токена.
/// </summary>
public sealed class StorageServiceHostTests
{
private const string ServiceTokenEnvKey = "DEAL_SERVICE_TOKEN";
private const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT";
private const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY";
private const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY";
private const string ServiceTokenMetadataKey = "service-token";
private const string ValidToken = "storage-test-token";
private const int RpcDeadlineSeconds = 10;
/// <summary>
/// Health отвечает SERVING — хост поднялся.
/// </summary>
[Fact]
public async Task HealthCheck_ReturnsServing()
{
await RunHostScenarioAsync(
ValidToken,
async channel =>
{
var health = new Health.HealthClient(channel);
HealthCheckResponse response = await health.CheckAsync(
new HealthCheckRequest(),
deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds));
Assert.Equal(HealthCheckResponse.Types.ServingStatus.Serving, response.Status);
});
}
/// <summary>
/// Stat без токена отклоняется.
/// </summary>
[Fact]
public async Task Stat_WithoutToken_IsUnauthenticated()
{
await RunHostScenarioAsync(
ValidToken,
channel => AssertRejectedAsync(channel, tokenHeader: null));
}
/// <summary>
/// Stat с неверным токеном отклоняется.
/// </summary>
[Fact]
public async Task Stat_WithWrongToken_IsUnauthenticated()
{
await RunHostScenarioAsync(
ValidToken,
channel => AssertRejectedAsync(channel, tokenHeader: "wrong-token"));
}
private static async Task AssertRejectedAsync(GrpcChannel channel, string? tokenHeader)
{
var client = new StorageService.StorageServiceClient(channel);
Metadata metadata = new();
if (tokenHeader is not null)
{
metadata.Add(ServiceTokenMetadataKey, tokenHeader);
}
var callOptions = new CallOptions(metadata, deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds));
AsyncUnaryCall<StatReply> call = client.StatAsync(new StatRequest { Id = "x" }, callOptions);
RpcException exception = await Assert.ThrowsAsync<RpcException>(() => call.ResponseAsync);
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
}
private static async Task RunHostScenarioAsync(string? serviceToken, Func<GrpcChannel, Task> scenario)
{
string? originalToken = Environment.GetEnvironmentVariable(ServiceTokenEnvKey);
string? originalEndpoint = Environment.GetEnvironmentVariable(EndpointEnvKey);
string? originalAccess = Environment.GetEnvironmentVariable(AccessKeyEnvKey);
string? originalSecret = Environment.GetEnvironmentVariable(SecretKeyEnvKey);
Environment.SetEnvironmentVariable(ServiceTokenEnvKey, serviceToken);
Environment.SetEnvironmentVariable(EndpointEnvKey, "127.0.0.1:9000");
Environment.SetEnvironmentVariable(AccessKeyEnvKey, "test");
Environment.SetEnvironmentVariable(SecretKeyEnvKey, "test");
WebApplication? app = null;
GrpcChannel? channel = null;
try
{
int port = FreeTcpPort();
app = StorageServiceHost.Create(port);
await app.StartAsync();
channel = GrpcChannel.ForAddress($"http://127.0.0.1:{port}");
await scenario(channel);
}
finally
{
if (channel is not null)
{
channel.Dispose();
}
if (app is not null)
{
await app.StopAsync();
await app.DisposeAsync();
}
Environment.SetEnvironmentVariable(ServiceTokenEnvKey, originalToken);
Environment.SetEnvironmentVariable(EndpointEnvKey, originalEndpoint);
Environment.SetEnvironmentVariable(AccessKeyEnvKey, originalAccess);
Environment.SetEnvironmentVariable(SecretKeyEnvKey, originalSecret);
}
}
private static int FreeTcpPort()
{
using var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
return ((IPEndPoint)listener.LocalEndpoint).Port;
}
}
@@ -0,0 +1,81 @@
using Deal.Storage.Services;
namespace Deal.Storage.Tests.Services;
/// <summary>
/// Тесты определения типа и MIME по содержимому.
/// </summary>
public sealed class FileKindSnifferTests
{
/// <summary>
/// PNG определяется как изображение.
/// </summary>
[Fact]
public void Png_IsImage()
{
byte[] head = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
(string kind, string mime) = FileKindSniffer.Detect(head, null);
Assert.Equal("image", kind);
Assert.Equal("image/png", mime);
}
/// <summary>
/// PDF определяется как документ.
/// </summary>
[Fact]
public void Pdf_IsDocument()
{
byte[] head = [0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E, 0x37];
(string kind, string mime) = FileKindSniffer.Detect(head, null);
Assert.Equal("document", kind);
Assert.Equal("application/pdf", mime);
}
/// <summary>
/// MP4 определяется как видео.
/// </summary>
[Fact]
public void Mp4_IsVideo()
{
byte[] head = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32];
(string kind, string mime) = FileKindSniffer.Detect(head, null);
Assert.Equal("video", kind);
Assert.Equal("video/mp4", mime);
}
/// <summary>
/// MP3 определяется как аудио.
/// </summary>
[Fact]
public void Mp3_IsAudio()
{
byte[] head = [0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00];
(string kind, string mime) = FileKindSniffer.Detect(head, null);
Assert.Equal("audio", kind);
Assert.Equal("audio/mpeg", mime);
}
/// <summary>
/// Неизвестное содержимое с известным MIME берёт тип из MIME.
/// </summary>
[Fact]
public void UnknownWithDeclaredText_IsDocument()
{
byte[] head = [0x00, 0x01, 0x02, 0x03];
(string kind, string mime) = FileKindSniffer.Detect(head, "text/plain");
Assert.Equal("document", kind);
Assert.Equal("text/plain", mime);
}
/// <summary>
/// Полностью неизвестное содержимое — «other».
/// </summary>
[Fact]
public void UnknownWithoutDeclared_IsOther()
{
byte[] head = [0x00, 0x01, 0x02, 0x03];
(string kind, string mime) = FileKindSniffer.Detect(head, null);
Assert.Equal(FileKindSniffer.KindOther, kind);
Assert.Equal("application/octet-stream", mime);
}
}
+76
View File
@@ -0,0 +1,76 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.0.31903.59
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage", "Deal.Storage\Deal.Storage.csproj", "{0728C329-E3A0-4EDD-8305-FE558C01C085}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Grpc.Hosting", "..\grpc-hosting\Deal.Grpc.Hosting\Deal.Grpc.Hosting.csproj", "{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Proto", "..\contracts\Deal.Proto.csproj", "{A3470473-FEE8-476C-96C4-0038F920A22A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage.Tests", "Deal.Storage.Tests\Deal.Storage.Tests.csproj", "{49D721BA-5380-421F-8072-4B7B0EE4480F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.ActiveCfg = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.Build.0 = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.ActiveCfg = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.Build.0 = Debug|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.Build.0 = Release|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.ActiveCfg = Release|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.Build.0 = Release|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.ActiveCfg = Release|Any CPU
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.Build.0 = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.ActiveCfg = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.Build.0 = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.ActiveCfg = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.Build.0 = Debug|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.Build.0 = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.ActiveCfg = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.Build.0 = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.ActiveCfg = Release|Any CPU
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.Build.0 = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.ActiveCfg = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.Build.0 = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.ActiveCfg = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.Build.0 = Debug|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.Build.0 = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.ActiveCfg = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.Build.0 = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.ActiveCfg = Release|Any CPU
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.Build.0 = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.ActiveCfg = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.Build.0 = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.ActiveCfg = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.Build.0 = Debug|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.Build.0 = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.ActiveCfg = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.Build.0 = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.ActiveCfg = Release|Any CPU
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<AssemblyName>Deal.Storage</AssemblyName>
<RootNamespace>Deal.Storage</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Grpc.AspNetCore" Version="2.83.0" />
<PackageReference Include="Grpc.AspNetCore.HealthChecks" Version="2.83.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Minio" Version="7.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\grpc-hosting\Deal.Grpc.Hosting\Deal.Grpc.Hosting.csproj" />
<ProjectReference Include="..\..\contracts\Deal.Proto.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,26 @@
# storage-service: общий gRPC-сервис данных (вложения источников).
#
# КОНТЕКСТ СБОРКИ — корень репозитория: Deal.Storage.csproj ссылается на src/contracts/Deal.Proto.csproj
# и src/grpc-hosting вне каталога сервиса. Запуск из корня:
# docker build -f src/storage-service/Deal.Storage/Dockerfile .
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
WORKDIR /repo
COPY src/contracts/Deal.Proto.csproj src/contracts/
COPY src/grpc-hosting/Deal.Grpc.Hosting/Deal.Grpc.Hosting.csproj src/grpc-hosting/Deal.Grpc.Hosting/
COPY src/storage-service/Directory.Build.props src/storage-service/
COPY src/storage-service/Deal.Storage/Deal.Storage.csproj src/storage-service/Deal.Storage/
RUN dotnet restore src/storage-service/Deal.Storage/Deal.Storage.csproj
COPY src/contracts/ src/contracts/
COPY src/grpc-hosting/ src/grpc-hosting/
COPY src/storage-service/Deal.Storage/ src/storage-service/Deal.Storage/
RUN dotnet publish src/storage-service/Deal.Storage/Deal.Storage.csproj -c Release -o /app/publish
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS final
WORKDIR /app
EXPOSE 5104
COPY --from=build /app/publish .
COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.35 /ko-app/grpc-health-probe /bin/grpc_health_probe
ENTRYPOINT ["dotnet", "Deal.Storage.dll"]
@@ -0,0 +1,12 @@
namespace Deal.Storage.Models;
/// <summary>
/// Дескриптор сохранённого объекта.
/// </summary>
public sealed record StoredObject(
string Id,
string Ref,
string Kind,
string MimeType,
string FileName,
long Size);
@@ -0,0 +1,73 @@
using Microsoft.Extensions.Configuration;
namespace Deal.Storage.Options;
/// <summary>
/// Настройки объектного хранилища сервиса данных.
/// </summary>
public sealed class StorageOptions
{
/// <summary>
/// Значение бакета по умолчанию.
/// </summary>
public const string DefaultBucketName = "deal-attachments";
/// <summary>
/// Env-ключ адреса хранилища (host:port).
/// </summary>
public const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT";
/// <summary>
/// Env-ключ ключа доступа.
/// </summary>
public const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY";
/// <summary>
/// Env-ключ секрета доступа.
/// </summary>
public const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY";
/// <summary>
/// Env-ключ имени бакета.
/// </summary>
public const string BucketEnvKey = "DEAL_STORAGE_BUCKET";
/// <summary>
/// Env-ключ флага TLS (1/true — https).
/// </summary>
public const string SecureEnvKey = "DEAL_STORAGE_SECURE";
public string Endpoint { get; init; } = string.Empty;
public string AccessKey { get; init; } = string.Empty;
public string SecretKey { get; init; } = string.Empty;
public string Bucket { get; init; } = DefaultBucketName;
public bool Secure { get; init; }
/// <summary>
/// Читает настройки из конфигурации хоста.
/// </summary>
public static StorageOptions FromConfiguration(IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(configuration);
string bucket = (configuration[BucketEnvKey] ?? string.Empty).Trim();
return new StorageOptions
{
Endpoint = (configuration[EndpointEnvKey] ?? string.Empty).Trim(),
AccessKey = (configuration[AccessKeyEnvKey] ?? string.Empty).Trim(),
SecretKey = configuration[SecretKeyEnvKey] ?? string.Empty,
Bucket = bucket.Length == 0 ? DefaultBucketName : bucket,
Secure = IsEnabled(configuration[SecureEnvKey]),
};
}
/// <summary>
/// Разбирает значение флага TLS.
/// </summary>
public static bool IsEnabled(string? rawValue)
=> string.Equals(rawValue, "1", StringComparison.Ordinal)
|| string.Equals(rawValue, "true", StringComparison.OrdinalIgnoreCase);
}
@@ -0,0 +1,31 @@
using Deal.Grpc.Hosting.Interceptors;
using Deal.Grpc.Hosting.Models;
using Deal.Grpc.Hosting.Options;
using Deal.Grpc.Hosting.Services;
using Deal.Storage;
const int defaultGrpcPort = 5104;
const string storageProcessName = "storage";
int grpcPort = GrpcHostEnvironment.ResolveGrpcPort(defaultGrpcPort);
int metricsPort = DealMetricsHosting.ResolveMetricsPort(DealMetricsHosting.DefaultMetricsPort);
WebApplication app = StorageServiceHost.Create(
grpcPort,
configureBuilder: builder =>
{
DealLogging.Configure(builder, storageProcessName);
DealMetricsHosting.AddDealMetrics(builder, metricsPort);
});
DealMetricsHosting.MapDealMetrics(app);
MtlsOptions mtlsOptions = MtlsOptions.FromConfiguration(app.Configuration);
GrpcHostEnvironment.RequireMtlsInProduction(mtlsOptions);
app.Logger.LogInformation(
"storage-service стартует: gRPC {Transport} 0.0.0.0:{Port} (health /grpc.health.v1.Health/Check)",
mtlsOptions.Enabled ? "mTLS (TLS + клиентский сертификат)" : "plaintext + service-token",
grpcPort);
await app.RunAsync();
@@ -0,0 +1,138 @@
namespace Deal.Storage.Services;
/// <summary>
/// Определяет тип и MIME объекта по первым байтам содержимого.
/// </summary>
public static class FileKindSniffer
{
/// <summary>
/// Тип «прочее».
/// </summary>
public const string KindOther = "other";
private const string DefaultMime = "application/octet-stream";
/// <summary>
/// Определяет тип и MIME по заголовку файла.
/// </summary>
/// <param name="head">Первые байты содержимого.</param>
/// <param name="contentType">MIME, заявленный загрузчиком (используется как подсказка).</param>
/// <returns>Пара «тип, MIME».</returns>
public static (string Kind, string MimeType) Detect(ReadOnlySpan<byte> head, string? contentType)
{
if (StartsWith(head, 0x89, 0x50, 0x4E, 0x47))
{
return ("image", "image/png");
}
if (StartsWith(head, 0xFF, 0xD8, 0xFF))
{
return ("image", "image/jpeg");
}
if (StartsWith(head, 0x47, 0x49, 0x46, 0x38))
{
return ("image", "image/gif");
}
if (StartsWith(head, 0x42, 0x4D))
{
return ("image", "image/bmp");
}
if (StartsWith(head, 0x49, 0x49, 0x2A, 0x00) || StartsWith(head, 0x4D, 0x4D, 0x00, 0x2A))
{
return ("image", "image/tiff");
}
if (StartsWith(head, 0x25, 0x50, 0x44, 0x46))
{
return ("document", "application/pdf");
}
if (StartsWith(head, 0x50, 0x4B, 0x03, 0x04) || StartsWith(head, 0x50, 0x4B, 0x05, 0x06))
{
return ("archive", "application/zip");
}
if (StartsWith(head, 0x52, 0x61, 0x72, 0x21))
{
return ("archive", "application/vnd.rar");
}
if (StartsWith(head, 0x37, 0x7A, 0xBC, 0xAF))
{
return ("archive", "application/x-7z-compressed");
}
if (StartsWith(head, 0x1F, 0x8B))
{
return ("archive", "application/gzip");
}
if (StartsWith(head, 0x1A, 0x45, 0xDF, 0xA3))
{
return ("video", "video/webm");
}
if (head.Length >= 12 && StartsWith(head, 0x52, 0x49, 0x46, 0x46))
{
if (StartsWith(head[8..], 0x57, 0x45, 0x42, 0x50))
{
return ("image", "image/webp");
}
if (StartsWith(head[8..], 0x57, 0x41, 0x56, 0x45))
{
return ("audio", "audio/wav");
}
if (StartsWith(head[8..], 0x41, 0x56, 0x49, 0x20))
{
return ("video", "video/x-msvideo");
}
}
if (head.Length >= 8 && StartsWith(head[4..], 0x66, 0x74, 0x79, 0x70))
{
return ("video", "video/mp4");
}
if (StartsWith(head, 0x49, 0x44, 0x33) || StartsWith(head, 0xFF, 0xFB))
{
return ("audio", "audio/mpeg");
}
if (StartsWith(head, 0x4F, 0x67, 0x67, 0x53))
{
return ("audio", "audio/ogg");
}
if (StartsWith(head, 0x66, 0x4C, 0x61, 0x43))
{
return ("audio", "audio/flac");
}
string declared = (contentType ?? string.Empty).Trim();
if (declared.Length == 0)
{
return (KindOther, DefaultMime);
}
return (KindFromMime(declared), declared);
}
// Определяет общий тип по префиксу MIME-типа.
private static string KindFromMime(string mime)
{
if (mime.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) return "image";
if (mime.StartsWith("video/", StringComparison.OrdinalIgnoreCase)) return "video";
if (mime.StartsWith("audio/", StringComparison.OrdinalIgnoreCase)) return "audio";
if (mime.StartsWith("text/", StringComparison.OrdinalIgnoreCase)) return "document";
return KindOther;
}
// Проверяет, начинается ли последовательность с заданных байтов.
private static bool StartsWith(ReadOnlySpan<byte> data, params byte[] prefix)
=> data.Length >= prefix.Length && data[..prefix.Length].SequenceEqual(prefix);
}
@@ -0,0 +1,195 @@
using Deal.Storage.Options;
using Microsoft.Extensions.Logging;
using Minio;
using Minio.DataModel;
using Minio.DataModel.Args;
using Minio.Exceptions;
namespace Deal.Storage.Services;
/// <summary>
/// Объектное хранилище сервиса данных поверх MinIO.
/// </summary>
public sealed class MinioObjectStore
{
private const string DefaultContentType = "application/octet-stream";
private readonly IMinioClient _client;
private readonly string _endpoint;
private readonly string _bucket;
private readonly ILogger<MinioObjectStore> _logger;
private readonly SemaphoreSlim _bucketCheckGate = new(1, 1);
private bool _bucketChecked;
/// <summary>
/// Создаёт хранилище поверх настроек.
/// </summary>
public MinioObjectStore(StorageOptions options, ILogger<MinioObjectStore> logger)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
if (string.IsNullOrWhiteSpace(options.Endpoint)
|| string.IsNullOrWhiteSpace(options.AccessKey)
|| string.IsNullOrWhiteSpace(options.SecretKey))
{
throw new InvalidOperationException(
"MinioObjectStore требует заполненные настройки хранилища (Endpoint и ключи доступа).");
}
_endpoint = options.Endpoint;
_bucket = options.Bucket;
_logger = logger;
_client = new MinioClient()
.WithEndpoint(options.Endpoint)
.WithCredentials(options.AccessKey, options.SecretKey)
.WithSSL(options.Secure)
.Build();
}
/// <summary>
/// Возвращает подпись режима для стартового лога.
/// </summary>
public override string ToString() => $"MinioObjectStore (endpoint: {_endpoint}; bucket: {_bucket})";
/// <summary>
/// Сохраняет объект, возвращает его размер.
/// </summary>
public async Task<long> PutAsync(string key, Stream content, string contentType, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(content);
await EnsureBucketAsync(ct);
if (content.CanSeek && content.Position != 0)
{
content.Position = 0;
}
using MemoryStream buffer = new();
await content.CopyToAsync(buffer, ct);
buffer.Position = 0;
await _client.PutObjectAsync(
new PutObjectArgs()
.WithBucket(_bucket)
.WithObject(key)
.WithStreamData(buffer)
.WithObjectSize(buffer.Length)
.WithContentType(string.IsNullOrWhiteSpace(contentType) ? DefaultContentType : contentType),
ct);
return buffer.Length;
}
/// <summary>
/// Возвращает содержимое объекта потоком либо null.
/// </summary>
public async Task<Stream?> GetAsync(string key, CancellationToken ct)
{
MemoryStream buffer = new();
try
{
await _client.GetObjectAsync(
new GetObjectArgs()
.WithBucket(_bucket)
.WithObject(key)
.WithCallbackStream(async (stream, token) => await stream.CopyToAsync(buffer, token)),
ct);
}
catch (ObjectNotFoundException)
{
buffer.Dispose();
return null;
}
catch (Exception)
{
buffer.Dispose();
throw;
}
buffer.Position = 0;
return buffer;
}
/// <summary>
/// Возвращает размер и MIME объекта либо null.
/// </summary>
public async Task<(long Size, string ContentType)?> StatAsync(string key, CancellationToken ct)
{
try
{
ObjectStat stat = await _client.StatObjectAsync(
new StatObjectArgs().WithBucket(_bucket).WithObject(key),
ct);
return (stat.Size, stat.ContentType ?? string.Empty);
}
catch (ObjectNotFoundException)
{
return null;
}
}
/// <summary>
/// Удаляет объект (отсутствующий — успех).
/// </summary>
public async Task DeleteAsync(string key, CancellationToken ct)
{
try
{
await _client.RemoveObjectAsync(
new RemoveObjectArgs().WithBucket(_bucket).WithObject(key),
ct);
}
catch (MinioException exception)
{
_logger.LogWarning(
exception,
"Не удалось удалить объект «{Key}» из бакета «{Bucket}»: {Message}",
key,
_bucket,
exception.Message);
}
}
private async Task EnsureBucketAsync(CancellationToken ct)
{
if (_bucketChecked)
{
return;
}
await _bucketCheckGate.WaitAsync(ct);
try
{
if (_bucketChecked)
{
return;
}
try
{
bool exists = await _client.BucketExistsAsync(new BucketExistsArgs().WithBucket(_bucket), ct);
if (!exists)
{
await _client.MakeBucketAsync(new MakeBucketArgs().WithBucket(_bucket), ct);
}
}
catch (MinioException exception)
{
_logger.LogWarning(
exception,
"Не удалось проверить/создать бакет «{Bucket}»: {Message}",
_bucket,
exception.Message);
}
_bucketChecked = true;
}
finally
{
_bucketCheckGate.Release();
}
}
}
@@ -0,0 +1,47 @@
using Deal.Grpc.Hosting.Services;
using Deal.Storage.Options;
using Deal.Storage.Services;
namespace Deal.Storage;
/// <summary>
/// Собирает WebApplication gRPC-хоста сервиса данных.
/// </summary>
public static class StorageServiceHost
{
/// <summary>
/// Создаёт (не запускает) хост.
/// </summary>
/// <param name="grpcPort">TCP-порт Kestrel.</param>
/// <param name="args">Аргументы командной строки.</param>
/// <param name="configureServices">Опциональный хук DI для тестов.</param>
/// <param name="configureBuilder">Опциональный хук конфигурации билдера (логи/метрики).</param>
/// <returns>Собранный хост.</returns>
public static WebApplication Create(
int grpcPort,
string[]? args = null,
Action<IServiceCollection>? configureServices = null,
Action<WebApplicationBuilder>? configureBuilder = null)
{
WebApplicationBuilder builder = WebApplication.CreateBuilder(args ?? []);
var mtlsCertificates = GrpcServer.LoadMtlsCertificates(builder);
GrpcServer.ConfigureKestrelHttp2Endpoint(builder, grpcPort, mtlsCertificates);
builder.Services.AddDealGrpcServer();
builder.Services.AddReadyHealthCheck("хост storage-service готов");
StorageOptions storageOptions = StorageOptions.FromConfiguration(builder.Configuration);
builder.Services.AddSingleton(storageOptions);
builder.Services.AddSingleton<MinioObjectStore>();
configureServices?.Invoke(builder.Services);
configureBuilder?.Invoke(builder);
WebApplication app = builder.Build();
app.MapGrpcService<StorageServiceImpl>();
app.MapGrpcHealthChecksService();
return app;
}
}
@@ -0,0 +1,135 @@
using Deal.Grpc.Storage;
using Deal.Storage.Services;
using Grpc.Core;
namespace Deal.Storage;
/// <summary>
/// Реализация сервиса данных.
/// </summary>
public sealed class StorageServiceImpl : StorageService.StorageServiceBase
{
private const int HeadSize = 512;
private const int DownloadChunkSize = 64 * 1024;
private readonly MinioObjectStore _store;
private readonly ILogger<StorageServiceImpl> _logger;
/// <summary>
/// Создаёт реализацию поверх хранилища.
/// </summary>
public StorageServiceImpl(MinioObjectStore store, ILogger<StorageServiceImpl> logger)
{
ArgumentNullException.ThrowIfNull(store);
ArgumentNullException.ThrowIfNull(logger);
_store = store;
_logger = logger;
}
/// <inheritdoc />
public override async Task<UploadReply> Upload(
IAsyncStreamReader<UploadRequest> requestStream,
ServerCallContext context)
{
CancellationToken ct = context.CancellationToken;
await using MemoryStream buffer = new();
byte[] head = new byte[HeadSize];
int headLength = 0;
UploadMeta? meta = null;
while (await requestStream.MoveNext(ct))
{
UploadRequest message = requestStream.Current;
if (meta is null && message.Meta is not null)
{
meta = message.Meta;
}
if (message.Data.Length == 0)
{
continue;
}
if (headLength < HeadSize)
{
int take = Math.Min(HeadSize - headLength, message.Data.Length);
message.Data.Span[..take].CopyTo(head.AsSpan(headLength));
headLength += take;
}
await buffer.WriteAsync(message.Data.Memory, ct);
}
if (meta is null || string.IsNullOrWhiteSpace(meta.TenantId))
{
throw new RpcException(new Status(StatusCode.InvalidArgument, "tenant_id обязателен"));
}
(string kind, string mimeType) = FileKindSniffer.Detect(head.AsSpan(0, headLength), meta.ContentType);
string key = $"{meta.TenantId.Trim()}/{Guid.NewGuid():N}";
buffer.Position = 0;
long size = await _store.PutAsync(key, buffer, mimeType, ct);
_logger.LogInformation("Объект сохранён: {Key} ({Kind}, {Size} байт)", key, kind, size);
return new UploadReply
{
Id = key,
Ref = key,
Kind = kind,
MimeType = mimeType,
FileName = meta.FileName ?? string.Empty,
Size = size,
};
}
/// <inheritdoc />
public override async Task Download(
DownloadRequest request,
IServerStreamWriter<DownloadChunk> responseStream,
ServerCallContext context)
{
CancellationToken ct = context.CancellationToken;
await using Stream? content = await _store.GetAsync(request.Id, ct);
if (content is null)
{
throw new RpcException(new Status(StatusCode.NotFound, "объект не найден"));
}
byte[] chunk = new byte[DownloadChunkSize];
int read;
while ((read = await content.ReadAsync(chunk, ct)) > 0)
{
await responseStream.WriteAsync(new DownloadChunk { Data = Google.Protobuf.ByteString.CopyFrom(chunk, 0, read) }, ct);
}
}
/// <inheritdoc />
public override async Task<StatReply> Stat(StatRequest request, ServerCallContext context)
{
(long Size, string ContentType)? stat = await _store.StatAsync(request.Id, context.CancellationToken);
if (stat is null)
{
return new StatReply { Found = false };
}
return new StatReply
{
Found = true,
Info = new UploadReply
{
Id = request.Id,
Ref = request.Id,
MimeType = stat.Value.ContentType,
Size = stat.Value.Size,
},
};
}
/// <inheritdoc />
public override async Task<DeleteReply> Delete(DeleteRequest request, ServerCallContext context)
{
await _store.DeleteAsync(request.Id, context.CancellationToken);
return new DeleteReply();
}
}
+11
View File
@@ -0,0 +1,11 @@
<Project>
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>latest</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<AnalysisLevel>latest</AnalysisLevel>
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
</PropertyGroup>
</Project>