Добавить общий Storage-сервис данных
Новый сервис Deal.Storage (src/storage-service): gRPC Upload/ Download/Stat/Delete, определение типа контент-снифингом, MinIO-бэкенд, токен-валидация через общий интерцептор. Контракт storage.proto в Deal.Proto; sln сервиса; подключение в compose.dev и эндпоинт в core; тесты (снифер + хост/токен).
This commit is contained in:
+28
-1
@@ -1,7 +1,7 @@
|
|||||||
# compose.dev.yml — dev-стек Дейла (этап 6, Task 20; Ruling 12).
|
# compose.dev.yml — dev-стек Дейла (этап 6, Task 20; Ruling 12).
|
||||||
#
|
#
|
||||||
# Полный стек: postgres + minio (хранилища), три автономных сервиса этапа 6
|
# Полный стек: postgres + minio (хранилища), три автономных сервиса этапа 6
|
||||||
# (telegram-service :5101, ai-service :5102, ml-service :5103) и core (deal-api:
|
# (telegram-service :5101, ai-service :5102, ml-service :5103, storage-service :5104) и core (deal-api:
|
||||||
# HTTP :5080 + gRPC-ингресс :5082).
|
# HTTP :5080 + gRPC-ингресс :5082).
|
||||||
#
|
#
|
||||||
# Режимы интеграций core — флагами SERVICES__{ML,AI,TELEGRAM}__USELOCAL (Ruling 6):
|
# Режимы интеграций core — флагами SERVICES__{ML,AI,TELEGRAM}__USELOCAL (Ruling 6):
|
||||||
@@ -88,6 +88,7 @@ services:
|
|||||||
Services__Ml__Endpoint: http://ml-service:5103
|
Services__Ml__Endpoint: http://ml-service:5103
|
||||||
Services__Ai__Endpoint: http://ai-service:5102
|
Services__Ai__Endpoint: http://ai-service:5102
|
||||||
Services__Telegram__Endpoint: http://telegram-service:5101
|
Services__Telegram__Endpoint: http://telegram-service:5101
|
||||||
|
Services__Storage__Endpoint: http://storage-service:5104
|
||||||
Storage__Minio__Endpoint: minio:9000
|
Storage__Minio__Endpoint: minio:9000
|
||||||
Storage__Minio__AccessKey: deal_minio
|
Storage__Minio__AccessKey: deal_minio
|
||||||
Storage__Minio__SecretKey: deal_minio_secret
|
Storage__Minio__SecretKey: deal_minio_secret
|
||||||
@@ -186,6 +187,32 @@ services:
|
|||||||
timeout: 3s
|
timeout: 3s
|
||||||
retries: 10
|
retries: 10
|
||||||
|
|
||||||
|
# storage-service — общий gRPC-сервис данных (вложения источников). Отдельный процесс, отдельное sln
|
||||||
|
# (src/storage-service/Deal.Storage.sln). Загрузка/выгрузка/стат/удаление объектов; тип объекта
|
||||||
|
# определяет сервис. Бэкенд — MinIO. Защита — общий service-token (DEAL_SERVICE_TOKEN).
|
||||||
|
storage-service:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: src/storage-service/Deal.Storage/Dockerfile
|
||||||
|
container_name: deal-storage-service
|
||||||
|
environment:
|
||||||
|
GRPC_PORT: "5104"
|
||||||
|
DEAL_SERVICE_TOKEN: ${DEAL_SERVICE_TOKEN:-deal_dev_service_token}
|
||||||
|
DEAL_STORAGE_ENDPOINT: minio:9000
|
||||||
|
DEAL_STORAGE_ACCESS_KEY: deal_minio
|
||||||
|
DEAL_STORAGE_SECRET_KEY: deal_minio_secret
|
||||||
|
DEAL_STORAGE_BUCKET: deal-attachments
|
||||||
|
DEAL_STORAGE_SECURE: "false"
|
||||||
|
ports:
|
||||||
|
- "5104:5104"
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "/bin/grpc_health_probe", "-addr=localhost:5104"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 10
|
||||||
|
|
||||||
# Prometheus (профиль observability, этап 12/пакет A) — сбор /metrics всех 4 процессов (:9464)
|
# Prometheus (профиль observability, этап 12/пакет A) — сбор /metrics всех 4 процессов (:9464)
|
||||||
# внутри dev-сети. Подъём: docker compose -f deploy/compose.dev.yml --profile observability up -d.
|
# внутри dev-сети. Подъём: docker compose -f deploy/compose.dev.yml --profile observability up -d.
|
||||||
# Конфиг — общий deploy/observability/prometheus.yml (те же имена сервисов и таргеты). UI — 9090.
|
# Конфиг — общий deploy/observability/prometheus.yml (те же имена сервисов и таргеты). UI — 9090.
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
<Protobuf Include="telegram.proto" GrpcServices="Both" />
|
<Protobuf Include="telegram.proto" GrpcServices="Both" />
|
||||||
<Protobuf Include="ml.proto" GrpcServices="Both" />
|
<Protobuf Include="ml.proto" GrpcServices="Both" />
|
||||||
<Protobuf Include="ai.proto" GrpcServices="Both" />
|
<Protobuf Include="ai.proto" GrpcServices="Both" />
|
||||||
|
<Protobuf Include="storage.proto" GrpcServices="Both" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
//
|
||||||
|
// Контракт общего сервиса данных: загрузка/выгрузка/стат/удаление объектов вложений.
|
||||||
|
//
|
||||||
|
// tenant-id — id тенанта (metadata); объект хранится в пространстве тенанта;
|
||||||
|
// service-token — общий токен сервисов (env DEAL_SERVICE_TOKEN); неверный/пустой →
|
||||||
|
// UNAUTHENTICATED (интерцептор хоста).
|
||||||
|
//
|
||||||
|
// Тип объекта (image/video/audio/document/archive/other) определяет сервис —
|
||||||
|
// контракт типы вложений не задаёт.
|
||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
package deal.storage.v1;
|
||||||
|
|
||||||
|
option csharp_namespace = "Deal.Grpc.Storage";
|
||||||
|
|
||||||
|
service StorageService {
|
||||||
|
// Поток: первое сообщение несёт meta, далее — data.
|
||||||
|
rpc Upload(stream UploadRequest) returns (UploadReply);
|
||||||
|
|
||||||
|
// Поток содержимого объекта частями.
|
||||||
|
rpc Download(DownloadRequest) returns (stream DownloadChunk);
|
||||||
|
|
||||||
|
// Дескриптор объекта (без содержимого).
|
||||||
|
rpc Stat(StatRequest) returns (StatReply);
|
||||||
|
|
||||||
|
// Удаление объекта (отсутствующий — успех).
|
||||||
|
rpc Delete(DeleteRequest) returns (DeleteReply);
|
||||||
|
}
|
||||||
|
|
||||||
|
message UploadRequest {
|
||||||
|
UploadMeta meta = 1;
|
||||||
|
bytes data = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message UploadMeta {
|
||||||
|
string tenant_id = 1;
|
||||||
|
string file_name = 2;
|
||||||
|
string content_type = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message UploadReply {
|
||||||
|
// Идентификатор объекта — ключ в хранилище.
|
||||||
|
string id = 1;
|
||||||
|
// Ссылка на скачивание/отображение.
|
||||||
|
string ref = 2;
|
||||||
|
// Тип, определённый сервисом (image/video/audio/document/archive/other).
|
||||||
|
string kind = 3;
|
||||||
|
string mime_type = 4;
|
||||||
|
string file_name = 5;
|
||||||
|
int64 size = 6;
|
||||||
|
optional int32 width = 7;
|
||||||
|
optional int32 height = 8;
|
||||||
|
optional double duration_sec = 9;
|
||||||
|
optional string preview_ref = 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DownloadRequest {
|
||||||
|
string id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DownloadChunk {
|
||||||
|
bytes data = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message StatRequest {
|
||||||
|
string id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message StatReply {
|
||||||
|
bool found = 1;
|
||||||
|
UploadReply info = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteRequest {
|
||||||
|
string id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteReply {}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// Тесты storage-service поднимают Kestrel-хост и меняют env-переменные процесса на время сценария —
|
||||||
|
// параллельный прогон классов дал бы гонки на env, поэтому тесты сериализованы.
|
||||||
|
[assembly: CollectionBehavior(DisableTestParallelization = true)]
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<IsPackable>false</IsPackable>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||||
|
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||||
|
<PackageReference Include="xunit" Version="2.9.3" />
|
||||||
|
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Grpc.Net.Client" Version="2.83.0" />
|
||||||
|
<PackageReference Include="Grpc.HealthCheck" Version="2.83.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\Deal.Storage\Deal.Storage.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Using Include="Xunit" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
using System.Net;
|
||||||
|
using System.Net.Sockets;
|
||||||
|
using Deal.Grpc.Storage;
|
||||||
|
using Grpc.Core;
|
||||||
|
using Grpc.Health.V1;
|
||||||
|
using Grpc.Net.Client;
|
||||||
|
using Microsoft.AspNetCore.Builder;
|
||||||
|
|
||||||
|
namespace Deal.Storage.Tests.Grpc;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Тесты хоста storage-service: health и проверка токена.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class StorageServiceHostTests
|
||||||
|
{
|
||||||
|
private const string ServiceTokenEnvKey = "DEAL_SERVICE_TOKEN";
|
||||||
|
private const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT";
|
||||||
|
private const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY";
|
||||||
|
private const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY";
|
||||||
|
private const string ServiceTokenMetadataKey = "service-token";
|
||||||
|
private const string ValidToken = "storage-test-token";
|
||||||
|
private const int RpcDeadlineSeconds = 10;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Health отвечает SERVING — хост поднялся.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task HealthCheck_ReturnsServing()
|
||||||
|
{
|
||||||
|
await RunHostScenarioAsync(
|
||||||
|
ValidToken,
|
||||||
|
async channel =>
|
||||||
|
{
|
||||||
|
var health = new Health.HealthClient(channel);
|
||||||
|
HealthCheckResponse response = await health.CheckAsync(
|
||||||
|
new HealthCheckRequest(),
|
||||||
|
deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds));
|
||||||
|
Assert.Equal(HealthCheckResponse.Types.ServingStatus.Serving, response.Status);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stat без токена отклоняется.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Stat_WithoutToken_IsUnauthenticated()
|
||||||
|
{
|
||||||
|
await RunHostScenarioAsync(
|
||||||
|
ValidToken,
|
||||||
|
channel => AssertRejectedAsync(channel, tokenHeader: null));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stat с неверным токеном отклоняется.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task Stat_WithWrongToken_IsUnauthenticated()
|
||||||
|
{
|
||||||
|
await RunHostScenarioAsync(
|
||||||
|
ValidToken,
|
||||||
|
channel => AssertRejectedAsync(channel, tokenHeader: "wrong-token"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task AssertRejectedAsync(GrpcChannel channel, string? tokenHeader)
|
||||||
|
{
|
||||||
|
var client = new StorageService.StorageServiceClient(channel);
|
||||||
|
Metadata metadata = new();
|
||||||
|
if (tokenHeader is not null)
|
||||||
|
{
|
||||||
|
metadata.Add(ServiceTokenMetadataKey, tokenHeader);
|
||||||
|
}
|
||||||
|
|
||||||
|
var callOptions = new CallOptions(metadata, deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds));
|
||||||
|
AsyncUnaryCall<StatReply> call = client.StatAsync(new StatRequest { Id = "x" }, callOptions);
|
||||||
|
RpcException exception = await Assert.ThrowsAsync<RpcException>(() => call.ResponseAsync);
|
||||||
|
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task RunHostScenarioAsync(string? serviceToken, Func<GrpcChannel, Task> scenario)
|
||||||
|
{
|
||||||
|
string? originalToken = Environment.GetEnvironmentVariable(ServiceTokenEnvKey);
|
||||||
|
string? originalEndpoint = Environment.GetEnvironmentVariable(EndpointEnvKey);
|
||||||
|
string? originalAccess = Environment.GetEnvironmentVariable(AccessKeyEnvKey);
|
||||||
|
string? originalSecret = Environment.GetEnvironmentVariable(SecretKeyEnvKey);
|
||||||
|
|
||||||
|
Environment.SetEnvironmentVariable(ServiceTokenEnvKey, serviceToken);
|
||||||
|
Environment.SetEnvironmentVariable(EndpointEnvKey, "127.0.0.1:9000");
|
||||||
|
Environment.SetEnvironmentVariable(AccessKeyEnvKey, "test");
|
||||||
|
Environment.SetEnvironmentVariable(SecretKeyEnvKey, "test");
|
||||||
|
|
||||||
|
WebApplication? app = null;
|
||||||
|
GrpcChannel? channel = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int port = FreeTcpPort();
|
||||||
|
app = StorageServiceHost.Create(port);
|
||||||
|
await app.StartAsync();
|
||||||
|
|
||||||
|
channel = GrpcChannel.ForAddress($"http://127.0.0.1:{port}");
|
||||||
|
await scenario(channel);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (channel is not null)
|
||||||
|
{
|
||||||
|
channel.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (app is not null)
|
||||||
|
{
|
||||||
|
await app.StopAsync();
|
||||||
|
await app.DisposeAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
Environment.SetEnvironmentVariable(ServiceTokenEnvKey, originalToken);
|
||||||
|
Environment.SetEnvironmentVariable(EndpointEnvKey, originalEndpoint);
|
||||||
|
Environment.SetEnvironmentVariable(AccessKeyEnvKey, originalAccess);
|
||||||
|
Environment.SetEnvironmentVariable(SecretKeyEnvKey, originalSecret);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int FreeTcpPort()
|
||||||
|
{
|
||||||
|
using var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||||
|
listener.Start();
|
||||||
|
return ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
using Deal.Storage.Services;
|
||||||
|
|
||||||
|
namespace Deal.Storage.Tests.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Тесты определения типа и MIME по содержимому.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class FileKindSnifferTests
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// PNG определяется как изображение.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Png_IsImage()
|
||||||
|
{
|
||||||
|
byte[] head = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, null);
|
||||||
|
Assert.Equal("image", kind);
|
||||||
|
Assert.Equal("image/png", mime);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// PDF определяется как документ.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Pdf_IsDocument()
|
||||||
|
{
|
||||||
|
byte[] head = [0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E, 0x37];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, null);
|
||||||
|
Assert.Equal("document", kind);
|
||||||
|
Assert.Equal("application/pdf", mime);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// MP4 определяется как видео.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Mp4_IsVideo()
|
||||||
|
{
|
||||||
|
byte[] head = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, null);
|
||||||
|
Assert.Equal("video", kind);
|
||||||
|
Assert.Equal("video/mp4", mime);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// MP3 определяется как аудио.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Mp3_IsAudio()
|
||||||
|
{
|
||||||
|
byte[] head = [0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, null);
|
||||||
|
Assert.Equal("audio", kind);
|
||||||
|
Assert.Equal("audio/mpeg", mime);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Неизвестное содержимое с известным MIME берёт тип из MIME.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void UnknownWithDeclaredText_IsDocument()
|
||||||
|
{
|
||||||
|
byte[] head = [0x00, 0x01, 0x02, 0x03];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, "text/plain");
|
||||||
|
Assert.Equal("document", kind);
|
||||||
|
Assert.Equal("text/plain", mime);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Полностью неизвестное содержимое — «other».
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void UnknownWithoutDeclared_IsOther()
|
||||||
|
{
|
||||||
|
byte[] head = [0x00, 0x01, 0x02, 0x03];
|
||||||
|
(string kind, string mime) = FileKindSniffer.Detect(head, null);
|
||||||
|
Assert.Equal(FileKindSniffer.KindOther, kind);
|
||||||
|
Assert.Equal("application/octet-stream", mime);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
|
||||||
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
|
# Visual Studio Version 17
|
||||||
|
VisualStudioVersion = 17.0.31903.59
|
||||||
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage", "Deal.Storage\Deal.Storage.csproj", "{0728C329-E3A0-4EDD-8305-FE558C01C085}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Grpc.Hosting", "..\grpc-hosting\Deal.Grpc.Hosting\Deal.Grpc.Hosting.csproj", "{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Proto", "..\contracts\Deal.Proto.csproj", "{A3470473-FEE8-476C-96C4-0038F920A22A}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage.Tests", "Deal.Storage.Tests\Deal.Storage.Tests.csproj", "{49D721BA-5380-421F-8072-4B7B0EE4480F}"
|
||||||
|
EndProject
|
||||||
|
Global
|
||||||
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
Debug|x64 = Debug|x64
|
||||||
|
Debug|x86 = Debug|x86
|
||||||
|
Release|Any CPU = Release|Any CPU
|
||||||
|
Release|x64 = Release|x64
|
||||||
|
Release|x86 = Release|x86
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.Build.0 = Debug|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.ActiveCfg = Release|Any CPU
|
||||||
|
{0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.Build.0 = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.Build.0 = Debug|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.ActiveCfg = Release|Any CPU
|
||||||
|
{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.Build.0 = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.Build.0 = Debug|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.ActiveCfg = Release|Any CPU
|
||||||
|
{A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.Build.0 = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.Build.0 = Debug|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.ActiveCfg = Release|Any CPU
|
||||||
|
{49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.Build.0 = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
|
HideSolutionNode = FALSE
|
||||||
|
EndGlobalSection
|
||||||
|
EndGlobal
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<AssemblyName>Deal.Storage</AssemblyName>
|
||||||
|
<RootNamespace>Deal.Storage</RootNamespace>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Grpc.AspNetCore" Version="2.83.0" />
|
||||||
|
<PackageReference Include="Grpc.AspNetCore.HealthChecks" Version="2.83.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Minio" Version="7.0.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\..\grpc-hosting\Deal.Grpc.Hosting\Deal.Grpc.Hosting.csproj" />
|
||||||
|
<ProjectReference Include="..\..\contracts\Deal.Proto.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# storage-service: общий gRPC-сервис данных (вложения источников).
|
||||||
|
#
|
||||||
|
# КОНТЕКСТ СБОРКИ — корень репозитория: Deal.Storage.csproj ссылается на src/contracts/Deal.Proto.csproj
|
||||||
|
# и src/grpc-hosting вне каталога сервиса. Запуск из корня:
|
||||||
|
# docker build -f src/storage-service/Deal.Storage/Dockerfile .
|
||||||
|
|
||||||
|
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||||
|
WORKDIR /repo
|
||||||
|
|
||||||
|
COPY src/contracts/Deal.Proto.csproj src/contracts/
|
||||||
|
COPY src/grpc-hosting/Deal.Grpc.Hosting/Deal.Grpc.Hosting.csproj src/grpc-hosting/Deal.Grpc.Hosting/
|
||||||
|
COPY src/storage-service/Directory.Build.props src/storage-service/
|
||||||
|
COPY src/storage-service/Deal.Storage/Deal.Storage.csproj src/storage-service/Deal.Storage/
|
||||||
|
RUN dotnet restore src/storage-service/Deal.Storage/Deal.Storage.csproj
|
||||||
|
|
||||||
|
COPY src/contracts/ src/contracts/
|
||||||
|
COPY src/grpc-hosting/ src/grpc-hosting/
|
||||||
|
COPY src/storage-service/Deal.Storage/ src/storage-service/Deal.Storage/
|
||||||
|
RUN dotnet publish src/storage-service/Deal.Storage/Deal.Storage.csproj -c Release -o /app/publish
|
||||||
|
|
||||||
|
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS final
|
||||||
|
WORKDIR /app
|
||||||
|
EXPOSE 5104
|
||||||
|
COPY --from=build /app/publish .
|
||||||
|
COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.35 /ko-app/grpc-health-probe /bin/grpc_health_probe
|
||||||
|
ENTRYPOINT ["dotnet", "Deal.Storage.dll"]
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
namespace Deal.Storage.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Дескриптор сохранённого объекта.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record StoredObject(
|
||||||
|
string Id,
|
||||||
|
string Ref,
|
||||||
|
string Kind,
|
||||||
|
string MimeType,
|
||||||
|
string FileName,
|
||||||
|
long Size);
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
|
||||||
|
namespace Deal.Storage.Options;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Настройки объектного хранилища сервиса данных.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class StorageOptions
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Значение бакета по умолчанию.
|
||||||
|
/// </summary>
|
||||||
|
public const string DefaultBucketName = "deal-attachments";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Env-ключ адреса хранилища (host:port).
|
||||||
|
/// </summary>
|
||||||
|
public const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Env-ключ ключа доступа.
|
||||||
|
/// </summary>
|
||||||
|
public const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Env-ключ секрета доступа.
|
||||||
|
/// </summary>
|
||||||
|
public const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Env-ключ имени бакета.
|
||||||
|
/// </summary>
|
||||||
|
public const string BucketEnvKey = "DEAL_STORAGE_BUCKET";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Env-ключ флага TLS (1/true — https).
|
||||||
|
/// </summary>
|
||||||
|
public const string SecureEnvKey = "DEAL_STORAGE_SECURE";
|
||||||
|
|
||||||
|
public string Endpoint { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
public string AccessKey { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
public string SecretKey { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
public string Bucket { get; init; } = DefaultBucketName;
|
||||||
|
|
||||||
|
public bool Secure { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Читает настройки из конфигурации хоста.
|
||||||
|
/// </summary>
|
||||||
|
public static StorageOptions FromConfiguration(IConfiguration configuration)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(configuration);
|
||||||
|
string bucket = (configuration[BucketEnvKey] ?? string.Empty).Trim();
|
||||||
|
return new StorageOptions
|
||||||
|
{
|
||||||
|
Endpoint = (configuration[EndpointEnvKey] ?? string.Empty).Trim(),
|
||||||
|
AccessKey = (configuration[AccessKeyEnvKey] ?? string.Empty).Trim(),
|
||||||
|
SecretKey = configuration[SecretKeyEnvKey] ?? string.Empty,
|
||||||
|
Bucket = bucket.Length == 0 ? DefaultBucketName : bucket,
|
||||||
|
Secure = IsEnabled(configuration[SecureEnvKey]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Разбирает значение флага TLS.
|
||||||
|
/// </summary>
|
||||||
|
public static bool IsEnabled(string? rawValue)
|
||||||
|
=> string.Equals(rawValue, "1", StringComparison.Ordinal)
|
||||||
|
|| string.Equals(rawValue, "true", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
using Deal.Grpc.Hosting.Interceptors;
|
||||||
|
using Deal.Grpc.Hosting.Models;
|
||||||
|
using Deal.Grpc.Hosting.Options;
|
||||||
|
using Deal.Grpc.Hosting.Services;
|
||||||
|
using Deal.Storage;
|
||||||
|
|
||||||
|
const int defaultGrpcPort = 5104;
|
||||||
|
const string storageProcessName = "storage";
|
||||||
|
|
||||||
|
int grpcPort = GrpcHostEnvironment.ResolveGrpcPort(defaultGrpcPort);
|
||||||
|
int metricsPort = DealMetricsHosting.ResolveMetricsPort(DealMetricsHosting.DefaultMetricsPort);
|
||||||
|
|
||||||
|
WebApplication app = StorageServiceHost.Create(
|
||||||
|
grpcPort,
|
||||||
|
configureBuilder: builder =>
|
||||||
|
{
|
||||||
|
DealLogging.Configure(builder, storageProcessName);
|
||||||
|
DealMetricsHosting.AddDealMetrics(builder, metricsPort);
|
||||||
|
});
|
||||||
|
|
||||||
|
DealMetricsHosting.MapDealMetrics(app);
|
||||||
|
|
||||||
|
MtlsOptions mtlsOptions = MtlsOptions.FromConfiguration(app.Configuration);
|
||||||
|
GrpcHostEnvironment.RequireMtlsInProduction(mtlsOptions);
|
||||||
|
|
||||||
|
app.Logger.LogInformation(
|
||||||
|
"storage-service стартует: gRPC {Transport} 0.0.0.0:{Port} (health /grpc.health.v1.Health/Check)",
|
||||||
|
mtlsOptions.Enabled ? "mTLS (TLS + клиентский сертификат)" : "plaintext + service-token",
|
||||||
|
grpcPort);
|
||||||
|
|
||||||
|
await app.RunAsync();
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
namespace Deal.Storage.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Определяет тип и MIME объекта по первым байтам содержимого.
|
||||||
|
/// </summary>
|
||||||
|
public static class FileKindSniffer
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Тип «прочее».
|
||||||
|
/// </summary>
|
||||||
|
public const string KindOther = "other";
|
||||||
|
|
||||||
|
private const string DefaultMime = "application/octet-stream";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Определяет тип и MIME по заголовку файла.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="head">Первые байты содержимого.</param>
|
||||||
|
/// <param name="contentType">MIME, заявленный загрузчиком (используется как подсказка).</param>
|
||||||
|
/// <returns>Пара «тип, MIME».</returns>
|
||||||
|
public static (string Kind, string MimeType) Detect(ReadOnlySpan<byte> head, string? contentType)
|
||||||
|
{
|
||||||
|
if (StartsWith(head, 0x89, 0x50, 0x4E, 0x47))
|
||||||
|
{
|
||||||
|
return ("image", "image/png");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0xFF, 0xD8, 0xFF))
|
||||||
|
{
|
||||||
|
return ("image", "image/jpeg");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x47, 0x49, 0x46, 0x38))
|
||||||
|
{
|
||||||
|
return ("image", "image/gif");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x42, 0x4D))
|
||||||
|
{
|
||||||
|
return ("image", "image/bmp");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x49, 0x49, 0x2A, 0x00) || StartsWith(head, 0x4D, 0x4D, 0x00, 0x2A))
|
||||||
|
{
|
||||||
|
return ("image", "image/tiff");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x25, 0x50, 0x44, 0x46))
|
||||||
|
{
|
||||||
|
return ("document", "application/pdf");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x50, 0x4B, 0x03, 0x04) || StartsWith(head, 0x50, 0x4B, 0x05, 0x06))
|
||||||
|
{
|
||||||
|
return ("archive", "application/zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x52, 0x61, 0x72, 0x21))
|
||||||
|
{
|
||||||
|
return ("archive", "application/vnd.rar");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x37, 0x7A, 0xBC, 0xAF))
|
||||||
|
{
|
||||||
|
return ("archive", "application/x-7z-compressed");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x1F, 0x8B))
|
||||||
|
{
|
||||||
|
return ("archive", "application/gzip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x1A, 0x45, 0xDF, 0xA3))
|
||||||
|
{
|
||||||
|
return ("video", "video/webm");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (head.Length >= 12 && StartsWith(head, 0x52, 0x49, 0x46, 0x46))
|
||||||
|
{
|
||||||
|
if (StartsWith(head[8..], 0x57, 0x45, 0x42, 0x50))
|
||||||
|
{
|
||||||
|
return ("image", "image/webp");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head[8..], 0x57, 0x41, 0x56, 0x45))
|
||||||
|
{
|
||||||
|
return ("audio", "audio/wav");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head[8..], 0x41, 0x56, 0x49, 0x20))
|
||||||
|
{
|
||||||
|
return ("video", "video/x-msvideo");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (head.Length >= 8 && StartsWith(head[4..], 0x66, 0x74, 0x79, 0x70))
|
||||||
|
{
|
||||||
|
return ("video", "video/mp4");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x49, 0x44, 0x33) || StartsWith(head, 0xFF, 0xFB))
|
||||||
|
{
|
||||||
|
return ("audio", "audio/mpeg");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x4F, 0x67, 0x67, 0x53))
|
||||||
|
{
|
||||||
|
return ("audio", "audio/ogg");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(head, 0x66, 0x4C, 0x61, 0x43))
|
||||||
|
{
|
||||||
|
return ("audio", "audio/flac");
|
||||||
|
}
|
||||||
|
|
||||||
|
string declared = (contentType ?? string.Empty).Trim();
|
||||||
|
if (declared.Length == 0)
|
||||||
|
{
|
||||||
|
return (KindOther, DefaultMime);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (KindFromMime(declared), declared);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Определяет общий тип по префиксу MIME-типа.
|
||||||
|
private static string KindFromMime(string mime)
|
||||||
|
{
|
||||||
|
if (mime.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) return "image";
|
||||||
|
if (mime.StartsWith("video/", StringComparison.OrdinalIgnoreCase)) return "video";
|
||||||
|
if (mime.StartsWith("audio/", StringComparison.OrdinalIgnoreCase)) return "audio";
|
||||||
|
if (mime.StartsWith("text/", StringComparison.OrdinalIgnoreCase)) return "document";
|
||||||
|
return KindOther;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Проверяет, начинается ли последовательность с заданных байтов.
|
||||||
|
private static bool StartsWith(ReadOnlySpan<byte> data, params byte[] prefix)
|
||||||
|
=> data.Length >= prefix.Length && data[..prefix.Length].SequenceEqual(prefix);
|
||||||
|
}
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
using Deal.Storage.Options;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Minio;
|
||||||
|
using Minio.DataModel;
|
||||||
|
using Minio.DataModel.Args;
|
||||||
|
using Minio.Exceptions;
|
||||||
|
|
||||||
|
namespace Deal.Storage.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Объектное хранилище сервиса данных поверх MinIO.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class MinioObjectStore
|
||||||
|
{
|
||||||
|
private const string DefaultContentType = "application/octet-stream";
|
||||||
|
|
||||||
|
private readonly IMinioClient _client;
|
||||||
|
private readonly string _endpoint;
|
||||||
|
private readonly string _bucket;
|
||||||
|
private readonly ILogger<MinioObjectStore> _logger;
|
||||||
|
|
||||||
|
private readonly SemaphoreSlim _bucketCheckGate = new(1, 1);
|
||||||
|
|
||||||
|
private bool _bucketChecked;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Создаёт хранилище поверх настроек.
|
||||||
|
/// </summary>
|
||||||
|
public MinioObjectStore(StorageOptions options, ILogger<MinioObjectStore> logger)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(options);
|
||||||
|
ArgumentNullException.ThrowIfNull(logger);
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(options.Endpoint)
|
||||||
|
|| string.IsNullOrWhiteSpace(options.AccessKey)
|
||||||
|
|| string.IsNullOrWhiteSpace(options.SecretKey))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"MinioObjectStore требует заполненные настройки хранилища (Endpoint и ключи доступа).");
|
||||||
|
}
|
||||||
|
|
||||||
|
_endpoint = options.Endpoint;
|
||||||
|
_bucket = options.Bucket;
|
||||||
|
_logger = logger;
|
||||||
|
_client = new MinioClient()
|
||||||
|
.WithEndpoint(options.Endpoint)
|
||||||
|
.WithCredentials(options.AccessKey, options.SecretKey)
|
||||||
|
.WithSSL(options.Secure)
|
||||||
|
.Build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Возвращает подпись режима для стартового лога.
|
||||||
|
/// </summary>
|
||||||
|
public override string ToString() => $"MinioObjectStore (endpoint: {_endpoint}; bucket: {_bucket})";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Сохраняет объект, возвращает его размер.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<long> PutAsync(string key, Stream content, string contentType, CancellationToken ct)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(content);
|
||||||
|
await EnsureBucketAsync(ct);
|
||||||
|
|
||||||
|
if (content.CanSeek && content.Position != 0)
|
||||||
|
{
|
||||||
|
content.Position = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
using MemoryStream buffer = new();
|
||||||
|
await content.CopyToAsync(buffer, ct);
|
||||||
|
buffer.Position = 0;
|
||||||
|
|
||||||
|
await _client.PutObjectAsync(
|
||||||
|
new PutObjectArgs()
|
||||||
|
.WithBucket(_bucket)
|
||||||
|
.WithObject(key)
|
||||||
|
.WithStreamData(buffer)
|
||||||
|
.WithObjectSize(buffer.Length)
|
||||||
|
.WithContentType(string.IsNullOrWhiteSpace(contentType) ? DefaultContentType : contentType),
|
||||||
|
ct);
|
||||||
|
|
||||||
|
return buffer.Length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Возвращает содержимое объекта потоком либо null.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<Stream?> GetAsync(string key, CancellationToken ct)
|
||||||
|
{
|
||||||
|
MemoryStream buffer = new();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _client.GetObjectAsync(
|
||||||
|
new GetObjectArgs()
|
||||||
|
.WithBucket(_bucket)
|
||||||
|
.WithObject(key)
|
||||||
|
.WithCallbackStream(async (stream, token) => await stream.CopyToAsync(buffer, token)),
|
||||||
|
ct);
|
||||||
|
}
|
||||||
|
catch (ObjectNotFoundException)
|
||||||
|
{
|
||||||
|
buffer.Dispose();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
buffer.Dispose();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.Position = 0;
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Возвращает размер и MIME объекта либо null.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<(long Size, string ContentType)?> StatAsync(string key, CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ObjectStat stat = await _client.StatObjectAsync(
|
||||||
|
new StatObjectArgs().WithBucket(_bucket).WithObject(key),
|
||||||
|
ct);
|
||||||
|
return (stat.Size, stat.ContentType ?? string.Empty);
|
||||||
|
}
|
||||||
|
catch (ObjectNotFoundException)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Удаляет объект (отсутствующий — успех).
|
||||||
|
/// </summary>
|
||||||
|
public async Task DeleteAsync(string key, CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _client.RemoveObjectAsync(
|
||||||
|
new RemoveObjectArgs().WithBucket(_bucket).WithObject(key),
|
||||||
|
ct);
|
||||||
|
}
|
||||||
|
catch (MinioException exception)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Не удалось удалить объект «{Key}» из бакета «{Bucket}»: {Message}",
|
||||||
|
key,
|
||||||
|
_bucket,
|
||||||
|
exception.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task EnsureBucketAsync(CancellationToken ct)
|
||||||
|
{
|
||||||
|
if (_bucketChecked)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await _bucketCheckGate.WaitAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (_bucketChecked)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
bool exists = await _client.BucketExistsAsync(new BucketExistsArgs().WithBucket(_bucket), ct);
|
||||||
|
if (!exists)
|
||||||
|
{
|
||||||
|
await _client.MakeBucketAsync(new MakeBucketArgs().WithBucket(_bucket), ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (MinioException exception)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Не удалось проверить/создать бакет «{Bucket}»: {Message}",
|
||||||
|
_bucket,
|
||||||
|
exception.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
_bucketChecked = true;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_bucketCheckGate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
using Deal.Grpc.Hosting.Services;
|
||||||
|
using Deal.Storage.Options;
|
||||||
|
using Deal.Storage.Services;
|
||||||
|
|
||||||
|
namespace Deal.Storage;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Собирает WebApplication gRPC-хоста сервиса данных.
|
||||||
|
/// </summary>
|
||||||
|
public static class StorageServiceHost
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Создаёт (не запускает) хост.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="grpcPort">TCP-порт Kestrel.</param>
|
||||||
|
/// <param name="args">Аргументы командной строки.</param>
|
||||||
|
/// <param name="configureServices">Опциональный хук DI для тестов.</param>
|
||||||
|
/// <param name="configureBuilder">Опциональный хук конфигурации билдера (логи/метрики).</param>
|
||||||
|
/// <returns>Собранный хост.</returns>
|
||||||
|
public static WebApplication Create(
|
||||||
|
int grpcPort,
|
||||||
|
string[]? args = null,
|
||||||
|
Action<IServiceCollection>? configureServices = null,
|
||||||
|
Action<WebApplicationBuilder>? configureBuilder = null)
|
||||||
|
{
|
||||||
|
WebApplicationBuilder builder = WebApplication.CreateBuilder(args ?? []);
|
||||||
|
|
||||||
|
var mtlsCertificates = GrpcServer.LoadMtlsCertificates(builder);
|
||||||
|
GrpcServer.ConfigureKestrelHttp2Endpoint(builder, grpcPort, mtlsCertificates);
|
||||||
|
builder.Services.AddDealGrpcServer();
|
||||||
|
builder.Services.AddReadyHealthCheck("хост storage-service готов");
|
||||||
|
|
||||||
|
StorageOptions storageOptions = StorageOptions.FromConfiguration(builder.Configuration);
|
||||||
|
builder.Services.AddSingleton(storageOptions);
|
||||||
|
builder.Services.AddSingleton<MinioObjectStore>();
|
||||||
|
|
||||||
|
configureServices?.Invoke(builder.Services);
|
||||||
|
configureBuilder?.Invoke(builder);
|
||||||
|
|
||||||
|
WebApplication app = builder.Build();
|
||||||
|
|
||||||
|
app.MapGrpcService<StorageServiceImpl>();
|
||||||
|
app.MapGrpcHealthChecksService();
|
||||||
|
|
||||||
|
return app;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
using Deal.Grpc.Storage;
|
||||||
|
using Deal.Storage.Services;
|
||||||
|
using Grpc.Core;
|
||||||
|
|
||||||
|
namespace Deal.Storage;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Реализация сервиса данных.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class StorageServiceImpl : StorageService.StorageServiceBase
|
||||||
|
{
|
||||||
|
private const int HeadSize = 512;
|
||||||
|
private const int DownloadChunkSize = 64 * 1024;
|
||||||
|
|
||||||
|
private readonly MinioObjectStore _store;
|
||||||
|
private readonly ILogger<StorageServiceImpl> _logger;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Создаёт реализацию поверх хранилища.
|
||||||
|
/// </summary>
|
||||||
|
public StorageServiceImpl(MinioObjectStore store, ILogger<StorageServiceImpl> logger)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(store);
|
||||||
|
ArgumentNullException.ThrowIfNull(logger);
|
||||||
|
_store = store;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override async Task<UploadReply> Upload(
|
||||||
|
IAsyncStreamReader<UploadRequest> requestStream,
|
||||||
|
ServerCallContext context)
|
||||||
|
{
|
||||||
|
CancellationToken ct = context.CancellationToken;
|
||||||
|
await using MemoryStream buffer = new();
|
||||||
|
byte[] head = new byte[HeadSize];
|
||||||
|
int headLength = 0;
|
||||||
|
UploadMeta? meta = null;
|
||||||
|
|
||||||
|
while (await requestStream.MoveNext(ct))
|
||||||
|
{
|
||||||
|
UploadRequest message = requestStream.Current;
|
||||||
|
if (meta is null && message.Meta is not null)
|
||||||
|
{
|
||||||
|
meta = message.Meta;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.Data.Length == 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headLength < HeadSize)
|
||||||
|
{
|
||||||
|
int take = Math.Min(HeadSize - headLength, message.Data.Length);
|
||||||
|
message.Data.Span[..take].CopyTo(head.AsSpan(headLength));
|
||||||
|
headLength += take;
|
||||||
|
}
|
||||||
|
|
||||||
|
await buffer.WriteAsync(message.Data.Memory, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (meta is null || string.IsNullOrWhiteSpace(meta.TenantId))
|
||||||
|
{
|
||||||
|
throw new RpcException(new Status(StatusCode.InvalidArgument, "tenant_id обязателен"));
|
||||||
|
}
|
||||||
|
|
||||||
|
(string kind, string mimeType) = FileKindSniffer.Detect(head.AsSpan(0, headLength), meta.ContentType);
|
||||||
|
string key = $"{meta.TenantId.Trim()}/{Guid.NewGuid():N}";
|
||||||
|
buffer.Position = 0;
|
||||||
|
long size = await _store.PutAsync(key, buffer, mimeType, ct);
|
||||||
|
|
||||||
|
_logger.LogInformation("Объект сохранён: {Key} ({Kind}, {Size} байт)", key, kind, size);
|
||||||
|
|
||||||
|
return new UploadReply
|
||||||
|
{
|
||||||
|
Id = key,
|
||||||
|
Ref = key,
|
||||||
|
Kind = kind,
|
||||||
|
MimeType = mimeType,
|
||||||
|
FileName = meta.FileName ?? string.Empty,
|
||||||
|
Size = size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override async Task Download(
|
||||||
|
DownloadRequest request,
|
||||||
|
IServerStreamWriter<DownloadChunk> responseStream,
|
||||||
|
ServerCallContext context)
|
||||||
|
{
|
||||||
|
CancellationToken ct = context.CancellationToken;
|
||||||
|
await using Stream? content = await _store.GetAsync(request.Id, ct);
|
||||||
|
if (content is null)
|
||||||
|
{
|
||||||
|
throw new RpcException(new Status(StatusCode.NotFound, "объект не найден"));
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[] chunk = new byte[DownloadChunkSize];
|
||||||
|
int read;
|
||||||
|
while ((read = await content.ReadAsync(chunk, ct)) > 0)
|
||||||
|
{
|
||||||
|
await responseStream.WriteAsync(new DownloadChunk { Data = Google.Protobuf.ByteString.CopyFrom(chunk, 0, read) }, ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override async Task<StatReply> Stat(StatRequest request, ServerCallContext context)
|
||||||
|
{
|
||||||
|
(long Size, string ContentType)? stat = await _store.StatAsync(request.Id, context.CancellationToken);
|
||||||
|
if (stat is null)
|
||||||
|
{
|
||||||
|
return new StatReply { Found = false };
|
||||||
|
}
|
||||||
|
|
||||||
|
return new StatReply
|
||||||
|
{
|
||||||
|
Found = true,
|
||||||
|
Info = new UploadReply
|
||||||
|
{
|
||||||
|
Id = request.Id,
|
||||||
|
Ref = request.Id,
|
||||||
|
MimeType = stat.Value.ContentType,
|
||||||
|
Size = stat.Value.Size,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override async Task<DeleteReply> Delete(DeleteRequest request, ServerCallContext context)
|
||||||
|
{
|
||||||
|
await _store.DeleteAsync(request.Id, context.CancellationToken);
|
||||||
|
return new DeleteReply();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
<Project>
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<LangVersion>latest</LangVersion>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||||
|
<AnalysisLevel>latest</AnalysisLevel>
|
||||||
|
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||||
|
</PropertyGroup>
|
||||||
|
</Project>
|
||||||
Reference in New Issue
Block a user