diff --git a/deploy/compose.dev.yml b/deploy/compose.dev.yml index b9b7843..fe83b57 100644 --- a/deploy/compose.dev.yml +++ b/deploy/compose.dev.yml @@ -1,7 +1,7 @@ # compose.dev.yml — dev-стек Дейла (этап 6, Task 20; Ruling 12). # # Полный стек: postgres + minio (хранилища), три автономных сервиса этапа 6 -# (telegram-service :5101, ai-service :5102, ml-service :5103) и core (deal-api: +# (telegram-service :5101, ai-service :5102, ml-service :5103, storage-service :5104) и core (deal-api: # HTTP :5080 + gRPC-ингресс :5082). # # Режимы интеграций core — флагами SERVICES__{ML,AI,TELEGRAM}__USELOCAL (Ruling 6): @@ -88,6 +88,7 @@ services: Services__Ml__Endpoint: http://ml-service:5103 Services__Ai__Endpoint: http://ai-service:5102 Services__Telegram__Endpoint: http://telegram-service:5101 + Services__Storage__Endpoint: http://storage-service:5104 Storage__Minio__Endpoint: minio:9000 Storage__Minio__AccessKey: deal_minio Storage__Minio__SecretKey: deal_minio_secret @@ -186,6 +187,32 @@ services: timeout: 3s retries: 10 + # storage-service — общий gRPC-сервис данных (вложения источников). Отдельный процесс, отдельное sln + # (src/storage-service/Deal.Storage.sln). Загрузка/выгрузка/стат/удаление объектов; тип объекта + # определяет сервис. Бэкенд — MinIO. Защита — общий service-token (DEAL_SERVICE_TOKEN). + storage-service: + build: + context: .. + dockerfile: src/storage-service/Deal.Storage/Dockerfile + container_name: deal-storage-service + environment: + GRPC_PORT: "5104" + DEAL_SERVICE_TOKEN: ${DEAL_SERVICE_TOKEN:-deal_dev_service_token} + DEAL_STORAGE_ENDPOINT: minio:9000 + DEAL_STORAGE_ACCESS_KEY: deal_minio + DEAL_STORAGE_SECRET_KEY: deal_minio_secret + DEAL_STORAGE_BUCKET: deal-attachments + DEAL_STORAGE_SECURE: "false" + ports: + - "5104:5104" + depends_on: + - minio + healthcheck: + test: ["CMD", "/bin/grpc_health_probe", "-addr=localhost:5104"] + interval: 5s + timeout: 3s + retries: 10 + # Prometheus (профиль observability, этап 12/пакет A) — сбор /metrics всех 4 процессов (:9464) # внутри dev-сети. Подъём: docker compose -f deploy/compose.dev.yml --profile observability up -d. # Конфиг — общий deploy/observability/prometheus.yml (те же имена сервисов и таргеты). UI — 9090. diff --git a/src/contracts/Deal.Proto.csproj b/src/contracts/Deal.Proto.csproj index b2400b6..a1535e8 100644 --- a/src/contracts/Deal.Proto.csproj +++ b/src/contracts/Deal.Proto.csproj @@ -41,6 +41,7 @@ + diff --git a/src/contracts/storage.proto b/src/contracts/storage.proto new file mode 100644 index 0000000..edf3bc7 --- /dev/null +++ b/src/contracts/storage.proto @@ -0,0 +1,78 @@ +// +// Контракт общего сервиса данных: загрузка/выгрузка/стат/удаление объектов вложений. +// +// tenant-id — id тенанта (metadata); объект хранится в пространстве тенанта; +// service-token — общий токен сервисов (env DEAL_SERVICE_TOKEN); неверный/пустой → +// UNAUTHENTICATED (интерцептор хоста). +// +// Тип объекта (image/video/audio/document/archive/other) определяет сервис — +// контракт типы вложений не задаёт. +syntax = "proto3"; + +package deal.storage.v1; + +option csharp_namespace = "Deal.Grpc.Storage"; + +service StorageService { + // Поток: первое сообщение несёт meta, далее — data. + rpc Upload(stream UploadRequest) returns (UploadReply); + + // Поток содержимого объекта частями. + rpc Download(DownloadRequest) returns (stream DownloadChunk); + + // Дескриптор объекта (без содержимого). + rpc Stat(StatRequest) returns (StatReply); + + // Удаление объекта (отсутствующий — успех). + rpc Delete(DeleteRequest) returns (DeleteReply); +} + +message UploadRequest { + UploadMeta meta = 1; + bytes data = 2; +} + +message UploadMeta { + string tenant_id = 1; + string file_name = 2; + string content_type = 3; +} + +message UploadReply { + // Идентификатор объекта — ключ в хранилище. + string id = 1; + // Ссылка на скачивание/отображение. + string ref = 2; + // Тип, определённый сервисом (image/video/audio/document/archive/other). + string kind = 3; + string mime_type = 4; + string file_name = 5; + int64 size = 6; + optional int32 width = 7; + optional int32 height = 8; + optional double duration_sec = 9; + optional string preview_ref = 10; +} + +message DownloadRequest { + string id = 1; +} + +message DownloadChunk { + bytes data = 1; +} + +message StatRequest { + string id = 1; +} + +message StatReply { + bool found = 1; + UploadReply info = 2; +} + +message DeleteRequest { + string id = 1; +} + +message DeleteReply {} diff --git a/src/storage-service/Deal.Storage.Tests/AssemblyInfo.cs b/src/storage-service/Deal.Storage.Tests/AssemblyInfo.cs new file mode 100644 index 0000000..a88bdbf --- /dev/null +++ b/src/storage-service/Deal.Storage.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +// Тесты storage-service поднимают Kestrel-хост и меняют env-переменные процесса на время сценария — +// параллельный прогон классов дал бы гонки на env, поэтому тесты сериализованы. +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/storage-service/Deal.Storage.Tests/Deal.Storage.Tests.csproj b/src/storage-service/Deal.Storage.Tests/Deal.Storage.Tests.csproj new file mode 100644 index 0000000..b58b1e7 --- /dev/null +++ b/src/storage-service/Deal.Storage.Tests/Deal.Storage.Tests.csproj @@ -0,0 +1,31 @@ + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/storage-service/Deal.Storage.Tests/Grpc/StorageServiceHostTests.cs b/src/storage-service/Deal.Storage.Tests/Grpc/StorageServiceHostTests.cs new file mode 100644 index 0000000..6c29f49 --- /dev/null +++ b/src/storage-service/Deal.Storage.Tests/Grpc/StorageServiceHostTests.cs @@ -0,0 +1,128 @@ +using System.Net; +using System.Net.Sockets; +using Deal.Grpc.Storage; +using Grpc.Core; +using Grpc.Health.V1; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Builder; + +namespace Deal.Storage.Tests.Grpc; + +/// +/// Тесты хоста storage-service: health и проверка токена. +/// +public sealed class StorageServiceHostTests +{ + private const string ServiceTokenEnvKey = "DEAL_SERVICE_TOKEN"; + private const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT"; + private const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY"; + private const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY"; + private const string ServiceTokenMetadataKey = "service-token"; + private const string ValidToken = "storage-test-token"; + private const int RpcDeadlineSeconds = 10; + + /// + /// Health отвечает SERVING — хост поднялся. + /// + [Fact] + public async Task HealthCheck_ReturnsServing() + { + await RunHostScenarioAsync( + ValidToken, + async channel => + { + var health = new Health.HealthClient(channel); + HealthCheckResponse response = await health.CheckAsync( + new HealthCheckRequest(), + deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds)); + Assert.Equal(HealthCheckResponse.Types.ServingStatus.Serving, response.Status); + }); + } + + /// + /// Stat без токена отклоняется. + /// + [Fact] + public async Task Stat_WithoutToken_IsUnauthenticated() + { + await RunHostScenarioAsync( + ValidToken, + channel => AssertRejectedAsync(channel, tokenHeader: null)); + } + + /// + /// Stat с неверным токеном отклоняется. + /// + [Fact] + public async Task Stat_WithWrongToken_IsUnauthenticated() + { + await RunHostScenarioAsync( + ValidToken, + channel => AssertRejectedAsync(channel, tokenHeader: "wrong-token")); + } + + private static async Task AssertRejectedAsync(GrpcChannel channel, string? tokenHeader) + { + var client = new StorageService.StorageServiceClient(channel); + Metadata metadata = new(); + if (tokenHeader is not null) + { + metadata.Add(ServiceTokenMetadataKey, tokenHeader); + } + + var callOptions = new CallOptions(metadata, deadline: DateTime.UtcNow.AddSeconds(RpcDeadlineSeconds)); + AsyncUnaryCall call = client.StatAsync(new StatRequest { Id = "x" }, callOptions); + RpcException exception = await Assert.ThrowsAsync(() => call.ResponseAsync); + Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode); + } + + private static async Task RunHostScenarioAsync(string? serviceToken, Func scenario) + { + string? originalToken = Environment.GetEnvironmentVariable(ServiceTokenEnvKey); + string? originalEndpoint = Environment.GetEnvironmentVariable(EndpointEnvKey); + string? originalAccess = Environment.GetEnvironmentVariable(AccessKeyEnvKey); + string? originalSecret = Environment.GetEnvironmentVariable(SecretKeyEnvKey); + + Environment.SetEnvironmentVariable(ServiceTokenEnvKey, serviceToken); + Environment.SetEnvironmentVariable(EndpointEnvKey, "127.0.0.1:9000"); + Environment.SetEnvironmentVariable(AccessKeyEnvKey, "test"); + Environment.SetEnvironmentVariable(SecretKeyEnvKey, "test"); + + WebApplication? app = null; + GrpcChannel? channel = null; + try + { + int port = FreeTcpPort(); + app = StorageServiceHost.Create(port); + await app.StartAsync(); + + channel = GrpcChannel.ForAddress($"http://127.0.0.1:{port}"); + await scenario(channel); + } + finally + { + if (channel is not null) + { + channel.Dispose(); + } + + if (app is not null) + { + await app.StopAsync(); + await app.DisposeAsync(); + } + + Environment.SetEnvironmentVariable(ServiceTokenEnvKey, originalToken); + Environment.SetEnvironmentVariable(EndpointEnvKey, originalEndpoint); + Environment.SetEnvironmentVariable(AccessKeyEnvKey, originalAccess); + Environment.SetEnvironmentVariable(SecretKeyEnvKey, originalSecret); + } + } + + private static int FreeTcpPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } +} diff --git a/src/storage-service/Deal.Storage.Tests/Services/FileKindSnifferTests.cs b/src/storage-service/Deal.Storage.Tests/Services/FileKindSnifferTests.cs new file mode 100644 index 0000000..26ac27a --- /dev/null +++ b/src/storage-service/Deal.Storage.Tests/Services/FileKindSnifferTests.cs @@ -0,0 +1,81 @@ +using Deal.Storage.Services; + +namespace Deal.Storage.Tests.Services; + +/// +/// Тесты определения типа и MIME по содержимому. +/// +public sealed class FileKindSnifferTests +{ + /// + /// PNG определяется как изображение. + /// + [Fact] + public void Png_IsImage() + { + byte[] head = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + (string kind, string mime) = FileKindSniffer.Detect(head, null); + Assert.Equal("image", kind); + Assert.Equal("image/png", mime); + } + + /// + /// PDF определяется как документ. + /// + [Fact] + public void Pdf_IsDocument() + { + byte[] head = [0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E, 0x37]; + (string kind, string mime) = FileKindSniffer.Detect(head, null); + Assert.Equal("document", kind); + Assert.Equal("application/pdf", mime); + } + + /// + /// MP4 определяется как видео. + /// + [Fact] + public void Mp4_IsVideo() + { + byte[] head = [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32]; + (string kind, string mime) = FileKindSniffer.Detect(head, null); + Assert.Equal("video", kind); + Assert.Equal("video/mp4", mime); + } + + /// + /// MP3 определяется как аудио. + /// + [Fact] + public void Mp3_IsAudio() + { + byte[] head = [0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00]; + (string kind, string mime) = FileKindSniffer.Detect(head, null); + Assert.Equal("audio", kind); + Assert.Equal("audio/mpeg", mime); + } + + /// + /// Неизвестное содержимое с известным MIME берёт тип из MIME. + /// + [Fact] + public void UnknownWithDeclaredText_IsDocument() + { + byte[] head = [0x00, 0x01, 0x02, 0x03]; + (string kind, string mime) = FileKindSniffer.Detect(head, "text/plain"); + Assert.Equal("document", kind); + Assert.Equal("text/plain", mime); + } + + /// + /// Полностью неизвестное содержимое — «other». + /// + [Fact] + public void UnknownWithoutDeclared_IsOther() + { + byte[] head = [0x00, 0x01, 0x02, 0x03]; + (string kind, string mime) = FileKindSniffer.Detect(head, null); + Assert.Equal(FileKindSniffer.KindOther, kind); + Assert.Equal("application/octet-stream", mime); + } +} diff --git a/src/storage-service/Deal.Storage.sln b/src/storage-service/Deal.Storage.sln new file mode 100644 index 0000000..72a5f16 --- /dev/null +++ b/src/storage-service/Deal.Storage.sln @@ -0,0 +1,76 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage", "Deal.Storage\Deal.Storage.csproj", "{0728C329-E3A0-4EDD-8305-FE558C01C085}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Grpc.Hosting", "..\grpc-hosting\Deal.Grpc.Hosting\Deal.Grpc.Hosting.csproj", "{67C12483-BC9B-46D8-A66E-B5510AE3E9C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Proto", "..\contracts\Deal.Proto.csproj", "{A3470473-FEE8-476C-96C4-0038F920A22A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Deal.Storage.Tests", "Deal.Storage.Tests\Deal.Storage.Tests.csproj", "{49D721BA-5380-421F-8072-4B7B0EE4480F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.ActiveCfg = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x64.Build.0 = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.ActiveCfg = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Debug|x86.Build.0 = Debug|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|Any CPU.Build.0 = Release|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.ActiveCfg = Release|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x64.Build.0 = Release|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.ActiveCfg = Release|Any CPU + {0728C329-E3A0-4EDD-8305-FE558C01C085}.Release|x86.Build.0 = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x64.Build.0 = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Debug|x86.Build.0 = Debug|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|Any CPU.Build.0 = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.ActiveCfg = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x64.Build.0 = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.ActiveCfg = Release|Any CPU + {67C12483-BC9B-46D8-A66E-B5510AE3E9C2}.Release|x86.Build.0 = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.ActiveCfg = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x64.Build.0 = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.ActiveCfg = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Debug|x86.Build.0 = Debug|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|Any CPU.Build.0 = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.ActiveCfg = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x64.Build.0 = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.ActiveCfg = Release|Any CPU + {A3470473-FEE8-476C-96C4-0038F920A22A}.Release|x86.Build.0 = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.ActiveCfg = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x64.Build.0 = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.ActiveCfg = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Debug|x86.Build.0 = Debug|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|Any CPU.Build.0 = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.ActiveCfg = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x64.Build.0 = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.ActiveCfg = Release|Any CPU + {49D721BA-5380-421F-8072-4B7B0EE4480F}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/storage-service/Deal.Storage/Deal.Storage.csproj b/src/storage-service/Deal.Storage/Deal.Storage.csproj new file mode 100644 index 0000000..e8dbacd --- /dev/null +++ b/src/storage-service/Deal.Storage/Deal.Storage.csproj @@ -0,0 +1,26 @@ + + + + Deal.Storage + Deal.Storage + + + + + + + + + + + + + + + + + + + + + diff --git a/src/storage-service/Deal.Storage/Dockerfile b/src/storage-service/Deal.Storage/Dockerfile new file mode 100644 index 0000000..a259d61 --- /dev/null +++ b/src/storage-service/Deal.Storage/Dockerfile @@ -0,0 +1,26 @@ +# storage-service: общий gRPC-сервис данных (вложения источников). +# +# КОНТЕКСТ СБОРКИ — корень репозитория: Deal.Storage.csproj ссылается на src/contracts/Deal.Proto.csproj +# и src/grpc-hosting вне каталога сервиса. Запуск из корня: +# docker build -f src/storage-service/Deal.Storage/Dockerfile . + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /repo + +COPY src/contracts/Deal.Proto.csproj src/contracts/ +COPY src/grpc-hosting/Deal.Grpc.Hosting/Deal.Grpc.Hosting.csproj src/grpc-hosting/Deal.Grpc.Hosting/ +COPY src/storage-service/Directory.Build.props src/storage-service/ +COPY src/storage-service/Deal.Storage/Deal.Storage.csproj src/storage-service/Deal.Storage/ +RUN dotnet restore src/storage-service/Deal.Storage/Deal.Storage.csproj + +COPY src/contracts/ src/contracts/ +COPY src/grpc-hosting/ src/grpc-hosting/ +COPY src/storage-service/Deal.Storage/ src/storage-service/Deal.Storage/ +RUN dotnet publish src/storage-service/Deal.Storage/Deal.Storage.csproj -c Release -o /app/publish + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS final +WORKDIR /app +EXPOSE 5104 +COPY --from=build /app/publish . +COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.35 /ko-app/grpc-health-probe /bin/grpc_health_probe +ENTRYPOINT ["dotnet", "Deal.Storage.dll"] diff --git a/src/storage-service/Deal.Storage/Models/StoredObject.cs b/src/storage-service/Deal.Storage/Models/StoredObject.cs new file mode 100644 index 0000000..eca3161 --- /dev/null +++ b/src/storage-service/Deal.Storage/Models/StoredObject.cs @@ -0,0 +1,12 @@ +namespace Deal.Storage.Models; + +/// +/// Дескриптор сохранённого объекта. +/// +public sealed record StoredObject( + string Id, + string Ref, + string Kind, + string MimeType, + string FileName, + long Size); diff --git a/src/storage-service/Deal.Storage/Options/StorageOptions.cs b/src/storage-service/Deal.Storage/Options/StorageOptions.cs new file mode 100644 index 0000000..c536bb2 --- /dev/null +++ b/src/storage-service/Deal.Storage/Options/StorageOptions.cs @@ -0,0 +1,73 @@ +using Microsoft.Extensions.Configuration; + +namespace Deal.Storage.Options; + +/// +/// Настройки объектного хранилища сервиса данных. +/// +public sealed class StorageOptions +{ + /// + /// Значение бакета по умолчанию. + /// + public const string DefaultBucketName = "deal-attachments"; + + /// + /// Env-ключ адреса хранилища (host:port). + /// + public const string EndpointEnvKey = "DEAL_STORAGE_ENDPOINT"; + + /// + /// Env-ключ ключа доступа. + /// + public const string AccessKeyEnvKey = "DEAL_STORAGE_ACCESS_KEY"; + + /// + /// Env-ключ секрета доступа. + /// + public const string SecretKeyEnvKey = "DEAL_STORAGE_SECRET_KEY"; + + /// + /// Env-ключ имени бакета. + /// + public const string BucketEnvKey = "DEAL_STORAGE_BUCKET"; + + /// + /// Env-ключ флага TLS (1/true — https). + /// + public const string SecureEnvKey = "DEAL_STORAGE_SECURE"; + + public string Endpoint { get; init; } = string.Empty; + + public string AccessKey { get; init; } = string.Empty; + + public string SecretKey { get; init; } = string.Empty; + + public string Bucket { get; init; } = DefaultBucketName; + + public bool Secure { get; init; } + + /// + /// Читает настройки из конфигурации хоста. + /// + public static StorageOptions FromConfiguration(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + string bucket = (configuration[BucketEnvKey] ?? string.Empty).Trim(); + return new StorageOptions + { + Endpoint = (configuration[EndpointEnvKey] ?? string.Empty).Trim(), + AccessKey = (configuration[AccessKeyEnvKey] ?? string.Empty).Trim(), + SecretKey = configuration[SecretKeyEnvKey] ?? string.Empty, + Bucket = bucket.Length == 0 ? DefaultBucketName : bucket, + Secure = IsEnabled(configuration[SecureEnvKey]), + }; + } + + /// + /// Разбирает значение флага TLS. + /// + public static bool IsEnabled(string? rawValue) + => string.Equals(rawValue, "1", StringComparison.Ordinal) + || string.Equals(rawValue, "true", StringComparison.OrdinalIgnoreCase); +} diff --git a/src/storage-service/Deal.Storage/Program.cs b/src/storage-service/Deal.Storage/Program.cs new file mode 100644 index 0000000..bb34656 --- /dev/null +++ b/src/storage-service/Deal.Storage/Program.cs @@ -0,0 +1,31 @@ +using Deal.Grpc.Hosting.Interceptors; +using Deal.Grpc.Hosting.Models; +using Deal.Grpc.Hosting.Options; +using Deal.Grpc.Hosting.Services; +using Deal.Storage; + +const int defaultGrpcPort = 5104; +const string storageProcessName = "storage"; + +int grpcPort = GrpcHostEnvironment.ResolveGrpcPort(defaultGrpcPort); +int metricsPort = DealMetricsHosting.ResolveMetricsPort(DealMetricsHosting.DefaultMetricsPort); + +WebApplication app = StorageServiceHost.Create( + grpcPort, + configureBuilder: builder => + { + DealLogging.Configure(builder, storageProcessName); + DealMetricsHosting.AddDealMetrics(builder, metricsPort); + }); + +DealMetricsHosting.MapDealMetrics(app); + +MtlsOptions mtlsOptions = MtlsOptions.FromConfiguration(app.Configuration); +GrpcHostEnvironment.RequireMtlsInProduction(mtlsOptions); + +app.Logger.LogInformation( + "storage-service стартует: gRPC {Transport} 0.0.0.0:{Port} (health /grpc.health.v1.Health/Check)", + mtlsOptions.Enabled ? "mTLS (TLS + клиентский сертификат)" : "plaintext + service-token", + grpcPort); + +await app.RunAsync(); diff --git a/src/storage-service/Deal.Storage/Services/FileKindSniffer.cs b/src/storage-service/Deal.Storage/Services/FileKindSniffer.cs new file mode 100644 index 0000000..c2be36d --- /dev/null +++ b/src/storage-service/Deal.Storage/Services/FileKindSniffer.cs @@ -0,0 +1,138 @@ +namespace Deal.Storage.Services; + +/// +/// Определяет тип и MIME объекта по первым байтам содержимого. +/// +public static class FileKindSniffer +{ + /// + /// Тип «прочее». + /// + public const string KindOther = "other"; + + private const string DefaultMime = "application/octet-stream"; + + /// + /// Определяет тип и MIME по заголовку файла. + /// + /// Первые байты содержимого. + /// MIME, заявленный загрузчиком (используется как подсказка). + /// Пара «тип, MIME». + public static (string Kind, string MimeType) Detect(ReadOnlySpan head, string? contentType) + { + if (StartsWith(head, 0x89, 0x50, 0x4E, 0x47)) + { + return ("image", "image/png"); + } + + if (StartsWith(head, 0xFF, 0xD8, 0xFF)) + { + return ("image", "image/jpeg"); + } + + if (StartsWith(head, 0x47, 0x49, 0x46, 0x38)) + { + return ("image", "image/gif"); + } + + if (StartsWith(head, 0x42, 0x4D)) + { + return ("image", "image/bmp"); + } + + if (StartsWith(head, 0x49, 0x49, 0x2A, 0x00) || StartsWith(head, 0x4D, 0x4D, 0x00, 0x2A)) + { + return ("image", "image/tiff"); + } + + if (StartsWith(head, 0x25, 0x50, 0x44, 0x46)) + { + return ("document", "application/pdf"); + } + + if (StartsWith(head, 0x50, 0x4B, 0x03, 0x04) || StartsWith(head, 0x50, 0x4B, 0x05, 0x06)) + { + return ("archive", "application/zip"); + } + + if (StartsWith(head, 0x52, 0x61, 0x72, 0x21)) + { + return ("archive", "application/vnd.rar"); + } + + if (StartsWith(head, 0x37, 0x7A, 0xBC, 0xAF)) + { + return ("archive", "application/x-7z-compressed"); + } + + if (StartsWith(head, 0x1F, 0x8B)) + { + return ("archive", "application/gzip"); + } + + if (StartsWith(head, 0x1A, 0x45, 0xDF, 0xA3)) + { + return ("video", "video/webm"); + } + + if (head.Length >= 12 && StartsWith(head, 0x52, 0x49, 0x46, 0x46)) + { + if (StartsWith(head[8..], 0x57, 0x45, 0x42, 0x50)) + { + return ("image", "image/webp"); + } + + if (StartsWith(head[8..], 0x57, 0x41, 0x56, 0x45)) + { + return ("audio", "audio/wav"); + } + + if (StartsWith(head[8..], 0x41, 0x56, 0x49, 0x20)) + { + return ("video", "video/x-msvideo"); + } + } + + if (head.Length >= 8 && StartsWith(head[4..], 0x66, 0x74, 0x79, 0x70)) + { + return ("video", "video/mp4"); + } + + if (StartsWith(head, 0x49, 0x44, 0x33) || StartsWith(head, 0xFF, 0xFB)) + { + return ("audio", "audio/mpeg"); + } + + if (StartsWith(head, 0x4F, 0x67, 0x67, 0x53)) + { + return ("audio", "audio/ogg"); + } + + if (StartsWith(head, 0x66, 0x4C, 0x61, 0x43)) + { + return ("audio", "audio/flac"); + } + + string declared = (contentType ?? string.Empty).Trim(); + if (declared.Length == 0) + { + return (KindOther, DefaultMime); + } + + return (KindFromMime(declared), declared); + } + + // Определяет общий тип по префиксу MIME-типа. + private static string KindFromMime(string mime) + { + if (mime.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) return "image"; + if (mime.StartsWith("video/", StringComparison.OrdinalIgnoreCase)) return "video"; + if (mime.StartsWith("audio/", StringComparison.OrdinalIgnoreCase)) return "audio"; + if (mime.StartsWith("text/", StringComparison.OrdinalIgnoreCase)) return "document"; + return KindOther; + } + + // Проверяет, начинается ли последовательность с заданных байтов. + private static bool StartsWith(ReadOnlySpan data, params byte[] prefix) + => data.Length >= prefix.Length && data[..prefix.Length].SequenceEqual(prefix); +} diff --git a/src/storage-service/Deal.Storage/Services/MinioObjectStore.cs b/src/storage-service/Deal.Storage/Services/MinioObjectStore.cs new file mode 100644 index 0000000..e665a3e --- /dev/null +++ b/src/storage-service/Deal.Storage/Services/MinioObjectStore.cs @@ -0,0 +1,195 @@ +using Deal.Storage.Options; +using Microsoft.Extensions.Logging; +using Minio; +using Minio.DataModel; +using Minio.DataModel.Args; +using Minio.Exceptions; + +namespace Deal.Storage.Services; + +/// +/// Объектное хранилище сервиса данных поверх MinIO. +/// +public sealed class MinioObjectStore +{ + private const string DefaultContentType = "application/octet-stream"; + + private readonly IMinioClient _client; + private readonly string _endpoint; + private readonly string _bucket; + private readonly ILogger _logger; + + private readonly SemaphoreSlim _bucketCheckGate = new(1, 1); + + private bool _bucketChecked; + + /// + /// Создаёт хранилище поверх настроек. + /// + public MinioObjectStore(StorageOptions options, ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + if (string.IsNullOrWhiteSpace(options.Endpoint) + || string.IsNullOrWhiteSpace(options.AccessKey) + || string.IsNullOrWhiteSpace(options.SecretKey)) + { + throw new InvalidOperationException( + "MinioObjectStore требует заполненные настройки хранилища (Endpoint и ключи доступа)."); + } + + _endpoint = options.Endpoint; + _bucket = options.Bucket; + _logger = logger; + _client = new MinioClient() + .WithEndpoint(options.Endpoint) + .WithCredentials(options.AccessKey, options.SecretKey) + .WithSSL(options.Secure) + .Build(); + } + + /// + /// Возвращает подпись режима для стартового лога. + /// + public override string ToString() => $"MinioObjectStore (endpoint: {_endpoint}; bucket: {_bucket})"; + + /// + /// Сохраняет объект, возвращает его размер. + /// + public async Task PutAsync(string key, Stream content, string contentType, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(content); + await EnsureBucketAsync(ct); + + if (content.CanSeek && content.Position != 0) + { + content.Position = 0; + } + + using MemoryStream buffer = new(); + await content.CopyToAsync(buffer, ct); + buffer.Position = 0; + + await _client.PutObjectAsync( + new PutObjectArgs() + .WithBucket(_bucket) + .WithObject(key) + .WithStreamData(buffer) + .WithObjectSize(buffer.Length) + .WithContentType(string.IsNullOrWhiteSpace(contentType) ? DefaultContentType : contentType), + ct); + + return buffer.Length; + } + + /// + /// Возвращает содержимое объекта потоком либо null. + /// + public async Task GetAsync(string key, CancellationToken ct) + { + MemoryStream buffer = new(); + try + { + await _client.GetObjectAsync( + new GetObjectArgs() + .WithBucket(_bucket) + .WithObject(key) + .WithCallbackStream(async (stream, token) => await stream.CopyToAsync(buffer, token)), + ct); + } + catch (ObjectNotFoundException) + { + buffer.Dispose(); + return null; + } + catch (Exception) + { + buffer.Dispose(); + throw; + } + + buffer.Position = 0; + return buffer; + } + + /// + /// Возвращает размер и MIME объекта либо null. + /// + public async Task<(long Size, string ContentType)?> StatAsync(string key, CancellationToken ct) + { + try + { + ObjectStat stat = await _client.StatObjectAsync( + new StatObjectArgs().WithBucket(_bucket).WithObject(key), + ct); + return (stat.Size, stat.ContentType ?? string.Empty); + } + catch (ObjectNotFoundException) + { + return null; + } + } + + /// + /// Удаляет объект (отсутствующий — успех). + /// + public async Task DeleteAsync(string key, CancellationToken ct) + { + try + { + await _client.RemoveObjectAsync( + new RemoveObjectArgs().WithBucket(_bucket).WithObject(key), + ct); + } + catch (MinioException exception) + { + _logger.LogWarning( + exception, + "Не удалось удалить объект «{Key}» из бакета «{Bucket}»: {Message}", + key, + _bucket, + exception.Message); + } + } + + private async Task EnsureBucketAsync(CancellationToken ct) + { + if (_bucketChecked) + { + return; + } + + await _bucketCheckGate.WaitAsync(ct); + try + { + if (_bucketChecked) + { + return; + } + + try + { + bool exists = await _client.BucketExistsAsync(new BucketExistsArgs().WithBucket(_bucket), ct); + if (!exists) + { + await _client.MakeBucketAsync(new MakeBucketArgs().WithBucket(_bucket), ct); + } + } + catch (MinioException exception) + { + _logger.LogWarning( + exception, + "Не удалось проверить/создать бакет «{Bucket}»: {Message}", + _bucket, + exception.Message); + } + + _bucketChecked = true; + } + finally + { + _bucketCheckGate.Release(); + } + } +} diff --git a/src/storage-service/Deal.Storage/StorageServiceHost.cs b/src/storage-service/Deal.Storage/StorageServiceHost.cs new file mode 100644 index 0000000..15bb8a8 --- /dev/null +++ b/src/storage-service/Deal.Storage/StorageServiceHost.cs @@ -0,0 +1,47 @@ +using Deal.Grpc.Hosting.Services; +using Deal.Storage.Options; +using Deal.Storage.Services; + +namespace Deal.Storage; + +/// +/// Собирает WebApplication gRPC-хоста сервиса данных. +/// +public static class StorageServiceHost +{ + /// + /// Создаёт (не запускает) хост. + /// + /// TCP-порт Kestrel. + /// Аргументы командной строки. + /// Опциональный хук DI для тестов. + /// Опциональный хук конфигурации билдера (логи/метрики). + /// Собранный хост. + public static WebApplication Create( + int grpcPort, + string[]? args = null, + Action? configureServices = null, + Action? configureBuilder = null) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args ?? []); + + var mtlsCertificates = GrpcServer.LoadMtlsCertificates(builder); + GrpcServer.ConfigureKestrelHttp2Endpoint(builder, grpcPort, mtlsCertificates); + builder.Services.AddDealGrpcServer(); + builder.Services.AddReadyHealthCheck("хост storage-service готов"); + + StorageOptions storageOptions = StorageOptions.FromConfiguration(builder.Configuration); + builder.Services.AddSingleton(storageOptions); + builder.Services.AddSingleton(); + + configureServices?.Invoke(builder.Services); + configureBuilder?.Invoke(builder); + + WebApplication app = builder.Build(); + + app.MapGrpcService(); + app.MapGrpcHealthChecksService(); + + return app; + } +} diff --git a/src/storage-service/Deal.Storage/StorageServiceImpl.cs b/src/storage-service/Deal.Storage/StorageServiceImpl.cs new file mode 100644 index 0000000..99f422a --- /dev/null +++ b/src/storage-service/Deal.Storage/StorageServiceImpl.cs @@ -0,0 +1,135 @@ +using Deal.Grpc.Storage; +using Deal.Storage.Services; +using Grpc.Core; + +namespace Deal.Storage; + +/// +/// Реализация сервиса данных. +/// +public sealed class StorageServiceImpl : StorageService.StorageServiceBase +{ + private const int HeadSize = 512; + private const int DownloadChunkSize = 64 * 1024; + + private readonly MinioObjectStore _store; + private readonly ILogger _logger; + + /// + /// Создаёт реализацию поверх хранилища. + /// + public StorageServiceImpl(MinioObjectStore store, ILogger logger) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(logger); + _store = store; + _logger = logger; + } + + /// + public override async Task Upload( + IAsyncStreamReader requestStream, + ServerCallContext context) + { + CancellationToken ct = context.CancellationToken; + await using MemoryStream buffer = new(); + byte[] head = new byte[HeadSize]; + int headLength = 0; + UploadMeta? meta = null; + + while (await requestStream.MoveNext(ct)) + { + UploadRequest message = requestStream.Current; + if (meta is null && message.Meta is not null) + { + meta = message.Meta; + } + + if (message.Data.Length == 0) + { + continue; + } + + if (headLength < HeadSize) + { + int take = Math.Min(HeadSize - headLength, message.Data.Length); + message.Data.Span[..take].CopyTo(head.AsSpan(headLength)); + headLength += take; + } + + await buffer.WriteAsync(message.Data.Memory, ct); + } + + if (meta is null || string.IsNullOrWhiteSpace(meta.TenantId)) + { + throw new RpcException(new Status(StatusCode.InvalidArgument, "tenant_id обязателен")); + } + + (string kind, string mimeType) = FileKindSniffer.Detect(head.AsSpan(0, headLength), meta.ContentType); + string key = $"{meta.TenantId.Trim()}/{Guid.NewGuid():N}"; + buffer.Position = 0; + long size = await _store.PutAsync(key, buffer, mimeType, ct); + + _logger.LogInformation("Объект сохранён: {Key} ({Kind}, {Size} байт)", key, kind, size); + + return new UploadReply + { + Id = key, + Ref = key, + Kind = kind, + MimeType = mimeType, + FileName = meta.FileName ?? string.Empty, + Size = size, + }; + } + + /// + public override async Task Download( + DownloadRequest request, + IServerStreamWriter responseStream, + ServerCallContext context) + { + CancellationToken ct = context.CancellationToken; + await using Stream? content = await _store.GetAsync(request.Id, ct); + if (content is null) + { + throw new RpcException(new Status(StatusCode.NotFound, "объект не найден")); + } + + byte[] chunk = new byte[DownloadChunkSize]; + int read; + while ((read = await content.ReadAsync(chunk, ct)) > 0) + { + await responseStream.WriteAsync(new DownloadChunk { Data = Google.Protobuf.ByteString.CopyFrom(chunk, 0, read) }, ct); + } + } + + /// + public override async Task Stat(StatRequest request, ServerCallContext context) + { + (long Size, string ContentType)? stat = await _store.StatAsync(request.Id, context.CancellationToken); + if (stat is null) + { + return new StatReply { Found = false }; + } + + return new StatReply + { + Found = true, + Info = new UploadReply + { + Id = request.Id, + Ref = request.Id, + MimeType = stat.Value.ContentType, + Size = stat.Value.Size, + }, + }; + } + + /// + public override async Task Delete(DeleteRequest request, ServerCallContext context) + { + await _store.DeleteAsync(request.Id, context.CancellationToken); + return new DeleteReply(); + } +} diff --git a/src/storage-service/Directory.Build.props b/src/storage-service/Directory.Build.props new file mode 100644 index 0000000..b2a7175 --- /dev/null +++ b/src/storage-service/Directory.Build.props @@ -0,0 +1,11 @@ + + + net10.0 + latest + enable + enable + true + latest + true + +