7.1 IImageUrlFetcher
Add IImageUrlFetcher with HttpClient-backed implementation that enforces UrlFetch.TimeoutSeconds, MaxRedirects, and MaxBytes. Surfaces failures as ImageFetchException so the URL-ingestion endpoint can map them to 502. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -177,7 +177,7 @@ Each type lives in `src/FrameProcessor/Domain/`. Tests in `tests/FrameProcessor.
|
|||||||
|
|
||||||
## Phase 7 — URL-fetch ingestion
|
## Phase 7 — URL-fetch ingestion
|
||||||
|
|
||||||
### [ ] 7.1 `IImageUrlFetcher`
|
### [x] 7.1 `IImageUrlFetcher`
|
||||||
- Constructor takes `HttpClient` (via `IHttpClientFactory`) configured with `UrlFetch.TimeoutSeconds` and `MaxRedirects`.
|
- Constructor takes `HttpClient` (via `IHttpClientFactory`) configured with `UrlFetch.TimeoutSeconds` and `MaxRedirects`.
|
||||||
- `Task<Stream> FetchAsync(Uri, CancellationToken)` — streams response, aborts if content exceeds `MaxBytes`.
|
- `Task<Stream> FetchAsync(Uri, CancellationToken)` — streams response, aborts if content exceeds `MaxBytes`.
|
||||||
- Throws a typed exception (`ImageFetchException`) for timeout / non-2xx / too-large / redirect-loop.
|
- Throws a typed exception (`ImageFetchException`) for timeout / non-2xx / too-large / redirect-loop.
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ using FrameProcessor.Configuration;
|
|||||||
using FrameProcessor.ImagePipeline;
|
using FrameProcessor.ImagePipeline;
|
||||||
using FrameProcessor.Mqtt;
|
using FrameProcessor.Mqtt;
|
||||||
using FrameProcessor.Storage;
|
using FrameProcessor.Storage;
|
||||||
|
using FrameProcessor.UrlFetch;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
@@ -42,6 +44,21 @@ builder.Services.AddSingleton<ImageStore>();
|
|||||||
builder.Services.AddSingleton<MqttPublisher>();
|
builder.Services.AddSingleton<MqttPublisher>();
|
||||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<MqttPublisher>());
|
builder.Services.AddHostedService(sp => sp.GetRequiredService<MqttPublisher>());
|
||||||
|
|
||||||
|
builder.Services.AddHttpClient<IImageUrlFetcher, ImageUrlFetcher>((sp, client) =>
|
||||||
|
{
|
||||||
|
var opts = sp.GetRequiredService<IOptions<UrlFetchOptions>>().Value;
|
||||||
|
client.Timeout = TimeSpan.FromSeconds(opts.TimeoutSeconds);
|
||||||
|
})
|
||||||
|
.ConfigurePrimaryHttpMessageHandler(sp =>
|
||||||
|
{
|
||||||
|
var opts = sp.GetRequiredService<IOptions<UrlFetchOptions>>().Value;
|
||||||
|
return new HttpClientHandler
|
||||||
|
{
|
||||||
|
AllowAutoRedirect = true,
|
||||||
|
MaxAutomaticRedirections = opts.MaxRedirects,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
|
|
||||||
// Eagerly resolve FramesRegistry so an invalid frames.json fails startup fast.
|
// Eagerly resolve FramesRegistry so an invalid frames.json fails startup fast.
|
||||||
|
|||||||
20
src/FrameProcessor/UrlFetch/IImageUrlFetcher.cs
Normal file
20
src/FrameProcessor/UrlFetch/IImageUrlFetcher.cs
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
namespace FrameProcessor.UrlFetch;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Fetches a remote image into an in-memory stream for the URL-ingestion endpoint
|
||||||
|
/// (SPEC.md §4.2). Enforces the guardrails from SPEC.md §8 — max response size,
|
||||||
|
/// timeout, and max redirect hops — surfacing failures as <see cref="ImageFetchException"/>
|
||||||
|
/// so callers can map them to <c>502 Bad Gateway</c>.
|
||||||
|
/// </summary>
|
||||||
|
public interface IImageUrlFetcher
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Fetch the image at <paramref name="uri"/>. The returned stream is positioned
|
||||||
|
/// at zero and owned by the caller.
|
||||||
|
/// </summary>
|
||||||
|
/// <exception cref="ImageFetchException">
|
||||||
|
/// Timeout, non-2xx status, response exceeding the configured maximum, or
|
||||||
|
/// excessive redirect hops.
|
||||||
|
/// </exception>
|
||||||
|
Task<Stream> FetchAsync(Uri uri, CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
18
src/FrameProcessor/UrlFetch/ImageFetchException.cs
Normal file
18
src/FrameProcessor/UrlFetch/ImageFetchException.cs
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
namespace FrameProcessor.UrlFetch;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Thrown by <see cref="IImageUrlFetcher"/> when a remote image fetch fails in a way
|
||||||
|
/// the caller should surface as <c>502 Bad Gateway</c> per SPEC.md §4.2 — timeout,
|
||||||
|
/// non-2xx status, response exceeding <c>UrlFetch.MaxBytes</c>, or redirect-loop /
|
||||||
|
/// excess redirect hops.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ImageFetchException : Exception
|
||||||
|
{
|
||||||
|
public ImageFetchException(string message) : base(message)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public ImageFetchException(string message, Exception innerException) : base(message, innerException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
84
src/FrameProcessor/UrlFetch/ImageUrlFetcher.cs
Normal file
84
src/FrameProcessor/UrlFetch/ImageUrlFetcher.cs
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
using FrameProcessor.Configuration;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace FrameProcessor.UrlFetch;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// <see cref="IImageUrlFetcher"/> implementation backed by an <see cref="HttpClient"/>
|
||||||
|
/// from <see cref="IHttpClientFactory"/>. The client is configured with
|
||||||
|
/// <see cref="UrlFetchOptions.TimeoutSeconds"/>; redirect-hop enforcement lives on the
|
||||||
|
/// primary message handler (see <c>Program.cs</c>).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ImageUrlFetcher : IImageUrlFetcher
|
||||||
|
{
|
||||||
|
private const int CopyBufferSize = 81_920;
|
||||||
|
|
||||||
|
private readonly HttpClient _httpClient;
|
||||||
|
private readonly IOptions<UrlFetchOptions> _options;
|
||||||
|
|
||||||
|
public ImageUrlFetcher(HttpClient httpClient, IOptions<UrlFetchOptions> options)
|
||||||
|
{
|
||||||
|
_httpClient = httpClient;
|
||||||
|
_options = options;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<Stream> FetchAsync(Uri uri, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
HttpResponseMessage response;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
response = await _httpClient
|
||||||
|
.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
throw new ImageFetchException($"Timed out fetching {uri}", ex);
|
||||||
|
}
|
||||||
|
catch (HttpRequestException ex)
|
||||||
|
{
|
||||||
|
// HttpClient surfaces redirect-limit overflows here as well as transport errors.
|
||||||
|
throw new ImageFetchException($"Request to {uri} failed: {ex.Message}", ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
using (response)
|
||||||
|
{
|
||||||
|
if (!response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
throw new ImageFetchException(
|
||||||
|
$"Request to {uri} returned HTTP {(int)response.StatusCode}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var maxBytes = _options.Value.MaxBytes;
|
||||||
|
if (response.Content.Headers.ContentLength is long advertised && advertised > maxBytes)
|
||||||
|
{
|
||||||
|
throw new ImageFetchException(
|
||||||
|
$"Response Content-Length {advertised} exceeds maximum {maxBytes} bytes");
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var source = await response.Content
|
||||||
|
.ReadAsStreamAsync(cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
|
var buffer = new MemoryStream();
|
||||||
|
var chunk = new byte[CopyBufferSize];
|
||||||
|
long total = 0;
|
||||||
|
int read;
|
||||||
|
while ((read = await source.ReadAsync(chunk.AsMemory(0, chunk.Length), cancellationToken)
|
||||||
|
.ConfigureAwait(false)) > 0)
|
||||||
|
{
|
||||||
|
total += read;
|
||||||
|
if (total > maxBytes)
|
||||||
|
{
|
||||||
|
buffer.Dispose();
|
||||||
|
throw new ImageFetchException(
|
||||||
|
$"Response exceeded maximum size of {maxBytes} bytes");
|
||||||
|
}
|
||||||
|
buffer.Write(chunk, 0, read);
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.Position = 0;
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user