diff --git a/docs/Setup/Configuration.md b/docs/Setup/Configuration.md index 75a03556..6caac9c0 100644 --- a/docs/Setup/Configuration.md +++ b/docs/Setup/Configuration.md @@ -123,7 +123,7 @@ Browser spellchecking uses the `` value in | EnableTagDiscoveryPanel | boolean | The default value is `true`. Enables the Tag Discovery Panel, which helps users discover topics by browsing popular tags. | | EnableBrokenLinkChecker | boolean | The default value is `true`. Checks all external links of published blog posts once a day and lists broken ones on the "Broken links" admin page. | | ShowCodeBlockLanguage | boolean | The default value is `true`. Shows the language of a fenced code block (e.g. `csharp`) in the header of the code block. | -| EnableMarkdownForLlms | boolean | The default value is `false`. Serves published blog posts as markdown to clients (like LLMs) that ask for `text/markdown`. See [Markdown for LLMs](#markdown-for-llms). | +| EnableMarkdownForLlms | boolean | The default value is `false`. Serves published blog posts as markdown to clients (like LLMs) that ask for `text/markdown` or append `.md` to the url, and serves `/llms.txt`. See [Markdown for LLMs](#markdown-for-llms). | ### Markdown for LLMs @@ -153,4 +153,7 @@ The content... - Markdown is only served if `text/markdown` is weighted at least as high as HTML in the `Accept` header (`text/html`, `text/*` or `*/*`), so `text/markdown, text/html;q=0.9` returns markdown, while `text/html, text/markdown;q=0.9` and a regular browser request return HTML. - Only published blog posts are served. Unpublished, scheduled and unknown blog posts return `404` with a markdown body. - Blog post responses carry `Vary: Accept` so caches and CDNs keep the HTML and markdown variants apart. -- If the setting is `false` or missing, the `Accept` header is ignored and HTML is always returned. +- Appending `.md` to a blog post url (`/blogPost/{id}.md` or `/blogPost/{id}/{slug}.md`) returns markdown regardless of the `Accept` header, for agents that can't set headers. +- HTML blog post pages advertise the markdown version via a `Link: ; rel="alternate"; type="text/markdown"` header and a `` tag. +- `/llms.txt` (see [llmstxt.org](https://llmstxt.org)) lists the blog name, introduction and all published blog posts with links to their markdown version. +- If the setting is `false` or missing, the `Accept` header is ignored and HTML is always returned, `/blogPost/{id}/{slug}.md` renders the HTML page (the slug is ignored) while `/blogPost/{id}.md` shows the not found page, no `Link` header or `` tag is emitted and `/llms.txt` returns `404`. diff --git a/src/LinkDotNet.Blog.Domain/BlogPost.cs b/src/LinkDotNet.Blog.Domain/BlogPost.cs index 9d0319f1..b846b2da 100644 --- a/src/LinkDotNet.Blog.Domain/BlogPost.cs +++ b/src/LinkDotNet.Blog.Domain/BlogPost.cs @@ -36,18 +36,20 @@ public sealed partial class BlogPost : Entity public int ReadingTimeInMinutes { get; private set; } - public string Slug => GenerateSlug(); + public string Slug => CreateSlug(Title); public string? AuthorName { get; private set; } - private string GenerateSlug() + public static string CreateSlug(string title) { - if (string.IsNullOrWhiteSpace(Title)) + ArgumentNullException.ThrowIfNull(title); + + if (string.IsNullOrWhiteSpace(title)) { - return Title; + return title; } - var normalizedTitle = Title.Normalize(NormalizationForm.FormD); + var normalizedTitle = title.Normalize(NormalizationForm.FormD); var stringBuilder = new StringBuilder(); foreach (var c in normalizedTitle.Where(c => CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark)) diff --git a/src/LinkDotNet.Blog.Web/Controller/LlmsTxtController.cs b/src/LinkDotNet.Blog.Web/Controller/LlmsTxtController.cs new file mode 100644 index 00000000..288a2ff3 --- /dev/null +++ b/src/LinkDotNet.Blog.Web/Controller/LlmsTxtController.cs @@ -0,0 +1,91 @@ +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using LinkDotNet.Blog.Domain; +using LinkDotNet.Blog.Infrastructure.Persistence; +using LinkDotNet.Blog.Web.Features; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Extensions.Options; + +namespace LinkDotNet.Blog.Web.Controller; + +[Route("llms.txt")] +[EnableRateLimiting("ip")] +public sealed partial class LlmsTxtController : ControllerBase +{ + private readonly IOptions applicationConfiguration; + private readonly IOptions introductionConfiguration; + private readonly IRepository blogPostRepository; + + public LlmsTxtController( + IOptions applicationConfiguration, + IOptions introductionConfiguration, + IRepository blogPostRepository) + { + this.applicationConfiguration = applicationConfiguration; + this.introductionConfiguration = introductionConfiguration; + this.blogPostRepository = blogPostRepository; + } + + [ResponseCache(Duration = 1200)] + [HttpGet] + public async Task GetLlmsTxt() + { + if (!applicationConfiguration.Value.EnableMarkdownForLlms) + { + return NotFound(); + } + + var baseUrl = $"{Request.Scheme}://{Request.Host}{Request.PathBase}"; + var sb = new StringBuilder(); + sb.Append("# ").Append(applicationConfiguration.Value.BlogName).Append("\n\n"); + var description = MarkdownConverter.ToPlainString(introductionConfiguration.Value.Description); + if (description is not null) + { + sb.Append("> ").Append(EscapeMarkdown(description)).Append("\n\n"); + } + + sb.Append("## Blog posts\n\n"); + var blogPosts = await blogPostRepository.GetAllByProjectionAsync( + b => new LlmsTxtEntry { Id = b.Id, Title = b.Title, ShortDescription = b.ShortDescription }, + f => f.IsPublished, + b => b.UpdatedDate); + foreach (var blogPost in blogPosts) + { + sb.Append("- [").Append(EscapeMarkdown(blogPost.Title)).Append("](").Append(baseUrl).Append("/blogPost/").Append(blogPost.Id); + var slug = BlogPost.CreateSlug(blogPost.Title); + if (!string.IsNullOrEmpty(slug)) + { + sb.Append('/').Append(slug); + } + + sb.Append(".md)"); + var shortDescription = MarkdownConverter.ToPlainString(blogPost.ShortDescription); + if (shortDescription is not null) + { + sb.Append(": ").Append(EscapeMarkdown(shortDescription)); + } + + sb.Append('\n'); + } + + return Content(sb.ToString(), "text/plain; charset=utf-8", Encoding.UTF8); + } + + private static string EscapeMarkdown(string text) + => MarkdownSpecialCharacters().Replace(text.ReplaceLineEndings(" "), @"\$0"); + + [GeneratedRegex(@"[\\`*_\[\]<>&]")] + private static partial Regex MarkdownSpecialCharacters(); + + // Member-init instead of a positional record: RavenDB rejects constructor projections + private sealed class LlmsTxtEntry + { + public required string Id { get; init; } + + public required string Title { get; init; } + + public required string ShortDescription { get; init; } + } +} diff --git a/src/LinkDotNet.Blog.Web/Features/Components/OgData.razor b/src/LinkDotNet.Blog.Web/Features/Components/OgData.razor index e153e382..3f7bca0f 100644 --- a/src/LinkDotNet.Blog.Web/Features/Components/OgData.razor +++ b/src/LinkDotNet.Blog.Web/Features/Components/OgData.razor @@ -19,6 +19,7 @@ { } + @ChildContent @code { @@ -37,6 +38,10 @@ [Parameter] public string? CanonicalRelativeUrl { get; set; } + // HeadContent is last-one-wins, so pages add extra head tags through here instead of their own HeadContent + [Parameter] + public RenderFragment? ChildContent { get; set; } + [Parameter] public string Robots { get; set; } = "max-image-preview:large"; diff --git a/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/BlogPostMarkdownMiddleware.cs b/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/BlogPostMarkdownMiddleware.cs index 4ef645f5..f6d66df7 100644 --- a/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/BlogPostMarkdownMiddleware.cs +++ b/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/BlogPostMarkdownMiddleware.cs @@ -14,6 +14,7 @@ namespace LinkDotNet.Blog.Web.Features.ShowBlogPost; public sealed class BlogPostMarkdownMiddleware { private const string MarkdownMediaType = "text/markdown"; + private const string MarkdownSuffix = ".md"; private static readonly string[] HtmlMediaTypes = ["text/html", "text/*", "*/*"]; private readonly RequestDelegate next; @@ -41,15 +42,17 @@ public async Task InvokeAsync( } context.Response.Headers.Append(HeaderNames.Vary, HeaderNames.Accept); - if (!PrefersMarkdown(context.Request.GetTypedHeaders().Accept)) + if (!IsMarkdownUrl(context.Request) && !PrefersMarkdown(context.Request.GetTypedHeaders().Accept)) { + var markdownUrl = context.Request.PathBase.Add(context.Request.Path).ToUriComponent().TrimEnd('/') + MarkdownSuffix; + context.Response.Headers.Append(HeaderNames.Link, $"<{markdownUrl}>; rel=\"alternate\"; type=\"{MarkdownMediaType}\""); await next(context); return; } // A content type keeps UseStatusCodePagesWithReExecute from swapping the 404 for the HTML NotFound page context.Response.ContentType = $"{MarkdownMediaType}; charset=utf-8"; - var blogPost = await blogPostRepository.GetByIdAsync(context.Request.RouteValues["blogPostId"]?.ToString() ?? string.Empty); + var blogPost = await blogPostRepository.GetByIdAsync(GetBlogPostId(context.Request)); if (blogPost is not { IsPublished: true }) { context.Response.StatusCode = StatusCodes.Status404NotFound; @@ -78,6 +81,17 @@ public static bool PrefersMarkdown(IList accept) .Select(a => (double?)(a.Quality ?? 1)) .Max(); + private static bool IsMarkdownUrl(HttpRequest request) + => request.Path.Value?.EndsWith(MarkdownSuffix, StringComparison.OrdinalIgnoreCase) == true; + + private static string GetBlogPostId(HttpRequest request) + { + var blogPostId = request.RouteValues["blogPostId"]?.ToString() ?? string.Empty; + return request.RouteValues["slug"] is null && IsMarkdownUrl(request) + ? blogPostId[..^MarkdownSuffix.Length] + : blogPostId; + } + private static bool IsBlogPostPage(HttpContext context) => context.GetEndpoint()?.Metadata.GetMetadata()?.Type == typeof(ShowBlogPostPage); diff --git a/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/ShowBlogPostPage.razor b/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/ShowBlogPostPage.razor index a9c5a8ac..1e6b750d 100644 --- a/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/ShowBlogPostPage.razor +++ b/src/LinkDotNet.Blog.Web/Features/ShowBlogPost/ShowBlogPostPage.razor @@ -32,7 +32,12 @@ else if (BlogPost is not null) AbsolutePreviewImageUrl="@OgDataImage" Description="@(Markdown.ToPlainText(BlogPost.ShortDescription))" Keywords="@BlogPost.TagsAsString" - CanonicalRelativeUrl="@BlogPostCanoncialUrl" /> + CanonicalRelativeUrl="@BlogPostCanoncialUrl"> + @if (AppConfiguration.Value.EnableMarkdownForLlms) + { + + } + @* Rendered in the body on purpose: HeadOutlet does not emit script tags, so a JSON-LD block placed in HeadContent never reaches the page. *@ diff --git a/tests/LinkDotNet.Blog.IntegrationTests/BlogPostMarkdownTests.cs b/tests/LinkDotNet.Blog.IntegrationTests/BlogPostMarkdownTests.cs index 9508202d..a539fdd1 100644 --- a/tests/LinkDotNet.Blog.IntegrationTests/BlogPostMarkdownTests.cs +++ b/tests/LinkDotNet.Blog.IntegrationTests/BlogPostMarkdownTests.cs @@ -46,6 +46,78 @@ public async Task ShouldServeMarkdownWithAndWithoutSlug(string urlTemplate) body.ShouldEndWith("---\n\n# My Blog Post\n\nHello"); } + [Theory] + [InlineData("/blogPost/{0}.md")] + [InlineData("/blogPost/{0}/my-blog-post.md")] + [InlineData("/blogPost/{0}/a-wrong-slug.MD")] + [InlineData("/blogPost/{0}/.md")] + public async Task ShouldServeMarkdownForMdSuffixRegardlessOfAccept(string urlTemplate) + { + var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle("My Blog Post").WithContent("Hello").Build()); + + using var response = await GetAsync(client, string.Format(urlTemplate, blogPost.Id), "text/html"); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + response.Content.Headers.ContentType!.ToString().ShouldBe(MarkdownContentType); + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + body.ShouldEndWith("# My Blog Post\n\nHello"); + } + + [Fact] + public async Task ShouldReturnNotFoundForMdSuffixOfUnpublishedBlogPost() + { + var blogPost = await SeedAsync(factory, new BlogPostBuilder().IsPublished(false).Build()); + + using var response = await GetAsync(client, $"/blogPost/{blogPost.Id}.md", null); + + await ShouldBeMarkdownNotFoundAsync(response); + } + + [Theory] + [InlineData("/blogPost/{0}", "/blogPost/{0}.md")] + [InlineData("/blogPost/{0}/some-slug/", "/blogPost/{0}/some-slug.md")] + public async Task ShouldAdvertiseMarkdownVersionOnHtmlResponse(string urlTemplate, string linkTemplate) + { + var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle("Linked Post").Build()); + + using var response = await GetAsync(client, string.Format(urlTemplate, blogPost.Id), null); + + await ShouldBeHtmlBlogPostAsync(response, "Linked Post"); + response.Headers.GetValues("Link").ShouldContain($"<{string.Format(linkTemplate, blogPost.Id)}>; rel=\"alternate\"; type=\"text/markdown\""); + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + body.ShouldContain($""); + } + + [Fact] + public async Task ShouldListPublishedBlogPostsInLlmsTxt() + { + var published = await SeedAsync(factory, new BlogPostBuilder().WithTitle("Listed Post").WithShortDescription("**Short**\nsummary").Build()); + await SeedAsync(factory, new BlogPostBuilder().WithTitle("Hidden Draft").IsPublished(false).Build()); + + using var response = await GetAsync(client, "/llms.txt", null); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + response.Content.Headers.ContentType!.ToString().ShouldBe("text/plain; charset=utf-8"); + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + body.ShouldStartWith("# "); + body.ShouldContain($"- [Listed Post](http://localhost/blogPost/{published.Id}/listed-post.md): Short summary\n"); + body.ShouldNotContain("Hidden Draft"); + } + + [Theory] + [InlineData("Why ] matters", @"Why \] matters", "why-matters")] + [InlineData("Generic & [more]", @"Generic \ \& \[more\]", "generic-t-more")] + [InlineData("Under_score *star* `tick` \\ slash", @"Under\_score \*star\* \`tick\` \\ slash", "underscore-star-tick-slash")] + public async Task ShouldEscapeMarkdownInLlmsTxt(string title, string escapedTitle, string slug) + { + var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle(title).WithShortDescription("`List`").Build()); + + using var response = await GetAsync(client, "/llms.txt", null); + + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + body.ShouldContain($"- [{escapedTitle}](http://localhost/blogPost/{blogPost.Id}/{slug}.md): List\\\n"); + } + [Theory] [InlineData("text/markdown")] [InlineData("TEXT/MARKDOWN")] @@ -208,6 +280,15 @@ public async Task ShouldServeHtmlWhenDisabled(string? enableMarkdownForLlms) await ShouldBeHtmlBlogPostAsync(response, "Disabled Post"); response.Headers.Vary.ShouldNotContain("Accept"); + response.Headers.Contains("Link").ShouldBeFalse(); + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + body.ShouldNotContain("text/markdown"); + + using var mdSuffixResponse = await GetAsync(disabledClient, $"/blogPost/{blogPost.Id}/disabled-post.md", null); + await ShouldBeHtmlBlogPostAsync(mdSuffixResponse, "Disabled Post"); + + using var llmsTxtResponse = await GetAsync(disabledClient, "/llms.txt", null); + llmsTxtResponse.StatusCode.ShouldBe(HttpStatusCode.NotFound); } public void Dispose() => client.Dispose();