From d8b64f1006000deef38ce447f875b978c19009a2 Mon Sep 17 00:00:00 2001
From: gusmar2017 <gustavomarquez3@gmail.com>
Date: Mon, 13 Jul 2026 18:10:34 -0500
Subject: [PATCH] feat: internal linking hubs for gated servers (indexing
 recovery slice 4)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Ensures every isIndexable() (Slice 2 gated) server is reachable in <=3
clicks from the homepage:

- Category hubs (/categories/[category]) now link the FULL gated set per
  category via new getIndexableServersByCategory(), not the old 200-row
  raw-active-servers query.
- New /categories index page (crawlable hub-of-hubs; fills the
  previously-dangling breadcrumb target referenced by generateCategoryJsonLd).
- RelatedServersForCategory now sources from the gated query and is wired
  onto every server detail page (previously only used on category hubs and
  blog posts) — self-excludes the current server, category-scoped.
- Homepage "Featured Servers" now uses new getIndexableTopServers() instead
  of the unfiltered getTopServers(), so the top-N-by-stars linking surface
  never resurfaces a thin/non-gated page.
- /servers pagination confirmed already SSR-crawlable (real next/link
  hrefs via PaginationLink, passed through the client filter wrapper as
  children) — documented with a comment, no behavior change needed.
- Homepage footer links to /categories in two places for redundant crawl
  paths.

Click-depth: home -> category hub (1) -> server (2); home -> Featured
Servers top-6 (1); home -> /servers paginated, stars-desc default fronts
the gated cluster (2-3). Category-less gated servers fall back to the
/servers and Featured Servers paths.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 apps/web/app/categories/[category]/page.tsx   |  14 +-
 apps/web/app/categories/page.tsx              | 144 ++++++++++++++++++
 apps/web/app/page.tsx                         |  24 ++-
 apps/web/app/servers/[slug]/page.tsx          |  49 ++----
 apps/web/app/servers/page.tsx                 |  10 +-
 .../components/RelatedServersForCategory.tsx  |  22 ++-
 apps/web/lib/queries.ts                       |  85 +++++++++++
 7 files changed, 300 insertions(+), 48 deletions(-)
 create mode 100644 apps/web/app/categories/page.tsx

diff --git a/apps/web/app/categories/[category]/page.tsx b/apps/web/app/categories/[category]/page.tsx
index 3cb8884..db5e876 100644
--- a/apps/web/app/categories/[category]/page.tsx
+++ b/apps/web/app/categories/[category]/page.tsx
@@ -1,4 +1,4 @@
-import { getServersByCategory, getCategoryCount } from '@/lib/queries';
+import { getIndexableServersByCategory, getCategoryCount } from '@/lib/queries';
 import { generateCategoryMetadata, generateCategoryJsonLd } from '@/lib/metadata';
 import { getQualityStatus } from '@/lib/quality-status';
 import { safeJsonLd } from '@/lib/json-ld';
@@ -45,10 +45,14 @@ export default async function CategoryPage({
   if (!(CATEGORIES as readonly string[]).includes(category)) notFound();
 
   const label = CATEGORY_LABELS[category as keyof typeof CATEGORY_LABELS] || category;
-  // Fetch servers (up to 200 for display) and the true total count in parallel.
-  // The true count feeds JSON-LD numberOfItems; servers feeds the card grid.
+  // Indexing-recovery Slice 4: this hub must link every isIndexable() (gated)
+  // server in the category — getIndexableServersByCategory() is the full,
+  // uncapped, pre-filtered set (not getServersByCategory()'s 200-row raw
+  // cap), so a large category can never silently drop gated servers from
+  // its own hub page. categoryCount (JSON-LD numberOfItems) intentionally
+  // still reflects the true active-server count, not just the gated subset.
   const [servers, categoryCount] = await Promise.all([
-    getServersByCategory(category),
+    getIndexableServersByCategory(category),
     getCategoryCount(category),
   ]);
 
@@ -90,7 +94,7 @@ export default async function CategoryPage({
           </p>
           <div className="flex flex-wrap items-center gap-x-4 gap-y-1">
             <p className="text-neutral-500 text-lg">
-              {categoryCount} servers in this category
+              {servers.length} servers in this category
             </p>
             <time
               dateTime={dateModified}
diff --git a/apps/web/app/categories/page.tsx b/apps/web/app/categories/page.tsx
new file mode 100644
index 0000000..8eeeee7
--- /dev/null
+++ b/apps/web/app/categories/page.tsx
@@ -0,0 +1,144 @@
+/**
+ * Categories index — /categories
+ *
+ * Crawlable hub-of-hubs: lists every category with a real <Link> to its hub
+ * page (/categories/[category]). Indexing-recovery Slice 4 (internal
+ * linking): this is the second hop on the home -> category hub -> server
+ * path, and gives category hubs a stable, dedicated landing point distinct
+ * from the homepage's "Browse by Category" filter cards (which link to
+ * /servers?category=X, not the canonical hub URL). generateCategoryJsonLd's
+ * BreadcrumbList on every category page already references
+ * `${SITE_URL}/categories` as "Categories" — this page fills that
+ * previously-dangling breadcrumb target.
+ *
+ * Server Component: fully SSR'd, no client JS required to read the list.
+ */
+
+import Link from "next/link";
+import type { Metadata } from "next";
+import { Navbar } from "@/components/ui/navbar";
+import { safeJsonLd } from "@/lib/json-ld";
+import { getIndexableServersByCategory } from "@/lib/queries";
+import { CATEGORIES, CATEGORY_LABELS, CATEGORY_DESCRIPTIONS, SITE_URL, SITE_NAME } from "@mcpfind/shared";
+import type { Category } from "@mcpfind/shared";
+import { IconServer, IconArrowRight } from "@tabler/icons-react";
+
+export const revalidate = 3600;
+
+export const metadata: Metadata = {
+  title: `Browse MCP Server Categories | ${SITE_NAME}`,
+  description:
+    "Browse Model Context Protocol servers by category — databases, cloud, devtools, AI & ML, security, and more.",
+  alternates: { canonical: `${SITE_URL}/categories` },
+  openGraph: {
+    title: `Browse MCP Server Categories | ${SITE_NAME}`,
+    description:
+      "Browse Model Context Protocol servers by category — databases, cloud, devtools, AI & ML, security, and more.",
+    url: `${SITE_URL}/categories`,
+    siteName: SITE_NAME,
+    type: "website",
+  },
+};
+
+export default async function CategoriesIndexPage() {
+  // Gated (isIndexable()) count per category — degrades to 0 per-category
+  // rather than failing the whole page if Supabase is unavailable (CI/build).
+  const counts = await Promise.all(
+    CATEGORIES.map(async (cat) => {
+      try {
+        const servers = await getIndexableServersByCategory(cat);
+        return [cat, servers.length] as const;
+      } catch {
+        return [cat, 0] as const;
+      }
+    })
+  );
+  const countMap = new Map(counts);
+
+  return (
+    <div className="min-h-screen bg-black text-white">
+      <Navbar variant="sticky" />
+
+      <script
+        type="application/ld+json"
+        dangerouslySetInnerHTML={{
+          __html: safeJsonLd({
+            "@context": "https://schema.org",
+            "@graph": [
+              {
+                "@type": "CollectionPage",
+                "@id": `${SITE_URL}/categories`,
+                name: "MCP Server Categories",
+                url: `${SITE_URL}/categories`,
+                description: "Browse Model Context Protocol servers by category.",
+                breadcrumb: { "@id": `${SITE_URL}/categories#breadcrumb` },
+                mainEntity: {
+                  "@type": "ItemList",
+                  numberOfItems: CATEGORIES.length,
+                  itemListElement: CATEGORIES.map((cat, i) => ({
+                    "@type": "ListItem",
+                    position: i + 1,
+                    url: `${SITE_URL}/categories/${cat}`,
+                    name: CATEGORY_LABELS[cat as Category],
+                  })),
+                },
+              },
+              {
+                "@type": "BreadcrumbList",
+                "@id": `${SITE_URL}/categories#breadcrumb`,
+                itemListElement: [
+                  { "@type": "ListItem", position: 1, name: "Home", item: SITE_URL },
+                  { "@type": "ListItem", position: 2, name: "Categories", item: `${SITE_URL}/categories` },
+                ],
+              },
+            ],
+          }),
+        }}
+      />
+
+      <main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 pt-28 pb-12">
+        <div className="mb-10">
+          <h1 className="text-4xl font-bold bg-clip-text text-transparent bg-gradient-to-r from-white to-neutral-400 mb-2">
+            Browse MCP Server Categories
+          </h1>
+          <p className="text-neutral-400 text-base max-w-2xl">
+            Every MCP server in the directory, organized by category. Pick a
+            category to see all servers that clear our quality bar.
+          </p>
+        </div>
+
+        <ul className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4 list-none p-0 m-0">
+          {CATEGORIES.map((cat) => (
+            <li key={cat} className="contents" role="listitem">
+              <Link
+                href={`/categories/${cat}`}
+                className="group flex flex-col gap-2 p-5 rounded-xl bg-neutral-900 border border-neutral-800 hover:border-neutral-700 hover:bg-neutral-800/80 transition-all duration-200"
+              >
+                <div className="flex items-center justify-between">
+                  <div className="flex items-center gap-2.5">
+                    <div className="w-8 h-8 rounded-lg bg-neutral-800 group-hover:bg-neutral-700 flex items-center justify-center transition-colors duration-200">
+                      <IconServer size={16} className="text-neutral-400" />
+                    </div>
+                    <span className="font-semibold text-neutral-100 group-hover:text-white transition-colors duration-200">
+                      {CATEGORY_LABELS[cat as Category]}
+                    </span>
+                  </div>
+                  <IconArrowRight
+                    size={16}
+                    className="text-neutral-600 group-hover:text-neutral-400 transition-colors duration-200"
+                  />
+                </div>
+                <p className="text-neutral-500 text-sm line-clamp-2">
+                  {CATEGORY_DESCRIPTIONS[cat as Category]}
+                </p>
+                <span className="text-neutral-600 text-xs mt-1">
+                  {countMap.get(cat) ?? 0} servers
+                </span>
+              </Link>
+            </li>
+          ))}
+        </ul>
+      </main>
+    </div>
+  );
+}
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index 14c21d3..8b36886 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -9,7 +9,7 @@ import { Navbar } from "@/components/ui/navbar";
 import { HeroSearch } from "@/components/ui/hero-search";
 import { safeJsonLd } from "@/lib/json-ld";
 import { getAllPosts } from "@/lib/blog";
-import { getTopServers, getServerCount, listServers } from "@/lib/queries";
+import { getIndexableTopServers, getServerCount, listServers } from "@/lib/queries";
 import { getQualityStatus } from "@/lib/quality-status";
 import { CATEGORIES, CATEGORY_LABELS, SITE_URL, FALLBACK_SERVER_COUNT_DISPLAY } from "@mcpfind/shared";
 import type { Category, ServerListItem } from "@mcpfind/shared";
@@ -169,9 +169,13 @@ function RecentServersSkeleton() {
 // ── Async server components (data-heavy, streamed below-the-fold) ───────────
 
 async function FeaturedServersSection() {
+  // Indexing-recovery Slice 4: homepage top-N linking surface must only
+  // link isIndexable() (gated) servers — getIndexableTopServers() is the
+  // gated equivalent of getTopServers(), pre-filtered by the Slice 2 quality
+  // bar so this never resurfaces a thin/non-gated server page.
   let featuredServers: ServerListItem[] = [];
   try {
-    featuredServers = await getTopServers(6);
+    featuredServers = await getIndexableTopServers(6);
   } catch {
     // Supabase not available
   }
@@ -616,9 +620,15 @@ export default async function HomePage() {
             {/* Browse — first 5 categories */}
             {/* Links point to /categories/<slug> so crawlers index the canonical */}
             {/* category pages. The interactive filter cards above are unchanged. */}
+            {/* Header links to /categories, the crawlable hub-of-hubs index. */}
             <div>
               <h4 className="text-sm font-semibold text-neutral-300 mb-4">
-                Browse
+                <Link
+                  href="/categories"
+                  className="hover:text-white transition-colors duration-200"
+                >
+                  Browse
+                </Link>
               </h4>
               <ul className="space-y-2">
                 {CATEGORIES.slice(0, 5).map((cat: string) => (
@@ -667,6 +677,14 @@ export default async function HomePage() {
                     Browse Servers
                   </Link>
                 </li>
+                <li>
+                  <Link
+                    href="/categories"
+                    className="text-neutral-500 hover:text-neutral-300 text-sm transition-colors duration-200"
+                  >
+                    All Categories
+                  </Link>
+                </li>
                 <li>
                   <Link
                     href="/submit"
diff --git a/apps/web/app/servers/[slug]/page.tsx b/apps/web/app/servers/[slug]/page.tsx
index afd735f..c4c62bd 100644
--- a/apps/web/app/servers/[slug]/page.tsx
+++ b/apps/web/app/servers/[slug]/page.tsx
@@ -1,4 +1,4 @@
-import { getServerBySlug, getServersByCategory, getIndexableServerSlugs } from "@/lib/queries";
+import { getServerBySlug, getIndexableServerSlugs } from "@/lib/queries";
 import { generateServerMetadata, generateServerJsonLd } from "@/lib/metadata";
 import { getQualityStatus } from "@/lib/quality-status";
 import { isIndexable } from "@/lib/indexable";
@@ -44,6 +44,7 @@ const ReadmeSection = dynamic(
 import { ServerCard } from "@/components/ui/server-card";
 import { formatNumber } from "@/components/ui/stat-badge";
 import { RelatedArticles } from "@/components/related-articles";
+import { RelatedServersForCategory } from "@/components/RelatedServersForCategory";
 import { StaleServerBadge } from "@/components/StaleServerBadge";
 import { VerifiedServerBadge } from "@/components/VerifiedServerBadge";
 import { Navbar } from "@/components/ui/navbar";
@@ -182,12 +183,6 @@ export default async function ServerDetailPage({
 
   const qualityStatus = getQualityStatus(slug);
 
-  const relatedServers = server.category
-    ? (await getServersByCategory(server.category))
-        .filter((s) => s.id !== server.id)
-        .slice(0, 4)
-    : [];
-
   // Build install config for Claude Desktop (primary)
   let claudeConfig: string | null = null;
   let installCommand: string | null = null;
@@ -757,36 +752,20 @@ export default async function ServerDetailPage({
                 </ul>
               </div>
             )}
-
-            {/* Related Servers */}
-            {relatedServers.length > 0 && (
-              <div className="rounded-xl bg-neutral-900 border border-neutral-800 p-5 space-y-3">
-                <h2 className="text-sm font-semibold text-neutral-300 uppercase tracking-wider">
-                  Related Servers
-                </h2>
-                <ul className="space-y-2" role="list">
-                  {relatedServers.map((related) => (
-                    <li key={related.id}>
-                      <Link
-                        href={`/servers/${related.slug}`}
-                        className="flex flex-col gap-0.5 p-3 rounded-lg bg-neutral-800/50 hover:bg-neutral-800 border border-neutral-700/50 hover:border-neutral-700 transition-all duration-200 group"
-                      >
-                        <span className="text-white text-sm font-medium group-hover:text-blue-300 transition-colors duration-200 line-clamp-1">
-                          {related.name}
-                        </span>
-                        {related.description && (
-                          <span className="text-neutral-500 text-xs line-clamp-1">
-                            {related.description}
-                          </span>
-                        )}
-                      </Link>
-                    </li>
-                  ))}
-                </ul>
-              </div>
-            )}
           </aside>
         </div>
+
+        {/* Related servers — gated (isIndexable()) servers in the same
+            category only; self-excluded via currentSlug. Indexing-recovery
+            Slice 4: this is a required internal-linking surface on every
+            server detail page so gated servers stay reachable in <=3 clicks
+            from the homepage. */}
+        <RelatedServersForCategory
+          category={server.category ?? ""}
+          currentSlug={server.slug}
+          includeDegraded={false}
+          limit={6}
+        />
       </div>
     </div>
   );
diff --git a/apps/web/app/servers/page.tsx b/apps/web/app/servers/page.tsx
index 6b11a69..59e4aca 100644
--- a/apps/web/app/servers/page.tsx
+++ b/apps/web/app/servers/page.tsx
@@ -161,7 +161,15 @@ export default async function ServersPage({
                 </div>
               )}
 
-              {/* Pagination */}
+              {/* Pagination — indexing-recovery Slice 4: PaginationLink renders a
+                  real next/link <a href="/servers?page=N"> in the server-rendered
+                  HTML (not a client-side-only click handler), and this whole
+                  block lives inside the children passed through the "use client"
+                  ServersFilters wrapper, so it stays in the initial SSR payload.
+                  Default sort is stars-desc, which fronts the isIndexable()
+                  (gated) core (see queries.ts _getIndexableSitemapRows comment),
+                  so crawlers walking Prev/Next reach the gated set within the
+                  first pages without needing a JS-executing crawler. */}
               {result.totalPages > 1 && (
                 <div className="flex items-center justify-between mt-10 pt-6 border-t border-neutral-900">
                   <div className="flex items-center gap-2">
diff --git a/apps/web/components/RelatedServersForCategory.tsx b/apps/web/components/RelatedServersForCategory.tsx
index b5b2821..9317d69 100644
--- a/apps/web/components/RelatedServersForCategory.tsx
+++ b/apps/web/components/RelatedServersForCategory.tsx
@@ -16,11 +16,18 @@
  * This is intentionally a Server Component so it renders at build/request
  * time with no client-side JS overhead. All data fetching is SSR-only.
  * Emits data-conversion attributes for GA4 funnel tracking.
+ *
+ * Indexing-recovery Slice 4: the candidate pool is sourced from
+ * getIndexableServersByCategory(), which is pre-filtered to isIndexable()
+ * (Slice 2's quality gate) — this block must NEVER link a non-gated/thin
+ * server, regardless of includeDegraded. The separate quality_status
+ * manifest (STALE/BROKEN/HEALTHY) is layered on top purely for visual
+ * muting within the already-gated pool, not as the base filter.
  */
 
 import Link from "next/link";
 import { IconServer, IconArrowRight } from "@tabler/icons-react";
-import { getServersByCategory } from "@/lib/queries";
+import { getIndexableServersByCategory } from "@/lib/queries";
 import { getQualityStatus } from "@/lib/quality-status";
 import { ServerCard } from "@/components/ui/server-card";
 import { CATEGORY_LABELS } from "@mcpfind/shared";
@@ -53,17 +60,24 @@ export async function RelatedServersForCategory({
   if (!category) return null;
 
   // Guard: degrade gracefully when Supabase credentials are absent (CI / static builds).
+  // getIndexableServersByCategory() is pre-filtered to isIndexable() — every
+  // row here has already cleared Slice 2's quality gate.
   let allServers;
   try {
-    allServers = await getServersByCategory(category);
+    allServers = await getIndexableServersByCategory(category);
   } catch {
     return null;
   }
 
   // Build status map — one getQualityStatus call per server (avoids 2N lookups).
+  // This is the separate manifest-driven visual-muting gate, layered on top
+  // of the already-isIndexable()-gated pool from allServers.
   const statusMap = new Map(allServers.map((s) => [s.slug, getQualityStatus(s.slug)]));
 
-  // Filter candidates: correct category, not self, status gate.
+  // Filter candidates: correct category (defensive — query is already
+  // category-scoped), not self, and (when includeDegraded is false) the
+  // manifest status gate. The isIndexable() gate itself was already applied
+  // by getIndexableServersByCategory(), so it is never re-checked here.
   const candidates = allServers.filter((server) => {
     if (server.category !== category) return false;
     if (currentSlug && server.slug === currentSlug) return false;
@@ -71,7 +85,7 @@ export async function RelatedServersForCategory({
     if (!includeDegraded) {
       return status === "HEALTHY";
     }
-    // includeDegraded: include all statuses (undefined treated as available)
+    // includeDegraded: include all manifest statuses (undefined treated as available)
     return true;
   });
 
diff --git a/apps/web/lib/queries.ts b/apps/web/lib/queries.ts
index 084b53a..388e6c8 100644
--- a/apps/web/lib/queries.ts
+++ b/apps/web/lib/queries.ts
@@ -183,6 +183,51 @@ export const getTopServers = cache(
     )()
 );
 
+// Columns needed for both list display (ServerListItem) and the isIndexable()
+// signal check — SERVER_LIST_COLUMNS plus readme_content (excluded from the
+// list columns as a large blob, but required to evaluate signal 1).
+const INDEXABLE_LIST_COLUMNS = `${SERVER_LIST_COLUMNS},readme_content`;
+
+type IndexableListRow = ServerListItem & IndexableServerInput;
+
+/**
+ * Gated top-N servers by github_stars, filtered to isIndexable() — the
+ * homepage "top servers" linking surface (Slice 4, internal linking). Unlike
+ * getTopServers(), this never surfaces a thin/non-gated server as a link.
+ *
+ * Scans in SUPABASE_MAX-row windows (same pagination technique as
+ * _getIndexableSitemapRows) so the star-ordered scan isn't truncated by
+ * Supabase's 1,000-row cap before enough indexable rows are found.
+ */
+export const getIndexableTopServers = cache(
+  (limit: number): Promise<ServerListItem[]> =>
+    unstable_cache(
+      async () => {
+        const SUPABASE_MAX = 1000;
+        const results: ServerListItem[] = [];
+        for (let offset = 0; results.length < limit; offset += SUPABASE_MAX) {
+          const { data } = await supabase
+            .from('servers')
+            .select(INDEXABLE_LIST_COLUMNS)
+            .eq('registry_status', 'active')
+            .order('github_stars', { ascending: false })
+            .range(offset, offset + SUPABASE_MAX - 1);
+          if (!data || data.length === 0) break;
+          for (const row of data as IndexableListRow[]) {
+            if (isIndexable(row)) {
+              results.push(row as ServerListItem);
+              if (results.length >= limit) break;
+            }
+          }
+          if (data.length < SUPABASE_MAX) break;
+        }
+        return results.slice(0, limit);
+      },
+      ['indexable-top-servers', String(limit)],
+      { tags: ['servers'], revalidate: 3600 }
+    )()
+);
+
 // Columns needed to evaluate isIndexable() in addition to the sitemap's own
 // slug/canonical_slug/updated_at fields. readme_content is the one signal not
 // already in SERVER_LIST_COLUMNS (excluded there as a large blob) — safe to
@@ -318,6 +363,46 @@ export const getServersByCategory = cache(
     )()
 );
 
+/**
+ * Gated (isIndexable()) servers for a category, ordered by github_stars
+ * desc — the source of truth for category-hub internal linking (Slice 4).
+ * Unlike getServersByCategory(), this never links a thin/non-gated server,
+ * so it is safe to use for category hub pages and the related-servers block
+ * without re-checking isIndexable() at the call site.
+ *
+ * Paginates past Supabase's 1,000-row cap the same way
+ * _getIndexableSitemapRows/getIndexableServerSlugs do, so large categories
+ * are fully scanned rather than silently truncated at the raw-row cap.
+ */
+export const getIndexableServersByCategory = cache(
+  (category: string): Promise<ServerListItem[]> =>
+    unstable_cache(
+      async () => {
+        const SUPABASE_MAX = 1000;
+        const results: ServerListItem[] = [];
+        for (let offset = 0; ; offset += SUPABASE_MAX) {
+          const { data } = await supabase
+            .from('servers')
+            .select(INDEXABLE_LIST_COLUMNS)
+            .eq('category', category)
+            .eq('registry_status', 'active')
+            .order('github_stars', { ascending: false })
+            .range(offset, offset + SUPABASE_MAX - 1);
+          if (!data || data.length === 0) break;
+          for (const row of data as IndexableListRow[]) {
+            if (isIndexable(row)) {
+              results.push(row as ServerListItem);
+            }
+          }
+          if (data.length < SUPABASE_MAX) break;
+        }
+        return results;
+      },
+      ['indexable-servers-by-category', category],
+      { tags: ['servers', `category-${category}`], revalidate: 3600 }
+    )()
+);
+
 /**
  * Returns the true count of active servers for a given category.
  * Used in JSON-LD numberOfItems to reflect the real category size,