Merge pull request #2706 from appwrite/fix-SER-1068-search-console-issues

fix: Improve SEO canonicalization and block non-indexable pages

Author: HarshMN2345

server/sitemap.js

@@ -27,12 +27,31 @@ export async function sitemaps() {
     console.info('Preparing Sitemap...');
     const { manifest } = await import('../build/server/manifest.js');
     const threads = collectThreads().map((id) => `/threads/${id}`);
+
+    // Internal/auth paths to exclude
+    const INTERNAL_PATHS = ['/console/login', '/console/register', '/v1/'];
+
     const otherRoutes = manifest._.routes
         .filter((r) => r.params.length === 0)
         .map((r) => r.id)
-        .filter(
-            (id) => !id.startsWith('/threads/') && !id.endsWith('.json') && !id.endsWith('.xml')
-        );
+        .filter((id) => {
+            // Exclude threads (handled separately), JSON/XML endpoints
+            if (id.startsWith('/threads/') || id.endsWith('.json') || id.endsWith('.xml')) {
+                return false;
+            }
+
+            // Exclude any docs references that are not the canonical \"cloud\" version
+            if (id.startsWith('/docs/references/') && !id.startsWith('/docs/references/cloud/')) {
+                return false;
+            }
+
+            // Exclude internal/auth paths
+            if (INTERNAL_PATHS.some((path) => id.startsWith(path))) {
+                return false;
+            }
+
+            return true;
+        });
 
     mkdirSync(SITEMAP_DIR, { recursive: true });
     mkdirSync(THREADS_DIR, { recursive: true });

src/hooks.server.ts

@@ -222,12 +222,43 @@ const initSession: Handle = async ({ event, resolve }) => {
     return resolve(event);
 };
 
+/**
+ * SEO optimization: noindex internal/auth pages and staging subdomains
+ */
+const NOINDEX_PATHS = [
+    /^\/console\/login\/?$/,
+    /^\/console\/register\/?$/,
+    /^\/v1\/storage\//,
+    /^\/v1\//
+];
+
+// Block any staging/preview subdomains (e.g., *.cloud.appwrite.io, stage.*, etc.)
+const NOINDEX_HOSTS = [/\.cloud\.appwrite\.io$/i, /^stage\./i, /^fra\./i];
+
+const seoOptimization: Handle = async ({ event, resolve }) => {
+    const { url } = event;
+
+    // Check if this is a path or host that should not be indexed
+    const shouldNoindex =
+        NOINDEX_PATHS.some((re) => re.test(url.pathname)) ||
+        NOINDEX_HOSTS.some((re) => re.test(url.hostname));
+
+    const response = await resolve(event);
+
+    if (shouldNoindex) {
+        response.headers.set('x-robots-tag', 'noindex, nofollow');
+    }
+
+    return response;
+};
+
 export const handle = sequence(
     Sentry.sentryHandle(),
     markdownHandler,
     redirecter,
     wwwRedirecter,
     securityheaders,
-    initSession
+    initSession,
+    seoOptimization
 );
 export const handleError = Sentry.handleErrorWithSentry();

src/lib/utils/canonical.ts

@@ -1,24 +1,20 @@
 import { DEFAULT_HOST } from './metadata';
 
-/**
- * Canonical version for API references - should always point to this version
- */
-const CANONICAL_VERSION = 'cloud';
-
 /**
  * Normalizes the version in a docs reference path to the canonical version
  */
 function normalizeDocsVersion(pathname: string): string {
-    // Match patterns like /docs/references/1.7.x/... or /docs/references/0.15.x/... or /docs/references/1.8.x/...
+    // Match patterns like /docs/references/1.7.x/... or /docs/references/0.15.x/... or /docs/references/cloud/...
     const versionPattern = /^\/docs\/references\/([^/]+)\//;
     const match = pathname.match(versionPattern);
 
     if (match) {
         const currentVersion = match[1];
-        // Always point to cloud version for canonical URLs
-        // 'cloud' maps to 1.8.x server-side, and is the preferred canonical version
-        if (currentVersion !== CANONICAL_VERSION) {
-            return pathname.replace(versionPattern, `/docs/references/${CANONICAL_VERSION}/`);
+        // Use "cloud" as the single canonical docs version so we don't have to
+        // maintain a list of explicit versions. Anything that isn't "cloud"
+        // will point canonically to the /cloud/ variant.
+        if (currentVersion !== 'cloud') {
+            return pathname.replace(versionPattern, `/docs/references/cloud/`);
         }
     }
 
@@ -28,7 +24,7 @@ function normalizeDocsVersion(pathname: string): string {
 /**
  * Builds a canonical URL from a page URL, applying smart normalization:
  * - Fixes prerender origin issue (uses DEFAULT_HOST during prerendering)
- * - For docs references: always points to cloud version
+ * - For docs references: always points to the chosen canonical version
  * - Query strings are already stripped by using pathname (original behavior preserved)
  */
 export function getCanonicalUrl(url: URL): string {

src/routes/docs/references/[version]/+layout.svelte

@@ -0,0 +1,18 @@
+<script lang="ts">
+    import { page } from '$app/state';
+    import type { Snippet } from 'svelte';
+
+    const { children }: { children: Snippet } = $props();
+
+    // Check if the current version is not the canonical "cloud" version
+    const isNotCloud = $derived(page.params?.version && page.params.version !== 'cloud');
+</script>
+
+<svelte:head>
+    {#if isNotCloud}
+        <!-- Noindex old documentation versions to avoid duplicates -->
+        <meta name="robots" content="noindex, follow" />
+    {/if}
+</svelte:head>
+
+{@render children()}

src/routes/robots.txt/+server.ts

@@ -1,7 +1,28 @@
 import type { RequestHandler } from '@sveltejs/kit';
 
 const follow = `# robotstxt.org/
-User-agent: *`;
+User-agent: *
+
+# Block tracking parameters to avoid duplicate indexing
+Disallow: /*?utm_
+Disallow: /*&utm_
+Disallow: /*?ref=
+Disallow: /*&ref=
+Disallow: /*?trk=
+Disallow: /*&trk=
+Disallow: /*?adobe_mc=
+Disallow: /*&adobe_mc=
+
+# Block internal/auth pages
+Disallow: /console/login
+Disallow: /console/register
+Disallow: /v1/
+
+# Block all versioned docs references
+Disallow: /docs/references/*/
+
+# Allow canonical cloud docs
+Allow: /docs/references/cloud/`;
 
 const nofollow = `# robotstxt.org/
 User-agent: *