33. Robots, AI Web Scrapers & Bot Mitigation Architecture

// 1. RFC 9309 Robots & AI Crawler Governance Engine
export const AI_SCRAPERS = ['GPTBot', 'ClaudeBot', 'PerplexityBot', 'Bytespider', 'CCBot', 'Google-Extended'] as const;
export function evaluateCrawlerPermission(userAgent: string, path: string): { allow: boolean; reason: string } {
  const isAiScraper = AI_SCRAPERS.some(bot => userAgent.toLowerCase().includes(bot.toLowerCase()));
  
  if (isAiScraper && path.startsWith('/labs/')) {
    return { allow: false, reason: 'EU Copyright Directive Art 4 TDM Reservation' };
  }
  return { allow: true, reason: 'Public Documentation Read Allowed' };
}
// 2. Client-Side Proof-of-Work (PoW) Anti-Bot Challenge (Altcha standard)
export async function solvePowChallenge(challenge: string, difficulty: number): Promise<number> {
  let nonce = 0;
  const encoder = new TextEncoder();
  while (true) {
    const data = encoder.encode(`${challenge}:${nonce}`);
    const hashBuffer = await crypto.subtle.digest('SHA-256', data);
    const hashArray = new Uint8Array(hashBuffer);
    
    // Check leading zero bits
    let leadingZeros = 0;
    for (const byte of hashArray) {
      if (byte === 0) leadingZeros += 8;
      else {
        leadingZeros += Math.clz32(byte) - 24;
        break;
      }
    }
    if (leadingZeros >= difficulty) return nonce;
    nonce++;
  }
}