#!/usr/bin/env node
/**
 * outsidein - check your public pages the way a stranger sees them.
 *
 * Every platform shows the owner a different reality than it shows the public,
 * and the owner's version is always the flattering one. This fetches your pages
 * with no cookies, no auth header and no session, follows the redirects, and
 * tells you what an anonymous visitor actually gets.
 *
 *   node outsidein.js urls.txt
 *   node outsidein.js https://example.com/a https://example.com/b
 *   node outsidein.js --links https://example.com   (also check every link inside)
 *   node outsidein.js --json urls.txt               (machine readable)
 *
 * Exits non-zero if anything is broken, so it can go in a release script.
 *
 * No dependencies. Node 18+ (uses global fetch).
 */
'use strict';

const fs = require('fs');

const UA = 'Mozilla/5.0 (compatible; outsidein/1.0; +https://github.com/)';
const TIMEOUT_MS = 20000;
const CONCURRENCY = 6;

// A 200 does not mean the page exists. Plenty of sites serve their error
// template with a 200, and a storefront will happily return the shop front
// instead of the product you deleted. These are the tells.
const SOFT_404 = [
  /\bpage not found\b/i,
  /\b404\b[^\d]{0,20}(not found|error)/i,
  /\bthis page (?:does ?n[o']t exist|is no longer available)\b/i,
  /\bno longer available\b/i,
  /\bsorry, we (?:could ?n[o']t|can ?n[o']t) find\b/i,
  /\bthe page you (?:requested|were looking for)\b[^.]{0,40}\bnot\b/i,
];

/** A clean fetch: no credentials, no cache, redirects followed. */
async function getAnonymously(url) {
  const started = Date.now();
  const ctrl = new AbortController();
  // The timer is always cleared. Leaving it alive keeps the event loop busy,
  // and on Windows that makes Node abort on the way out.
  const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
  try {
    const res = await fetch(url, {
      redirect: 'follow',
      credentials: 'omit',
      cache: 'no-store',
      headers: { 'User-Agent': UA, Accept: 'text/html,application/xhtml+xml,*/*' },
      signal: ctrl.signal,
    });
    const type = res.headers.get('content-type') || '';
    // The body is read on errors too: that is where the signal lives for whether
    // a 403 comes from bot protection or from the site itself.
    const body = /text|html|json|xml|csv/i.test(type) ? await res.text() : '';
    return { url, status: res.status, finalUrl: res.url, type, body,
      server: res.headers.get('server') || '', ms: Date.now() - started };
  } catch (e) {
    return { url, status: 0, finalUrl: url, type: '', body: '', server: '',
      ms: Date.now() - started, error: e.name === 'AbortError' ? 'timeout' : e.message };
  } finally {
    clearTimeout(timer);
  }
}

const titleOf = (html) => {
  const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
  return m ? m[1].replace(/\s+/g, ' ').trim().slice(0, 80) : '';
};

const isNoindex = (html) =>
  /<meta[^>]+name=["']robots["'][^>]*content=["'][^"']*noindex/i.test(html);

/** Visible text of the document. Body only: the <head> of a modern site is tens
 *  of kilobytes of CSS and preloads, and measuring over it produces a false
 *  positive on every real page. */
function visibleText(html) {
  const m = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
  return (m ? m[1] : html)
    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
    .replace(/<!--[\s\S]*?-->/g, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/gi, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

const isClientRendered = (html) =>
  /<div[^>]+id=["'](root|app|__next|__nuxt)["']/i.test(html) ||
  (html.match(/<script\b/gi) || []).length > 8;

function contentProblem(r) {
  if (r.status !== 200 || !/html/i.test(r.type)) return null;
  const text = visibleText(r.body);
  const title = titleOf(r.body);

  // The title is the reliable signal. Searching the whole body marked as broken
  // any page that merely *talks about* 404s - including an article of mine on
  // exactly that subject.
  if (SOFT_404.some((re) => re.test(title))) {
    return { level: 'FAIL', note: `soft 404: replies 200 but the title says "${title}"` };
  }
  if (text.length < 400 && SOFT_404.some((re) => re.test(text.slice(0, 300)))) {
    return { level: 'FAIL', note: 'soft 404: replies 200 with an error page' };
  }
  if (text.length < 120) {
    // Empty can mean broken, or it can mean rendered in the browser. Say suspect,
    // not guilty: a checker that accuses too much never gets read twice.
    return { level: 'WARN', note: isClientRendered(r.body)
      ? 'empty without JavaScript - a crawler sees nothing here'
      : 'reachable but no visible content came back' };
  }
  return null;
}

/** Redirects that change nothing (http->https, www, trailing slash) are not
 *  news. Only landing somewhere genuinely different is. */
function sameDestination(a, b) {
  try {
    const x = new URL(a), y = new URL(b);
    const host = (u) => u.hostname.replace(/^www\./, '');
    const path = (u) => u.pathname.replace(/\/+$/, '');
    return host(x) === host(y) && path(x) === path(y);
  } catch { return a === b; }
}

// Signs that the thing refusing you is bot protection, not a broken site.
// Telling them apart matters: without this the tool calls sites broken that work
// perfectly for a person, which is worse than not checking at all.
const BOT_WALL = [
  /just a moment/i, /checking your browser/i, /enable javascript and cookies/i,
  /blocked by network security/i, /access denied/i, /are you a robot/i,
  /cf-browser-verification/i, /captcha/i, /request blocked/i, /attention required/i,
];
const looksLikeBotWall = (r) =>
  BOT_WALL.some((re) => re.test(r.body.slice(0, 4000))) ||
  /cloudflare|akamai|imperva|perimeterx|datadome/i.test(r.server || '');

function verdict(r) {
  // A network failure from here does not prove the page is broken: it can be DNS,
  // TLS, geoblocking, or simply not being let in. That is "I do not know".
  if (r.error || r.status === 0)
    return { level: 'WARN', note: `could not check from here (${r.error || 'no response'}) - open it yourself` };
  if (r.status === 429)
    return { level: 'WARN', note: 'rate-limited me, not necessarily broken - retry slower' };
  if (r.status === 404) return { level: 'FAIL', note: 'not found for the public' };
  if (r.status === 401 || r.status === 403) {
    return looksLikeBotWall(r)
      ? { level: 'WARN', note: `${r.status} to automated clients (bot protection) - a person is probably fine` }
      : { level: 'FAIL', note: `blocked (${r.status}) - visible only when logged in?` };
  }
  if (r.status >= 500) return { level: 'FAIL', note: `server error ${r.status}` };
  if (r.status >= 400) return { level: 'FAIL', note: `error ${r.status}` };
  const problem = contentProblem(r);
  if (problem) return problem;
  if (isNoindex(r.body)) return { level: 'WARN', note: 'live but marked noindex' };
  if (!sameDestination(r.url, r.finalUrl))
    return { level: 'WARN', note: `redirected to ${r.finalUrl}` };
  return { level: 'OK', note: '' };
}

/** Outbound links of a page, absolute and deduplicated. */
function linksIn(html, base) {
  const out = new Set();
  for (const m of html.matchAll(/<a\b[^>]*href=["']([^"'#]+)["']/gi)) {
    const href = m[1].trim();
    if (/^(mailto:|tel:|javascript:|data:)/i.test(href)) continue;
    try { out.add(new URL(href, base).toString()); } catch { /* broken href, ignored */ }
  }
  return [...out];
}

async function pool(items, worker, limit = CONCURRENCY) {
  const results = new Array(items.length);
  let i = 0;
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (i < items.length) {
      const n = i++;
      results[n] = await worker(items[n], n);
    }
  }));
  return results;
}

function readTargets(args) {
  const urls = [];
  for (const a of args) {
    if (/^https?:\/\//i.test(a)) { urls.push(a); continue; }
    if (!fs.existsSync(a)) { console.error(`no such file or url: ${a}`); process.exit(2); }
    for (const line of fs.readFileSync(a, 'utf8').split('\n')) {
      const s = line.trim();
      if (s && !s.startsWith('#')) urls.push(s);
    }
  }
  return [...new Set(urls)];
}

const COLOUR = process.stdout.isTTY && !process.env.NO_COLOR;
const paint = (s, c) => (COLOUR ? `[${c}m${s}[0m` : s);
const badge = (l) => (l === 'OK' ? paint('  OK  ', 32)
  : l === 'WARN' ? paint(' WARN ', 33) : paint(' FAIL ', 31));

(async () => {
  const args = process.argv.slice(2);
  const asJson = args.includes('--json');
  const withLinks = args.includes('--links');
  const targets = readTargets(args.filter((a) => !a.startsWith('--')));

  if (!targets.length) {
    console.error('usage: outsidein.js <urls.txt | url ...> [--links] [--json]');
    process.exit(2);
  }

  const rows = [];
  const checked = await pool(targets, getAnonymously);

  for (const r of checked) {
    const v = verdict(r);
    rows.push({ url: r.url, status: r.status, level: v.level, note: v.note,
      title: titleOf(r.body), ms: r.ms, from: null });
  }

  if (withLinks) {
    for (const r of checked) {
      if (!/html/i.test(r.type) || !r.body) continue;
      const links = linksIn(r.body, r.finalUrl).filter((u) => !targets.includes(u));
      const sub = await pool(links, getAnonymously);
      for (const s of sub) {
        const v = verdict(s);
        // On somebody else's page, only what is actually broken is interesting.
        if (v.level === 'OK') continue;
        rows.push({ url: s.url, status: s.status, level: v.level, note: v.note,
          title: titleOf(s.body), ms: s.ms, from: r.url });
      }
    }
  }

  const bad = rows.filter((r) => r.level === 'FAIL').length;
  const warn = rows.filter((r) => r.level === 'WARN').length;

  if (asJson) {
    console.log(JSON.stringify({ checked: rows.length, failed: bad, warned: warn, rows }, null, 2));
  } else {
    console.log('');
    for (const r of rows) {
      const code = r.status ? String(r.status) : '---';
      console.log(`${badge(r.level)} ${code.padStart(3)}  ${r.url}`);
      if (r.title) console.log(`             ${paint(r.title, 90)}`);
      if (r.note) console.log(`             ${paint('-> ' + r.note, r.level === 'FAIL' ? 31 : 33)}`);
      if (r.from) console.log(`             ${paint('linked from ' + r.from, 90)}`);
    }
    console.log('');
    console.log(`${rows.length} checked, ${bad} broken, ${warn} worth a look, ` +
      `all of it without a session.`);
    console.log('');
  }

  // exitCode rather than exit(): lets Node close the sockets still open instead
  // of dying mid-flight, which on Windows aborts the runtime itself.
  process.exitCode = bad ? 1 : 0;
})();
