/**
 * Comprueba si las webs declaradas por paquetes de npm siguen en pie.
 *
 * Usa el mismo criterio que outsidein, incluida la parte que costo aprender: un
 * fallo de red o un 403 de muro antibot NO es "roto", es "no lo se". Sin esa
 * distincion este barrido acusaria a cientos de proyectos que funcionan.
 *
 * Educado por diseno: una peticion por dominio a la vez, pausa entre tandas, y
 * nada que no haga un navegador.
 *
 *   node tools/check-homepages.js data/npm-homepages.csv data/npm-homepages-checked.csv
 */
const fs = require('fs');
const path = require('path');

const IN = process.argv[2];
const OUT = process.argv[3] || path.join(__dirname, '..', 'data', 'npm-homepages-checked.csv');
const CONCURRENCY = 5;
const TIMEOUT_MS = 15000;

const UA = 'Mozilla/5.0 (compatible; outsidein/1.0)';
const SOFT_404 = [/\bpage not found\b/i, /\b404\b[^\d]{0,20}(not found|error)/i,
  /\bthis page (?:does ?n[o']t exist|is no longer available)\b/i, /\bno longer available\b/i,
  /\bsorry, we (?:could ?n[o']t|can ?n[o']t) find\b/i, /\bdomain (?:is )?for sale\b/i,
  /\bbuy this domain\b/i, /\bparked (?:free )?(?:domain|courtesy)\b/i];
const BOT_WALL = [/just a moment/i, /checking your browser/i, /enable javascript and cookies/i,
  /blocked by network security/i, /access denied/i, /are you a robot/i, /captcha/i,
  /attention required/i, /request blocked/i];

const q = (v) => '"' + String(v == null ? '' : v).replace(/"/g, '""') + '"';
const titleOf = (h) => { const m = h.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
  return m ? m[1].replace(/\s+/g, ' ').trim().slice(0, 90) : ''; };

async function check(url) {
  const ctrl = new AbortController();
  const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
  try {
    const res = await fetch(url, { redirect: 'follow', credentials: 'omit', cache: 'no-store',
      headers: { 'User-Agent': UA, Accept: 'text/html,*/*' }, signal: ctrl.signal });
    const type = res.headers.get('content-type') || '';
    const body = /text|html/i.test(type) ? await res.text() : '';
    const title = titleOf(body);
    const server = res.headers.get('server') || '';
    const botWall = BOT_WALL.some((re) => re.test(body.slice(0, 4000)))
      || /cloudflare|akamai|imperva|datadome/i.test(server);

    let verdict;
    if (res.status === 404) verdict = 'gone';
    else if (res.status === 410) verdict = 'gone';
    else if (res.status === 403 || res.status === 401) verdict = botWall ? 'unknown' : 'blocked';
    else if (res.status === 429) verdict = 'unknown';
    else if (res.status >= 500) verdict = 'server-error';
    else if (res.status >= 400) verdict = 'error';
    else if (SOFT_404.some((re) => re.test(title))) verdict = 'soft-404';
    else verdict = 'ok';

    return { status: res.status, finalUrl: res.url, title, verdict,
      movedHost: (() => { try { return new URL(url).hostname !== new URL(res.url).hostname; } catch { return false; } })() };
  } catch (e) {
    return { status: 0, finalUrl: url, title: '', verdict: 'unknown',
      error: (e.name === 'AbortError' ? 'timeout' : e.message).slice(0, 60), movedHost: false };
  } finally { clearTimeout(timer); }
}

function parseCsv(text) {
  const rows = []; let row = [], cell = '', inQ = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i];
    if (inQ) { if (c === '"' && text[i + 1] === '"') { cell += '"'; i++; } else if (c === '"') inQ = false; else cell += c; }
    else if (c === '"') inQ = true;
    else if (c === ',') { row.push(cell); cell = ''; }
    else if (c === '\n') { row.push(cell); rows.push(row); row = []; cell = ''; }
    else if (c !== '\r') cell += c;
  }
  if (cell || row.length) { row.push(cell); rows.push(row); }
  return rows;
}

(async () => {
  const rows = parseCsv(fs.readFileSync(IN, 'utf8')).slice(1).filter((r) => r[0]);
  if (!fs.existsSync(OUT)) fs.writeFileSync(OUT, 'package,homepage,host,status,verdict,final_url,title,error\n');
  const done = new Set(fs.readFileSync(OUT, 'utf8').split('\n').slice(1)
    .map((l) => (l.match(/^"([^"]*)"/) || [])[1]).filter(Boolean));

  const todo = rows.filter((r) => !done.has(r[0]));
  console.log(`por comprobar: ${todo.length} (hechos: ${done.size})`);

  let i = 0;
  await Promise.all(Array.from({ length: CONCURRENCY }, async () => {
    while (i < todo.length) {
      const n = i++;
      const [pkg, , home, host] = todo[n];
      const r = await check(home);
      fs.appendFileSync(OUT, [pkg, home, host, r.status, r.verdict, r.finalUrl, r.title, r.error || '']
        .map(q).join(',') + '\n');
      if (n % 50 === 0) console.log(`  ${n}/${todo.length}`);
      await new Promise((s) => setTimeout(s, 400));
    }
  }));

  const out = parseCsv(fs.readFileSync(OUT, 'utf8')).slice(1).filter((r) => r[0]);
  const by = {};
  for (const r of out) by[r[4]] = (by[r[4]] || 0) + 1;
  console.log('\n' + out.length + ' webs comprobadas');
  Object.entries(by).sort((a, b) => b[1] - a[1])
    .forEach(([k, v]) => console.log(`  ${String(v).padStart(5)}  ${k}  (${(v / out.length * 100).toFixed(1)}%)`));
})();
