import fs from 'node:fs/promises';
import path from 'node:path';
import * as cheerio from 'cheerio';
import { sites, initialPages } from './sites.mjs';
import { root, cacheRoot, resource, canonical, routeOf, blockedPath, pool, writeJSON, launchBrowser } from './lib.mjs';

const selected = process.argv.slice(2);
const refresh = selected.includes('--refresh');
const selectedSites = sites.filter(s => !selected.some(a => !a.startsWith('--')) || selected.includes(s.id));
let browser;
if (!selected.includes('--api-only')) browser = await launchBrowser();

try {
  for (const site of selectedSites) {
    const folder = path.join(cacheRoot, 'sites', site.id);
    const report = { site: site.id, origin: site.origin, capturedAt: new Date().toISOString(), pages: [], api: {}, failures: [] };
    const api = {};
    async function getAPI(route, required = false) {
      if (route in api) return api[route];
      try {
        const response = await resource(site.origin + '/console/v1' + route, { refresh });
        const json = JSON.parse(response.body.toString());
        if (response.status !== 200 || json.status !== 200) throw new Error(`${response.status}: ${json.msg || 'API unavailable'}`);
        api[route] = json.data;
        report.api[route] = { source: response.url, fetchedAt: response.fetchedAt };
        return json.data;
      } catch (e) {
        report.failures.push({ kind: 'api', route, required, message: e.message.slice(0,300) });
        api[route] = null;
        return null;
      }
    }
    await pool(['/common', '/news/type', '/announcement/type', '/help', '/product/4', '/product/4/mf_cloud/order_page'], 4, route => getAPI(route, ['/common','/help','/product/4/mf_cloud/order_page'].includes(route)));
    const contents = {};
    for (const type of ['news', 'announcement']) {
      const list = [];
      let count = 0;
      for (let page = 1; page === 1 || list.length < count; page++) {
        const data = await getAPI(`/${type}?page=${page}&limit=100`, true);
        if (!data) break;
        const chunk = data.list || [];
        list.push(...chunk);
        count = Number(data.count ?? list.length);
        if (!chunk.length) break;
      }
      contents[type] = await pool(list, 4, async item => ({ ...item, detail: await getAPI(`/${type}/${item.id}`, true) }));
    }
    contents.help = await pool((api['/help']?.list || []).flatMap(type => (type.helps || []).map(item => ({ ...item, categoryId: type.id, category: type.name }))), 4, async item => ({ ...item, detail: await getAPI(`/help/${item.id}`, true) }));
    const order = api['/product/4/mf_cloud/order_page'];
    const lines = (order?.data_center || []).flatMap(country => country.city.flatMap(city => city.area.flatMap(area => area.line || [])));
    await pool(lines, 4, line => getAPI(`/product/4/mf_cloud/line/${line.id}`));
    await writeJSON(path.join(folder, 'data.json'), { site, capturedAt: report.capturedAt, api, contents });
    console.log(`${site.id}: content collected; news=${contents.news.length}, announcements=${contents.announcement.length}, documents=${contents.help.length}`);
    if (!browser) { await writeJSON(path.join(folder, 'capture-report.json'), report); continue; }

    const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, deviceScaleFactor: 1, ignoreHTTPSErrors: true, locale: 'zh-CN', serviceWorkers: 'block' });
    let activeDownloads = 0;
    const waiting = [];
    const acquire = async () => { if (activeDownloads >= 8) await new Promise(resolve => waiting.push(resolve)); activeDownloads++; };
    const release = () => { activeDownloads--; waiting.shift()?.(); };
    await context.route('**/*', async route => {
      const request = route.request();
      const url = request.url();
      if (!url.startsWith('http')) return route.continue();
      if (request.method() !== 'GET' || /translate\.|translate\/|analytics|googletagmanager|baidu\.com\/hm|\/account(?:[/?]|$)|\/certification\//i.test(url)) return route.abort();
      await acquire();
      try {
        const u = new URL(url);
        if (site.hosts.includes(u.hostname)) { u.protocol = 'https:'; u.host = new URL(site.origin).host; }
        const r = await resource(u.href);
        await route.fulfill({ status: r.status, contentType: r.type || 'application/octet-stream', body: r.body, headers: { 'access-control-allow-origin': '*' } });
      } catch { await route.abort().catch(() => {}); }
      finally { release(); }
    });

    const pending = new Map();
    const seen = new Set();
    const pages = [];
    const add = value => {
      try {
        const url = canonical(value, site);
        const u = new URL(url);
        if (!site.hosts.includes(u.hostname) || blockedPath(u.pathname) || /helpTotal\.htm/.test(u.pathname)) return;
        if (u.pathname !== '/' && !/\.html$/.test(u.pathname)) return;
        if (u.searchParams.has('search')) return;
        if (!seen.has(url)) pending.set(url, routeOf(url, site));
      } catch { /* Non-HTTP navigation is kept as a normal contact link. */ }
    };
    initialPages.forEach(add);
    for (const type of ['news','announcement']) {
      const short = type === 'announcement' ? 'announce' : 'news';
      contents[type].forEach(item => add(`/${short}-details.html?id=${item.id}`));
      (api[`/${type}/type`]?.list || []).forEach(item => add(`/${short}-classify.html?id=${item.id}`));
    }
    while (pending.size) {
      const batch = [...pending].slice(0,3);
      await pool(batch, 3, async ([url, file]) => {
        pending.delete(url); seen.add(url);
        const page = await context.newPage();
        try {
          const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 120000 });
          if (!response || response.status() !== 200) throw new Error(`HTTP ${response?.status()}`);
          await page.waitForLoadState('networkidle', { timeout: 20000 }).catch(() => {});
          await page.evaluate(async () => {
            for (let y = 0; y < Math.min(document.body.scrollHeight, 24000); y += 800) { window.scrollTo(0,y); await new Promise(r => setTimeout(r, 70)); }
            window.scrollTo(0,0);
          });
          await page.waitForTimeout(400);
          const html = await page.content();
          if (/当前访问路由未定义|系统发生错误/.test(await page.title())) throw new Error('Source error page');
          const $ = cheerio.load(html);
          $('a[href]').each((_, el) => { const href = $(el).attr('href')?.trim(); if (href && !/^(?:#|javascript:|mailto:|tel:|data:)/i.test(href)) add(new URL(href, url).href); });
          pages.push({ url, file, html, title: $('title').text(), capturedAt: new Date().toISOString() });
          if (['index.html','dedicated.html','about.html'].includes(file)) {
            await fs.mkdir(path.join(folder,'screenshots'), { recursive: true });
            await page.screenshot({ path: path.join(folder,'screenshots',file+'.png'), fullPage: true });
          }
          report.pages.push({ url, file, status: 200, title: $('title').text() });
          console.log(`${site.id}: ${file} captured (${pages.length} pages)`);
          await writeJSON(path.join(folder, 'pages.json'), pages);
          await writeJSON(path.join(folder, 'capture-report.json'), report);
        } catch (e) {
          report.failures.push({ kind: 'page', url, file, message: e.message.slice(0,300) });
          console.log(`${site.id}: source unavailable: ${url}: ${e.message.slice(0,100)}`);
        } finally { await page.close(); }
      });
      if (seen.size > 250) throw new Error(`Unexpected navigation expansion for ${site.id}; inspect capture-report.json`);
    }
    await context.close();
    await writeJSON(path.join(folder, 'pages.json'), pages);
    await writeJSON(path.join(folder, 'capture-report.json'), report);
    console.log(`${site.id}: capture finished; ${pages.length} pages; ${report.failures.length} reported source failures`);
  }
} finally { await browser?.close(); }