135 lines
7.8 KiB
JavaScript
135 lines
7.8 KiB
JavaScript
import fs from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import * as cheerio from 'cheerio';
|
|
import { sites, initialPages } from './sites.mjs';
|
|
import { root, cacheRoot, resource, canonical, routeOf, blockedPath, pool, writeJSON, launchBrowser } from './lib.mjs';
|
|
|
|
const selected = process.argv.slice(2);
|
|
const refresh = selected.includes('--refresh');
|
|
const selectedSites = sites.filter(s => !selected.some(a => !a.startsWith('--')) || selected.includes(s.id));
|
|
let browser;
|
|
if (!selected.includes('--api-only')) browser = await launchBrowser();
|
|
|
|
try {
|
|
for (const site of selectedSites) {
|
|
const folder = path.join(cacheRoot, 'sites', site.id);
|
|
const report = { site: site.id, origin: site.origin, capturedAt: new Date().toISOString(), pages: [], api: {}, failures: [] };
|
|
const api = {};
|
|
async function getAPI(route, required = false) {
|
|
if (route in api) return api[route];
|
|
try {
|
|
const response = await resource(site.origin + '/console/v1' + route, { refresh });
|
|
const json = JSON.parse(response.body.toString());
|
|
if (response.status !== 200 || json.status !== 200) throw new Error(`${response.status}: ${json.msg || 'API unavailable'}`);
|
|
api[route] = json.data;
|
|
report.api[route] = { source: response.url, fetchedAt: response.fetchedAt };
|
|
return json.data;
|
|
} catch (e) {
|
|
report.failures.push({ kind: 'api', route, required, message: e.message.slice(0,300) });
|
|
api[route] = null;
|
|
return null;
|
|
}
|
|
}
|
|
await pool(['/common', '/news/type', '/announcement/type', '/help', '/product/4', '/product/4/mf_cloud/order_page'], 4, route => getAPI(route, ['/common','/help','/product/4/mf_cloud/order_page'].includes(route)));
|
|
const contents = {};
|
|
for (const type of ['news', 'announcement']) {
|
|
const list = [];
|
|
let count = 0;
|
|
for (let page = 1; page === 1 || list.length < count; page++) {
|
|
const data = await getAPI(`/${type}?page=${page}&limit=100`, true);
|
|
if (!data) break;
|
|
const chunk = data.list || [];
|
|
list.push(...chunk);
|
|
count = Number(data.count ?? list.length);
|
|
if (!chunk.length) break;
|
|
}
|
|
contents[type] = await pool(list, 4, async item => ({ ...item, detail: await getAPI(`/${type}/${item.id}`, true) }));
|
|
}
|
|
contents.help = await pool((api['/help']?.list || []).flatMap(type => (type.helps || []).map(item => ({ ...item, categoryId: type.id, category: type.name }))), 4, async item => ({ ...item, detail: await getAPI(`/help/${item.id}`, true) }));
|
|
const order = api['/product/4/mf_cloud/order_page'];
|
|
const lines = (order?.data_center || []).flatMap(country => country.city.flatMap(city => city.area.flatMap(area => area.line || [])));
|
|
await pool(lines, 4, line => getAPI(`/product/4/mf_cloud/line/${line.id}`));
|
|
await writeJSON(path.join(folder, 'data.json'), { site, capturedAt: report.capturedAt, api, contents });
|
|
console.log(`${site.id}: content collected; news=${contents.news.length}, announcements=${contents.announcement.length}, documents=${contents.help.length}`);
|
|
if (!browser) { await writeJSON(path.join(folder, 'capture-report.json'), report); continue; }
|
|
|
|
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, deviceScaleFactor: 1, ignoreHTTPSErrors: true, locale: 'zh-CN', serviceWorkers: 'block' });
|
|
let activeDownloads = 0;
|
|
const waiting = [];
|
|
const acquire = async () => { if (activeDownloads >= 8) await new Promise(resolve => waiting.push(resolve)); activeDownloads++; };
|
|
const release = () => { activeDownloads--; waiting.shift()?.(); };
|
|
await context.route('**/*', async route => {
|
|
const request = route.request();
|
|
const url = request.url();
|
|
if (!url.startsWith('http')) return route.continue();
|
|
if (request.method() !== 'GET' || /translate\.|translate\/|analytics|googletagmanager|baidu\.com\/hm|\/account(?:[/?]|$)|\/certification\//i.test(url)) return route.abort();
|
|
await acquire();
|
|
try {
|
|
const u = new URL(url);
|
|
if (site.hosts.includes(u.hostname)) { u.protocol = 'https:'; u.host = new URL(site.origin).host; }
|
|
const r = await resource(u.href);
|
|
await route.fulfill({ status: r.status, contentType: r.type || 'application/octet-stream', body: r.body, headers: { 'access-control-allow-origin': '*' } });
|
|
} catch { await route.abort().catch(() => {}); }
|
|
finally { release(); }
|
|
});
|
|
|
|
const pending = new Map();
|
|
const seen = new Set();
|
|
const pages = [];
|
|
const add = value => {
|
|
try {
|
|
const url = canonical(value, site);
|
|
const u = new URL(url);
|
|
if (!site.hosts.includes(u.hostname) || blockedPath(u.pathname) || /helpTotal\.htm/.test(u.pathname)) return;
|
|
if (u.pathname !== '/' && !/\.html$/.test(u.pathname)) return;
|
|
if (u.searchParams.has('search')) return;
|
|
if (!seen.has(url)) pending.set(url, routeOf(url, site));
|
|
} catch { /* Non-HTTP navigation is kept as a normal contact link. */ }
|
|
};
|
|
initialPages.forEach(add);
|
|
for (const type of ['news','announcement']) {
|
|
const short = type === 'announcement' ? 'announce' : 'news';
|
|
contents[type].forEach(item => add(`/${short}-details.html?id=${item.id}`));
|
|
(api[`/${type}/type`]?.list || []).forEach(item => add(`/${short}-classify.html?id=${item.id}`));
|
|
}
|
|
while (pending.size) {
|
|
const batch = [...pending].slice(0,3);
|
|
await pool(batch, 3, async ([url, file]) => {
|
|
pending.delete(url); seen.add(url);
|
|
const page = await context.newPage();
|
|
try {
|
|
const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 120000 });
|
|
if (!response || response.status() !== 200) throw new Error(`HTTP ${response?.status()}`);
|
|
await page.waitForLoadState('networkidle', { timeout: 20000 }).catch(() => {});
|
|
await page.evaluate(async () => {
|
|
for (let y = 0; y < Math.min(document.body.scrollHeight, 24000); y += 800) { window.scrollTo(0,y); await new Promise(r => setTimeout(r, 70)); }
|
|
window.scrollTo(0,0);
|
|
});
|
|
await page.waitForTimeout(400);
|
|
const html = await page.content();
|
|
if (/当前访问路由未定义|系统发生错误/.test(await page.title())) throw new Error('Source error page');
|
|
const $ = cheerio.load(html);
|
|
$('a[href]').each((_, el) => { const href = $(el).attr('href')?.trim(); if (href && !/^(?:#|javascript:|mailto:|tel:|data:)/i.test(href)) add(new URL(href, url).href); });
|
|
pages.push({ url, file, html, title: $('title').text(), capturedAt: new Date().toISOString() });
|
|
if (['index.html','dedicated.html','about.html'].includes(file)) {
|
|
await fs.mkdir(path.join(folder,'screenshots'), { recursive: true });
|
|
await page.screenshot({ path: path.join(folder,'screenshots',file+'.png'), fullPage: true });
|
|
}
|
|
report.pages.push({ url, file, status: 200, title: $('title').text() });
|
|
console.log(`${site.id}: ${file} captured (${pages.length} pages)`);
|
|
await writeJSON(path.join(folder, 'pages.json'), pages);
|
|
await writeJSON(path.join(folder, 'capture-report.json'), report);
|
|
} catch (e) {
|
|
report.failures.push({ kind: 'page', url, file, message: e.message.slice(0,300) });
|
|
console.log(`${site.id}: source unavailable: ${url}: ${e.message.slice(0,100)}`);
|
|
} finally { await page.close(); }
|
|
});
|
|
if (seen.size > 250) throw new Error(`Unexpected navigation expansion for ${site.id}; inspect capture-report.json`);
|
|
}
|
|
await context.close();
|
|
await writeJSON(path.join(folder, 'pages.json'), pages);
|
|
await writeJSON(path.join(folder, 'capture-report.json'), report);
|
|
console.log(`${site.id}: capture finished; ${pages.length} pages; ${report.failures.length} reported source failures`);
|
|
}
|
|
} finally { await browser?.close(); }
|