import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import fs from 'node:fs/promises'; import path from 'node:path'; import { createHash } from 'node:crypto'; const exec = promisify(execFile); export const root = process.cwd(); export const cacheRoot = path.join(root, '.cache'); export const sha = (s) => createHash('sha256').update(s).digest('hex'); export async function jsonFile(file, fallback) { try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch (error) { if (error.code === 'ENOENT' && fallback !== undefined) return fallback; throw error; } } export async function write(file, value) { await fs.mkdir(path.dirname(file), { recursive: true }); await fs.writeFile(file, value); } export async function writeJSON(file, value) { await write(file, JSON.stringify(value, null, 2) + '\n'); } export async function pool(values, limit, callback) { let index = 0; const results = new Array(values.length); await Promise.all(Array.from({ length: Math.min(limit, values.length) }, async () => { while (index < values.length) { const i = index++; results[i] = await callback(values[i], i); } })); return results; } const inFlight = new Map(); export async function resource(url, { refresh = false } = {}) { url = new URL(url).href; if (inFlight.has(url)) return inFlight.get(url); const task = (async () => { const key = sha(url); const folder = path.join(cacheRoot, 'http'); const metaFile = path.join(folder, key + '.json'); const dataFile = path.join(folder, key + '.bin'); const old = await jsonFile(metaFile, null); if (old && !refresh && old.status >= 200 && old.status < 400) return { ...old, body: await fs.readFile(dataFile) }; await fs.mkdir(folder, { recursive: true }); let lastError; for (let attempt = 0; attempt < 3; attempt++) { const temporary = dataFile + '.download'; try { const { stdout } = await exec('curl', ['-sS', '-L', '--compressed', '--connect-timeout', '10', '--max-time', '35', '-A', 'Mozilla/5.0 (compatible; StaticArchive/1.0)', '-o', temporary, '-w', '%{http_code}\n%{url_effective}\n%{content_type}', url], { maxBuffer: 1024 * 1024 }); const [code, effectiveUrl, type] = stdout.trim().split('\n'); const meta = { url, effectiveUrl, type: type || '', status: Number(code), fetchedAt: new Date().toISOString() }; const body = await fs.readFile(temporary); if (meta.status >= 500 && attempt < 2) continue; await fs.rename(temporary, dataFile); await writeJSON(metaFile, meta); return { ...meta, body }; } catch (error) { lastError = error; } } throw new Error(`Download failed: ${url}: ${lastError?.message}`); })(); inFlight.set(url, task); try { return await task; } finally { inFlight.delete(url); } } export function canonical(url, site) { const u = new URL(url, site.origin); if (site.hosts.includes(u.hostname)) { u.protocol = 'https:'; u.host = new URL(site.origin).host; } u.hash = ''; u.searchParams.delete('lang'); u.searchParams.delete('language'); if (['/index.php', '/index.html'].includes(u.pathname)) u.pathname = '/'; u.searchParams.sort(); return u.href; } export const blockedPath = (p) => /\/(?:console|clientarea|admin|cart)(?:\/|\.|$)/i.test(p) || /\/(?:login|regist(?:er)?|signup|home|finance|account|security|ticket|logout|reset_password|forget_password|shopping[_-]?car)(?:\/|\.|$)/i.test(p) || /\/plugin\/\d+\/(?!helpTotal)(?:.*)/i.test(p) || /\/cart[^/]*\.html$/i.test(p); export function routeOf(url, site) { const u = new URL(canonical(url, site)); if (u.pathname === '/') return 'index.html'; if (/\/cart\/goods\.htm/.test(u.pathname)) return 'cloud.html'; const id = u.searchParams.get('id'); if (/helpTotal\.htm/.test(u.pathname) && id) return `docs/${id}.html`; if (id && /news-details/.test(u.pathname)) return `news/${id}.html`; if (id && /announce-details/.test(u.pathname)) return `announce/${id}.html`; let name = u.pathname.replace(/^\//, '').replace(/\.(?:php|htm)$/, '.html'); if (u.search) { const slug = [...u.searchParams].filter(([key]) => key !== 'title').map(([key, value]) => `${key}-${value.replace(/[^a-zA-Z0-9_-]/g, '')}`).join('-'); if (slug) name = name.replace(/\.html$/, '') + '-' + slug + '.html'; } if (!path.extname(name)) name = name.replace(/\/$/, '') + '/index.html'; return name; } export const relative = (from, to) => path.posix.relative(path.posix.dirname(from), to) || path.posix.basename(to); export const escapeHTML = (s) => String(s ?? '').replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c]); export async function launchBrowser() { process.env.PLAYWRIGHT_BROWSERS_PATH = '0'; const { chromium } = await import('playwright'); return chromium.launch({ headless: true, args: ['--no-sandbox', '--disable-dev-shm-usage'] }); }