95 lines
4.8 KiB
JavaScript
95 lines
4.8 KiB
JavaScript
import { execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import fs from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { createHash } from 'node:crypto';
|
|
|
|
const exec = promisify(execFile);
|
|
export const root = process.cwd();
|
|
export const cacheRoot = path.join(root, '.cache');
|
|
export const sha = (s) => createHash('sha256').update(s).digest('hex');
|
|
export async function jsonFile(file, fallback) {
|
|
try { return JSON.parse(await fs.readFile(file, 'utf8')); }
|
|
catch (error) { if (error.code === 'ENOENT' && fallback !== undefined) return fallback; throw error; }
|
|
}
|
|
export async function write(file, value) {
|
|
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
await fs.writeFile(file, value);
|
|
}
|
|
export async function writeJSON(file, value) { await write(file, JSON.stringify(value, null, 2) + '\n'); }
|
|
export async function pool(values, limit, callback) {
|
|
let index = 0;
|
|
const results = new Array(values.length);
|
|
await Promise.all(Array.from({ length: Math.min(limit, values.length) }, async () => {
|
|
while (index < values.length) { const i = index++; results[i] = await callback(values[i], i); }
|
|
}));
|
|
return results;
|
|
}
|
|
|
|
const inFlight = new Map();
|
|
export async function resource(url, { refresh = false } = {}) {
|
|
url = new URL(url).href;
|
|
if (inFlight.has(url)) return inFlight.get(url);
|
|
const task = (async () => {
|
|
const key = sha(url);
|
|
const folder = path.join(cacheRoot, 'http');
|
|
const metaFile = path.join(folder, key + '.json');
|
|
const dataFile = path.join(folder, key + '.bin');
|
|
const old = await jsonFile(metaFile, null);
|
|
if (old && !refresh && old.status >= 200 && old.status < 400) return { ...old, body: await fs.readFile(dataFile) };
|
|
await fs.mkdir(folder, { recursive: true });
|
|
let lastError;
|
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
const temporary = dataFile + '.download';
|
|
try {
|
|
const { stdout } = await exec('curl', ['-sS', '-L', '--compressed', '--connect-timeout', '10', '--max-time', '35', '-A', 'Mozilla/5.0 (compatible; StaticArchive/1.0)', '-o', temporary, '-w', '%{http_code}\n%{url_effective}\n%{content_type}', url], { maxBuffer: 1024 * 1024 });
|
|
const [code, effectiveUrl, type] = stdout.trim().split('\n');
|
|
const meta = { url, effectiveUrl, type: type || '', status: Number(code), fetchedAt: new Date().toISOString() };
|
|
const body = await fs.readFile(temporary);
|
|
if (meta.status >= 500 && attempt < 2) continue;
|
|
await fs.rename(temporary, dataFile);
|
|
await writeJSON(metaFile, meta);
|
|
return { ...meta, body };
|
|
} catch (error) { lastError = error; }
|
|
}
|
|
throw new Error(`Download failed: ${url}: ${lastError?.message}`);
|
|
})();
|
|
inFlight.set(url, task);
|
|
try { return await task; } finally { inFlight.delete(url); }
|
|
}
|
|
|
|
export function canonical(url, site) {
|
|
const u = new URL(url, site.origin);
|
|
if (site.hosts.includes(u.hostname)) { u.protocol = 'https:'; u.host = new URL(site.origin).host; }
|
|
u.hash = '';
|
|
u.searchParams.delete('lang');
|
|
u.searchParams.delete('language');
|
|
if (['/index.php', '/index.html'].includes(u.pathname)) u.pathname = '/';
|
|
u.searchParams.sort();
|
|
return u.href;
|
|
}
|
|
export const blockedPath = (p) => /\/(?:console|clientarea|admin|cart)(?:\/|\.|$)/i.test(p) || /\/(?:login|regist(?:er)?|signup|home|finance|account|security|ticket|logout|reset_password|forget_password|shopping[_-]?car)(?:\/|\.|$)/i.test(p) || /\/plugin\/\d+\/(?!helpTotal)(?:.*)/i.test(p) || /\/cart[^/]*\.html$/i.test(p);
|
|
export function routeOf(url, site) {
|
|
const u = new URL(canonical(url, site));
|
|
if (u.pathname === '/') return 'index.html';
|
|
if (/\/cart\/goods\.htm/.test(u.pathname)) return 'cloud.html';
|
|
const id = u.searchParams.get('id');
|
|
if (/helpTotal\.htm/.test(u.pathname) && id) return `docs/${id}.html`;
|
|
if (id && /news-details/.test(u.pathname)) return `news/${id}.html`;
|
|
if (id && /announce-details/.test(u.pathname)) return `announce/${id}.html`;
|
|
let name = u.pathname.replace(/^\//, '').replace(/\.(?:php|htm)$/, '.html');
|
|
if (u.search) {
|
|
const slug = [...u.searchParams].filter(([key]) => key !== 'title').map(([key, value]) => `${key}-${value.replace(/[^a-zA-Z0-9_-]/g, '')}`).join('-');
|
|
if (slug) name = name.replace(/\.html$/, '') + '-' + slug + '.html';
|
|
}
|
|
if (!path.extname(name)) name = name.replace(/\/$/, '') + '/index.html';
|
|
return name;
|
|
}
|
|
export const relative = (from, to) => path.posix.relative(path.posix.dirname(from), to) || path.posix.basename(to);
|
|
export const escapeHTML = (s) => String(s ?? '').replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c]);
|
|
export async function launchBrowser() {
|
|
process.env.PLAYWRIGHT_BROWSERS_PATH = '0';
|
|
const { chromium } = await import('playwright');
|
|
return chromium.launch({ headless: true, args: ['--no-sandbox', '--disable-dev-shm-usage'] });
|
|
}
|