import https from "node:https"; import { lookup } from "node:dns/promises"; import { BlockList, isIP } from "node:net"; import type { Readable } from "node:stream"; import { fileTypeFromBuffer } from "file-type"; import sharp from "sharp"; import { z } from "zod"; export const SourceImportKind = z.enum(["product_page", "fabric_page", "fabric_card", "fabric_sample"]); export type SourceImportKind = z.infer; /** * Rodzaje pobierania szersze niż wiersze source_imports — "product_image" to * zdjęcia galerii vendora pobierane wewnątrz joba import_product. Nie jest * osobnym rodzajem importu (kolumna kind w DB ma CHECK na SourceImportKind). */ export const SourceFetchKind = z.enum(["product_page", "fabric_page", "fabric_card", "fabric_sample", "product_image", "leg_page", "leg_image"]); export type SourceFetchKind = z.infer; export const SOURCE_FETCH_VERSION = "source-fetch-v1"; const PRODUCT_HOSTS = new Set(["bobochicparis.com", "www.bobochicparis.com"]); const PRODUCT_IMAGE_HOSTS = new Set(["bobochicparis.com", "www.bobochicparis.com", "cdn.bobochicparis.com"]); const FABRIC_HOSTS = new Set(["sic.com.pl", "www.sic.com.pl", "davis.pl", "www.davis.pl", "pim.davis.pl", "lechfabrics.com", "www.lechfabrics.com"]); const LEG_HOSTS = new Set(["mir-tex24.pl", "www.mir-tex24.pl"]); const IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"]); const LIMITS = { timeoutMs: 30000, maxRedirects: 3, maxBytes: 25 * 1024 * 1024 }; export class SourceFetchError extends Error { constructor(public readonly code: string, message: string, public readonly retryable = false) { super(message); } } export function normalizeSourceUrl(raw: string, kind: SourceFetchKind) { const invalid = () => new SourceFetchError("invalid_url", "Niedozwolony URL źródła. Wymagany HTTPS, obsługiwany dostawca i publiczny adres bez danych logowania."); if (!SourceFetchKind.safeParse(kind).success || raw.length > 2048 || /[\s\\\u0000-\u001f\u007f]/.test(raw) || /%(?:0[0-9a-f]|1[0-9a-f]|7f|5c)/i.test(raw)) throw invalid(); let url: URL; try { url = new URL(raw); } catch { throw invalid(); } const hosts = kind === "product_page" ? PRODUCT_HOSTS : kind === "product_image" ? PRODUCT_IMAGE_HOSTS : kind.startsWith("leg_") ? LEG_HOSTS : FABRIC_HOSTS; if (url.protocol !== "https:" || !hosts.has(url.hostname) || url.port || url.username || url.password) throw invalid(); for (const [key, value] of url.searchParams) { if (!["v", "id_product_attribute", "lang"].includes(key) || !/^[\w.-]{1,64}$/.test(value) || url.searchParams.getAll(key).length !== 1) throw invalid(); } if (url.hash && (kind !== "product_page" || url.hash.length > 512 || !/^#\/[\w/,.-]*$/.test(url.hash))) throw invalid(); const requestedUrl = url.href; const fragment = url.hash || null; url.hash = ""; return { requestedUrl, fetchUrl: url.href, fragment }; } const blocked = new BlockList(); for (const [network, bits] of [ ["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8], ["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.0.2.0", 24], ["192.88.99.0", 24], ["192.168.0.0", 16], ["198.18.0.0", 15], ["198.51.100.0", 24], ["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4], ] as const) blocked.addSubnet(network, bits, "ipv4"); for (const [network, bits] of [["2001::", 23], ["2001:db8::", 32], ["2002::", 16], ["3fff::", 20]] as const) blocked.addSubnet(network, bits, "ipv6"); const globalV6 = new BlockList(); globalV6.addSubnet("2000::", 3, "ipv6"); export function isPublicAddress(address: string): boolean { const family = isIP(address); return family === 4 ? !blocked.check(address, "ipv4") : family === 6 && globalV6.check(address, "ipv6") && !blocked.check(address, "ipv6"); } interface Address { address: string; family: number } export interface SourceResponse { status: number; headers: Record; body: Readable } interface RequestInput { url: URL; address: Address; signal: AbortSignal } interface FetchDependencies { resolve: (hostname: string) => Promise; request: (input: RequestInput) => Promise; } export interface FetchedSource { buffer: Buffer; mime: string; finalUrl: string; redirects: Array<{ from: string; to: string; status: number }>; fetchedAt: string; } export type SourceFetcher = (url: string, kind: SourceFetchKind, signal?: AbortSignal) => Promise; function requestPinned({ url, address, signal }: RequestInput): Promise { return new Promise((resolve, reject) => { const req = https.request(url, { method: "GET", agent: false, family: address.family, servername: url.hostname, rejectUnauthorized: true, signal, maxHeaderSize: 16384, lookup: (_hostname, _options, callback) => callback(null, address.address, address.family), headers: { "User-Agent": "VilmalSourceImporter/1.0", Accept: "text/html,application/xhtml+xml,application/pdf,image/jpeg,image/png,image/webp", "Accept-Encoding": "identity" }, }, (res) => { const headers: Record = {}; for (const key of ["content-type", "content-length", "content-encoding", "location"]) { const value = res.headers[key]; headers[key] = Array.isArray(value) ? value.join(",") : value; } resolve({ status: res.statusCode ?? 0, headers, body: res }); }); req.once("error", reject); req.end(); }); } function abortable(promise: Promise, signal: AbortSignal): Promise { if (signal.aborted) { void promise.catch(() => {}); return Promise.reject(signal.reason); } return new Promise((resolve, reject) => { const abort = () => reject(signal.reason); signal.addEventListener("abort", abort, { once: true }); promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort)); }); } export async function validateSourceBytes(buffer: Buffer, declaredMime: string, kind: SourceFetchKind): Promise { if (!buffer.length) throw new SourceFetchError("invalid_mime", "Źródło zwróciło pusty plik."); const mime = declaredMime.split(";")[0]!.trim().toLowerCase(); if (kind.endsWith("_page")) { const prefix = buffer.subarray(0, 2048).toString("utf8"); if (!["text/html", "application/xhtml+xml"].includes(mime) || !/^\s*(?:]|<\?xml[\s]|