import { BaseParser, tools } from 'a-parser-types';

declare const require: any;
const fs = require('node:fs');
const path = require('node:path');

// ============================================================================
// Core Wayback CDX Interfaces & Helper Functions (Embedded)
// ============================================================================

export interface Capture {
    timestamp: string;
    original: string;
}

export interface SelectionOptions {
    fromYear: number;
    toYear: number;
    excludeQueryStrings: boolean;
    excludePathParts: string[];
    stripWww: boolean;
}

export interface ParsedCdx {
    rowsReceived: number;
    captures: Capture[];
}

function cleanTimestamp(value: unknown): string {
    const timestamp = String(value ?? '');
    return /^\d{14}$/.test(timestamp) ? timestamp : '';
}

export function normalizeQueryPattern(input: string, stripWww = true): string {
    const raw = String(input ?? '').trim();
    if (!raw) throw new Error('Query is empty');

    if (raw.startsWith('*.')) {
        throw new Error('Subdomain wildcards are not supported in prefix mode');
    }

    const cleanInput = raw.replace(/\*+$/, '');

    let url: URL;
    try {
        url = new URL(cleanInput.includes('://') ? cleanInput : `http://${cleanInput}`);
    } catch {
        throw new Error(`Invalid query URL: ${raw}`);
    }

    let hostname = url.hostname.toLowerCase().replace(/\.$/, '');
    if (stripWww) {
        hostname = hostname.replace(/^www\./, '');
    }
    if (!hostname || hostname.includes('..')) {
        throw new Error(`Invalid hostname: ${raw}`);
    }

    const port = (url.protocol === 'http:' && url.port === '80') ||
        (url.protocol === 'https:' && url.port === '443') ? '' : url.port;
    const hostWithPort = port ? `${hostname}:${port}` : hostname;

    let pathname = url.pathname.replace(/\/{2,}/g, '/');

    if (!pathname || pathname === '/') {
        return `${hostWithPort}/`;
    }

    // Exact page file e.g. /news.html or /about.php
    if (/\.[a-zA-Z0-9]{2,5}$/.test(pathname)) {
        return `${hostWithPort}${pathname}`;
    }

    // Folder prefix e.g. /articles/ or /articles
    const folderPath = pathname.endsWith('/') ? pathname : `${pathname}/`;
    return `${hostWithPort}${folderPath}`;
}

export function buildCdxUrl(
    targetPattern: string,
    fromYear: number,
    toYear: number,
    limit: number,
): string {
    const params = [
        `url=${encodeURIComponent(targetPattern)}`,
        'matchType=prefix',
        'collapse=urlkey',
        'output=json',
        'fl=timestamp,original',
        'filter=statuscode%3A200',
        'filter=mimetype%3Atext%2Fhtml',
        `from=${fromYear}`,
        `to=${toYear}`,
        `limit=${limit}`,
    ];
    return `https://web.archive.org/cdx/search/cdx?${params.join('&')}`;
}

export function parseCdx(data: string): ParsedCdx {
    let rows: unknown;
    try {
        rows = JSON.parse(data);
    } catch (error) {
        throw new Error(`CDX returned invalid JSON: ${(error as Error).message}`);
    }
    if (Array.isArray(rows) && rows.length === 0) {
        return { rowsReceived: 0, captures: [] };
    }
    if (!Array.isArray(rows) || !Array.isArray(rows[0])) {
        throw new Error('CDX returned no tabular data');
    }

    const header = rows[0].map(String);
    const timestampIndex = header.indexOf('timestamp');
    const originalIndex = header.indexOf('original');

    if ([timestampIndex, originalIndex].includes(-1)) {
        throw new Error(`Unexpected CDX header: ${header.join(',')}`);
    }

    const captures: Capture[] = [];
    for (const rawRow of rows.slice(1)) {
        if (!Array.isArray(rawRow)) continue;
        const timestamp = cleanTimestamp(rawRow[timestampIndex]);
        const original = String(rawRow[originalIndex] ?? '');

        if (!timestamp || !original) {
            continue;
        }
        captures.push({ timestamp, original });
    }
    return { rowsReceived: rows.length - 1, captures };
}

function normalizedOriginalKey(original: string, stripWww: boolean, excludeQueryStrings: boolean): string {
    const url = new URL(original);
    let hostname = url.hostname.toLowerCase();
    if (stripWww) hostname = hostname.replace(/^www\./, '');
    const port = (url.protocol === 'http:' && url.port === '80') ||
        (url.protocol === 'https:' && url.port === '443') ? '' : url.port;
    let pathname = url.pathname.replace(/\/{2,}/g, '/');
    if (pathname.length > 1) pathname = pathname.replace(/\/$/, '');
    const queryString = excludeQueryStrings ? '' : url.search;
    return `${hostname}${port ? `:${port}` : ''}${pathname}${queryString}`;
}

export function selectCaptures(
    captures: Capture[],
    options: SelectionOptions,
): Capture[] {
    const from = String(options.fromYear).padStart(4, '0');
    const to = String(options.toYear).padStart(4, '0');
    const excluded = options.excludePathParts
        .map((value) => value.trim().toLowerCase())
        .filter(Boolean);
    const latestByKey = new Map<string, Capture>();

    for (const capture of captures) {
        if (capture.timestamp.slice(0, 4) < from || capture.timestamp.slice(0, 4) > to) {
            continue;
        }

        let url: URL;
        try {
            url = new URL(capture.original);
        } catch {
            continue;
        }

        if (options.excludeQueryStrings && url.search) {
            continue;
        }

        const pathLower = url.pathname.toLowerCase();
        if (excluded.some((fragment) => pathLower.includes(fragment))) {
            continue;
        }

        const key = normalizedOriginalKey(capture.original, options.stripWww, options.excludeQueryStrings);
        latestByKey.set(key, capture);
    }

    const selected = Array.from(latestByKey.values());
    selected.sort((a, b) => a.original.localeCompare(b.original));
    return selected;
}

export function buildReplayUrl(timestamp: string, original: string, useIdReplay: boolean): string {
    const mode = useIdReplay ? 'id_' : '';
    return `https://web.archive.org/web/${timestamp}${mode}/${original}`;
}

export function positiveInteger(value: unknown, fallback: number, min = 1, max = Number.MAX_SAFE_INTEGER): number {
    const parsed = Number(value);
    if (!Number.isFinite(parsed) || parsed < min) return fallback;
    return Math.min(Math.floor(parsed), max);
}

// ============================================================================
// WaybackCDX Main Parser Class
// ============================================================================

interface ArticleResult {
    success?: number | boolean;
    code?: number | string;
    title?: string;
    content?: string;
    textContent?: string;
    length?: number | string;
    excerpt?: string;
    byline?: string;
    siteName?: string;
}

interface AcceptedArticle {
    title: string;
    content: string;
    text_content: string;
    length: number;
    excerpt: string;
    byline: string;
    site_name: string;
    timestamp: string;
    original_url: string;
    replay_url: string;
    markdown_file?: string;
}

export class JS_WaybackCDX extends BaseParser {
    static defaultConf: typeof BaseParser.defaultConf = {
        version: '0.6.1',
        useproxy: 1,
        cdxUseProxy: 0,
        proxyretries: 8,
        timeout: 60,
        max_size: 20971520,
        fromYear: 2020,
        toYear: 2026,
        maxArticles: 500,
        minLength: 10,
        markdownFiles: 1,
        results: {
            flat: [
                ['status', 'Run status'],
                ['error', 'Fatal error'],
                ['target', 'Normalized query target pattern'],
                ['cdx_rows', 'CDX rows received'],
                ['cdx_truncated', 'CDX response reached limit'],
                ['usable_urls', 'Unique snapshots discovered'],
                ['attempted', 'Article extraction requests attempted'],
                ['accepted', 'Accepted articles meeting quality criteria'],
                ['rejected', 'Rejected captures'],
                ['markdown_dir', 'Directory containing per-article Markdown files'],
                ['files_written', 'Markdown files written'],
            ],
            arrays: {
                articles: ['Accepted articles as JSON', [
                    ['json', 'Serialized article'],
                ]],
                urls: ['Discovered replay URLs', [
                    ['replay_url', 'Wayback replay URL'],
                    ['timestamp', 'Capture timestamp (YYYYMMDDhhmmss)'],
                    ['original_url', 'Original URL'],
                ]],
            },
        },
        results_format: "$articles.format('$json\\n')",
    };

    static editableConf: typeof BaseParser.editableConf = [
        ['fromYear', ['textfield', 'From year (e.g. 2020)']],
        ['toYear', ['textfield', 'To year (e.g. current year)']],
        ['maxArticles', ['textfield', 'Maximum articles to extract and save']],
        ['minLength', ['textfield', 'Minimum extracted text length (characters)']],
        ['markdownFiles', ['checkbox', 'Save articles as individual .md files with YAML frontmatter']],
        ['cdxUseProxy', ['checkbox', 'Use proxy for CDX index requests (unchecked: direct fast connection)']],
    ];

    static parserOptions: typeof BaseParser.parserOptions = [
        ['fromYear', 'From year (fromYear)',
            ['textfield', 'From year']
        ],
        ['toYear', 'To year (toYear)',
            ['textfield', 'To year']
        ],
        ['maxArticles', 'Maximum articles (maxArticles)',
            ['textfield', 'Maximum articles']
        ],
        ['minLength', 'Minimum text length (minLength)',
            ['textfield', 'Minimum text length']
        ],
        ['markdownFiles', 'Save as Markdown files (markdownFiles)',
            ['checkbox', 'Save as Markdown files']
        ],
        ['cdxUseProxy', 'CDX: Use proxy (cdxUseProxy)',
            ['checkbox', 'Use proxy for CDX index requests']
        ],
    ];

    async parse(set, results) {
        this.initializeResults(results);

        try {
            // Level 1: Subquery for an individual article capture
            if (set.lvl === 1 || set.timestamp) {
                return await this.processQueuedCapture(set, results);
            }

            // Level 0: CDX Discovery and queueing subqueries
            return await this.discoverAndQueueCaptures(set, results);
        } catch (error) {
            results.status = 'failed';
            results.error = error instanceof Error ? error.message : String(error);
            results.success = 0;
            this.logger.put(`Wayback extraction failed: ${results.error}`);
            return results;
        }
    }

    private async discoverAndQueueCaptures(set, results) {
        let rawQuery = String(set.query ?? '').trim();
        let fromYear = positiveInteger(this.conf.fromYear, 2020, 1996, 2100);
        let toYear = positiveInteger(this.conf.toYear, new Date().getUTCFullYear(), fromYear, 2100);
        let maxArticles = positiveInteger(this.conf.maxArticles, 500, 1, 100000);

        // Support space-separated per-query syntax:
        // "domain [limit]" or "domain [year] [limit]" or "domain [fromYear] [toYear] [limit]"
        const queryParts = rawQuery.split(/\s+/);
        if (queryParts.length >= 2) {
            rawQuery = queryParts[0];
            if (queryParts.length === 2 && /^\d+$/.test(queryParts[1])) {
                maxArticles = parseInt(queryParts[1], 10);
            } else if (queryParts.length === 3 && /^\d{4}$/.test(queryParts[1]) && /^\d+$/.test(queryParts[2])) {
                fromYear = parseInt(queryParts[1], 10);
                toYear = parseInt(queryParts[1], 10);
                maxArticles = parseInt(queryParts[2], 10);
            } else if (queryParts.length >= 4 && /^\d{4}$/.test(queryParts[1]) && /^\d{4}$/.test(queryParts[2]) && /^\d+$/.test(queryParts[3])) {
                fromYear = parseInt(queryParts[1], 10);
                toYear = parseInt(queryParts[2], 10);
                maxArticles = parseInt(queryParts[3], 10);
            }
        }

        const targetPattern = normalizeQueryPattern(rawQuery, true);
        results.target = targetPattern;

        const minLength = positiveInteger(this.conf.minLength, 10, 0, 1000000);
        const saveMarkdown = Boolean(this.conf.markdownFiles);
        const useProxy = Boolean(this.conf.useproxy);
        const cdxUseProxy = Boolean(this.conf.cdxUseProxy);
        const proxyRetries = positiveInteger(this.conf.proxyretries, 8, 0, 20);
        const timeoutSeconds = positiveInteger(this.conf.timeout, 60, 5, 300);

        // Proportional CDX query limit based directly on requested article count
        const cdxLimit = Math.min(100000, Math.max(1, maxArticles * 2));

        const cdxUrl = buildCdxUrl(targetPattern, fromYear, toYear, cdxLimit);
        const maxAttempts = Math.max(1, proxyRetries + 1);

        let parsed = null;
        let lastError: Error | null = null;

        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                this.logger.put(`Fetching CDX index for ${targetPattern} (attempt ${attempt}/${maxAttempts}, proxy=${cdxUseProxy ? 1 : 0}, limit=${cdxLimit})`);

                const response = await this.request('GET', cdxUrl, {}, {
                    use_proxy: cdxUseProxy ? 1 : 0,
                    proxyretries: cdxUseProxy ? proxyRetries : 0,
                    timeout: timeoutSeconds,
                    max_size: this.conf.max_size,
                    headers: {
                        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
                        'accept': 'application/json, text/plain, */*',
                    },
                });

                if (!response || !response.success || !response.data) {
                    throw new Error(`CDX API request failed or returned empty response (success=${response?.success ?? 0}, code=${response?.code ?? 0})`);
                }

                const dataStr = String(response.data).trim();

                // Check for HTML error pages (Cloudflare/Varnish 502/503/504 Gateway Time-out)
                if (dataStr.startsWith('<') || dataStr.toLowerCase().includes('<html') || dataStr.toLowerCase().includes('<!doctype')) {
                    throw new Error(`CDX returned HTML gateway error page (${response.code || 504})`);
                }

                // Validate JSON array structure
                if (!dataStr.startsWith('[') || !dataStr.endsWith(']')) {
                    throw new Error('CDX response was truncated (incomplete JSON stream)');
                }

                parsed = parseCdx(dataStr);
                break;
            } catch (err) {
                lastError = err instanceof Error ? err : new Error(String(err));
                this.logger.put(`CDX attempt ${attempt}/${maxAttempts} failed: ${lastError.message}`);
                if (attempt < maxAttempts) {
                    if (cdxUseProxy && typeof this.proxy !== 'undefined' && typeof this.proxy.next === 'function') {
                        try { await this.proxy.next(); } catch {}
                    }
                    await new Promise((resolve) => setTimeout(resolve, 1500 * attempt));
                }
            }
        }

        if (!parsed) {
            throw lastError || new Error('All CDX retry attempts failed');
        }

        results.cdx_rows = parsed.rowsReceived;
        results.cdx_truncated = parsed.rowsReceived >= cdxLimit ? 1 : 0;

        const selected = selectCaptures(parsed.captures, {
            fromYear,
            toYear,
            excludeQueryStrings: true,
            excludePathParts: [],
            stripWww: true,
        });

        results.usable_urls = selected.length;

        let taskId = set.task_id;
        if (!taskId && typeof tools !== 'undefined' && tools?.task?.id) {
            try {
                taskId = await tools.task.id();
            } catch {
                taskId = '';
            }
        }
        const safeTaskId = String(taskId || Date.now()).replace(/[^a-zA-Z0-9_-]/g, '_');
        let markdownDir = '';
        if (saveMarkdown) {
            markdownDir = path.join('results', `wayback-md-${safeTaskId}`);
            fs.mkdirSync(markdownDir, { recursive: true });
            results.markdown_dir = markdownDir;
        }

        if (!selected.length) {
            results.status = 'completed';
            results.success = 1;
            return results;
        }

        // Limit candidate queue to exact requested maxArticles
        const candidates = selected.slice(0, maxArticles);

        if (this.query && typeof this.query.add === 'function') {
            for (let i = 0; i < candidates.length; i++) {
                const cap = candidates[i];
                await this.query.add({
                    query: cap.original,
                    timestamp: cap.timestamp,
                    minLength,
                    saveMarkdown,
                    markdownDir,
                    taskId: safeTaskId,
                    lvl: 1,
                });
            }
            results.status = 'pages-queued';
            results.success = 1;
            results.SKIP = 1;
            this.logger.put(`Wayback queued ${candidates.length} subqueries for ${targetPattern}`);
            return results;
        }

        // Direct synchronous fallback
        for (const capture of candidates) {
            await this.processCaptureDirect(capture, minLength, saveMarkdown, markdownDir, results);
        }

        results.status = 'completed';
        results.success = 1;
        return results;
    }

    private async processQueuedCapture(set, results) {
        const timestamp = String(set.timestamp ?? '');
        const originalUrl = String(set.query ?? '');

        if (!timestamp || !originalUrl) {
            results.status = 'skipped-invalid';
            results.success = 1;
            results.SKIP = 1;
            return results;
        }

        results.attempted = 1;

        const minLength = positiveInteger(set.minLength ?? this.conf.minLength, 10, 0, 1000000);
        const saveMarkdown = set.saveMarkdown !== undefined ? Boolean(set.saveMarkdown) : Boolean(this.conf.markdownFiles);
        const markdownDir = String(set.markdownDir || path.join('results', `wayback-md-${set.taskId || 'default'}`));

        const replayUrl = buildReplayUrl(timestamp, originalUrl, true);
        if (results.urls && typeof results.urls.addElement === 'function') {
            results.urls.addElement({
                replay_url: replayUrl,
                timestamp: timestamp,
                original_url: originalUrl,
            });
        }

        let articleData: ArticleResult | null = null;
        try {
            articleData = await this.parser.request(
                'HTML::ArticleExtractor',
                'default',
                {
                    useproxy: this.conf.useproxy,
                    proxyretries: this.conf.proxyretries,
                    timeout: this.conf.timeout,
                },
                replayUrl,
            );
        } catch (extractorErr) {
            this.logger.put(`Extractor error for ${replayUrl}: ${extractorErr}`);
            results.rejected = 1;
            results.status = 'rejected-fetch-error';
            results.success = 1;
            results.SKIP = 1;
            return results;
        }

        if (!articleData || articleData.success === 0 || articleData.code === 596) {
            results.rejected = 1;
            results.status = 'rejected-fetch-error';
            results.success = 1;
            results.SKIP = 1;
            return results;
        }

        const title = String(articleData.title ?? '').trim();
        const content = String(articleData.content ?? '').trim();
        const textContent = String(articleData.textContent ?? '').trim();
        const length = Number(articleData.length) || textContent.length;

        if (!title || length < minLength || (!content && !textContent)) {
            results.rejected = 1;
            results.status = 'rejected-too-short';
            results.success = 1;
            results.SKIP = 1;
            return results;
        }

        const acceptedItem: AcceptedArticle = {
            title,
            content,
            text_content: textContent,
            length,
            excerpt: String(articleData.excerpt ?? '').trim(),
            byline: String(articleData.byline ?? '').trim(),
            site_name: String(articleData.siteName ?? '').trim(),
            timestamp,
            original_url: originalUrl,
            replay_url: replayUrl,
        };

        if (saveMarkdown && markdownDir) {
            try {
                fs.mkdirSync(markdownDir, { recursive: true });
                const safeTitle = title.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50);
                const fileName = `${timestamp}_${safeTitle}.md`;
                const filePath = path.join(markdownDir, fileName);

                const frontMatter = [
                    '---',
                    `title: ${JSON.stringify(title)}`,
                    `author: ${JSON.stringify(acceptedItem.byline)}`,
                    `site_name: ${JSON.stringify(acceptedItem.site_name)}`,
                    `length: ${length}`,
                    `timestamp: ${JSON.stringify(timestamp)}`,
                    `original_url: ${JSON.stringify(originalUrl)}`,
                    `archive_url: ${JSON.stringify(replayUrl)}`,
                    '---',
                    '',
                    `# ${title}`,
                    '',
                    `**Author:** ${acceptedItem.byline || 'Unknown'}  `,
                    `**Source:** ${acceptedItem.site_name || 'Web Archive'}  `,
                    `**Archive Replay:** [${replayUrl}](${replayUrl})`,
                    '',
                    '---',
                    '',
                    textContent || content,
                    '',
                ].join('\n');

                fs.writeFileSync(filePath, frontMatter, 'utf8');
                acceptedItem.markdown_file = fileName;
                results.files_written = 1;
            } catch (writeErr) {
                this.logger.put(`Failed to write markdown file: ${writeErr}`);
            }
        }

        if (results.articles && typeof results.articles.addElement === 'function') {
            results.articles.addElement({ json: JSON.stringify(acceptedItem) });
        } else if (Array.isArray(results.articles)) {
            results.articles.push({ json: JSON.stringify(acceptedItem) });
        }

        results.accepted = 1;
        results.status = 'completed';
        results.success = 1;
        return results;
    }

    private async processCaptureDirect(capture: any, minLength: number, saveMarkdown: boolean, markdownDir: string, results) {
        results.attempted = (results.attempted || 0) + 1;
        const replayUrl = buildReplayUrl(capture.timestamp, capture.original, true);
        let articleData: ArticleResult | null = null;
        try {
            articleData = await this.parser.request(
                'HTML::ArticleExtractor',
                'default',
                {
                    useproxy: this.conf.useproxy,
                    proxyretries: this.conf.proxyretries,
                    timeout: this.conf.timeout,
                },
                replayUrl,
            );
        } catch {
            results.rejected = (results.rejected || 0) + 1;
            return;
        }

        if (!articleData || articleData.success === 0 || articleData.code === 596) {
            results.rejected = (results.rejected || 0) + 1;
            return;
        }

        const title = String(articleData.title ?? '').trim();
        const content = String(articleData.content ?? '').trim();
        const textContent = String(articleData.textContent ?? '').trim();
        const length = Number(articleData.length) || textContent.length;

        if (!title || length < minLength || (!content && !textContent)) {
            results.rejected = (results.rejected || 0) + 1;
            return;
        }

        const acceptedItem: AcceptedArticle = {
            title,
            content,
            text_content: textContent,
            length,
            excerpt: String(articleData.excerpt ?? '').trim(),
            byline: String(articleData.byline ?? '').trim(),
            site_name: String(articleData.siteName ?? '').trim(),
            timestamp: capture.timestamp,
            original_url: capture.original,
            replay_url: replayUrl,
        };

        if (saveMarkdown && markdownDir) {
            try {
                fs.mkdirSync(markdownDir, { recursive: true });
                const safeTitle = title.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50);
                const fileName = `${capture.timestamp}_${safeTitle}.md`;
                const filePath = path.join(markdownDir, fileName);

                const frontMatter = [
                    '---',
                    `title: ${JSON.stringify(title)}`,
                    `author: ${JSON.stringify(acceptedItem.byline)}`,
                    `site_name: ${JSON.stringify(acceptedItem.site_name)}`,
                    `length: ${length}`,
                    `timestamp: ${JSON.stringify(capture.timestamp)}`,
                    `original_url: ${JSON.stringify(capture.original)}`,
                    `archive_url: ${JSON.stringify(replayUrl)}`,
                    '---',
                    '',
                    `# ${title}`,
                    '',
                    `**Author:** ${acceptedItem.byline || 'Unknown'}  `,
                    `**Source:** ${acceptedItem.site_name || 'Web Archive'}  `,
                    `**Archive Replay:** [${replayUrl}](${replayUrl})`,
                    '',
                    '---',
                    '',
                    textContent || content,
                    '',
                ].join('\n');

                fs.writeFileSync(filePath, frontMatter, 'utf8');
                acceptedItem.markdown_file = fileName;
                results.files_written = (results.files_written || 0) + 1;
            } catch {}
        }

        if (results.articles && typeof results.articles.addElement === 'function') {
            results.articles.addElement({ json: JSON.stringify(acceptedItem) });
        }

        results.accepted = (results.accepted || 0) + 1;
    }

    private initializeResults(results) {
        results.status = 'initialized';
        results.error = '';
        results.target = '';
        results.cdx_rows = 0;
        results.cdx_truncated = 0;
        results.usable_urls = 0;
        results.attempted = 0;
        results.accepted = 0;
        results.rejected = 0;
        results.markdown_dir = '';
        results.files_written = 0;
    }
}