import { WebExtractor } from '@anisirji/web-extractor';

const extractor = new WebExtractor({
  apiKey: 'your-firecrawl-api-key'
});

// Extract single page
const page = await extractor.extractPage('https://example.com');

console.log(page.title);
console.log(page.content);
console.log(page.metadata.wordCount);

const result = await extractor.extractWebsite('https://docs.example.com', {
  maxPages: 20,
  includeSubdomains: false,
  titlePrefix: 'Docs',
  maxDepth: 3
});

console.log(`Extracted ${result.pages.length} pages`);
console.log(`Success rate: ${result.stats.successRate}%`);
console.log(`Total words: ${result.stats.totalWords}`);

for (const page of result.pages) {
  console.log(`${page.title} - ${page.url}`);
}

new WebExtractor(config: WebExtractorConfig)

await extractor.extractPage('https://example.com', {
  onlyMainContent: true,  // Extract only main content
  format: 'markdown',     // 'markdown' | 'html' | 'text'
  waitFor: 1000          // Wait time before extraction (ms)
});

await extractor.extractWebsite('https://example.com', {
  maxPages: 10,                    // Maximum pages to scrape
  includeSubdomains: false,        // Include subdomains
  titlePrefix: 'My Site',          // Prefix for all titles
  maxDepth: 3,                     // Maximum crawl depth
  followExternalLinks: false,      // Follow external links
  includePatterns: [/\/docs\//],   // URL patterns to include
  excludePatterns: [/\/blog\//],   // URL patterns to exclude
  onlyMainContent: true,           // Extract only main content
  format: 'markdown'               // Output format
});

import {
  normalizeUrl,
  validateUrl,
  deduplicateUrls,
  isSameDomain,
  extractDomain
} from '@anisirji/web-extractor';

// Normalize URL
const normalized = normalizeUrl('https://Example.com/path/?b=2&a=1#hash', {
  lowercase: true,           // Convert to lowercase
  removeTrailingSlash: true, // Remove trailing slash
  removeFragment: true,      // Remove #hash
  sortQueryParams: true      // Sort query params
});
// => 'https://example.com/path?a=1&b=2'

// Validate URL
const urlObj = validateUrl('https://example.com'); // Returns URL object or throws

// Deduplicate URLs
const unique = deduplicateUrls([
  'https://example.com/page',
  'https://example.com/page/',
  'https://EXAMPLE.COM/page'
]);
// => ['https://example.com/page']

// Check same domain
isSameDomain('https://example.com', 'https://example.com/page'); // true
isSameDomain('https://example.com', 'https://other.com'); // false

// Extract domain
extractDomain('https://blog.example.com/page'); // => 'blog.example.com'

import {
  cleanContent,
  countWords,
  generateExcerpt,
  detectLanguage
} from '@anisirji/web-extractor';

// Clean content
const cleaned = cleanContent('  text\n\n\n\nmore text  ');
// => 'text\n\nmore text'

// Count words
countWords('Hello world from TermiX'); // => 4

// Generate excerpt
generateExcerpt('Very long content here...', 10);
// => 'Very long content here (first 10 words)...'

// Detect language
detectLanguage('This is an English text'); // => 'en'

const result = await extractor.extractWebsite('https://docs.example.com', {
  maxPages: 50,
  // Only include documentation pages
  includePatterns: [
    /\/docs\//,
    /\/api\//,
    /\/guides\//
  ],
  // Exclude blog and changelog
  excludePatterns: [
    /\/blog\//,
    /\/changelog\//
  ]
});

const result = await extractor.extractWebsite('https://example.com', {
  maxPages: 30
});

// Filter by word count
const substantialPages = result.pages.filter(
  page => page.metadata.wordCount > 500
);

// Group by language
const byLanguage = result.pages.reduce((acc, page) => {
  const lang = page.metadata.language || 'unknown';
  acc[lang] = acc[lang] || [];
  acc[lang].push(page);
  return acc;
}, {});

// Calculate reading time
const withReadingTime = result.pages.map(page => ({
  ...page,
  readingTimeMinutes: Math.ceil(page.metadata.wordCount / 200)
}));

const urls = [
  'https://example.com/page1',
  'https://example.com/page2',
  'https://example.com/page3'
];

const results = await Promise.allSettled(
  urls.map(url => extractor.extractPage(url))
);

const successful = results
  .filter(r => r.status === 'fulfilled')
  .map(r => r.value);

const failed = results
  .filter(r => r.status === 'rejected')
  .map((r, i) => ({ url: urls[i], error: r.reason }));

console.log(`Success: ${successful.length}, Failed: ${failed.length}`);

interface ExtractedPage {
  title: string;
  content: string;
  url: string;
  metadata: PageMetadata;
}

interface PageMetadata {
  scrapedAt: Date;
  sourceUrl: string;
  description?: string;
  wordCount: number;
  language?: string;
  statusCode?: number;
  [key: string]: any;  // Custom metadata
}

interface ExtractionResult {
  pages: ExtractedPage[];
  totalPages: number;
  failed: FailedExtraction[];
  stats: ExtractionStats;
}

interface ExtractionStats {
  duration: number;           // Total time in ms
  successRate: number;        // Success rate %
  totalWords: number;         // Total words extracted
  avgWordsPerPage: number;    // Average words per page
}