import { chunkTextIntelligently, ChunkingOptions } from 'intelligent-text-chunking';

const text = `
# Introduction
This is the introduction section with some content.

## Methodology
Here we describe our methodology in detail.

### Data Collection
We collected data from various sources.

## Results
Our results show significant improvements.
`;

const options: ChunkingOptions = {
  maxChunkSize: 500,
  overlapSize: 50,
  respectHeadingBoundaries: true
};

const chunks = chunkTextIntelligently(text, options);

console.log(`Generated ${chunks.length} chunks`);
chunks.forEach((chunk, index) => {
  console.log(`Chunk ${index + 1}:`);
  console.log(`  Heading: ${chunk.metadata.heading || 'None'}`);
  console.log(`  Level: ${chunk.metadata.headingLevel || 'N/A'}`);
  console.log(`  Words: ${chunk.metadata.wordCount}`);
  console.log(`  Text: ${chunk.text.substring(0, 100)}...`);
});

import { chunkPDFTextIntelligently } from 'intelligent-text-chunking';

// Extract text from PDF (using pdf2json or similar)
const pdfText = "Your PDF text content...";
const pageBreaks = [1000, 2000, 3000]; // Character positions of page breaks

const chunks = chunkPDFTextIntelligently(pdfText, pageBreaks, {
  maxChunkSize: 800,
  respectParagraphBoundaries: true
});

chunks.forEach(chunk => {
  console.log(`Page ${chunk.metadata.pageNumber}: ${chunk.text.substring(0, 50)}...`);
});

import { IntelligentChunker, ChunkingOptions } from 'intelligent-text-chunking';

const options: ChunkingOptions = {
  maxChunkSize: 1000,        // Maximum characters per chunk
  minChunkSize: 200,         // Minimum characters per chunk
  overlapSize: 100,          // Overlap between chunks
  respectSentenceBoundaries: true,    // Don't break mid-sentence
  respectParagraphBoundaries: true,  // Don't break mid-paragraph
  respectHeadingBoundaries: true,    // Don't break across headings
  preserveHeadingHierarchy: true,    // Maintain heading structure
  maxHeadingLevel: 6         // Maximum heading level to recognize
};

const chunker = new IntelligentChunker(options);
const chunks = chunker.chunkText(yourText);

interface ChunkingOptions {
  maxChunkSize?: number;           // Default: 1000
  minChunkSize?: number;           // Default: 200
  overlapSize?: number;            // Default: 100
  respectSentenceBoundaries?: boolean;    // Default: true
  respectParagraphBoundaries?: boolean;   // Default: true
  respectHeadingBoundaries?: boolean;     // Default: true
  preserveHeadingHierarchy?: boolean;     // Default: true
  maxHeadingLevel?: number;               // Default: 6
}

interface IntelligentChunk {
  text: string;
  metadata: ChunkMetadata;
}

interface ChunkMetadata {
  heading?: string;           // Detected heading text
  headingLevel?: number;      // Heading level (1-6)
  section?: string;           // Section name
  pageNumber?: number;        // Page number (for PDFs)
  chunkIndex: number;         // Index of this chunk
  totalChunks: number;       // Total number of chunks
  wordCount: number;         // Word count in chunk
  charCount: number;         // Character count in chunk
  startPosition: number;     // Start position in original text
  endPosition: number;       // End position in original text
}

const academicText = `
Abstract
This paper presents a novel approach to text processing.

1. Introduction
Text processing is a fundamental task in NLP.

1.1 Background
Previous work has shown...

2. Methodology
We propose a new algorithm...

3. Results
Our experiments demonstrate...

References
[1] Smith, J. (2023). Text Processing...
`;

const chunks = chunkTextIntelligently(academicText);
// Automatically detects Abstract, Introduction, Methodology, Results, References

const legalText = `
Article 1. Definitions
For the purposes of this agreement...

Section 2.1. Rights and Obligations
Each party shall have the right to...

Chapter III. Termination
This agreement may be terminated...
`;

const chunks = chunkTextIntelligently(legalText);
// Recognizes Article, Section, Chapter structure