readers

type AuthConfig struct {
	Type          string            // "bearer", "basic", "apikey", "oauth2", "custom"
	Token         string            // Bearer token or API key
	Username      string            // For basic auth
	Password      string            // For basic auth
	HeaderName    string            // Custom header name for API key
	HeaderValue   string            // Custom header value
	QueryParam    string            // Query parameter name for API key
	CustomHeaders map[string]string // Additional custom headers
}

type CSVReader struct {
	// contains filtered or unexported fields
}

type CSVReaderError struct {
	Op  string // Operation that failed (e.g., "read", "read_headers", "read_record")
	Err error  // Underlying error
}

type CSVReaderOptions struct {
	Comma            rune // Field delimiter (default ',')
	Comment          rune // Comment character (optional)
	FieldsPerRecord  int  // Number of expected fields per record (optional)
	LazyQuotes       bool // Allow lazy quotes in CSV
	TrimLeadingSpace bool // Trim leading space in fields
	HasHeaders       bool // Whether the first row is a header
}

type CSVReaderStats struct {
	RecordsRead     int64            // Total records read
	ReadDuration    time.Duration    // Total time spent reading
	LastReadTime    time.Time        // Time of last read
	NullValueCounts map[string]int64 // Count of null values per field
}

type HTTPReader struct {
	// contains filtered or unexported fields
}

type HTTPReaderError struct {
	Op         string // Operation that failed (e.g., "request", "auth", "parse", "pagination")
	StatusCode int    // HTTP status code if applicable
	URL        string // URL being accessed when error occurred
	Err        error  // Underlying error
}

type HTTPReaderOptions struct {
	Method           string            // HTTP method (default: GET)
	Headers          map[string]string // Additional headers
	QueryParams      map[string]string // Query parameters
	Body             io.Reader         // Request body for POST/PUT
	Auth             *AuthConfig       // Authentication configuration
	Pagination       *PaginationConfig // Pagination configuration
	Timeout          time.Duration     // Request timeout
	RetryAttempts    int               // Number of retry attempts
	RetryDelay       time.Duration     // Base delay between retries
	RateLimit        time.Duration     // Minimum time between requests
	ResponseFormat   string            // "json", "jsonl", "csv", "xml"
	DataPath         string            // JSON path to extract data array
	BufferSize       int               // Buffer size for response reading
	MaxResponseSize  int64             // Maximum response size in bytes
	FollowRedirects  bool              // Follow HTTP redirects
	ValidStatusCodes []int             // Valid HTTP status codes
	UserAgent        string            // User agent string
	AcceptEncoding   string            // Accept encoding header
	CustomClient     *http.Client      // Custom HTTP client
}

type HTTPReaderStats struct {
	RequestCount    int64            // Total HTTP requests made
	RecordsRead     int64            // Total records read
	BytesRead       int64            // Total bytes read
	ReadDuration    time.Duration    // Total time spent reading
	LastReadTime    time.Time        // Time of last read
	RetryCount      int64            // Number of retries performed
	RateLimitHits   int64            // Number of rate limit hits
	NullValueCounts map[string]int64 // Count of null values per field
	ResponseTimes   []time.Duration  // Response times for monitoring
}

type JSONReader struct {
	// contains filtered or unexported fields
}

type JSONReaderError struct {
	Op  string // Operation that failed (e.g., "read", "scan", "unmarshal")
	Err error  // Underlying error
}

type JSONReaderOptions struct {
	BufferSize int // Optional buffer size in bytes for scanning
}

type JSONReaderStats struct {
	RecordsRead     int64            // Total records read
	BytesRead       int64            // Total bytes read
	ReadDuration    time.Duration    // Total time spent reading
	LastReadTime    time.Time        // Time of last read
	NullValueCounts map[string]int64 // Count of null values per field
}

type MongoReadMode string

type MongoReader struct {
	// contains filtered or unexported fields
}

type MongoReaderError struct {
	Op         string // Operation that failed (e.g., "connect", "query", "decode", "aggregate")
	Collection string // Collection being accessed when error occurred
	Err        error  // Underlying error
}

type MongoReaderOptions struct {
	URI              string                   // MongoDB connection URI
	Database         string                   // Database name
	Collection       string                   // Collection name
	Mode             MongoReadMode            // Read mode
	Filter           bson.M                   // Query filter for find operations
	Projection       bson.M                   // Field projection
	Sort             bson.M                   // Sort specification
	Pipeline         []bson.M                 // Aggregation pipeline stages
	BatchSize        int32                    // Batch size for cursor
	Limit            int64                    // Maximum number of documents to read
	Skip             int64                    // Number of documents to skip
	Timeout          time.Duration            // Operation timeout
	MaxPoolSize      uint64                   // Connection pool size
	MinPoolSize      uint64                   // Minimum connections in pool
	MaxConnIdleTime  time.Duration            // Max idle time for connections
	ReadPreference   string                   // Read preference: primary, secondary, etc.
	ReadConcern      string                   // Read concern level
	AuthDatabase     string                   // Authentication database
	Username         string                   // Authentication username
	Password         string                   // Authentication password
	TLS              bool                     // Enable TLS
	TLSInsecure      bool                     // Skip TLS verification
	ReplicaSetName   string                   // Replica set name
	RetryReads       bool                     // Enable read retries
	RetryWrites      bool                     // Enable write retries
	Compressors      []string                 // Compression algorithms
	ZlibLevel        int                      // Zlib compression level
	CustomClientOpts []*options.ClientOptions // Custom client options
	AllowDiskUse     bool                     // Allow aggregation to use disk
	Hint             interface{}              // Index hint for queries
	MaxTimeMS        int64                    // Maximum execution time
	Comment          string                   // Query comment for profiling
	Collation        *options.Collation       // Collation options
}

type MongoReaderStats struct {
	RecordsRead     int64            // Total records read
	QueriesExecuted int64            // Total queries executed
	ReadDuration    time.Duration    // Total time spent reading
	LastReadTime    time.Time        // Time of last read
	BatchesRead     int64            // Number of batches processed
	BytesRead       int64            // Estimated bytes read
	NullValueCounts map[string]int64 // Count of null values per field
	ErrorCount      int64            // Number of errors encountered
}

type PaginationConfig struct {
	Type         string // "offset", "cursor", "page", "link_header", "none"
	LimitParam   string // Parameter name for limit/page size
	OffsetParam  string // Parameter name for offset
	PageParam    string // Parameter name for page number
	CursorParam  string // Parameter name for cursor
	PageSize     int    // Number of records per page
	MaxPages     int    // Maximum pages to fetch (0 = unlimited)
	NextURLField string // JSON field containing next page URL
	CursorField  string // JSON field containing next cursor
	TotalField   string // JSON field containing total count
	HasMoreField string // JSON field indicating more data available
}

type ParquetReader struct {
	// contains filtered or unexported fields
}

type ParquetReaderError struct {
	Op  string // Operation that failed (e.g., "read", "load_batch", "open_file", "schema")
	Err error  // Underlying error
}

type ParquetReaderOptions struct {
	BatchSize     int64
	Columns       []string
	UseArrowTypes bool              // Return native Arrow types vs Go primitives
	PreloadBatch  bool              // Preload next batch for better performance
	MemoryLimit   int64             // Memory usage limit for batches
	ParallelRead  bool              // Enable parallel column reading
	Metadata      map[string]string // Custom metadata filters
}

type PostgresReader struct {
	// contains filtered or unexported fields
}

type PostgresReaderError struct {
	Op  string // Operation that failed (e.g., "connect", "query", "scan", "read")
	Err error  // Underlying error
}

type PostgresReaderOption func(*PostgresReaderOptions)

type PostgresReaderOptions struct {
	DSN                 string            // Database connection string
	Query               string            // SQL query to execute
	Params              []interface{}     // Optional query parameters
	BatchSize           int               // Records to fetch per batch (used for cursor queries)
	ConnMaxLifetime     time.Duration     // Maximum connection lifetime
	ConnMaxIdleTime     time.Duration     // Maximum connection idle time
	MaxOpenConns        int               // Maximum open connections
	MaxIdleConns        int               // Maximum idle connections
	QueryTimeout        time.Duration     // Query execution timeout
	Metadata            map[string]string // Custom metadata
	UseCursor           bool              // Use server-side cursor for large results
	CursorName          string            // Name for the cursor (if UseCursor is true)
	HealthCheckInterval time.Duration
}

type PostgresReaderStats struct {
	RecordsRead     int64
	QueryDuration   time.Duration
	ReadDuration    time.Duration
	LastReadTime    time.Time
	NullValueCounts map[string]int64
	ConnectionTime  time.Duration
}

type ReaderOption func(*ParquetReaderOptions)

type ReaderOptionCSV func(*CSVReaderOptions)

type ReaderOptionHTTP func(*HTTPReaderOptions)

type ReaderOptionJSON func(*JSONReaderOptions)

type ReaderOptionMongo func(*MongoReaderOptions)

type ReaderOptionS3 func(*S3ReaderOptions)

type ReaderStats struct {
	RecordsRead     int64
	BatchesRead     int64
	BytesRead       int64
	ReadDuration    time.Duration
	LastReadTime    time.Time
	NullValueCounts map[string]int64
}

type S3Object struct {
	Key          string
	Size         int64
	LastModified time.Time
	ETag         string
	Metadata     map[string]string
}

type S3Reader struct {
	// contains filtered or unexported fields
}

type S3ReaderError struct {
	Op  string // Operation that failed (e.g., "list_objects", "get_object", "read")
	Err error  // Underlying error
}

type S3ReaderOptions struct {
	Bucket          string            // S3 bucket name
	Prefix          string            // Key prefix filter
	Suffix          string            // Key suffix filter (e.g., ".csv", ".json")
	MaxKeys         int32             // Maximum number of objects to list
	Region          string            // AWS region
	Profile         string            // AWS profile to use
	Credentials     aws.Credentials   // Explicit credentials
	EndpointURL     string            // Custom S3 endpoint (for S3-compatible services)
	ForcePathStyle  bool              // Use path-style addressing
	FilePattern     string            // Regex pattern for file matching
	Recursive       bool              // Process subdirectories recursively
	SortOrder       SortOrder         // Order to process files
	BatchSize       int               // Number of files to process in parallel
	BufferSize      int64             // Buffer size for reading objects
	Metadata        map[string]string // Custom metadata filters
	IncludeMetadata bool              // Include S3 object metadata in records
}

type S3ReaderStats struct {
	ObjectsListed  int64         // Total objects discovered
	ObjectsRead    int64         // Total objects successfully read
	RecordsRead    int64         // Total records read across all objects
	BytesRead      int64         // Total bytes read
	ReadDuration   time.Duration // Total time spent reading
	LastReadTime   time.Time     // Time of last read operation
	ObjectErrors   int64         // Number of objects that failed to read
	CurrentObject  string        // Currently processing object
	ProcessedFiles []string      // List of successfully processed files
}

type SortOrder string