scrapemate

type BrowserPage interface {
	// Goto navigates to a URL and waits for the specified state.
	// Returns the page response with status code, headers, and body.
	Goto(url string, waitUntil WaitUntilState) (*PageResponse, error)

	// URL returns the current page URL.
	URL() string

	// Content returns the full HTML content of the page.
	Content() (string, error)

	// Reload reloads the current page.
	Reload(waitUntil WaitUntilState) error

	// Screenshot takes a screenshot of the page.
	// If fullPage is true, captures the entire scrollable page.
	Screenshot(fullPage bool) ([]byte, error)

	// Eval executes JavaScript in the page context and returns the result.
	Eval(js string, args ...any) (any, error)

	// WaitForURL waits until the page URL matches the given pattern.
	WaitForURL(url string, timeout time.Duration) error

	// WaitForSelector waits for an element matching the selector to appear.
	WaitForSelector(selector string, timeout time.Duration) error

	// WaitForTimeout waits for the specified duration.
	// Note: This is generally discouraged in favor of waiting for specific conditions.
	WaitForTimeout(timeout time.Duration)

	// Locator creates a locator for finding elements matching the selector.
	Locator(selector string) Locator

	// Close closes the page and releases resources.
	Close() error

	// Unwrap returns the underlying page object (e.g., playwright.Page or *rod.Page).
	// This allows users to access library-specific features when needed.
	Unwrap() any
}

type Cacher interface {
	Close() error
	Get(ctx context.Context, key string) (Response, error)
	Set(ctx context.Context, key string, value *Response) error
}

type CsvCapable interface {
	CsvHeaders() []string
	CsvRow() []string
}

type HTMLParser interface {
	Parse(ctx context.Context, body []byte) (any, error)
}

type HTTPFetcher interface {
	Fetch(ctx context.Context, job IJob) Response
	Close() error
}

type IJob interface {
	fmt.Stringer
	// GetID returns the unique identifier of the job.
	GetID() string
	// GetParentID returns the parent id of the job
	GetParentID() string
	// GetMethod returns the http method to use
	GetMethod() string
	// GetBody returns the body of the request
	GetBody() []byte
	// GetURL returns the url to request
	GetURL() string
	// GetHeaders returns the headers to use
	GetHeaders() map[string]string
	// GetURLParams returns the url params to use
	GetURLParams() map[string]string
	// GetFullURL returns the full url to request
	// it includes the url params
	GetFullURL() string
	// GetTimeout returns the timeout of the job
	GetTimeout() time.Duration
	// GetPriority returns the priority of the job
	GetPriority() int
	// CheckResponse checks the response of the job
	DoCheckResponse(resp *Response) bool
	// GetActionOnResponse returns the action to perform on the response
	GetRetryPolicy() RetryPolicy
	// GetMaxRetries returns the max retries of the job
	GetMaxRetries() int
	// Process processes the job
	Process(ctx context.Context, resp *Response) (any, []IJob, error)
	// GetMaxRetryDelay returns the delay to wait before retrying
	GetMaxRetryDelay() time.Duration
	BrowserActions(ctx context.Context, page BrowserPage) Response
	// DoScreenshot takes a screenshot of the page
	// Only works if the scraper uses jsfetcher
	DoScreenshot() bool
	// GetCacheKey returns the key to use for caching
	GetCacheKey() string
	// UseInResults returns true if the job should be used in the results
	UseInResults() bool
	// ProcessOnFetchError returns true if the job should be processed even if the job failed
	ProcessOnFetchError() bool
}

type Job struct {
	// ID is an identifier for the job
	ID string
	// ParentID is the parent id of the job
	ParentID string
	// Method can be one valid HTTP method
	Method string
	// Body is the request's body
	Body []byte
	// URL is the url to sent a request
	URL string
	// Headers is the map of headers to use in HTTP
	Headers map[string]string
	// URLParams are the url parameters to use in the query string
	URLParams map[string]string
	// Timeout is the timeout of that job. By timeout we mean the time
	// it takes to finish a single crawl
	Timeout time.Duration
	// Priority is a number indicating the priority. By convention the higher
	// the priority
	Priority int
	// MaxRetries defines the maximum number of retries when a job fails
	MaxRetries int
	// CheckResponse is a function that takes as an input a Response and returns:
	// true: when the response is to be accepted
	// false: when the response is to be rejected
	// By default a response is accepted if status code is 200
	CheckResponse func(resp *Response) bool
	// RetryPolicy can be one of:
	// RetryJob: to retry the job untl it's successful
	// DiscardJob:for not accepted responses just discard them and do not retry the job
	// RefreshIP: Similar to RetryJob with an importan difference
	// 				Before the job is retried the IP is refreshed.
	RetryPolicy RetryPolicy
	// MaxRetryDelay By default when a job is rejected is retried with an exponential backof
	// for a MaxRetries numbers of time. If the sleep time between the retries is more than
	// MaxRetryDelay then it's capped to that. (Default is 2 seconds)
	MaxRetryDelay time.Duration
	// TakeScreenshot if true takes a screenshot of the page
	TakeScreenshot bool
	Response       Response
}

type JobProvider interface {
	Jobs(ctx context.Context) (<-chan IJob, <-chan error)
	// Push pushes a job to the job provider
	Push(ctx context.Context, job IJob) error
}

type Locator interface {
	// Click clicks on the first matching element.
	Click(timeout time.Duration) error
	// Count returns the number of matching elements.
	Count() (int, error)
	// First returns a locator for the first matching element.
	First() Locator
}

type PageResponse struct {
	// URL is the final URL after any redirects.
	URL string
	// StatusCode is the HTTP status code of the response.
	StatusCode int
	// Headers contains the response headers.
	Headers http.Header
	// Body contains the response body bytes.
	Body []byte
}

type Proxy struct {
	URL      string
	Username string
	Password string
}

type ProxyRotator interface {
	RoundTrip(req *http.Request) (*http.Response, error)
	Next() Proxy
	Proxies() []string
}

type Response struct {
	URL        string
	StatusCode int
	Headers    http.Header
	Duration   time.Duration
	Body       []byte
	Error      error
	Meta       map[string]any
	Screenshot []byte

	// Document is the parsed document
	// if you don't set an html parser the document will be nil
	// Since each html parser has it's own document type
	// the document is an interface
	// You need to cast it to the type of the parser you are using
	// For example if you are using goquery you need to cast it to *goquery.Document
	// If you are using the stdib parser net/html the it will be *html.Node
	Document any
}

type Result struct {
	Job  IJob
	Data any
}

type ResultWriter interface {
	Run(ctx context.Context, in <-chan Result) error
}

type RetryPolicy int

type ScrapeMate struct {
	// contains filtered or unexported fields
}

type WaitUntilState string