phishingclub/backend/vendor/github.com/enetx/surf/client.go

// Package surf provides a comprehensive HTTP client library with advanced features
// for web scraping, automation, and HTTP/3 support with various browser fingerprinting capabilities.
package surf

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net"
	"net/url"
	"reflect"
	"strings"

	"github.com/enetx/g"
	"github.com/enetx/http"
	"github.com/enetx/surf/header"
)

// Client represents a highly configurable HTTP client with middleware support,
// advanced transport options (HTTP/1.1, HTTP/2, HTTP/3), proxy handling,
// TLS fingerprinting, and comprehensive request/response processing capabilities.
type Client struct {
	cli       *http.Client           // Standard HTTP client for actual requests
	dialer    *net.Dialer            // Network dialer with optional custom DNS resolver
	builder   *Builder               // Associated builder for configuration
	transport http.RoundTripper      // HTTP transport (can be HTTP/1.1, HTTP/2, or HTTP/3)
	tlsConfig *tls.Config            // TLS configuration for secure connections
	reqMWs    *middleware[*Request]  // Priority-ordered request middlewares
	respMWs   *middleware[*Response] // Priority-ordered response middlewares
	boundary  func() g.String        // Custom boundary generator for multipart requests
}

// NewClient creates a new Client with sensible default settings including
// default dialer, TLS configuration, HTTP transport, and basic middleware.
func NewClient() *Client {
	cli := &Client{
		reqMWs:  newMiddleware[*Request](),
		respMWs: newMiddleware[*Response](),
	}

	defaultDialerMW(cli)
	defaultTLSConfigMW(cli)
	defaultTransportMW(cli)
	defaultClientMW(cli)
	redirectPolicyMW(cli)

	cli.reqMWs.add(0, defaultUserAgentMW)

	cli.respMWs.add(0, decodeBodyMW)

	return cli
}

// applyReqMW applies all registered request middlewares to the given request in priority order.
// Middlewares are sorted by priority before execution, and processing stops on first error.
func (c *Client) applyReqMW(req *Request) error {
	return c.reqMWs.run(req)
}

// applyRespMW applies all registered response middlewares to the given response in priority order.
// Middlewares are sorted by priority before execution, and processing stops on first error.
func (c *Client) applyRespMW(resp *Response) error {
	return c.respMWs.run(resp)
}

// CloseIdleConnections closes idle connections while keeping the client usable.
// Safe to call periodically to free resources during long-running operations.
func (c *Client) CloseIdleConnections() { c.cli.CloseIdleConnections() }

// Close completely shuts down the client and releases all resources.
// After calling Close, the client should not be used.
func (c *Client) Close() error {
	if closer, ok := c.transport.(interface{ Close() error }); ok {
		return closer.Close()
	}

	c.CloseIdleConnections()
	return nil
}

// GetClient returns http.Client used by the Client.
func (c *Client) GetClient() *http.Client { return c.cli }

// GetDialer returns the net.Dialer used by the Client.
func (c *Client) GetDialer() *net.Dialer { return c.dialer }

// GetTransport returns the http.transport used by the Client.
func (c *Client) GetTransport() http.RoundTripper { return c.transport }

// GetTLSConfig returns the tls.Config used by the Client.
func (c *Client) GetTLSConfig() *tls.Config { return c.tlsConfig }

// Builder returns a new Builder instance associated with this client.
// The builder allows for method chaining to configure various client options.
func (c *Client) Builder() *Builder {
	c.builder = &Builder{cli: c, cliMWs: newMiddleware[*Client]()}
	return c.builder
}

// Raw creates a new HTTP request using the provided raw data and scheme.
// The raw parameter should contain the raw HTTP request data as a string.
// The scheme parameter specifies the scheme (e.g., http, https) for the request.
func (c *Client) Raw(raw, scheme g.String) *Request {
	request := new(Request)

	req, err := http.ReadRequest(bufio.NewReader(raw.Trim().Append("\n\n").Reader()))
	if err != nil {
		request.err = err
		return request
	}

	req.RequestURI, req.URL.Scheme, req.URL.Host = "", scheme.Std(), req.Host

	request.request = req
	request.cli = c

	return request
}

// Get creates a new HTTP GET request for the specified URL.
// GET requests are used to retrieve data from a server.
func (c *Client) Get(rawURL g.String) *Request { return c.newRequest(http.MethodGet, rawURL) }

// Delete creates a new HTTP DELETE request for the specified URL.
// DELETE requests are used to remove a resource from a server.
func (c *Client) Delete(rawURL g.String) *Request { return c.newRequest(http.MethodDelete, rawURL) }

// Head creates a new HTTP HEAD request for the specified URL.
// HEAD requests are identical to GET but without the response body.
func (c *Client) Head(rawURL g.String) *Request { return c.newRequest(http.MethodHead, rawURL) }

// Post creates a new HTTP POST request for the specified URL.
// POST requests are used to submit data to a server.
func (c *Client) Post(rawURL g.String) *Request { return c.newRequest(http.MethodPost, rawURL) }

// Put creates a new HTTP PUT request for the specified URL.
// PUT requests are used to replace a resource on a server.
func (c *Client) Put(rawURL g.String) *Request { return c.newRequest(http.MethodPut, rawURL) }

// Patch creates a new HTTP PATCH request for the specified URL.
// PATCH requests are used to apply partial modifications to a resource.
func (c *Client) Patch(rawURL g.String) *Request { return c.newRequest(http.MethodPatch, rawURL) }

// Options creates a new HTTP OPTIONS request for the specified URL.
// OPTIONS requests are used to describe the communication options for a resource.
func (c *Client) Options(rawURL g.String) *Request { return c.newRequest(http.MethodOptions, rawURL) }

// Connect creates a new HTTP CONNECT request for the specified URL.
// CONNECT requests are used to establish a tunnel to the server.
func (c *Client) Connect(rawURL g.String) *Request { return c.newRequest(http.MethodConnect, rawURL) }

// Trace creates a new HTTP TRACE request for the specified URL.
// TRACE requests are used to perform a message loop-back test along the path to the target resource.
func (c *Client) Trace(rawURL g.String) *Request { return c.newRequest(http.MethodTrace, rawURL) }

// getCookies returns cookies for the specified URL.
func (c *Client) getCookies(rawURL g.String) []*http.Cookie {
	if c.cli.Jar == nil {
		return nil
	}

	parsedURL := parseURL(rawURL)
	if parsedURL.IsErr() {
		return nil
	}

	return c.cli.Jar.Cookies(parsedURL.Ok())
}

// setCookies sets cookies for the specified URL.
func (c *Client) setCookies(rawURL g.String, cookies []*http.Cookie) error {
	if c.cli.Jar == nil {
		return errors.New("cookie jar is not available")
	}

	parsedURL := parseURL(rawURL)
	if parsedURL.IsErr() {
		return parsedURL.Err()
	}

	c.cli.Jar.SetCookies(parsedURL.Ok(), cookies)

	return nil
}

// newRequest creates a new Request with the specified HTTP method and URL.
// It initializes the underlying http.Request and associates it with this client.
func (c *Client) newRequest(method string, rawURL g.String) *Request {
	request := new(Request)

	req, err := http.NewRequest(method, rawURL.Std(), nil)
	if err != nil {
		request.err = err
		return request
	}

	request.request = req
	request.cli = c
	return request
}

// Body sets the request body from various data types.
// Supported types include: []byte, string, g.String, g.Bytes, map[string]string,
// g.Map, g.MapOrd, and structs with json/xml tags.
// The Content-Type header is automatically detected and set based on the data.
// Returns the request for method chaining.
func (req *Request) Body(data any) *Request {
	if req.err != nil {
		return req
	}

	if data == nil {
		return req
	}

	body, contentType, err := buildBody(data)
	if err != nil {
		req.err = err
		return req
	}

	n := int64(-1)

	switch v := body.(type) {
	case *bytes.Reader:
		n = int64(v.Len())
	case *strings.Reader:
		n = int64(v.Len())
	case *bytes.Buffer:
		n = int64(v.Len())
	}

	req.request.ContentLength = n

	if n == 0 {
		req.request.Body = http.NoBody
	} else {
		if rc, ok := body.(io.ReadCloser); ok {
			req.request.Body = rc
		} else {
			req.request.Body = io.NopCloser(body)
		}
	}

	if contentType != "" {
		req.request.Header.Set(header.CONTENT_TYPE, contentType)
	}

	return req
}

// buildBody takes data of any type and, depending on its type, calls the appropriate method to
// build the request body.
// It returns an io.Reader, content type string, and an error if any.
func buildBody(data any) (io.Reader, string, error) {
	if data == nil {
		return nil, "", nil
	}

	if r, ok := data.(io.Reader); ok {
		return r, "", nil
	}

	switch d := data.(type) {
	case []byte:
		return buildByteBody(d)
	case g.Bytes:
		return buildByteBody(d)
	case string:
		return buildStringBody(d)
	case g.String:
		return buildStringBody(d)
	case map[string]string:
		return buildMapBody(d)
	case g.Map[string, string]:
		return buildMapBody(d)
	case g.Map[g.String, g.String]:
		return buildMapBody(d)
	case g.MapOrd[g.String, g.String]:
		return buildMapOrdBody(d)
	case g.MapOrd[string, string]:
		return buildMapOrdBody(d)
	default:
		return buildAnnotatedBody(data)
	}
}

// buildByteBody accepts a byte slice and returns an io.Reader, content type string, and an error
// if any.
// It detects the content type of the data and creates a bytes.Reader from the data.
func buildByteBody(data []byte) (io.Reader, string, error) {
	// raw data
	contentType := http.DetectContentType(data)
	reader := bytes.NewReader(data)

	return reader, contentType, nil
}

// buildStringBody accepts a string and returns an io.Reader, content type string, and an error if
// any.
// It detects the content type of the data and creates a strings.Reader from the data.
func buildStringBody[T ~string](data T) (io.Reader, string, error) {
	s := g.String(data)

	contentType := detectContentType(s.Bytes())

	if contentType == "text/plain; charset=utf-8" && isFormEncoded(s) {
		contentType = "application/x-www-form-urlencoded"
	}

	return s.Reader(), contentType, nil
}

// isFormEncoded checks if a string looks like valid URL-encoded form data.
// It verifies that the string consists of key=value pairs separated by &,
// where keys and values are non-empty and contain no whitespace.
func isFormEncoded(s g.String) bool {
	if s.IsEmpty() || !s.Contains("=") {
		return false
	}

	pairs := s.Split("&")

	for _, pair := range pairs {
		if pair.IsEmpty() {
			return false
		}

		eq := pair.IndexRune('=')
		if eq.Lte(0) {
			return false
		}

		if pair.ContainsAny(" \t\n\r") {
			return false
		}
	}

	return true
}

// detectContentType takes a string and returns the content type of the data by checking if it's a
// JSON or XML string.
func detectContentType(data []byte) string {
	var v any

	if json.Unmarshal(data, &v) == nil {
		return "application/json; charset=utf-8"
	} else if xml.Unmarshal(data, &v) == nil {
		return "application/xml; charset=utf-8"
	}

	// other types like pdf etc..
	return http.DetectContentType(data)
}

// buildMapBody accepts a map of string keys and values, and returns an io.Reader, content type
// string, and an error if any.
// It converts the map to a URL-encoded string and creates a strings.Reader from it.
func buildMapBody[T ~string, M ~map[T]T](m M) (io.Reader, string, error) {
	// post data map[string]string{"aaa": "bbb", "ddd": "ccc"}
	contentType := "application/x-www-form-urlencoded"
	form := make(url.Values)

	for key, value := range m {
		form.Add(string(key), string(value))
	}

	reader := g.String(form.Encode()).Reader()

	return reader, contentType, nil
}

// buildMapOrdBody takes an ordered map with string keys and values (g.MapOrd[T, T])
// and returns an io.Reader, a content type string, and an error if any.
// It encodes the map as an application/x-www-form-urlencoded string
// and creates a strings.Reader from the result.
//
// This is useful for building HTTP POST request bodies while preserving field order.
func buildMapOrdBody[T ~string](m g.MapOrd[T, T]) (io.Reader, string, error) {
	contentType := "application/x-www-form-urlencoded"

	var buf strings.Builder

	m.Iter().ForEach(func(key, value T) {
		if buf.Len() > 0 {
			buf.WriteByte('&')
		}
		buf.WriteString(url.QueryEscape(string(key)))
		buf.WriteByte('=')
		buf.WriteString(url.QueryEscape(string(value)))
	})

	reader := strings.NewReader(buf.String())

	return reader, contentType, nil
}

// buildAnnotatedBody accepts data of any type and returns an io.Reader, content type string, and
// an error if any. It detects the data format by checking the struct tags and encodes the data in
// the corresponding format (JSON or XML).
func buildAnnotatedBody(data any) (io.Reader, string, error) {
	var buf bytes.Buffer

	switch detectAnnotatedDataType(data) {
	case "json":
		if json.NewEncoder(&buf).Encode(data) == nil {
			return &buf, "application/json; charset=utf-8", nil
		}
	case "xml":
		if xml.NewEncoder(&buf).Encode(data) == nil {
			return &buf, "application/xml; charset=utf-8", nil
		}
	}

	return nil, "", errors.New("data type not detected")
}

// detectAnnotatedDataType takes data of any type and returns the data format as a string (either
// "json" or "xml") by checking the struct tags.
func detectAnnotatedDataType(data any) string {
	t := reflect.TypeOf(data)
	if t == nil {
		return ""
	}

	if t.Kind() == reflect.Pointer {
		t = t.Elem()
	}

	if t.Kind() != reflect.Struct {
		return ""
	}

	for i := range t.NumField() {
		tag := t.Field(i).Tag

		if _, ok := tag.Lookup("json"); ok {
			return "json"
		}

		if _, ok := tag.Lookup("xml"); ok {
			return "xml"
		}
	}

	return ""
}

// parseURL attempts to parse any supported rawURL type into a *url.URL.
// Returns an error if the type is unsupported or if parsing fails.
func parseURL(rawURL g.String) g.Result[*url.URL] {
	if rawURL.IsEmpty() {
		return g.Err[*url.URL](errors.New("URL is empty"))
	}

	parsedURL, err := url.Parse(rawURL.Std())
	if err != nil {
		return g.Err[*url.URL](fmt.Errorf("failed to parse URL '%s': %w", rawURL, err))
	}

	return g.Ok(parsedURL)
}