Module Values.WebCrawlerLimitsSource

The rate limits for the URLs that you want to crawl. You should be authorized to crawl the URLs.

Sourcetype nonrec t = {
  1. rateLimit : WebCrawlerLimitsRateLimitInteger.t option;
    (*

    The max rate at which pages are crawled, up to 300 per minute per host.

    *)
  2. maxPages : WebCrawlerLimitsMaxPagesInteger.t option;
    (*

    The max number of web pages crawled from your source URLs, up to 25,000 pages. If the web pages exceed this limit, the data source sync will fail and no web pages will be ingested.

    *)
}
Sourceval make : ?rateLimit:??? -> ?maxPages:??? -> unit -> t
Sourceval to_value : t -> [> `Structure of (string * [> `Integer of WebCrawlerLimitsRateLimitInteger.t ]) list ]
Sourceval to_query : t -> Awso.Client.Query.t
Sourceval of_xml : Awso.Xml.t -> t
Sourceval of_string : string -> t
Sourceval of_json : Yojson.Safe.t -> t
Sourceval to_json : t -> Yojson.Safe.t