[httpx] replace searx.poolrequests by searx.network

settings.yml:

* outgoing.networks:
   * can contains network definition
   * propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
     keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
   * retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
   * local_addresses can be "192.168.0.1/24" (it supports IPv6)
   * support_ipv4 & support_ipv6: both True by default
     see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
   * either a full network description
   * either reference an existing network

* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)

1 files changed, 66 insertions, 0 deletions

diff --git a/searx/network/raise_for_httperror.py b/searx/network/raise_for_httperror.py
new file mode 100644
index 00000000..bd12df9a
--- /dev/null
+++ b/searx/network/raise_for_httperror.py
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Raise exception for an HTTP response is an error.
+"""
+from searx.exceptions import (SearxEngineCaptchaException, SearxEngineTooManyRequestsException,
+                              SearxEngineAccessDeniedException)
+
+
+def is_cloudflare_challenge(resp):
+    if resp.status_code in [429, 503]:
+        if ('__cf_chl_jschl_tk__=' in resp.text)\
+           or ('/cdn-cgi/challenge-platform/' in resp.text
+               and 'orchestrate/jsch/v1' in resp.text
+               and 'window._cf_chl_enter(' in resp.text):
+            return True
+    if resp.status_code == 403 and '__cf_chl_captcha_tk__=' in resp.text:
+        return True
+    return False
+
+
+def is_cloudflare_firewall(resp):
+    return resp.status_code == 403 and '<span class="cf-error-code">1020</span>' in resp.text
+
+
+def raise_for_cloudflare_captcha(resp):
+    if resp.headers.get('Server', '').startswith('cloudflare'):
+        if is_cloudflare_challenge(resp):
+            # https://support.cloudflare.com/hc/en-us/articles/200170136-Understanding-Cloudflare-Challenge-Passage-Captcha-
+            # suspend for 2 weeks
+            raise SearxEngineCaptchaException(message='Cloudflare CAPTCHA', suspended_time=3600 * 24 * 15)
+
+        if is_cloudflare_firewall(resp):
+            raise SearxEngineAccessDeniedException(message='Cloudflare Firewall', suspended_time=3600 * 24)
+
+
+def raise_for_recaptcha(resp):
+    if resp.status_code == 503 \
+       and '"https://www.google.com/recaptcha/' in resp.text:
+        raise SearxEngineCaptchaException(message='ReCAPTCHA', suspended_time=3600 * 24 * 7)
+
+
+def raise_for_captcha(resp):
+    raise_for_cloudflare_captcha(resp)
+    raise_for_recaptcha(resp)
+
+
+def raise_for_httperror(resp):
+    """Raise exception for an HTTP response is an error.
+
+    Args:
+        resp (requests.Response): Response to check
+
+    Raises:
+        requests.HTTPError: raise by resp.raise_for_status()
+        searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403.
+        searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429.
+        searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected.
+    """
+    if resp.status_code and resp.status_code >= 400:
+        raise_for_captcha(resp)
+        if resp.status_code in (402, 403):
+            raise SearxEngineAccessDeniedException(message='HTTP error ' + str(resp.status_code),
+                                                   suspended_time=3600 * 24)
+        if resp.status_code == 429:
+            raise SearxEngineTooManyRequestsException()
+        resp.raise_for_status()