Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bb6a331

Browse files
asukaminato0721autofix-ci[bot]crazywoola
authored
change all to httpx (#26119)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: crazywoola <[email protected]>
1 parent 3922ad8 commit bb6a331

23 files changed

Lines changed: 231 additions & 172 deletions

File tree

‎api/core/extension/api_based_extension_requestor.py‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
from typing import cast
22

3-
import requests
3+
import httpx
44

55
from configs import dify_config
66
from models.api_based_extension import APIBasedExtensionPoint
77

88

99
class APIBasedExtensionRequestor:
10-
timeout: tuple[int, int] = (5, 60)
10+
timeout: httpx.Timeout = httpx.Timeout(60.0, connect=5.0)
1111
"""timeout for request connect and read"""
1212

1313
def __init__(self, api_endpoint: str, api_key: str):
@@ -27,25 +27,23 @@ def request(self, point: APIBasedExtensionPoint, params: dict):
2727
url = self.api_endpoint
2828

2929
try:
30-
# proxy support for security
31-
proxies = None
30+
mounts: dict[str, httpx.BaseTransport] | None = None
3231
if dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL:
33-
proxies = {
34-
"http": dify_config.SSRF_PROXY_HTTP_URL,
35-
"https": dify_config.SSRF_PROXY_HTTPS_URL,
32+
mounts = {
33+
"http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL),
34+
"https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL),
3635
}
3736

38-
response = requests.request(
39-
method="POST",
40-
url=url,
41-
json={"point": point.value, "params": params},
42-
headers=headers,
43-
timeout=self.timeout,
44-
proxies=proxies,
45-
)
46-
except requests.Timeout:
37+
with httpx.Client(mounts=mounts, timeout=self.timeout) as client:
38+
response = client.request(
39+
method="POST",
40+
url=url,
41+
json={"point": point.value, "params": params},
42+
headers=headers,
43+
)
44+
except httpx.TimeoutException:
4745
raise ValueError("request timeout")
48-
except requests.ConnectionError:
46+
except httpx.RequestError:
4947
raise ValueError("request connection error")
5048

5149
if response.status_code != 200:

‎api/core/plugin/impl/base.py‎

Lines changed: 71 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
import json
33
import logging
44
from collections.abc import Callable, Generator
5-
from typing import TypeVar
5+
from typing import Any, TypeVar
66

7-
import requests
7+
import httpx
88
from pydantic import BaseModel
9-
from requests.exceptions import HTTPError
109
from yarl import URL
1110

1211
from configs import dify_config
@@ -47,29 +46,56 @@ def _request(
4746
data: bytes | dict | str | None = None,
4847
params: dict | None = None,
4948
files: dict | None = None,
50-
stream: bool = False,
51-
) -> requests.Response:
49+
) -> httpx.Response:
5250
"""
5351
Make a request to the plugin daemon inner API.
5452
"""
55-
url = plugin_daemon_inner_api_baseurl / path
56-
headers = headers or {}
57-
headers["X-Api-Key"] = dify_config.PLUGIN_DAEMON_KEY
58-
headers["Accept-Encoding"] = "gzip, deflate, br"
53+
url, headers, prepared_data, params, files = self._prepare_request(path, headers, data, params, files)
5954

60-
if headers.get("Content-Type") == "application/json" and isinstance(data, dict):
61-
data = json.dumps(data)
55+
request_kwargs: dict[str, Any] = {
56+
"method": method,
57+
"url": url,
58+
"headers": headers,
59+
"params": params,
60+
"files": files,
61+
}
62+
if isinstance(prepared_data, dict):
63+
request_kwargs["data"] = prepared_data
64+
elif prepared_data is not None:
65+
request_kwargs["content"] = prepared_data
6266

6367
try:
64-
response = requests.request(
65-
method=method, url=str(url), headers=headers, data=data, params=params, stream=stream, files=files
66-
)
67-
except requests.ConnectionError:
68+
response = httpx.request(**request_kwargs)
69+
except httpx.RequestError:
6870
logger.exception("Request to Plugin Daemon Service failed")
6971
raise PluginDaemonInnerError(code=-500, message="Request to Plugin Daemon Service failed")
7072

7173
return response
7274

75+
def _prepare_request(
76+
self,
77+
path: str,
78+
headers: dict | None,
79+
data: bytes | dict | str | None,
80+
params: dict | None,
81+
files: dict | None,
82+
) -> tuple[str, dict, bytes | dict | str | None, dict | None, dict | None]:
83+
url = plugin_daemon_inner_api_baseurl / path
84+
prepared_headers = dict(headers or {})
85+
prepared_headers["X-Api-Key"] = dify_config.PLUGIN_DAEMON_KEY
86+
prepared_headers.setdefault("Accept-Encoding", "gzip, deflate, br")
87+
88+
prepared_data: bytes | dict | str | None = (
89+
data if isinstance(data, (bytes, str, dict)) or data is None else None
90+
)
91+
if isinstance(data, dict):
92+
if prepared_headers.get("Content-Type") == "application/json":
93+
prepared_data = json.dumps(data)
94+
else:
95+
prepared_data = data
96+
97+
return str(url), prepared_headers, prepared_data, params, files
98+
7399
def _stream_request(
74100
self,
75101
method: str,
@@ -78,17 +104,38 @@ def _stream_request(
78104
headers: dict | None = None,
79105
data: bytes | dict | None = None,
80106
files: dict | None = None,
81-
) -> Generator[bytes, None, None]:
107+
) -> Generator[str, None, None]:
82108
"""
83109
Make a stream request to the plugin daemon inner API
84110
"""
85-
response = self._request(method, path, headers, data, params, files, stream=True)
86-
for line in response.iter_lines(chunk_size=1024 * 8):
87-
line = line.decode("utf-8").strip()
88-
if line.startswith("data:"):
89-
line = line[5:].strip()
90-
if line:
91-
yield line
111+
url, headers, prepared_data, params, files = self._prepare_request(path, headers, data, params, files)
112+
113+
stream_kwargs: dict[str, Any] = {
114+
"method": method,
115+
"url": url,
116+
"headers": headers,
117+
"params": params,
118+
"files": files,
119+
}
120+
if isinstance(prepared_data, dict):
121+
stream_kwargs["data"] = prepared_data
122+
elif prepared_data is not None:
123+
stream_kwargs["content"] = prepared_data
124+
125+
try:
126+
with httpx.stream(**stream_kwargs) as response:
127+
for raw_line in response.iter_lines():
128+
if raw_line is None:
129+
continue
130+
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
131+
line = line.strip()
132+
if line.startswith("data:"):
133+
line = line[5:].strip()
134+
if line:
135+
yield line
136+
except httpx.RequestError:
137+
logger.exception("Stream request to Plugin Daemon Service failed")
138+
raise PluginDaemonInnerError(code=-500, message="Request to Plugin Daemon Service failed")
92139

93140
def _stream_request_with_model(
94141
self,
@@ -139,7 +186,7 @@ def _request_with_plugin_daemon_response(
139186
try:
140187
response = self._request(method, path, headers, data, params, files)
141188
response.raise_for_status()
142-
except HTTPError as e:
189+
except httpx.HTTPStatusError as e:
143190
logger.exception("Failed to request plugin daemon, status: %s, url: %s", e.response.status_code, path)
144191
raise e
145192
except Exception as e:

‎api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from typing import Any, cast
55
from urllib.parse import urlparse
66

7-
import requests
7+
from elasticsearch import ConnectionError as ElasticsearchConnectionError
88
from elasticsearch import Elasticsearch
99
from flask import current_app
1010
from packaging.version import parse as parse_version
@@ -138,7 +138,7 @@ def _init_client(self, config: ElasticSearchConfig) -> Elasticsearch:
138138
if not client.ping():
139139
raise ConnectionError("Failed to connect to Elasticsearch")
140140

141-
except requests.ConnectionError as e:
141+
except ElasticsearchConnectionError as e:
142142
raise ConnectionError(f"Vector database connection error: {str(e)}")
143143
except Exception as e:
144144
raise ConnectionError(f"Elasticsearch client initialization failed: {str(e)}")

‎api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55
from itertools import islice
66
from typing import TYPE_CHECKING, Any, Union
77

8+
import httpx
89
import qdrant_client
9-
import requests
1010
from flask import current_app
11+
from httpx import DigestAuth
1112
from pydantic import BaseModel
1213
from qdrant_client.http import models as rest
1314
from qdrant_client.http.models import (
@@ -19,7 +20,6 @@
1920
TokenizerType,
2021
)
2122
from qdrant_client.local.qdrant_local import QdrantLocal
22-
from requests.auth import HTTPDigestAuth
2323
from sqlalchemy import select
2424

2525
from configs import dify_config
@@ -504,10 +504,10 @@ def create_tidb_serverless_cluster(self, tidb_config: TidbConfig, display_name:
504504
}
505505
cluster_data = {"displayName": display_name, "region": region_object, "labels": labels}
506506

507-
response = requests.post(
507+
response = httpx.post(
508508
f"{tidb_config.api_url}/clusters",
509509
json=cluster_data,
510-
auth=HTTPDigestAuth(tidb_config.public_key, tidb_config.private_key),
510+
auth=DigestAuth(tidb_config.public_key, tidb_config.private_key),
511511
)
512512

513513
if response.status_code == 200:
@@ -527,10 +527,10 @@ def change_tidb_serverless_root_password(self, tidb_config: TidbConfig, cluster_
527527

528528
body = {"password": new_password}
529529

530-
response = requests.put(
530+
response = httpx.put(
531531
f"{tidb_config.api_url}/clusters/{cluster_id}/password",
532532
json=body,
533-
auth=HTTPDigestAuth(tidb_config.public_key, tidb_config.private_key),
533+
auth=DigestAuth(tidb_config.public_key, tidb_config.private_key),
534534
)
535535

536536
if response.status_code == 200:

‎api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_service.py‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
import uuid
33
from collections.abc import Sequence
44

5-
import requests
6-
from requests.auth import HTTPDigestAuth
5+
import httpx
6+
from httpx import DigestAuth
77

88
from configs import dify_config
99
from extensions.ext_database import db
@@ -49,7 +49,7 @@ def create_tidb_serverless_cluster(
4949
"rootPassword": password,
5050
}
5151

52-
response = requests.post(f"{api_url}/clusters", json=cluster_data, auth=HTTPDigestAuth(public_key, private_key))
52+
response = httpx.post(f"{api_url}/clusters", json=cluster_data, auth=DigestAuth(public_key, private_key))
5353

5454
if response.status_code == 200:
5555
response_data = response.json()
@@ -83,7 +83,7 @@ def delete_tidb_serverless_cluster(api_url: str, public_key: str, private_key: s
8383
:return: The response from the API.
8484
"""
8585

86-
response = requests.delete(f"{api_url}/clusters/{cluster_id}", auth=HTTPDigestAuth(public_key, private_key))
86+
response = httpx.delete(f"{api_url}/clusters/{cluster_id}", auth=DigestAuth(public_key, private_key))
8787

8888
if response.status_code == 200:
8989
return response.json()
@@ -102,7 +102,7 @@ def get_tidb_serverless_cluster(api_url: str, public_key: str, private_key: str,
102102
:return: The response from the API.
103103
"""
104104

105-
response = requests.get(f"{api_url}/clusters/{cluster_id}", auth=HTTPDigestAuth(public_key, private_key))
105+
response = httpx.get(f"{api_url}/clusters/{cluster_id}", auth=DigestAuth(public_key, private_key))
106106

107107
if response.status_code == 200:
108108
return response.json()
@@ -127,10 +127,10 @@ def change_tidb_serverless_root_password(
127127

128128
body = {"password": new_password, "builtinRole": "role_admin", "customRoles": []}
129129

130-
response = requests.patch(
130+
response = httpx.patch(
131131
f"{api_url}/clusters/{cluster_id}/sqlUsers/{account}",
132132
json=body,
133-
auth=HTTPDigestAuth(public_key, private_key),
133+
auth=DigestAuth(public_key, private_key),
134134
)
135135

136136
if response.status_code == 200:
@@ -161,9 +161,7 @@ def batch_update_tidb_serverless_cluster_status(
161161
tidb_serverless_list_map = {item.cluster_id: item for item in tidb_serverless_list}
162162
cluster_ids = [item.cluster_id for item in tidb_serverless_list]
163163
params = {"clusterIds": cluster_ids, "view": "BASIC"}
164-
response = requests.get(
165-
f"{api_url}/clusters:batchGet", params=params, auth=HTTPDigestAuth(public_key, private_key)
166-
)
164+
response = httpx.get(f"{api_url}/clusters:batchGet", params=params, auth=DigestAuth(public_key, private_key))
167165

168166
if response.status_code == 200:
169167
response_data = response.json()
@@ -224,8 +222,8 @@ def batch_create_tidb_serverless_cluster(
224222
clusters.append(cluster_data)
225223

226224
request_body = {"requests": clusters}
227-
response = requests.post(
228-
f"{api_url}/clusters:batchCreate", json=request_body, auth=HTTPDigestAuth(public_key, private_key)
225+
response = httpx.post(
226+
f"{api_url}/clusters:batchCreate", json=request_body, auth=DigestAuth(public_key, private_key)
229227
)
230228

231229
if response.status_code == 200:

‎api/core/rag/datasource/vdb/weaviate/weaviate_vector.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
import json
33
from typing import Any
44

5-
import requests
65
import weaviate # type: ignore
76
from pydantic import BaseModel, model_validator
87

@@ -45,8 +44,8 @@ def _init_client(self, config: WeaviateConfig) -> weaviate.Client:
4544
client = weaviate.Client(
4645
url=config.endpoint, auth_client_secret=auth_config, timeout_config=(5, 60), startup_period=None
4746
)
48-
except requests.ConnectionError:
49-
raise ConnectionError("Vector database connection error")
47+
except Exception as exc:
48+
raise ConnectionError("Vector database connection error") from exc
5049

5150
client.batch.configure(
5251
# `batch_size` takes an `int` value to enable auto-batching

‎api/core/rag/extractor/firecrawl/firecrawl_app.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import time
33
from typing import Any, cast
44

5-
import requests
5+
import httpx
66

77
from extensions.ext_storage import storage
88

@@ -104,18 +104,18 @@ def _extract_common_fields(self, item: dict[str, Any]) -> dict[str, Any]:
104104
def _prepare_headers(self) -> dict[str, Any]:
105105
return {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
106106

107-
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> requests.Response:
107+
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
108108
for attempt in range(retries):
109-
response = requests.post(url, headers=headers, json=data)
109+
response = httpx.post(url, headers=headers, json=data)
110110
if response.status_code == 502:
111111
time.sleep(backoff_factor * (2**attempt))
112112
else:
113113
return response
114114
return response
115115

116-
def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> requests.Response:
116+
def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
117117
for attempt in range(retries):
118-
response = requests.get(url, headers=headers)
118+
response = httpx.get(url, headers=headers)
119119
if response.status_code == 502:
120120
time.sleep(backoff_factor * (2**attempt))
121121
else:

0 commit comments

Comments
 (0)