OpenTTD Source: src/network/core/http_curl.cpp Source File

OpenTTD Source 15.3

http_curl.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
6 */
7
12#include "../../stdafx.h"
13#include "../../debug.h"
14#include "../../fileio_func.h"
15#include "../../rev.h"
16#include "../../thread.h"
17#include "../network_internal.h"
18
19#include "http.h"
20#include "http_shared.h"
21
22#include <atomic>
23#include <condition_variable>
24#include <curl/curl.h>
25#include <mutex>
26#include <queue>
27
28#include "../../safeguards.h"
29
30#if defined(UNIX)
32static constexpr std::initializer_list<std::string_view> _certificate_files = {
33 "/etc/ssl/certs/ca-certificates.crt"sv, // Debian/Ubuntu/Gentoo etc.
34 "/etc/pki/tls/certs/ca-bundle.crt"sv, // Fedora/RHEL 6
35 "/etc/ssl/ca-bundle.pem"sv, // OpenSUSE
36 "/etc/pki/tls/cacert.pem"sv, // OpenELEC
37 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"sv, // CentOS/RHEL 7
38 "/etc/ssl/cert.pem"sv, // Alpine Linux
39};
41static constexpr std::initializer_list<std::string_view> _certificate_directories = {
42 "/etc/ssl/certs"sv, // SLES10/SLES11, https://golang.org/issue/12139
43 "/etc/pki/tls/certs"sv, // Fedora/RHEL
44 "/system/etc/security/cacerts"sv, // Android
45};
46#endif /* UNIX */
47
48static std::vector<HTTPThreadSafeCallback *> _http_callbacks;
49static std::vector<HTTPThreadSafeCallback *> _new_http_callbacks;
50static std::mutex _http_callback_mutex;
51static std::mutex _new_http_callback_mutex;
52
54 class NetworkHTTPRequest {
55public:
63 NetworkHTTPRequest(std::string_view uri, HTTPCallback *callback, std::string &&data) :
64 uri(uri),
65 callback(callback),
66 data(std::move(data))
67 {
68 std::lock_guard<std::mutex> lock(_new_http_callback_mutex);
69 _new_http_callbacks.push_back(&this->callback);
70 }
71
72 ~NetworkHTTPRequest()
73 {
74 std::lock_guard<std::mutex> lock(_http_callback_mutex);
75 _http_callbacks.erase(std::remove(_http_callbacks.begin(), _http_callbacks.end(), &this->callback), _http_callbacks.end());
76 }
77
78 const std::string uri;
79 HTTPThreadSafeCallback callback;
80 const std::string data;
81};
82
83static std::thread _http_thread;
84static std::atomic<bool> _http_thread_exit = false;
85static std::queue<std::unique_ptr<NetworkHTTPRequest>> _http_requests;
86static std::mutex _http_mutex;
87static std::condition_variable _http_cv;
88#if defined(UNIX)
89static std::string _http_ca_file = "";
90static std::string _http_ca_path = "";
91#endif /* UNIX */
92
93/* static */ void NetworkHTTPSocketHandler::Connect(std::string_view uri, HTTPCallback *callback, std::string &&data)
94{
95#if defined(UNIX)
96 if (_http_ca_file.empty() && _http_ca_path.empty()) {
97 callback->OnFailure();
98 return;
99 }
100#endif /* UNIX */
101
102 std::lock_guard<std::mutex> lock(_http_mutex);
103 _http_requests.push(std::make_unique<NetworkHTTPRequest>(uri, callback, std::move(data)));
104 _http_cv.notify_one();
105}
106
107/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
108{
109 std::lock_guard<std::mutex> lock(_http_callback_mutex);
110
111 {
112 std::lock_guard<std::mutex> lock_new(_new_http_callback_mutex);
113 if (!_new_http_callbacks.empty()) {
114 /* We delay adding new callbacks, as HandleQueue() below might add a new callback. */
115 _http_callbacks.insert(_http_callbacks.end(), _new_http_callbacks.begin(), _new_http_callbacks.end());
116 _new_http_callbacks.clear();
117 }
118 }
119
120 for (auto &callback : _http_callbacks) {
121 callback->HandleQueue();
122 }
123}
124
125void CurlSetOption(CURL *curl, auto option, auto value)
126{
127 CURLcode res = curl_easy_setopt(curl, option, value);
128 if (res != CURLE_OK) {
129 Debug(net, 0, "Could not execute curl_easy_setopt for {} [{}]", option, res);
130 }
131}
132
133void HttpThread()
134{
135 CURL *curl = curl_easy_init();
136 assert(curl != nullptr);
137
138 for (;;) {
139 std::unique_lock<std::mutex> lock(_http_mutex);
140
141 /* Wait for a new request. */
142 while (_http_requests.empty() && !_http_thread_exit) {
143 _http_cv.wait(lock);
144 }
145 if (_http_thread_exit) break;
146
147 std::unique_ptr<NetworkHTTPRequest> request = std::move(_http_requests.front());
148 _http_requests.pop();
149
150 /* Release the lock, as we will take a while to process the request. */
151 lock.unlock();
152
153 /* Reset to default settings. */
154 curl_easy_reset(curl);
155 curl_slist *headers = nullptr;
156
157 if (_debug_net_level >= 5) {
158 CurlSetOption(curl, CURLOPT_VERBOSE, 1L);
159 }
160
161 /* Setup some default options. */
162 std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
163 CurlSetOption(curl, CURLOPT_USERAGENT, user_agent.c_str());
164 CurlSetOption(curl, CURLOPT_FOLLOWLOCATION, 1L);
165 CurlSetOption(curl, CURLOPT_MAXREDIRS, 5L);
166
167 /* Ensure we validate the certificate and hostname of the server. */
168#if defined(UNIX)
169 CurlSetOption(curl, CURLOPT_CAINFO, _http_ca_file.empty() ? nullptr : _http_ca_file.c_str());
170 CurlSetOption(curl, CURLOPT_CAPATH, _http_ca_path.empty() ? nullptr : _http_ca_path.c_str());
171#endif /* UNIX */
172 CurlSetOption(curl, CURLOPT_SSL_VERIFYHOST, 2);
173 CurlSetOption(curl, CURLOPT_SSL_VERIFYPEER, true);
174
175 /* Give the connection about 10 seconds to complete. */
176 CurlSetOption(curl, CURLOPT_CONNECTTIMEOUT, 10L);
177
178 /* Set a buffer of 100KiB, as the default of 16KiB seems a bit small. */
179 CurlSetOption(curl, CURLOPT_BUFFERSIZE, 100L * 1024L);
180
181 /* Fail our call if we don't receive a 2XX return value. */
182 CurlSetOption(curl, CURLOPT_FAILONERROR, 1L);
183
184 /* Prepare POST body and URI. */
185 if (!request->data.empty()) {
186 /* When the payload starts with a '{', it is a JSON payload. */
187 if (request->data.starts_with("{")) {
188 headers = curl_slist_append(headers, "Content-Type: application/json");
189 } else {
190 headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded");
191 }
192
193 CurlSetOption(curl, CURLOPT_POST, 1L);
194 CurlSetOption(curl, CURLOPT_POSTFIELDS, request->data.c_str());
195 CurlSetOption(curl, CURLOPT_HTTPHEADER, headers);
196 }
197 CurlSetOption(curl, CURLOPT_URL, request->uri.c_str());
198
199 /* Setup our (C-style) callback function which we pipe back into the callback. */
200 CurlSetOption(curl, CURLOPT_WRITEFUNCTION, +[](char *ptr, size_t size, size_t nmemb, void *userdata) -> size_t {
201 Debug(net, 6, "HTTP callback: {} bytes", size * nmemb);
202 HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
203
204 /* Copy the buffer out of CURL. OnReceiveData() will free it when done. */
205 std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size * nmemb);
206 std::copy_n(ptr, size * nmemb, buffer.get());
207 callback->OnReceiveData(std::move(buffer), size * nmemb);
208
209 return size * nmemb;
210 });
211 CurlSetOption(curl, CURLOPT_WRITEDATA, &request->callback);
212
213 /* Create a callback from which we can cancel. Sadly, there is no other
214 * thread-safe way to do this. If the connection went idle, it can take
215 * up to a second before this callback is called. There is little we can
216 * do about this. */
217 CurlSetOption(curl, CURLOPT_NOPROGRESS, 0L);
218 CurlSetOption(curl, CURLOPT_XFERINFOFUNCTION, +[](void *userdata, curl_off_t /*dltotal*/, curl_off_t /*dlnow*/, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) -> int {
219 const HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
220 return (callback->cancelled || _http_thread_exit) ? 1 : 0;
221 });
222 CurlSetOption(curl, CURLOPT_XFERINFODATA, &request->callback);
223
224 /* Perform the request. */
225 CURLcode res = curl_easy_perform(curl);
226
227 curl_slist_free_all(headers);
228
229 if (res == CURLE_OK) {
230 Debug(net, 1, "HTTP request succeeded");
231 request->callback.OnReceiveData(nullptr, 0);
232 } else {
233 long status_code = 0;
234 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status_code);
235
236 /* No need to be verbose about rate limiting. */
237 Debug(net, (request->callback.cancelled || _http_thread_exit || status_code == HTTP_429_TOO_MANY_REQUESTS) ? 1 : 0, "HTTP request failed: status_code: {}, error: {}", status_code, curl_easy_strerror(res));
238 request->callback.OnFailure();
239 }
240
241 /* Wait till the callback tells us all data is dequeued, or _http_thread_exit has been set. */
242 request->callback.WaitTillEmptyOrCondition([]() -> bool {
243 return _http_thread_exit;
244 });
245 }
246
247 curl_easy_cleanup(curl);
248}
249
250 void NetworkHTTPInitialize()
251{
252 curl_global_init(CURL_GLOBAL_DEFAULT);
253
254#if defined(UNIX)
255 /* Depending on the Linux distro, certificates can either be in
256 * a bundle or a folder, in a wide range of different locations.
257 * Try to find what location is used by this OS. */
258 for (auto &ca_file : _certificate_files) {
259 if (FileExists(ca_file)) {
260 _http_ca_file = ca_file;
261 break;
262 }
263 }
264 if (_http_ca_file.empty()) {
265 for (auto &ca_path : _certificate_directories) {
266 if (FileExists(ca_path)) {
267 _http_ca_path = ca_path;
268 break;
269 }
270 }
271 }
272 Debug(net, 3, "Using certificate file: {}", _http_ca_file.empty() ? "none" : _http_ca_file);
273 Debug(net, 3, "Using certificate path: {}", _http_ca_path.empty() ? "none" : _http_ca_path);
274
275 /* Tell the user why HTTPS will not be working. */
276 if (_http_ca_file.empty() && _http_ca_path.empty()) {
277 Debug(net, 0, "No certificate files or directories found, HTTPS will not work!");
278 }
279#endif /* UNIX */
280
281 _http_thread_exit = false;
282 StartNewThread(&_http_thread, "ottd:http", &HttpThread);
283}
284
285 void NetworkHTTPUninitialize()
286{
287 _http_thread_exit = true;
288
289 /* Ensure the callbacks are handled. This is mostly needed as we send
290 * a survey just before close, and that might be pending here. */
291 NetworkHTTPSocketHandler::HTTPReceive();
292
293 {
294 std::lock_guard<std::mutex> lock(_http_mutex);
295 _http_cv.notify_one();
296 }
297
298 if (_http_thread.joinable()) {
299 _http_thread.join();
300 }
301
302 curl_global_cleanup();
303}
HTTPThreadSafeCallback
Converts a HTTPCallback to a Thread-Safe variant.
Definition http_shared.h:22
HTTPThreadSafeCallback::OnReceiveData
void OnReceiveData(std::unique_ptr< char[]> data, size_t length)
Similar to HTTPCallback::OnReceiveData, but thread-safe.
Definition http_shared.h:48
NetworkHTTPRequest
Single HTTP request.
Definition http_curl.cpp:54
NetworkHTTPRequest::data
const std::string data
Data to send, if any.
Definition http_curl.cpp:80
NetworkHTTPRequest::~NetworkHTTPRequest
~NetworkHTTPRequest()
Destructor of the HTTP request.
Definition http_curl.cpp:72
NetworkHTTPRequest::NetworkHTTPRequest
NetworkHTTPRequest(std::string_view uri, HTTPCallback *callback, std::string &&data)
Create a new HTTP request.
Definition http_curl.cpp:63
NetworkHTTPRequest::callback
HTTPThreadSafeCallback callback
Callback to send data back on.
Definition http_curl.cpp:79
NetworkHTTPRequest::uri
const std::string uri
URI to connect to.
Definition http_curl.cpp:78
NetworkHTTPSocketHandler::Connect
static void Connect(std::string_view uri, HTTPCallback *callback, std::string &&data="")
Connect to the given URI.
Definition http_curl.cpp:93
NetworkHTTPSocketHandler::HTTPReceive
static void HTTPReceive()
Do the receiving for all HTTP connections.
Definition http_curl.cpp:107
Debug
#define Debug(category, level, format_string,...)
Output a line of debugging information.
Definition debug.h:37
FileExists
bool FileExists(std::string_view filename)
Test whether the given filename exists.
Definition fileio.cpp:132
http.h
Basic functions to send and receive HTTP packets.
NetworkHTTPInitialize
void NetworkHTTPInitialize()
Initialize the HTTP socket handler.
Definition http_curl.cpp:250
NetworkHTTPUninitialize
void NetworkHTTPUninitialize()
Uninitialize the HTTP socket handler.
Definition http_curl.cpp:285
http_shared.h
Shared functions for implementations of HTTP requests.
HTTPCallback
Callback for when the HTTP handler has something to tell us.
Definition http.h:20
HTTPCallback::OnFailure
virtual void OnFailure()=0
An error has occurred and the connection has been closed.
StartNewThread
bool StartNewThread(std::thread *thr, std::string_view name, TFn &&_Fx, TArgs &&... _Ax)
Start a new thread.
Definition thread.h:47
lock
std::mutex lock
synchronization for playback status fields
Definition win32_m.cpp:35