OpenTTD Source: src/network/core/http_curl.cpp Source File

OpenTTD Source 15.0-beta1

http_curl.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6 */
7
12#include "../../stdafx.h"
13#include "../../debug.h"
14#include "../../fileio_func.h"
15#include "../../rev.h"
16#include "../../thread.h"
17#include "../network_internal.h"
18
19#include "http.h"
20#include "http_shared.h"
21
22#include <atomic>
23#include <condition_variable>
24#include <curl/curl.h>
25#include <mutex>
26#include <queue>
27
28#include "../../safeguards.h"
29
30#if defined(UNIX)
32static auto _certificate_files = {
33 "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
34 "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
35 "/etc/ssl/ca-bundle.pem", // OpenSUSE
36 "/etc/pki/tls/cacert.pem", // OpenELEC
37 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
38 "/etc/ssl/cert.pem", // Alpine Linux
39};
41static auto _certificate_directories = {
42 "/etc/ssl/certs", // SLES10/SLES11, https://golang.org/issue/12139
43 "/etc/pki/tls/certs", // Fedora/RHEL
44 "/system/etc/security/cacerts", // Android
45};
46#endif /* UNIX */
47
48static std::vector<HTTPThreadSafeCallback *> _http_callbacks;
49static std::vector<HTTPThreadSafeCallback *> _new_http_callbacks;
50static std::mutex _http_callback_mutex;
51static std::mutex _new_http_callback_mutex;
52
54 class NetworkHTTPRequest {
55public:
63 NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const std::string &data) :
64 uri(uri),
65 callback(callback),
66 data(data)
67 {
68 std::lock_guard<std::mutex> lock(_new_http_callback_mutex);
69 _new_http_callbacks.push_back(&this->callback);
70 }
71
72 ~NetworkHTTPRequest()
73 {
74 std::lock_guard<std::mutex> lock(_http_callback_mutex);
75 _http_callbacks.erase(std::remove(_http_callbacks.begin(), _http_callbacks.end(), &this->callback), _http_callbacks.end());
76 }
77
78 const std::string uri;
79 HTTPThreadSafeCallback callback;
80 const std::string data;
81};
82
83static std::thread _http_thread;
84static std::atomic<bool> _http_thread_exit = false;
85static std::queue<std::unique_ptr<NetworkHTTPRequest>> _http_requests;
86static std::mutex _http_mutex;
87static std::condition_variable _http_cv;
88#if defined(UNIX)
89static std::string _http_ca_file = "";
90static std::string _http_ca_path = "";
91#endif /* UNIX */
92
93/* static */ void NetworkHTTPSocketHandler::Connect(const std::string &uri, HTTPCallback *callback, const std::string data)
94{
95#if defined(UNIX)
96 if (_http_ca_file.empty() && _http_ca_path.empty()) {
97 callback->OnFailure();
98 return;
99 }
100#endif /* UNIX */
101
102 std::lock_guard<std::mutex> lock(_http_mutex);
103 _http_requests.push(std::make_unique<NetworkHTTPRequest>(uri, callback, data));
104 _http_cv.notify_one();
105}
106
107/* static */ void NetworkHTTPSocketHandler::HTTPReceive()
108{
109 std::lock_guard<std::mutex> lock(_http_callback_mutex);
110
111 {
112 std::lock_guard<std::mutex> lock_new(_new_http_callback_mutex);
113 if (!_new_http_callbacks.empty()) {
114 /* We delay adding new callbacks, as HandleQueue() below might add a new callback. */
115 _http_callbacks.insert(_http_callbacks.end(), _new_http_callbacks.begin(), _new_http_callbacks.end());
116 _new_http_callbacks.clear();
117 }
118 }
119
120 for (auto &callback : _http_callbacks) {
121 callback->HandleQueue();
122 }
123}
124
125void HttpThread()
126{
127 CURL *curl = curl_easy_init();
128 assert(curl != nullptr);
129
130 for (;;) {
131 std::unique_lock<std::mutex> lock(_http_mutex);
132
133 /* Wait for a new request. */
134 while (_http_requests.empty() && !_http_thread_exit) {
135 _http_cv.wait(lock);
136 }
137 if (_http_thread_exit) break;
138
139 std::unique_ptr<NetworkHTTPRequest> request = std::move(_http_requests.front());
140 _http_requests.pop();
141
142 /* Release the lock, as we will take a while to process the request. */
143 lock.unlock();
144
145 /* Reset to default settings. */
146 curl_easy_reset(curl);
147 curl_slist *headers = nullptr;
148
149 if (_debug_net_level >= 5) {
150 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
151 }
152
153 /* Setup some default options. */
154 std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
155 curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent.c_str());
156 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
157 curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
158
159 /* Ensure we validate the certificate and hostname of the server. */
160#if defined(UNIX)
161 curl_easy_setopt(curl, CURLOPT_CAINFO, _http_ca_file.empty() ? nullptr : _http_ca_file.c_str());
162 curl_easy_setopt(curl, CURLOPT_CAPATH, _http_ca_path.empty() ? nullptr : _http_ca_path.c_str());
163#endif /* UNIX */
164 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2);
165 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, true);
166
167 /* Give the connection about 10 seconds to complete. */
168 curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
169
170 /* Set a buffer of 100KiB, as the default of 16KiB seems a bit small. */
171 curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 100L * 1024L);
172
173 /* Fail our call if we don't receive a 2XX return value. */
174 curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
175
176 /* Prepare POST body and URI. */
177 if (!request->data.empty()) {
178 /* When the payload starts with a '{', it is a JSON payload. */
179 if (request->data.starts_with("{")) {
180 headers = curl_slist_append(headers, "Content-Type: application/json");
181 } else {
182 headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded");
183 }
184
185 curl_easy_setopt(curl, CURLOPT_POST, 1L);
186 curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request->data.c_str());
187 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
188 }
189 curl_easy_setopt(curl, CURLOPT_URL, request->uri.c_str());
190
191 /* Setup our (C-style) callback function which we pipe back into the callback. */
192 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](char *ptr, size_t size, size_t nmemb, void *userdata) -> size_t {
193 Debug(net, 6, "HTTP callback: {} bytes", size * nmemb);
194 HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
195
196 /* Copy the buffer out of CURL. OnReceiveData() will free it when done. */
197 std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size * nmemb);
198 memcpy(buffer.get(), ptr, size * nmemb);
199 callback->OnReceiveData(std::move(buffer), size * nmemb);
200
201 return size * nmemb;
202 });
203 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &request->callback);
204
205 /* Create a callback from which we can cancel. Sadly, there is no other
206 * thread-safe way to do this. If the connection went idle, it can take
207 * up to a second before this callback is called. There is little we can
208 * do about this. */
209 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
210 curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, +[](void *userdata, curl_off_t /*dltotal*/, curl_off_t /*dlnow*/, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) -> int {
211 const HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
212 return (callback->cancelled || _http_thread_exit) ? 1 : 0;
213 });
214 curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &request->callback);
215
216 /* Perform the request. */
217 CURLcode res = curl_easy_perform(curl);
218
219 curl_slist_free_all(headers);
220
221 if (res == CURLE_OK) {
222 Debug(net, 1, "HTTP request succeeded");
223 request->callback.OnReceiveData(nullptr, 0);
224 } else {
225 long status_code = 0;
226 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status_code);
227
228 /* No need to be verbose about rate limiting. */
229 Debug(net, (request->callback.cancelled || _http_thread_exit || status_code == HTTP_429_TOO_MANY_REQUESTS) ? 1 : 0, "HTTP request failed: status_code: {}, error: {}", status_code, curl_easy_strerror(res));
230 request->callback.OnFailure();
231 }
232
233 /* Wait till the callback tells us all data is dequeued, or _http_thread_exit has been set. */
234 request->callback.WaitTillEmptyOrCondition([]() -> bool {
235 return _http_thread_exit;
236 });
237 }
238
239 curl_easy_cleanup(curl);
240}
241
242 void NetworkHTTPInitialize()
243{
244 curl_global_init(CURL_GLOBAL_DEFAULT);
245
246#if defined(UNIX)
247 /* Depending on the Linux distro, certificates can either be in
248 * a bundle or a folder, in a wide range of different locations.
249 * Try to find what location is used by this OS. */
250 for (auto &ca_file : _certificate_files) {
251 if (FileExists(ca_file)) {
252 _http_ca_file = ca_file;
253 break;
254 }
255 }
256 if (_http_ca_file.empty()) {
257 for (auto &ca_path : _certificate_directories) {
258 if (FileExists(ca_path)) {
259 _http_ca_path = ca_path;
260 break;
261 }
262 }
263 }
264 Debug(net, 3, "Using certificate file: {}", _http_ca_file.empty() ? "none" : _http_ca_file);
265 Debug(net, 3, "Using certificate path: {}", _http_ca_path.empty() ? "none" : _http_ca_path);
266
267 /* Tell the user why HTTPS will not be working. */
268 if (_http_ca_file.empty() && _http_ca_path.empty()) {
269 Debug(net, 0, "No certificate files or directories found, HTTPS will not work!");
270 }
271#endif /* UNIX */
272
273 _http_thread_exit = false;
274 StartNewThread(&_http_thread, "ottd:http", &HttpThread);
275}
276
277 void NetworkHTTPUninitialize()
278{
279 _http_thread_exit = true;
280
281 /* Ensure the callbacks are handled. This is mostly needed as we send
282 * a survey just before close, and that might be pending here. */
283 NetworkHTTPSocketHandler::HTTPReceive();
284
285 {
286 std::lock_guard<std::mutex> lock(_http_mutex);
287 _http_cv.notify_one();
288 }
289
290 if (_http_thread.joinable()) {
291 _http_thread.join();
292 }
293
294 curl_global_cleanup();
295}
HTTPThreadSafeCallback
Converts a HTTPCallback to a Thread-Safe variant.
Definition http_shared.h:22
HTTPThreadSafeCallback::OnReceiveData
void OnReceiveData(std::unique_ptr< char[]> data, size_t length)
Similar to HTTPCallback::OnReceiveData, but thread-safe.
Definition http_shared.h:48
NetworkHTTPRequest
Single HTTP request.
Definition http_curl.cpp:54
NetworkHTTPRequest::NetworkHTTPRequest
NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const std::string &data)
Create a new HTTP request.
Definition http_curl.cpp:63
NetworkHTTPRequest::data
const std::string data
Data to send, if any.
Definition http_curl.cpp:80
NetworkHTTPRequest::~NetworkHTTPRequest
~NetworkHTTPRequest()
Destructor of the HTTP request.
Definition http_curl.cpp:72
NetworkHTTPRequest::callback
HTTPThreadSafeCallback callback
Callback to send data back on.
Definition http_curl.cpp:79
NetworkHTTPRequest::uri
const std::string uri
URI to connect to.
Definition http_curl.cpp:78
NetworkHTTPSocketHandler::Connect
static void Connect(const std::string &uri, HTTPCallback *callback, const std::string data="")
Connect to the given URI.
Definition http_curl.cpp:93
NetworkHTTPSocketHandler::HTTPReceive
static void HTTPReceive()
Do the receiving for all HTTP connections.
Definition http_curl.cpp:107
Debug
#define Debug(category, level, format_string,...)
Ouptut a line of debugging information.
Definition debug.h:37
FileExists
bool FileExists(const std::string &filename)
Test whether the given filename exists.
Definition fileio.cpp:132
http.h
Basic functions to send and receive HTTP packets.
NetworkHTTPInitialize
void NetworkHTTPInitialize()
Initialize the HTTP socket handler.
Definition http_curl.cpp:242
NetworkHTTPUninitialize
void NetworkHTTPUninitialize()
Uninitialize the HTTP socket handler.
Definition http_curl.cpp:277
http_shared.h
Shared functions for implementations of HTTP requests.
HTTPCallback
Callback for when the HTTP handler has something to tell us.
Definition http.h:20
HTTPCallback::OnFailure
virtual void OnFailure()=0
An error has occurred and the connection has been closed.
StartNewThread
bool StartNewThread(std::thread *thr, const char *name, TFn &&_Fx, TArgs &&... _Ax)
Start a new thread.
Definition thread.h:47
lock
std::mutex lock
synchronization for playback status fields
Definition win32_m.cpp:35