diff --git a/Makefile b/Makefile index cd0448198..22f1002e1 100644 --- a/Makefile +++ b/Makefile @@ -193,6 +193,7 @@ include $(PATH_INTERNAL_C)/parts/input/game_controller/build.mk include $(PATH_INTERNAL_C)/parts/video/font/ttf/build.mk include $(PATH_INTERNAL_C)/parts/video/image/build.mk include $(PATH_INTERNAL_C)/parts/gui/build.mk +include $(PATH_INTERNAL_C)/parts/network/http/build.mk .PHONY: all clean @@ -341,6 +342,14 @@ else QBLIB_NAME := $(addsuffix 0,$(QBLIB_NAME)) endif +ifneq ($(filter y,$(DEP_HTTP)),) + EXE_LIBS += $(CURL_EXE_LIBS) + CXXFLAGS += $(CURL_CXXFLAGS) + CXXLIBS += $(CURL_CXXLIBS) + + LICENSE_IN_USE += libcurl +endif + ifneq ($(OS),osx) EXE_LIBS += $(QB_CORE_LIB) diff --git a/docs/build-system.md b/docs/build-system.md index 7fb8eb43e..a7539dea2 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -135,6 +135,7 @@ These flags controls whether certain dependencies are compiled in or not. All of | `DEP_CONSOLE` | On Windows, this gives the program console support (graphical support is still allowed) | | `DEP_CONSOLE_ONLY` | Same as `DEP_CONSOLE`, but also removes GLUT and graphics support. | | `DEP_AUDIO_MINIAUDIO` | Enables the miniaudio backend. Should not be used with the other `DEP_AUDIO` switches which enable the old backend. | +| `DEP_HTTP` | Enables http support via libcurl. Should only be used if `DEP_SOCKETS` is on. | Versioning ---------- diff --git a/internal/c/libqb.cpp b/internal/c/libqb.cpp index bf736ac01..e61f097e1 100644 --- a/internal/c/libqb.cpp +++ b/internal/c/libqb.cpp @@ -26,6 +26,7 @@ #include "audio.h" #include "image.h" #include "gui.h" +#include "http.h" int32 disableEvents = 0; diff --git a/internal/c/libqb/build.mk b/internal/c/libqb/build.mk index d6cfc5115..3350de34c 100644 --- a/internal/c/libqb/build.mk +++ b/internal/c/libqb/build.mk @@ -3,6 +3,9 @@ libqb-objs-y += $(PATH_LIBQB)/src/threading.o libqb-objs-y += $(PATH_LIBQB)/src/buffer.o libqb-objs-y += $(PATH_LIBQB)/src/filepath.o +libqb-objs-$(DEP_HTTP) += $(PATH_LIBQB)/src/http.o +libqb-objs-y$(DEP_HTTP) += $(PATH_LIBQB)/src/http-stub.o + libqb-objs-y += $(PATH_LIBQB)/src/threading-$(PLATFORM).o -CLEAN_LIST += $(libqb-objs-y) +CLEAN_LIST += $(libqb-objs-y) $(libqb-objs-yy) $(libqb-objs-) diff --git a/internal/c/libqb/include/http.h b/internal/c/libqb/include/http.h new file mode 100644 index 000000000..39b598ca5 --- /dev/null +++ b/internal/c/libqb/include/http.h @@ -0,0 +1,39 @@ +#ifndef INCLUDE_LIBQB_HTTP_H +#define INCLUDE_LIBQB_HTTP_H + +#include + +// Initialize the HTTP system +void libqb_http_init(); +void libqb_http_stop(); + +// All of these functions return 0 on success, and a negative error code on failure. + +// Handle is provided and should be unique. Used to identify this connection +int libqb_http_open(const char *url, int handle); +int libqb_http_close(int handle); + +int libqb_http_connected(int handle); + +// Get length of bytes waiting to be read. +// +// Note that more bytes may come in after calling function, but you're guarenteed to at least have this many bytes +int libqb_http_get_length(int handle, size_t *length); + +// Gets the value from the Content-Length HTTP header. If none was provided, returns an error +int libqb_http_get_content_length(int handle, uint64_t *length); + +// Returns the "effective url" as reported by curl, it indicates the location +// actually connected to after redirects and such. +// +// Returns NULL if it could not be resolved. Returned string is only valid for +// the life of this handle. +const char *libqb_http_get_url(int handle); + +// Reads up to length bytes into buf. Length is modified if less bytes than requested are returned +int libqb_http_get(int handle, char *buf, size_t *length); + +// Returns an error if less than length bytes are availiable to read +int libqb_http_get_fixed(int handle, char *buf, size_t length); + +#endif diff --git a/internal/c/libqb/src/http-stub.cpp b/internal/c/libqb/src/http-stub.cpp new file mode 100644 index 000000000..e382ff75d --- /dev/null +++ b/internal/c/libqb/src/http-stub.cpp @@ -0,0 +1,46 @@ + +#include +#include +#include +#include + +#include "http.h" + +void libqb_http_init() { +} + +void libqb_http_stop() { +} + +int libqb_http_open(const char *url, int handle) { + return -1; +} + +int libqb_http_close(int handle) { + return -1; +} + +int libqb_http_connected(int handle) { + return -1; +} + +int libqb_http_get_length(int handle, size_t *length) { + *length = 0; + return -1; +} + +int libqb_http_get(int handle, char *buf, size_t *length) { + return -1; +} + +int libqb_http_get_fixed(int id, char *buf, size_t length) { + return -1; +} + +int libqb_http_get_content_length(int id, uint64_t *ptr) { + return -1; +} + +const char *libqb_http_get_url(int handle) { + return NULL; +} diff --git a/internal/c/libqb/src/http.cpp b/internal/c/libqb/src/http.cpp new file mode 100644 index 000000000..afe9183dc --- /dev/null +++ b/internal/c/libqb/src/http.cpp @@ -0,0 +1,495 @@ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mutex.h" +#include "condvar.h" +#include "thread.h" +#include "completion.h" +#include "buffer.h" +#include "http.h" + +struct handle { + int id = 0; + + CURL *con = NULL; + + // Lock protects all the below members + struct libqb_mutex *io_lock; + + struct libqb_buffer out; + int closed = 0; + int err = 0; + + struct completion *response_started = NULL; + + // Reports whether the below info is valid. It won't be if the response + // hasn't started yet. + bool has_info = false; + + bool has_content_length = false; + uint64_t content_length = 0; + char *url = NULL; + + handle() { + io_lock = libqb_mutex_new(); + libqb_buffer_init(&out); + } + + ~handle() { + libqb_mutex_free(io_lock); + libqb_buffer_clear(&out); + + if (url) + free(url); + } +}; + +// Signals the curl thread that a new handle was added, it's CURL connection should be started. +struct add_handle { + int handle; + int err; + + struct completion added; +}; + +// Signals to the curl thread that a handle is finished and should be closed +struct close_handle { + int handle; + + struct completion closed; +}; + +struct curl_state { + CURLM *multi; + + // Lock protects all the below members + struct libqb_mutex *lock; + + struct std::unordered_map handle_table; + std::queue add_handle_queue; + std::queue close_handle_queue; + int stop_curl; + + curl_state() { + lock = libqb_mutex_new(); + } +}; + +// Fills out all of the 'info' fields in the handle, sets the has_info flag, +// and triggers the response_started completion if it exists. +// +// Handle should be locked when calling this function +static void __fillout_curl_info(struct handle *handle) { + if (handle->has_info) + return; + + handle->has_info = true; + + curl_off_t cl; + CURLcode res = curl_easy_getinfo(handle->con, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl); + + if (res != CURLE_OK || cl == -1) { + handle->has_content_length = false; + } else { + handle->has_content_length = true; + handle->content_length = cl; + } + + char *urlp = NULL; + + res = curl_easy_getinfo(handle->con, CURLINFO_EFFECTIVE_URL, &urlp); + + if (urlp) + handle->url = strdup(urlp); + + // If someone is waiting for the info then we let them know + if (handle->response_started) { + completion_finish(handle->response_started); + handle->response_started = NULL; + } +} + +// Processes the handle addition and deletion lists +static void process_handles(struct curl_state *state) { + std::list connectionsToDrop; + + { + libqb_mutex_guard guard(state->lock); + + for (; !state->add_handle_queue.empty(); state->add_handle_queue.pop()) { + struct add_handle *add = state->add_handle_queue.front(); + + curl_multi_add_handle(state->multi, state->handle_table[add->handle]->con); + + add->err = 0; + completion_finish(&add->added); + } + + for (; !state->close_handle_queue.empty(); state->close_handle_queue.pop()) { + struct close_handle *close = state->close_handle_queue.front(); + struct handle *handle = state->handle_table[close->handle]; + + // If this was already finished, then con will be NULL + if (handle->con) + connectionsToDrop.push_back(handle->con); + + state->handle_table.erase(close->handle); + + delete handle; + + completion_finish(&close->closed); + } + } + + for (CURL * const &con : connectionsToDrop) { + // Removing the connection can trigger the callbacks to get run. Due to + // that we have to call it without holding the lock, or we could + // deadlock. + curl_multi_remove_handle(state->multi, con); + curl_easy_cleanup(con); + } +} + +static void handle_messages(struct curl_state *state) { + CURLMsg *msg; + int left; + + while (msg = curl_multi_info_read(state->multi, &left), msg) { + if (msg->msg != CURLMSG_DONE) + continue; + + CURL *e = msg->easy_handle; + + struct handle *handle; + curl_easy_getinfo(e, CURLINFO_PRIVATE, &handle); + + handle->con = NULL; + + { + libqb_mutex_guard guard(handle->io_lock); + + handle->closed = 1; + + // In the event this connection had no data, we want to make sure + // to fill this out and trigger the completion if there is one. + __fillout_curl_info(handle); + + switch (msg->data.result) { + case CURLE_OK: + break; + + default: + handle->err = 1; + break; + } + } + + curl_multi_remove_handle(state->multi, e); + curl_easy_cleanup(e); + } +} + +#if CURL_AT_LEAST_VERSION(7, 68, 0) +static void curl_state_poll(struct curl_state *state) { + // We use a one second timeout to avoid any accidental deadlocks if we + // don't wakeup the thread. + curl_multi_poll(state->multi, NULL, 0, 1000, NULL); +} + +static void curl_state_wakeup(struct curl_state *state) { + curl_multi_wakeup(state->multi); +} +#else +// This is a workaround for libcurl version lacking the curl_multi_poll() and +// curl_multi_wakeup() functions. Unfortunately this old version is on OS X, +// so we need to support it +// +// We use curl_multi_wait() with a small timeout, and don't support wakeup (so +// commands have to wait for the timeout to trigger). +// +// If numfds from curl_multi_wait() is zero, then we have to do the timeout +// manually via usleep() +static void curl_state_poll(struct curl_state *state) { + int numfds = 0; + + curl_multi_wait(state->multi, NULL, 0, 50, &numfds); + + if (!numfds) + usleep(50 * 1000); +} + +static void curl_state_wakeup(struct curl_state *state) { + // NOP, curl_state_poll will timeout automatically +} +#endif + +static void libqb_curl_thread_handler(void *arg) { + struct curl_state *state = (struct curl_state *)arg; + int running_transfers = 0; + + while (!state->stop_curl) { + curl_state_poll(state); + + // Process handle additions and calls to close() + process_handles(state); + + // Process requests, performs any read/write operations + curl_multi_perform(state->multi, &running_transfers); + + // Handle any requests that finished + handle_messages(state); + } + + // FIXME: This should do graceful closing for uploads, but because we only support + // downloads at the moment throwing the data away doesn't matter. +} + +static struct curl_state curl_state; +static struct libqb_thread *curl_thread; + +// This callback services the data recevied from the http connection. +static size_t receive_http_block(void *ptr, size_t size, size_t nmemb, void *data) { + struct handle *handle = (struct handle *)data; + size_t length = size * nmemb; + + libqb_mutex_guard guard(handle->io_lock); + + libqb_buffer_write(&handle->out, (const char *)ptr, length); + + // The first time this connection starts to recieve data we fill out the + // connection info. + __fillout_curl_info(handle); + + return length; +} + +static bool is_valid_http_id(int id) +{ + return curl_state.handle_table.find(id) != curl_state.handle_table.end(); +} + +int libqb_http_get_length(int id, size_t *length) { + if (!is_valid_http_id(id)) + return -1; + + struct handle *handle = curl_state.handle_table[id]; + + libqb_mutex_guard guard(handle->io_lock); + + *length = libqb_buffer_length(&handle->out); + + return 0; +} + +// Waits for handle to have valid info, which is availiable after the +// connection headers have been recieved. +static void wait_for_info(struct handle *handle) { + struct completion comp; + + { + libqb_mutex_guard guard(handle->io_lock); + + if (handle->has_info) + return; + + completion_init(&comp); + handle->response_started = ∁ + } + + completion_wait(&comp); + completion_clear(&comp); +} + +int libqb_http_get_content_length(int id, uint64_t *length) { + if (!is_valid_http_id(id)) + return -1; + + struct handle *handle = curl_state.handle_table[id]; + + wait_for_info(handle); + + { + libqb_mutex_guard guard(handle->io_lock); + + if (!handle->has_content_length) + return -1; + + *length = handle->content_length; + return 0; + } +} + +const char *libqb_http_get_url(int id) { + if (!is_valid_http_id(id)) + return NULL; + + struct handle *handle = curl_state.handle_table[id]; + + wait_for_info(handle); + + { + libqb_mutex_guard guard(handle->io_lock); + + return handle->url; + } +} + +int libqb_http_get(int id, char *buf, size_t *length) { + if (!is_valid_http_id(id)) + return -1; + + struct handle *handle = curl_state.handle_table[id]; + + libqb_mutex_guard guard(handle->io_lock); + + *length = libqb_buffer_read(&handle->out, buf, *length); + + return 0; +} + +int libqb_http_get_fixed(int id, char *buf, size_t length) { + if (!is_valid_http_id(id)) + return -1; + + struct handle *handle = curl_state.handle_table[id]; + + libqb_mutex_guard guard(handle->io_lock); + + size_t total_length = libqb_buffer_length(&handle->out); + + if (total_length < length) + return -1; + + libqb_buffer_read(&handle->out, buf, length); + + return 0; +} + +int libqb_http_connected(int id) { + if (!is_valid_http_id(id)) + return -1; + + struct handle *handle = curl_state.handle_table[id]; + + libqb_mutex_guard guard(handle->io_lock); + + return !handle->closed; +} + +int libqb_http_open(const char *url, int id) { + struct handle *handle = new struct handle(); + + handle->id = id; + + handle->con = curl_easy_init(); + curl_easy_setopt(handle->con, CURLOPT_PRIVATE, handle); + curl_easy_setopt(handle->con, CURLOPT_URL, url); + + curl_easy_setopt(handle->con, CURLOPT_WRITEFUNCTION, receive_http_block); + curl_easy_setopt(handle->con, CURLOPT_WRITEDATA, handle); + + // Allow redirects to be followed + curl_easy_setopt(handle->con, CURLOPT_FOLLOWLOCATION, 1); + + // Only allow HTTP and HTTPS to be used + curl_easy_setopt(handle->con, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + curl_easy_setopt(handle->con, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + struct add_handle add; + struct completion info_comp; + + add.handle = id; + completion_init(&add.added); + completion_init(&info_comp); + + handle->response_started = &info_comp; + + { + libqb_mutex_guard guard(curl_state.lock); + + curl_state.handle_table.insert(std::make_pair(id, handle)); + curl_state.add_handle_queue.push(&add); + } + + // Kick the curl thread so that it processes our new handle + curl_state_wakeup(&curl_state); + + completion_wait(&add.added); + completion_clear(&add.added); + + // Wait for the Server's response to begin + completion_wait(&info_comp); + completion_clear(&info_comp); + + int errored = 0; + + { + libqb_mutex_guard guard(handle->io_lock); + + // If this is set, the connection bombed out with some kind of error + errored = handle->err; + } + + if (errored) { + libqb_http_close(id); + return -1; + } + + return 0; +} + +int libqb_http_close(int id) { + if (!is_valid_http_id(id)) + return -1; + + struct close_handle close; + + close.handle = id; + completion_init(&close.closed); + + { + libqb_mutex_guard guard(curl_state.lock); + + curl_state.close_handle_queue.push(&close); + } + + // Kick the curl thread so that it processes our close request + curl_state_wakeup(&curl_state); + + completion_wait(&close.closed); + completion_clear(&close.closed); + + return 0; +} + +void libqb_http_init() { + curl_global_init(CURL_GLOBAL_DEFAULT); + + curl_state.multi = curl_multi_init(); + + curl_thread = libqb_thread_new(); + libqb_thread_start(curl_thread, libqb_curl_thread_handler, &curl_state); +} + +void libqb_http_stop() { + { + libqb_mutex_guard guard(curl_state.lock); + + // Tell CURL to finish up requests + curl_state.stop_curl = 1; + } + + curl_state_wakeup(&curl_state); + + // Wait for curl to finish + libqb_thread_join(curl_thread); + + libqb_thread_free(curl_thread); +} diff --git a/internal/c/parts/network/http/build.mk b/internal/c/parts/network/http/build.mk new file mode 100644 index 000000000..cccb70a13 --- /dev/null +++ b/internal/c/parts/network/http/build.mk @@ -0,0 +1,38 @@ + +ifeq ($(OS),win) +# This version is only used for Windows, Linux and OSX use the library provided by their system + +CURL_LIB := $(PATH_INTERNAL_C)/parts/network/http/libcurl.a + +CURL_MAKE_FLAGS := CFG=-schannel +CURL_MAKE_FLAGS += "CURL_CFLAG_EXTRAS=-DCURL_STATICLIB -DHTTP_ONLY" +CURL_MAKE_FLAGS += CC=../../../../c_compiler/bin/gcc.exe +CURL_MAKE_FLAGS += AR=../../../../c_compiler/bin/ar.exe +CURL_MAKE_FLAGS += RANLIB=../../../../c_compiler/bin/ranlib.exe +CURL_MAKE_FLAGS += STRIP=../../../../c_compiler/bin/strip.exe +CURL_MAKE_FLAGS += libcurl_a_LIBRARY="../libcurl.a" +CURL_MAKE_FLAGS += ARCH=w$(BITS) + +$(CURL_LIB): + $(MAKE) -C $(PATH_INTERNAL_C)/parts/network/http/curl -f ./Makefile.m32 $(CURL_MAKE_FLAGS) "../libcurl.a" + +.PHONY: clean-curl-lib +clean-curl-lib: + $(MAKE) -C $(PATH_INTERNAL_C)/parts/network/http/curl -f ./Makefile.m32 $(CURL_MAKE_FLAGS) clean + +CLEAN_DEP_LIST += clean-curl-lib +CLEAN_LIST += $(CURL_LIB) + +CURL_EXE_LIBS := $(CURL_LIB) + +CURL_CXXFLAGS += -DCURL_STATICLIB +CURL_CXXFLAGS += -I$(PATH_INTERNAL_C)/parts/network/http/include + +CURL_CXXLIBS += -lcrypt32 + +else + +CURL_EXE_LIBS := +CURL_CXXLIBS := +CURL_CXXLIBS += -lcurl +endif diff --git a/tests/build.mk b/tests/build.mk index b697158d2..f69032c64 100644 --- a/tests/build.mk +++ b/tests/build.mk @@ -11,11 +11,25 @@ TEST_DEF_OBJS := tests/c/test.o # Defines the list of test sets TESTS += buffer +TESTS += http # Describe how to build each test buffer.src-y := ./tests/c/buffer.cpp \ $(PATH_LIBQB)/src/buffer.cpp +http.src-y := ./tests/c/http.cpp \ + $(PATH_LIBQB)/src/http.cpp \ + $(PATH_LIBQB)/src/buffer.cpp \ + $(PATH_LIBQB)/src/threading-$(PLATFORM).cpp \ + $(PATH_LIBQB)/src/threading.cpp + +http.cflags-y := $(CURL_CXXFLAGS) +http.libs-y := $(CURL_CXXLIBS) +http.exe-libs-y := $(CURL_EXE_LIBS) + +http.libs-$(lnx) += -lpthread +http.libs-$(win) += -lws2_32 + TEST_OBJS := $(TEST_DEF_OBJS) TEST_OBJS += $(foreach test,$(TESTS),$(filter ./tests/c/%,$($(test)).objs-y)) diff --git a/tests/c/http.cpp b/tests/c/http.cpp new file mode 100644 index 000000000..346c211ef --- /dev/null +++ b/tests/c/http.cpp @@ -0,0 +1,116 @@ + +#include +#include +#include +#include + +#include "test.h" +#include "http.h" + +const char *example_result = +"\n" +"\n" +"\n" +" Example Domain\n" +"\n" +" \n" +" \n" +" \n" +" \n" +"\n" +"\n" +"\n" +"
\n" +"

Example Domain

\n" +"

This domain is for use in illustrative examples in documents. You may use this\n" +" domain in literature without prior coordination or asking for permission.

\n" +"

More information...

\n" +"
\n" +"\n" +"\n" +; + +void test_http() { + size_t expected_result_len = strlen(example_result); + const char *urls[] = { + "http://www.example.com", + "https://www.example.com", + "HTTPS://WWW.EXAMPLE.COM", + "httP://wwW.example.com", + "www.example.com", + "http://www.example.com:80", + "https://www.example.com:443", + NULL + }; + + for (const char **url = urls; *url; url++) { + int c = libqb_http_open(*url, 1); + + while (libqb_http_connected(1)) + usleep(10); + + size_t buflen = 0; + int err = libqb_http_get_length(1, &buflen); + + test_assert_ints_with_name(*url, 0, err); + test_assert_ints_with_name(*url, expected_result_len, buflen); + + char buf[4096]; + + buflen = sizeof(buf); + err = libqb_http_get(1, buf, &buflen); + test_assert_ints_with_name(*url, 0, err); + test_assert_ints_with_name(*url, expected_result_len, buflen); + test_assert_buffers_with_name(*url, example_result, buf, expected_result_len); + + // Verify Content-Length header is read correctly + uint64_t len; + err = libqb_http_get_content_length(1, &len); + test_assert_ints_with_name(*url, 0, err); + + test_assert_ints_with_name(*url, expected_result_len, len); + + err = libqb_http_close(1); + + test_assert_ints_with_name(*url, 0, err); + } +} + +int main() { + libqb_http_init(); + + int ret; + struct unit_test tests[] = { + { test_http, "http" }, + }; + + ret = run_tests("http", tests, sizeof(tests) / sizeof(*tests)); + + return ret; +} diff --git a/tests/run_c_tests.sh b/tests/run_c_tests.sh index ba7b9584f..a32d6d6e2 100755 --- a/tests/run_c_tests.sh +++ b/tests/run_c_tests.sh @@ -4,7 +4,7 @@ result=0 -for test in buffer +for test in buffer http do ./tests/exes/cpp/${test}_test || result=1 done