#ifndef SOKOL_FETCH_INCLUDED /* sokol_fetch.h -- asynchronous data loading/streaming Project URL: https://github.com/floooh/sokol Do this: #define SOKOL_IMPL before you include this file in *one* C or C++ file to create the implementation. Optionally provide the following defines with your own implementations: SOKOL_ASSERT(c) - your own assert macro (default: assert(c)) SOKOL_MALLOC(s) - your own malloc function (default: malloc(s)) SOKOL_FREE(p) - your own free function (default: free(p)) SOKOL_LOG(msg) - your own logging function (default: puts(msg)) SOKOL_UNREACHABLE() - a guard macro for unreachable code (default: assert(false)) SOKOL_API_DECL - public function declaration prefix (default: extern) SOKOL_API_IMPL - public function implementation prefix (default: -) SFETCH_MAX_PATH - max length of UTF-8 filesystem path / URL (default: 1024 bytes) SFETCH_MAX_USERDATA_UINT64 - max size of embedded userdata in number of uint64_t, userdata will be copied into an 8-byte aligned memory region associated with each in-flight request, default value is 16 (== 128 bytes) SFETCH_MAX_CHANNELS - max number of IO channels (default is 16, also see sfetch_desc_t.num_channels) If sokol_fetch.h is compiled as a DLL, define the following before including the declaration or implementation: SOKOL_DLL On Windows, SOKOL_DLL will define SOKOL_API_DECL as __declspec(dllexport) or __declspec(dllimport) as needed. NOTE: The following documentation talks a lot about "IO threads". Actual threads are only used on platforms where threads are available. The web version (emscripten/wasm) doesn't use POSIX-style threads, but instead asynchronous Javascript calls chained together by callbacks. The actual source code differences between the two approaches have been kept to a minimum though. FEATURE OVERVIEW ================ - Asynchronously load complete files, or stream files incrementally via HTTP (on web platform), or the local file system (on native platforms) - Request / response-callback model, user code sends a request to initiate a file-load, sokol_fetch.h calls the response callback on the same thread when data is ready or user-code needs to respond otherwise - Not limited to the main-thread or a single thread: A sokol-fetch "context" can live on any thread, and multiple contexts can operate side-by-side on different threads. - Memory management for data buffers is under full control of user code. sokol_fetch.h won't allocate memory after it has been setup. - Automatic rate-limiting guarantees that only a maximum number of requests is processed at any one time, allowing a zero-allocation model, where all data is streamed into fixed-size, pre-allocated buffers. - Active Requests can be paused, continued and cancelled from anywhere in the user-thread which sent this request. TL;DR EXAMPLE CODE ================== This is the most-simple example code to load a single data file with a known maximum size: (1) initialize sokol-fetch with default parameters (but NOTE that the default setup parameters provide a safe-but-slow "serialized" operation): sfetch_setup(&(sfetch_desc_t){ 0 }); (2) send a fetch-request to load a file from the current directory into a buffer big enough to hold the entire file content: static uint8_t buf[MAX_FILE_SIZE]; sfetch_send(&(sfetch_request_t){ .path = "my_file.txt", .callback = response_callback, .buffer_ptr = buf, .buffer_size = sizeof(buf) }); (3) write a 'response-callback' function, this will be called whenever the user-code must respond to state changes of the request (most importantly when data has been loaded): void response_callback(const sfetch_response_t* response) { if (response->fetched) { // data has been loaded, and is available via // 'buffer_ptr' and 'fetched_size': const void* data = response->buffer_ptr; uint64_t num_bytes = response->fetched_size; } if (response->finished) { // the 'finished'-flag is the catch-all flag for when the request // is finished, no matter if loading was successful or failed, // so any cleanup-work should happen here... ... if (response->failed) { // 'failed' is true in (addition to 'finished') if something // went wrong (file doesn't exist, or less bytes could be // read from the file than expected) } } } (4) finally, call sfetch_shutdown() at the end of the application: sfetch_shutdown() There's many other loading-scenarios, for instance one doesn't have to provide a buffer upfront, this can also happen in the response callback. Or it's possible to stream huge files into small fixed-size buffer, complete with pausing and continuing the download. It's also possible to improve the 'pipeline throughput' by fetching multiple files in parallel, but at the same time limit the maximum number of requests that can be 'in-flight'. For how this all works, please read the following documentation sections :) API DOCUMENTATION ================= void sfetch_setup(const sfetch_desc_t* desc) -------------------------------------------- First call sfetch_setup(const sfetch_desc_t*) on any thread before calling any other sokol-fetch functions on the same thread. sfetch_setup() takes a pointer to an sfetch_desc_t struct with setup parameters. Parameters which should use their default values must be zero-initialized: - max_requests (uint32_t): The maximum number of requests that can be alive at any time, the default is 128. - num_channels (uint32_t): The number of "IO channels" used to parallelize and prioritize requests, the default is 1. - num_lanes (uint32_t): The number of "lanes" on a single channel. Each request which is currently 'inflight' on a channel occupies one lane until the request is finished. This is used for automatic rate-limiting (search below for CHANNELS AND LANES for more details). The default number of lanes is 1. For example, to setup sokol-fetch for max 1024 active requests, 4 channels, and 8 lanes per channel in C99: sfetch_setup(&(sfetch_desc_t){ .max_requests = 1024, .num_channels = 4, .num_lanes = 8 }); sfetch_setup() is the only place where sokol-fetch will allocate memory. NOTE that the default setup parameters of 1 channel and 1 lane per channel has a very poor 'pipeline throughput' since this essentially serializes IO requests (a new request will only be processed when the last one has finished), and since each request needs at least one roundtrip between the user- and IO-thread the throughput will be at most one request per frame. Search for LATENCY AND THROUGHPUT below for more information on how to increase throughput. NOTE that you can call sfetch_setup() on multiple threads, each thread will get its own thread-local sokol-fetch instance, which will work independently from sokol-fetch instances on other threads. void sfetch_shutdown(void) -------------------------- Call sfetch_shutdown() at the end of the application to stop any IO threads and free all memory that was allocated in sfetch_setup(). sfetch_handle_t sfetch_send(const sfetch_request_t* request) ------------------------------------------------------------ Call sfetch_send() to start loading data, the function takes a pointer to an sfetch_request_t struct with request parameters and returns a sfetch_handle_t identifying the request for later calls. At least a path/URL and callback must be provided: sfetch_handle_t h = sfetch_send(&(sfetch_request_t){ .path = "my_file.txt", .callback = my_response_callback }); sfetch_send() will return an invalid handle if no request can be allocated from the internal pool because all available request items are 'in-flight'. The sfetch_request_t struct contains the following parameters (optional parameters that are not provided must be zero-initialized): - path (const char*, required) Pointer to an UTF-8 encoded C string describing the filesystem path or HTTP URL. The string will be copied into an internal data structure, and passed "as is" (apart from any required encoding-conversions) to fopen(), CreateFileW() or XMLHttpRequest. The maximum length of the string is defined by the SFETCH_MAX_PATH configuration define, the default is 1024 bytes including the 0-terminator byte. - callback (sfetch_callback_t, required) Pointer to a response-callback function which is called when the request needs "user code attention". Search below for REQUEST STATES AND THE RESPONSE CALLBACK for detailed information about handling responses in the response callback. - channel (uint32_t, optional) Index of the IO channel where the request should be processed. Channels are used to parallelize and prioritize requests relative to each other. Search below for CHANNELS AND LANES for more information. The default channel is 0. - chunk_size (uint32_t, optional) The chunk_size member is used for streaming data incrementally in small chunks. After 'chunk_size' bytes have been loaded into to the streaming buffer, the response callback will be called with the buffer containing the fetched data for the current chunk. If chunk_size is 0 (the default), than the whole file will be loaded. Please search below for CHUNK SIZE AND HTTP COMPRESSION for important information how streaming works if the web server is serving compressed data. - buffer_ptr, buffer_size (void*, uint64_t, optional) This is a optional pointer/size pair describing a chunk of memory where data will be loaded into (if no buffer is provided upfront, this must happen in the response callback). If a buffer is provided, it must be big enough to either hold the entire file (if chunk_size is zero), or the *uncompressed* data for one downloaded chunk (if chunk_size is > 0). - user_data_ptr, user_data_size (const void*, uint32_t, both optional) user_data_ptr and user_data_size describe an optional POD (plain-old-data) associated with the request which will be copied(!) into an internal memory block. The maximum default size of this memory block is 128 bytes (but can be overridden by defining SFETCH_MAX_USERDATA_UINT64 before including the notification, note that this define is in "number of uint64_t", not number of bytes). The user-data block is 8-byte aligned, and will be copied via memcpy() (so don't put any C++ "smart members" in there). NOTE that request handles are strictly thread-local and only unique within the thread the handle was created on, and all function calls involving a request handle must happen on that same thread. bool sfetch_handle_valid(sfetch_handle_t request) ------------------------------------------------- This checks if the provided request handle is valid, and is associated with a currently active request. It will return false if: - sfetch_send() returned an invalid handle because it couldn't allocate a new request from the internal request pool (because they're all in flight) - the request associated with the handle is no longer alive (because it either finished successfully, or the request failed for some reason) void sfetch_dowork(void) ------------------------ Call sfetch_dowork(void) in regular intervals (for instance once per frame) on the same thread as sfetch_setup() to "turn the gears". If you are sending requests but never hear back from them in the response callback function, then the most likely reason is that you forgot to add the call to sfetch_dowork() in the per-frame function. sfetch_dowork() roughly performs the following work: - any new requests that have been sent with sfetch_send() since the last call to sfetch_dowork() will be dispatched to their IO channels and assigned a free lane. If all lanes on that channel are occupied by requests 'in flight', incoming requests must wait until a lane becomes avaible - for all new requests which have been enquened on a channel which don't already have a buffer assigned the response callback will be called with (response->dispatched == true) so that the response callback can inspect the dynamically assigned lane and bind a buffer to the request (search below for CHANNELS AND LANE for more info) - a state transition from "user side" to "IO thread side" happens for each new request that has been dispatched to a channel. - requests dispatched to a channel are either forwarded into that channel's worker thread (on native platforms), or cause an HTTP request to be sent via an asynchronous XMLHttpRequest (on the web platform) - for all requests which have finished their current IO operation a state transition from "IO thread side" to "user side" happens, and the response callback is called so that the fetched data can be processed. - requests which are completely finished (either because the entire file content has been loaded, or they are in the FAILED state) are freed (this just changes their state in the 'request pool', no actual memory is freed) - requests which are not yet finished are fed back into the 'incoming' queue of their channel, and the cycle starts again, this only happens for requests which perform data streaming (not load the entire file at once). void sfetch_cancel(sfetch_handle_t request) ------------------------------------------- This cancels a request in the next sfetch_dowork() call and invokes the response callback with (response.failed == true) and (response.finished == true) to give user-code a chance to do any cleanup work for the request. If sfetch_cancel() is called for a request that is no longer alive, nothing bad will happen (the call will simply do nothing). void sfetch_pause(sfetch_handle_t request) ------------------------------------------ This pauses an active request in the next sfetch_dowork() call and puts it into the PAUSED state. For all requests in PAUSED state, the response callback will be called in each call to sfetch_dowork() to give user-code a chance to CONTINUE the request (by calling sfetch_continue()). Pausing a request makes sense for dynamic rate-limiting in streaming scenarios (like video/audio streaming with a fixed number of streaming buffers. As soon as all available buffers are filled with download data, downloading more data must be prevented to allow video/audio playback to catch up and free up empty buffers for new download data. void sfetch_continue(sfetch_handle_t request) --------------------------------------------- Continues a paused request, counterpart to the sfetch_pause() function. void sfetch_bind_buffer(sfetch_handle_t request, void* buffer_ptr, uint64_t buffer_size) ---------------------------------------------------------------------------------------- This "binds" a new buffer (pointer/size pair) to an active request. The function *must* be called from inside the response-callback, and there must not already be another buffer bound. void* sfetch_unbind_buffer(sfetch_handle_t request) --------------------------------------------------- This removes the current buffer binding from the request and returns a pointer to the previous buffer (useful if the buffer was dynamically allocated and it must be freed). sfetch_unbind_buffer() *must* be called from inside the response callback. The usual code sequence to bind a different buffer in the response callback might look like this: void response_callback(const sfetch_response_t* response) { if (response.fetched) { ... // switch to a different buffer (in the FETCHED state it is // guaranteed that the request has a buffer, otherwise it // would have gone into the FAILED state void* old_buf_ptr = sfetch_unbind_buffer(response.handle); free(old_buf_ptr); void* new_buf_ptr = malloc(new_buf_size); sfetch_bind_buffer(response.handle, new_buf_ptr, new_buf_size); } if (response.finished) { // unbind and free the currently associated buffer, // the buffer pointer could be null if the request has failed // NOTE that it is legal to call free() with a nullptr, // this happens if the request failed to open its file // and never goes into the OPENED state void* buf_ptr = sfetch_unbind_buffer(response.handle); free(buf_ptr); } } sfetch_desc_t sfetch_desc(void) ------------------------------- sfetch_desc() returns a copy of the sfetch_desc_t struct passed to sfetch_setup(), with zero-inititialized values replaced with their default values. int sfetch_max_userdata_bytes(void) ----------------------------------- This returns the value of the SFETCH_MAX_USERDATA_UINT64 config define, but in number of bytes (so SFETCH_MAX_USERDATA_UINT64*8). int sfetch_max_path(void) ------------------------- Returns the value of the SFETCH_MAX_PATH config define. REQUEST STATES AND THE RESPONSE CALLBACK ======================================== A request goes through a number of states during its lifetime. Depending on the current state of a request, it will be 'owned' either by the "user-thread" (where the request was sent) or an IO thread. You can think of a request as "ping-ponging" between the IO thread and user thread, any actual IO work is done on the IO thread, while invocations of the response-callback happen on the user-thread. All state transitions and callback invocations happen inside the sfetch_dowork() function. An active request goes through the following states: ALLOCATED (user-thread) The request has been allocated in sfetch_send() and is waiting to be dispatched into its IO channel. When this happens, the request will transition into the DISPATCHED state. DISPATCHED (IO thread) The request has been dispatched into its IO channel, and a lane has been assigned to the request. If a buffer was provided in sfetch_send() the request will immediately transition into the FETCHING state and start loading data into the buffer. If no buffer was provided in sfetch_send(), the response callback will be called with (response->dispatched == true), so that the response callback can bind a buffer to the request. Binding the buffer in the response callback makes sense if the buffer isn't dynamically allocated, but instead a pre-allocated buffer must be selected from the request's channel and lane. Note that it isn't possible to get a file size in the response callback which would help with allocating a buffer of the right size, this is because it isn't possible in HTTP to query the file size before the entire file is downloaded (...when the web server serves files compressed). If opening the file failed, the request will transition into the FAILED state with the error code SFETCH_ERROR_FILE_NOT_FOUND. FETCHING (IO thread) While a request is in the FETCHING state, data will be loaded into the user-provided buffer. If no buffer was provided, the request will go into the FAILED state with the error code SFETCH_ERROR_NO_BUFFER. If a buffer was provided, but it is too small to contain the fetched data, the request will go into the FAILED state with error code SFETCH_ERROR_BUFFER_TOO_SMALL. If less data can be read from the file than expected, the request will go into the FAILED state with error code SFETCH_ERROR_UNEXPECTED_EOF. If loading data into the provided buffer works as expected, the request will go into the FETCHED state. FETCHED (user thread) The request goes into the FETCHED state either when the entire file has been loaded into the provided buffer (when request.chunk_size == 0), or a chunk has been loaded (and optionally decompressed) into the buffer (when request.chunk_size > 0). The response callback will be called so that the user-code can process the loaded data using the following sfetch_response_t struct members: - fetched_size: the number of bytes in the provided buffer - buffer_ptr: pointer to the start of fetched data - fetched_offset: the byte offset of the loaded data chunk in the overall file (this is only set to a non-zero value in a streaming scenario) Once all file data has been loaded, the 'finished' flag will be set in the response callback's sfetch_response_t argument. After the user callback returns, and all file data has been loaded (response.finished flag is set) the request has reached its end-of-life and will recycled. Otherwise, if there's still data to load (because streaming was requested by providing a non-zero request.chunk_size), the request will switch back to the FETCHING state to load the next chunk of data. Note that it is ok to associate a different buffer or buffer-size with the request by calling sfetch_bind_buffer() in the response-callback. To check in the response callback for the FETCHED state, and independently whether the request is finished: void response_callback(const sfetch_response_t* response) { if (response->fetched) { // request is in FETCHED state, the loaded data is available // in .buffer_ptr, and the number of bytes that have been // loaded in .fetched_size: const void* data = response->buffer_ptr; const uint64_t num_bytes = response->fetched_size; } if (response->finished) { // the finished flag is set either when all data // has been loaded, the request has been cancelled, // or the file operation has failed, this is where // any required per-request cleanup work should happen } } FAILED (user thread) A request will transition into the FAILED state in the following situations: - if the file doesn't exist or couldn't be opened for other reasons (SFETCH_ERROR_FILE_NOT_FOUND) - if no buffer is associated with the request in the FETCHING state (SFETCH_ERROR_NO_BUFFER) - if the provided buffer is too small to hold the entire file (if request.chunk_size == 0), or the (potentially decompressed) partial data chunk (SFETCH_ERROR_BUFFER_TOO_SMALL) - if less bytes could be read from the file then expected (SFETCH_ERROR_UNEXPECTED_EOF) - if a request has been cancelled via sfetch_cancel() (SFETCH_ERROR_CANCELLED) The response callback will be called once after a request goes into the FAILED state, with the 'response->finished' and 'response->failed' flags set to true. This gives the user-code a chance to cleanup any resources associated with the request. To check for the failed state in the response callback: void response_callback(const sfetch_response_t* response) { if (response->failed) { // specifically check for the failed state... } // or you can do a catch-all check via the finished-flag: if (response->finished) { if (response->failed) { // if more detailed error handling is needed: switch (response->error_code) { ... } } } } PAUSED (user thread) A request will transition into the PAUSED state after user-code calls the function sfetch_pause() on the request's handle. Usually this happens from within the response-callback in streaming scenarios when the data streaming needs to wait for a data decoder (like a video/audio player) to catch up. While a request is in PAUSED state, the response-callback will be called in each sfetch_dowork(), so that the user-code can either continue the request by calling sfetch_continue(), or cancel the request by calling sfetch_cancel(). When calling sfetch_continue() on a paused request, the request will transition into the FETCHING state. Otherwise if sfetch_cancel() is called, the request will switch into the FAILED state. To check for the PAUSED state in the response callback: void response_callback(const sfetch_response_t* response) { if (response->paused) { // we can check here whether the request should // continue to load data: if (should_continue(response->handle)) { sfetch_continue(response->handle); } } } CHUNK SIZE AND HTTP COMPRESSION =============================== TL;DR: for streaming scenarios, the provided chunk-size must be smaller than the provided buffer-size because the web server may decide to serve the data compressed and the chunk-size must be given in 'compressed bytes' while the buffer receives 'uncompressed bytes'. It's not possible in HTTP to query the uncompressed size for a compressed download until that download has finished. With vanilla HTTP, it is not possible to query the actual size of a file without downloading the entire file first (the Content-Length response header only provides the compressed size). Furthermore, for HTTP range-requests, the range is given on the compressed data, not the uncompressed data. So if the web server decides to server the data compressed, the content-length and range-request parameters don't correspond to the uncompressed data that's arriving in the sokol-fetch buffers, and there's no way from JS or WASM to either force uncompressed downloads (e.g. by setting the Accept-Encoding field), or access the compressed data. This has some implications for sokol_fetch.h, most notably that buffers can't be provided in the exactly right size, because that size can't be queried from HTTP before the data is actually downloaded. When downloading whole files at once, it is basically expected that you know the maximum files size upfront through other means (for instance through a separate meta-data-file which contains the file sizes and other meta-data for each file that needs to be loaded). For streaming downloads the situation is a bit more complicated. These use HTTP range-requests, and those ranges are defined on the (potentially) compressed data which the JS/WASM side doesn't have access to. However, the JS/WASM side only ever sees the uncompressed data, and it's not possible to query the uncompressed size of a range request before that range request has finished. If the provided buffer is too small to contain the uncompressed data, the request will fail with error code SFETCH_ERROR_BUFFER_TOO_SMALL. CHANNELS AND LANES ================== Channels and lanes are (somewhat artificial) concepts to manage parallelization, prioritization and rate-limiting. Channels can be used to parallelize message processing for better 'pipeline throughput', and to prioritize messages: user-code could reserve one channel for "small and big" streaming downloads, another channel for "regular" downloads and yet another high-priority channel which would only be used for small files which need to start loading immediately. Each channel comes with its own IO thread and message queues for pumping messages in and out of the thread. The channel where a request is processed is selected manually when sending a message: sfetch_send(&(sfetch_request_t){ .path = "my_file.txt", .callback = my_reponse_callback, .channel = 2 }); The number of channels is configured at startup in sfetch_setup() and cannot be changed afterwards. Channels are completely separate from each other, and a request will never "hop" from one channel to another. Each channel consists of a fixed number of "lanes" for automatic rate limiting: When a request is sent to a channel via sfetch_send(), a "free lane" will be picked and assigned to the request. The request will occupy this lane for its entire life time (also while it is paused). If all lanes of a channel are currently occupied, new requests will need to wait until a lane becomes unoccupied. Since the number of channels and lanes is known upfront, it is guaranteed that there will never be more than "num_channels * num_lanes" requests in flight at any one time. This guarantee eliminates unexpected load- and memory-spikes when many requests are sent in very short time, and it allows to pre-allocate a fixed number of memory buffers which can be reused for the entire "lifetime" of a sokol-fetch context. In the most simple scenario - when a maximum file size is known - buffers can be statically allocated like this: uint8_t buffer[NUM_CHANNELS][NUM_LANES][MAX_FILE_SIZE]; Then in the user callback pick a buffer by channel and lane, and associate it with the request like this: void response_callback(const sfetch_response_t* response) { if (response->dispatched) { void* ptr = buffer[response->channel][response->lane]; sfetch_bind_buffer(response->handle, ptr, MAX_FILE_SIZE); } ... } NOTES ON OPTIMIZING PIPELINE LATENCY AND THROUGHPUT =================================================== With the default configuration of 1 channel and 1 lane per channel, sokol_fetch.h will appear to have a shockingly bad loading performance if several files are loaded. This has two reasons: (1) all parallelization when loading data has been disabled. A new request will only be processed, when the last request has finished. (2) every invocation of the response-callback adds one frame of latency to the request, because callbacks will only be called from within sfetch_dowork() sokol-fetch takes a few shortcuts to improve step (2) and reduce the 'inherent latency' of a request: - if a buffer is provided upfront, the response-callback won't be called in the OPENED state, but start right with the FETCHED state where data has already been loaded into the buffer - there is no separate CLOSED state where the callback is invoked separately when loading has finished (or the request has failed), instead the finished and failed flags will be set as part of the last FETCHED invocation This means providing a big-enough buffer to fit the entire file is the best case, the response callback will only be called once, ideally in the next frame (or two calls to sfetch_dowork()). If no buffer is provided upfront, one frame of latency is added because the response callback needs to be invoked in the OPENED state so that the user code can bind a buffer. This means the best case for a request without an upfront-provided buffer is 2 frames (or 3 calls to sfetch_dowork()). That's about what can be done to improve the latency for a single request, but the really important step is to improve overall throughput. If you need to load thousands of files you don't want that to be completely serialized. The most important action to increase throughput is to increase the number of lanes per channel. This defines how many requests can be 'in flight' on a single channel at the same time. The guiding decision factor for how many lanes you can "afford" is the memory size you want to set aside for buffers. Each lane needs its own buffer so that the data loaded for one request doesn't scribble over the data loaded for another request. Here's a simple example of sending 4 requests without upfront buffer on a channel with 1, 2 and 4 lanes, each line is one frame: 1 LANE (8 frames): Lane 0: ------------- REQ 0 OPENED REQ 0 FETCHED REQ 1 OPENED REQ 1 FETCHED REQ 2 OPENED REQ 2 FETCHED REQ 3 OPENED REQ 3 FETCHED Note how the request don't overlap, so they can all use the same buffer. 2 LANES (4 frames): Lane 0: Lane 1: --------------------------------- REQ 0 OPENED REQ 1 OPENED REQ 0 FETCHED REQ 1 FETCHED REQ 2 OPENED REQ 3 OPENED REQ 2 FETCHED REQ 3 FETCHED This reduces the overall time to 4 frames, but now you need 2 buffers so that requests don't scribble over each other. 4 LANES (2 frames): Lane 0: Lane 1: Lane 2: Lane 3: ------------------------------------------------------------- REQ 0 OPENED REQ 1 OPENED REQ 2 OPENED REQ 3 OPENED REQ 0 FETCHED REQ 1 FETCHED REQ 2 FETCHED REQ 3 FETCHED Now we're down to the same 'best-case' latency as sending a single request. Apart from the memory requirements for the streaming buffers (which is under your control), you can be generous with the number of channels, they don't add any processing overhead. The last option for tweaking latency and throughput is channels. Each channel works independently from other channels, so while one channel is busy working through a large number of requests (or one very long streaming download), you can set aside a high-priority channel for requests that need to start as soon as possible. On platforms with threading support, each channel runs on its own thread, but this is mainly an implementation detail to work around the blocking traditional file IO functions, not for performance reasons. FUTURE PLANS / V2.0 IDEA DUMP ============================= - An optional polling API (as alternative to callback API) - Move buffer-management into the API? The "manual management" can be quite tricky especially for dynamic allocation scenarios, API support for buffer management would simplify cases like preventing that requests scribble over each other's buffers, or an automatic garbage collection for dynamically allocated buffers, or automatically falling back to dynamic allocation if static buffers aren't big enough. - Pluggable request handlers to load data from other "sources" (especially HTTP downloads on native platforms via e.g. libcurl would be useful) - I'm currently not happy how the user-data block is handled, this should getting and updating the user-data should be wrapped by API functions (similar to bind/unbind buffer) LICENSE ======= zlib/libpng license Copyright (c) 2019 Andre Weissflog This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. */ #define SOKOL_FETCH_INCLUDED (1) #include #include #ifndef SOKOL_API_DECL #if defined(_WIN32) && defined(SOKOL_DLL) && defined(SOKOL_IMPL) #define SOKOL_API_DECL __declspec(dllexport) #elif defined(_WIN32) && defined(SOKOL_DLL) #define SOKOL_API_DECL __declspec(dllimport) #else #define SOKOL_API_DECL extern #endif #endif #ifdef __cplusplus extern "C" { #endif /* configuration values for sfetch_setup() */ typedef struct sfetch_desc_t { uint32_t _start_canary; uint32_t max_requests; /* max number of active requests across all channels, default is 128 */ uint32_t num_channels; /* number of channels to fetch requests in parallel, default is 1 */ uint32_t num_lanes; /* max number of requests active on the same channel, default is 1 */ uint32_t _end_canary; } sfetch_desc_t; /* a request handle to identify an active fetch request, returned by sfetch_send() */ typedef struct sfetch_handle_t { uint32_t id; } sfetch_handle_t; /* error codes */ typedef enum { SFETCH_ERROR_NO_ERROR, SFETCH_ERROR_FILE_NOT_FOUND, SFETCH_ERROR_NO_BUFFER, SFETCH_ERROR_BUFFER_TOO_SMALL, SFETCH_ERROR_UNEXPECTED_EOF, SFETCH_ERROR_INVALID_HTTP_STATUS, SFETCH_ERROR_CANCELLED } sfetch_error_t; /* the response struct passed to the response callback */ typedef struct sfetch_response_t { sfetch_handle_t handle; /* request handle this response belongs to */ bool dispatched; /* true when request is in DISPATCHED state (lane has been assigned) */ bool fetched; /* true when request is in FETCHED state (fetched data is available) */ bool paused; /* request is currently in paused state */ bool finished; /* this is the last response for this request */ bool failed; /* request has failed (always set together with 'finished') */ bool cancelled; /* request was cancelled (always set together with 'finished') */ sfetch_error_t error_code; /* more detailed error code when failed is true */ uint32_t channel; /* the channel which processes this request */ uint32_t lane; /* the lane this request occupies on its channel */ const char* path; /* the original filesystem path of the request (FIXME: this is unsafe, wrap in API call?) */ void* user_data; /* pointer to read/write user-data area (FIXME: this is unsafe, wrap in API call?) */ uint32_t fetched_offset; /* current offset of fetched data chunk in file data */ uint32_t fetched_size; /* size of fetched data chunk in number of bytes */ void* buffer_ptr; /* pointer to buffer with fetched data */ uint32_t buffer_size; /* overall buffer size (may be >= than fetched_size!) */ } sfetch_response_t; /* response callback function signature */ typedef void(*sfetch_callback_t)(const sfetch_response_t*); /* request parameters passed to sfetch_send() */ typedef struct sfetch_request_t { uint32_t _start_canary; uint32_t channel; /* index of channel this request is assigned to (default: 0) */ const char* path; /* filesystem path or HTTP URL (required) */ sfetch_callback_t callback; /* response callback function pointer (required) */ void* buffer_ptr; /* buffer pointer where data will be loaded into (optional) */ uint32_t buffer_size; /* buffer size in number of bytes (optional) */ uint32_t chunk_size; /* number of bytes to load per stream-block (optional) */ const void* user_data_ptr; /* pointer to a POD user-data block which will be memcpy'd(!) (optional) */ uint32_t user_data_size; /* size of user-data block (optional) */ uint32_t _end_canary; } sfetch_request_t; /* setup sokol-fetch (can be called on multiple threads) */ SOKOL_API_DECL void sfetch_setup(const sfetch_desc_t* desc); /* discard a sokol-fetch context */ SOKOL_API_DECL void sfetch_shutdown(void); /* return true if sokol-fetch has been setup */ SOKOL_API_DECL bool sfetch_valid(void); /* get the desc struct that was passed to sfetch_setup() */ SOKOL_API_DECL sfetch_desc_t sfetch_desc(void); /* return the max userdata size in number of bytes (SFETCH_MAX_USERDATA_UINT64 * sizeof(uint64_t)) */ SOKOL_API_DECL int sfetch_max_userdata_bytes(void); /* return the value of the SFETCH_MAX_PATH implementation config value */ SOKOL_API_DECL int sfetch_max_path(void); /* send a fetch-request, get handle to request back */ SOKOL_API_DECL sfetch_handle_t sfetch_send(const sfetch_request_t* request); /* return true if a handle is valid *and* the request is alive */ SOKOL_API_DECL bool sfetch_handle_valid(sfetch_handle_t h); /* do per-frame work, moves requests into and out of IO threads, and invokes response-callbacks */ SOKOL_API_DECL void sfetch_dowork(void); /* bind a data buffer to a request (request must not currently have a buffer bound, must be called from response callback */ SOKOL_API_DECL void sfetch_bind_buffer(sfetch_handle_t h, void* buffer_ptr, uint32_t buffer_size); /* clear the 'buffer binding' of a request, returns previous buffer pointer (can be 0), must be called from response callback */ SOKOL_API_DECL void* sfetch_unbind_buffer(sfetch_handle_t h); /* cancel a request that's in flight (will call response callback with .cancelled + .finished) */ SOKOL_API_DECL void sfetch_cancel(sfetch_handle_t h); /* pause a request (will call response callback each frame with .paused) */ SOKOL_API_DECL void sfetch_pause(sfetch_handle_t h); /* continue a paused request */ SOKOL_API_DECL void sfetch_continue(sfetch_handle_t h); #ifdef __cplusplus } /* extern "C" */ #endif #endif // SOKOL_FETCH_INCLUDED /*--- IMPLEMENTATION ---------------------------------------------------------*/ #ifdef SOKOL_IMPL #define SOKOL_FETCH_IMPL_INCLUDED (1) #include /* memset, memcpy */ #ifndef SFETCH_MAX_PATH #define SFETCH_MAX_PATH (1024) #endif #ifndef SFETCH_MAX_USERDATA_UINT64 #define SFETCH_MAX_USERDATA_UINT64 (16) #endif #ifndef SFETCH_MAX_CHANNELS #define SFETCH_MAX_CHANNELS (16) #endif #ifndef SOKOL_API_IMPL #define SOKOL_API_IMPL #endif #ifndef SOKOL_DEBUG #ifndef NDEBUG #define SOKOL_DEBUG (1) #endif #endif #ifndef SOKOL_ASSERT #include #define SOKOL_ASSERT(c) assert(c) #endif #ifndef SOKOL_MALLOC #include #define SOKOL_MALLOC(s) malloc(s) #define SOKOL_FREE(p) free(p) #endif #ifndef SOKOL_LOG #ifdef SOKOL_DEBUG #include #define SOKOL_LOG(s) { SOKOL_ASSERT(s); puts(s); } #else #define SOKOL_LOG(s) #endif #endif #ifndef _SOKOL_PRIVATE #if defined(__GNUC__) #define _SOKOL_PRIVATE __attribute__((unused)) static #else #define _SOKOL_PRIVATE static #endif #endif #if defined(__EMSCRIPTEN__) #include #define _SFETCH_PLATFORM_EMSCRIPTEN (1) #define _SFETCH_PLATFORM_WINDOWS (0) #define _SFETCH_PLATFORM_POSIX (0) #define _SFETCH_HAS_THREADS (0) #elif defined(_WIN32) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include #define _SFETCH_PLATFORM_WINDOWS (1) #define _SFETCH_PLATFORM_EMSCRIPTEN (0) #define _SFETCH_PLATFORM_POSIX (0) #define _SFETCH_HAS_THREADS (1) #else #include #include /* fopen, fread, fseek, fclose */ #define _SFETCH_PLATFORM_POSIX (1) #define _SFETCH_PLATFORM_EMSCRIPTEN (0) #define _SFETCH_PLATFORM_WINDOWS (0) #define _SFETCH_HAS_THREADS (1) #endif /*=== private type definitions ===============================================*/ typedef struct _sfetch_path_t { char buf[SFETCH_MAX_PATH]; } _sfetch_path_t; typedef struct _sfetch_buffer_t { uint8_t* ptr; uint32_t size; } _sfetch_buffer_t; /* a thread with incoming and outgoing message queue syncing */ #if _SFETCH_PLATFORM_POSIX typedef struct { pthread_t thread; pthread_cond_t incoming_cond; pthread_mutex_t incoming_mutex; pthread_mutex_t outgoing_mutex; pthread_mutex_t running_mutex; pthread_mutex_t stop_mutex; bool stop_requested; bool valid; } _sfetch_thread_t; #elif _SFETCH_PLATFORM_WINDOWS typedef struct { HANDLE thread; HANDLE incoming_event; CRITICAL_SECTION incoming_critsec; CRITICAL_SECTION outgoing_critsec; CRITICAL_SECTION running_critsec; CRITICAL_SECTION stop_critsec; bool stop_requested; bool valid; } _sfetch_thread_t; #endif /* file handle abstraction */ #if _SFETCH_PLATFORM_POSIX typedef FILE* _sfetch_file_handle_t; #define _SFETCH_INVALID_FILE_HANDLE (0) typedef void*(*_sfetch_thread_func_t)(void*); #elif _SFETCH_PLATFORM_WINDOWS typedef HANDLE _sfetch_file_handle_t; #define _SFETCH_INVALID_FILE_HANDLE (INVALID_HANDLE_VALUE) typedef LPTHREAD_START_ROUTINE _sfetch_thread_func_t; #endif /* user-side per-request state */ typedef struct { bool pause; /* switch item to PAUSED state if true */ bool cont; /* switch item back to FETCHING if true */ bool cancel; /* cancel the request, switch into FAILED state */ /* transfer IO => user thread */ uint32_t fetched_offset; /* number of bytes fetched so far */ uint32_t fetched_size; /* size of last fetched chunk */ sfetch_error_t error_code; bool finished; /* user thread only */ uint32_t user_data_size; uint64_t user_data[SFETCH_MAX_USERDATA_UINT64]; } _sfetch_item_user_t; /* thread-side per-request state */ typedef struct { /* transfer IO => user thread */ uint32_t fetched_offset; uint32_t fetched_size; sfetch_error_t error_code; bool failed; bool finished; /* IO thread only */ #if _SFETCH_PLATFORM_EMSCRIPTEN uint32_t http_range_offset; #else _sfetch_file_handle_t file_handle; #endif uint32_t content_size; } _sfetch_item_thread_t; /* a request goes through the following states, ping-ponging between IO and user thread */ typedef enum _sfetch_state_t { _SFETCH_STATE_INITIAL, /* internal: request has just been initialized */ _SFETCH_STATE_ALLOCATED, /* internal: request has been allocated from internal pool */ _SFETCH_STATE_DISPATCHED, /* user thread: request has been dispatched to its IO channel */ _SFETCH_STATE_FETCHING, /* IO thread: waiting for data to be fetched */ _SFETCH_STATE_FETCHED, /* user thread: fetched data available */ _SFETCH_STATE_PAUSED, /* user thread: request has been paused via sfetch_pause() */ _SFETCH_STATE_FAILED, /* user thread: follow state or FETCHING if something went wrong */ } _sfetch_state_t; /* an internal request item */ #define _SFETCH_INVALID_LANE (0xFFFFFFFF) typedef struct { sfetch_handle_t handle; _sfetch_state_t state; uint32_t channel; uint32_t lane; uint32_t chunk_size; sfetch_callback_t callback; _sfetch_buffer_t buffer; /* updated by IO-thread, off-limits to user thread */ _sfetch_item_thread_t thread; /* accessible by user-thread, off-limits to IO thread */ _sfetch_item_user_t user; /* big stuff at the end */ _sfetch_path_t path; } _sfetch_item_t; /* a pool of internal per-request items */ typedef struct { uint32_t size; uint32_t free_top; _sfetch_item_t* items; uint32_t* free_slots; uint32_t* gen_ctrs; bool valid; } _sfetch_pool_t; /* a ringbuffer for pool-slot ids */ typedef struct { uint32_t head; uint32_t tail; uint32_t num; uint32_t* buf; } _sfetch_ring_t; /* an IO channel with its own IO thread */ struct _sfetch_t; typedef struct { struct _sfetch_t* ctx; /* back-pointer to thread-local _sfetch state pointer, since this isn't accessible from the IO threads */ _sfetch_ring_t free_lanes; _sfetch_ring_t user_sent; _sfetch_ring_t user_incoming; _sfetch_ring_t user_outgoing; #if _SFETCH_HAS_THREADS _sfetch_ring_t thread_incoming; _sfetch_ring_t thread_outgoing; _sfetch_thread_t thread; #endif void (*request_handler)(struct _sfetch_t* ctx, uint32_t slot_id); bool valid; } _sfetch_channel_t; /* the sfetch global state */ typedef struct _sfetch_t { bool setup; bool valid; bool in_callback; sfetch_desc_t desc; _sfetch_pool_t pool; _sfetch_channel_t chn[SFETCH_MAX_CHANNELS]; } _sfetch_t; #if _SFETCH_HAS_THREADS #if defined(_MSC_VER) static __declspec(thread) _sfetch_t* _sfetch; #else static __thread _sfetch_t* _sfetch; #endif #else static _sfetch_t* _sfetch; #endif /*=== general helper functions and macros =====================================*/ #define _sfetch_def(val, def) (((val) == 0) ? (def) : (val)) _SOKOL_PRIVATE _sfetch_t* _sfetch_ctx(void) { return _sfetch; } _SOKOL_PRIVATE void _sfetch_path_copy(_sfetch_path_t* dst, const char* src) { SOKOL_ASSERT(dst); if (src && (strlen(src) < SFETCH_MAX_PATH)) { #if defined(_MSC_VER) strncpy_s(dst->buf, SFETCH_MAX_PATH, src, (SFETCH_MAX_PATH-1)); #else strncpy(dst->buf, src, SFETCH_MAX_PATH); #endif dst->buf[SFETCH_MAX_PATH-1] = 0; } else { memset(dst->buf, 0, SFETCH_MAX_PATH); } } _SOKOL_PRIVATE _sfetch_path_t _sfetch_path_make(const char* str) { _sfetch_path_t res; _sfetch_path_copy(&res, str); return res; } _SOKOL_PRIVATE uint32_t _sfetch_make_id(uint32_t index, uint32_t gen_ctr) { return (gen_ctr<<16) | (index & 0xFFFF); } _SOKOL_PRIVATE sfetch_handle_t _sfetch_make_handle(uint32_t slot_id) { sfetch_handle_t h; h.id = slot_id; return h; } _SOKOL_PRIVATE uint32_t _sfetch_slot_index(uint32_t slot_id) { return slot_id & 0xFFFF; } /*=== a circular message queue ===============================================*/ _SOKOL_PRIVATE uint32_t _sfetch_ring_wrap(const _sfetch_ring_t* rb, uint32_t i) { return i % rb->num; } _SOKOL_PRIVATE void _sfetch_ring_discard(_sfetch_ring_t* rb) { SOKOL_ASSERT(rb); if (rb->buf) { SOKOL_FREE(rb->buf); rb->buf = 0; } rb->head = 0; rb->tail = 0; rb->num = 0; } _SOKOL_PRIVATE bool _sfetch_ring_init(_sfetch_ring_t* rb, uint32_t num_slots) { SOKOL_ASSERT(rb && (num_slots > 0)); SOKOL_ASSERT(0 == rb->buf); rb->head = 0; rb->tail = 0; /* one slot reserved to detect full vs empty */ rb->num = num_slots + 1; const size_t queue_size = rb->num * sizeof(sfetch_handle_t); rb->buf = (uint32_t*) SOKOL_MALLOC(queue_size); if (rb->buf) { memset(rb->buf, 0, queue_size); return true; } else { _sfetch_ring_discard(rb); return false; } } _SOKOL_PRIVATE bool _sfetch_ring_full(const _sfetch_ring_t* rb) { SOKOL_ASSERT(rb && rb->buf); return _sfetch_ring_wrap(rb, rb->head + 1) == rb->tail; } _SOKOL_PRIVATE bool _sfetch_ring_empty(const _sfetch_ring_t* rb) { SOKOL_ASSERT(rb && rb->buf); return rb->head == rb->tail; } _SOKOL_PRIVATE uint32_t _sfetch_ring_count(const _sfetch_ring_t* rb) { SOKOL_ASSERT(rb && rb->buf); uint32_t count; if (rb->head >= rb->tail) { count = rb->head - rb->tail; } else { count = (rb->head + rb->num) - rb->tail; } SOKOL_ASSERT(count < rb->num); return count; } _SOKOL_PRIVATE void _sfetch_ring_enqueue(_sfetch_ring_t* rb, uint32_t slot_id) { SOKOL_ASSERT(rb && rb->buf); SOKOL_ASSERT(!_sfetch_ring_full(rb)); SOKOL_ASSERT(rb->head < rb->num); rb->buf[rb->head] = slot_id; rb->head = _sfetch_ring_wrap(rb, rb->head + 1); } _SOKOL_PRIVATE uint32_t _sfetch_ring_dequeue(_sfetch_ring_t* rb) { SOKOL_ASSERT(rb && rb->buf); SOKOL_ASSERT(!_sfetch_ring_empty(rb)); SOKOL_ASSERT(rb->tail < rb->num); uint32_t slot_id = rb->buf[rb->tail]; rb->tail = _sfetch_ring_wrap(rb, rb->tail + 1); return slot_id; } _SOKOL_PRIVATE uint32_t _sfetch_ring_peek(const _sfetch_ring_t* rb, uint32_t index) { SOKOL_ASSERT(rb && rb->buf); SOKOL_ASSERT(!_sfetch_ring_empty(rb)); SOKOL_ASSERT(index < _sfetch_ring_count(rb)); uint32_t rb_index = _sfetch_ring_wrap(rb, rb->tail + index); return rb->buf[rb_index]; } /*=== request pool implementation ============================================*/ _SOKOL_PRIVATE void _sfetch_item_init(_sfetch_item_t* item, uint32_t slot_id, const sfetch_request_t* request) { SOKOL_ASSERT(item && (0 == item->handle.id)); SOKOL_ASSERT(request && request->path); memset(item, 0, sizeof(_sfetch_item_t)); item->handle.id = slot_id; item->state = _SFETCH_STATE_INITIAL; item->channel = request->channel; item->chunk_size = request->chunk_size; item->lane = _SFETCH_INVALID_LANE; item->callback = request->callback; item->buffer.ptr = (uint8_t*) request->buffer_ptr; item->buffer.size = request->buffer_size; item->path = _sfetch_path_make(request->path); #if !_SFETCH_PLATFORM_EMSCRIPTEN item->thread.file_handle = _SFETCH_INVALID_FILE_HANDLE; #endif if (request->user_data_ptr && (request->user_data_size > 0) && (request->user_data_size <= (SFETCH_MAX_USERDATA_UINT64*8))) { item->user.user_data_size = request->user_data_size; memcpy(item->user.user_data, request->user_data_ptr, request->user_data_size); } } _SOKOL_PRIVATE void _sfetch_item_discard(_sfetch_item_t* item) { SOKOL_ASSERT(item && (0 != item->handle.id)); memset(item, 0, sizeof(_sfetch_item_t)); } _SOKOL_PRIVATE void _sfetch_pool_discard(_sfetch_pool_t* pool) { SOKOL_ASSERT(pool); if (pool->free_slots) { SOKOL_FREE(pool->free_slots); pool->free_slots = 0; } if (pool->gen_ctrs) { SOKOL_FREE(pool->gen_ctrs); pool->gen_ctrs = 0; } if (pool->items) { SOKOL_FREE(pool->items); pool->items = 0; } pool->size = 0; pool->free_top = 0; pool->valid = false; } _SOKOL_PRIVATE bool _sfetch_pool_init(_sfetch_pool_t* pool, uint32_t num_items) { SOKOL_ASSERT(pool && (num_items > 0) && (num_items < ((1<<16)-1))); SOKOL_ASSERT(0 == pool->items); /* NOTE: item slot 0 is reserved for the special "invalid" item index 0*/ pool->size = num_items + 1; pool->free_top = 0; const size_t items_size = pool->size * sizeof(_sfetch_item_t); pool->items = (_sfetch_item_t*) SOKOL_MALLOC(items_size); /* generation counters indexable by pool slot index, slot 0 is reserved */ const size_t gen_ctrs_size = sizeof(uint32_t) * pool->size; pool->gen_ctrs = (uint32_t*) SOKOL_MALLOC(gen_ctrs_size); SOKOL_ASSERT(pool->gen_ctrs); /* NOTE: it's not a bug to only reserve num_items here */ const size_t free_slots_size = num_items * sizeof(int); pool->free_slots = (uint32_t*) SOKOL_MALLOC(free_slots_size); if (pool->items && pool->free_slots) { memset(pool->items, 0, items_size); memset(pool->gen_ctrs, 0, gen_ctrs_size); /* never allocate the 0-th item, this is the reserved 'invalid item' */ for (uint32_t i = pool->size - 1; i >= 1; i--) { pool->free_slots[pool->free_top++] = i; } pool->valid = true; } else { /* allocation error */ _sfetch_pool_discard(pool); } return pool->valid; } _SOKOL_PRIVATE uint32_t _sfetch_pool_item_alloc(_sfetch_pool_t* pool, const sfetch_request_t* request) { SOKOL_ASSERT(pool && pool->valid); if (pool->free_top > 0) { uint32_t slot_index = pool->free_slots[--pool->free_top]; SOKOL_ASSERT((slot_index > 0) && (slot_index < pool->size)); uint32_t slot_id = _sfetch_make_id(slot_index, ++pool->gen_ctrs[slot_index]); _sfetch_item_init(&pool->items[slot_index], slot_id, request); pool->items[slot_index].state = _SFETCH_STATE_ALLOCATED; return slot_id; } else { /* pool exhausted, return the 'invalid handle' */ return _sfetch_make_id(0, 0); } } _SOKOL_PRIVATE void _sfetch_pool_item_free(_sfetch_pool_t* pool, uint32_t slot_id) { SOKOL_ASSERT(pool && pool->valid); uint32_t slot_index = _sfetch_slot_index(slot_id); SOKOL_ASSERT((slot_index > 0) && (slot_index < pool->size)); SOKOL_ASSERT(pool->items[slot_index].handle.id == slot_id); #if defined(SOKOL_DEBUG) /* debug check against double-free */ for (uint32_t i = 0; i < pool->free_top; i++) { SOKOL_ASSERT(pool->free_slots[i] != slot_index); } #endif _sfetch_item_discard(&pool->items[slot_index]); pool->free_slots[pool->free_top++] = slot_index; SOKOL_ASSERT(pool->free_top <= (pool->size - 1)); } /* return pointer to item by handle without matching id check */ _SOKOL_PRIVATE _sfetch_item_t* _sfetch_pool_item_at(_sfetch_pool_t* pool, uint32_t slot_id) { SOKOL_ASSERT(pool && pool->valid); uint32_t slot_index = _sfetch_slot_index(slot_id); SOKOL_ASSERT((slot_index > 0) && (slot_index < pool->size)); return &pool->items[slot_index]; } /* return pointer to item by handle with matching id check */ _SOKOL_PRIVATE _sfetch_item_t* _sfetch_pool_item_lookup(_sfetch_pool_t* pool, uint32_t slot_id) { SOKOL_ASSERT(pool && pool->valid); if (0 != slot_id) { _sfetch_item_t* item = _sfetch_pool_item_at(pool, slot_id); if (item->handle.id == slot_id) { return item; } } return 0; } /*=== PLATFORM WRAPPER FUNCTIONS =============================================*/ #if _SFETCH_PLATFORM_POSIX _SOKOL_PRIVATE _sfetch_file_handle_t _sfetch_file_open(const _sfetch_path_t* path) { return fopen(path->buf, "rb"); } _SOKOL_PRIVATE void _sfetch_file_close(_sfetch_file_handle_t h) { fclose(h); } _SOKOL_PRIVATE bool _sfetch_file_handle_valid(_sfetch_file_handle_t h) { return h != _SFETCH_INVALID_FILE_HANDLE; } _SOKOL_PRIVATE uint32_t _sfetch_file_size(_sfetch_file_handle_t h) { fseek(h, 0, SEEK_END); return (uint32_t) ftell(h); } _SOKOL_PRIVATE bool _sfetch_file_read(_sfetch_file_handle_t h, uint32_t offset, uint32_t num_bytes, void* ptr) { fseek(h, offset, SEEK_SET); return num_bytes == fread(ptr, 1, num_bytes, h); } _SOKOL_PRIVATE bool _sfetch_thread_init(_sfetch_thread_t* thread, _sfetch_thread_func_t thread_func, void* thread_arg) { SOKOL_ASSERT(thread && !thread->valid && !thread->stop_requested); pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); pthread_mutex_init(&thread->incoming_mutex, &attr); pthread_mutexattr_destroy(&attr); pthread_mutexattr_init(&attr); pthread_mutex_init(&thread->outgoing_mutex, &attr); pthread_mutexattr_destroy(&attr); pthread_mutexattr_init(&attr); pthread_mutex_init(&thread->running_mutex, &attr); pthread_mutexattr_destroy(&attr); pthread_mutexattr_init(&attr); pthread_mutex_init(&thread->stop_mutex, &attr); pthread_mutexattr_destroy(&attr); pthread_condattr_t cond_attr; pthread_condattr_init(&cond_attr); pthread_cond_init(&thread->incoming_cond, &cond_attr); pthread_condattr_destroy(&cond_attr); /* FIXME: in debug mode, the threads should be named */ pthread_mutex_lock(&thread->running_mutex); int res = pthread_create(&thread->thread, 0, thread_func, thread_arg); thread->valid = (0 == res); pthread_mutex_unlock(&thread->running_mutex); return thread->valid; } _SOKOL_PRIVATE void _sfetch_thread_request_stop(_sfetch_thread_t* thread) { pthread_mutex_lock(&thread->stop_mutex); thread->stop_requested = true; pthread_mutex_unlock(&thread->stop_mutex); } _SOKOL_PRIVATE bool _sfetch_thread_stop_requested(_sfetch_thread_t* thread) { pthread_mutex_lock(&thread->stop_mutex); bool stop_requested = thread->stop_requested; pthread_mutex_unlock(&thread->stop_mutex); return stop_requested; } _SOKOL_PRIVATE void _sfetch_thread_join(_sfetch_thread_t* thread) { SOKOL_ASSERT(thread); if (thread->valid) { pthread_mutex_lock(&thread->incoming_mutex); _sfetch_thread_request_stop(thread); pthread_cond_signal(&thread->incoming_cond); pthread_mutex_unlock(&thread->incoming_mutex); pthread_join(thread->thread, 0); thread->valid = false; } pthread_mutex_destroy(&thread->stop_mutex); pthread_mutex_destroy(&thread->running_mutex); pthread_mutex_destroy(&thread->incoming_mutex); pthread_mutex_destroy(&thread->outgoing_mutex); pthread_cond_destroy(&thread->incoming_cond); } /* called when the thread-func is entered, this blocks the thread func until the _sfetch_thread_t object is fully initialized */ _SOKOL_PRIVATE void _sfetch_thread_entered(_sfetch_thread_t* thread) { pthread_mutex_lock(&thread->running_mutex); } /* called by the thread-func right before it is left */ _SOKOL_PRIVATE void _sfetch_thread_leaving(_sfetch_thread_t* thread) { pthread_mutex_unlock(&thread->running_mutex); } _SOKOL_PRIVATE void _sfetch_thread_enqueue_incoming(_sfetch_thread_t* thread, _sfetch_ring_t* incoming, _sfetch_ring_t* src) { /* called from user thread */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(incoming && incoming->buf); SOKOL_ASSERT(src && src->buf); if (!_sfetch_ring_empty(src)) { pthread_mutex_lock(&thread->incoming_mutex); while (!_sfetch_ring_full(incoming) && !_sfetch_ring_empty(src)) { _sfetch_ring_enqueue(incoming, _sfetch_ring_dequeue(src)); } pthread_cond_signal(&thread->incoming_cond); pthread_mutex_unlock(&thread->incoming_mutex); } } _SOKOL_PRIVATE uint32_t _sfetch_thread_dequeue_incoming(_sfetch_thread_t* thread, _sfetch_ring_t* incoming) { /* called from thread function */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(incoming && incoming->buf); pthread_mutex_lock(&thread->incoming_mutex); while (_sfetch_ring_empty(incoming) && !thread->stop_requested) { pthread_cond_wait(&thread->incoming_cond, &thread->incoming_mutex); } uint32_t item = 0; if (!thread->stop_requested) { item = _sfetch_ring_dequeue(incoming); } pthread_mutex_unlock(&thread->incoming_mutex); return item; } _SOKOL_PRIVATE bool _sfetch_thread_enqueue_outgoing(_sfetch_thread_t* thread, _sfetch_ring_t* outgoing, uint32_t item) { /* called from thread function */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(outgoing && outgoing->buf); SOKOL_ASSERT(0 != item); pthread_mutex_lock(&thread->outgoing_mutex); bool result = false; if (!_sfetch_ring_full(outgoing)) { _sfetch_ring_enqueue(outgoing, item); } pthread_mutex_unlock(&thread->outgoing_mutex); return result; } _SOKOL_PRIVATE void _sfetch_thread_dequeue_outgoing(_sfetch_thread_t* thread, _sfetch_ring_t* outgoing, _sfetch_ring_t* dst) { /* called from user thread */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(outgoing && outgoing->buf); SOKOL_ASSERT(dst && dst->buf); pthread_mutex_lock(&thread->outgoing_mutex); while (!_sfetch_ring_full(dst) && !_sfetch_ring_empty(outgoing)) { _sfetch_ring_enqueue(dst, _sfetch_ring_dequeue(outgoing)); } pthread_mutex_unlock(&thread->outgoing_mutex); } #endif /* _SFETCH_PLATFORM_POSIX */ #if _SFETCH_PLATFORM_WINDOWS _SOKOL_PRIVATE bool _sfetch_win32_utf8_to_wide(const char* src, wchar_t* dst, int dst_num_bytes) { SOKOL_ASSERT(src && dst && (dst_num_bytes > 1)); memset(dst, 0, dst_num_bytes); const int dst_chars = dst_num_bytes / sizeof(wchar_t); const int dst_needed = MultiByteToWideChar(CP_UTF8, 0, src, -1, 0, 0); if ((dst_needed > 0) && (dst_needed < dst_chars)) { MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, dst_chars); return true; } else { /* input string doesn't fit into destination buffer */ return false; } } _SOKOL_PRIVATE _sfetch_file_handle_t _sfetch_file_open(const _sfetch_path_t* path) { wchar_t w_path[SFETCH_MAX_PATH]; if (!_sfetch_win32_utf8_to_wide(path->buf, w_path, sizeof(w_path))) { SOKOL_LOG("_sfetch_file_open: error converting UTF-8 path to wide string"); return 0; } _sfetch_file_handle_t h = CreateFileW( w_path, /* lpFileName */ GENERIC_READ, /* dwDesiredAccess */ FILE_SHARE_READ, /* dwShareMode */ NULL, /* lpSecurityAttributes */ OPEN_EXISTING, /* dwCreationDisposition */ FILE_ATTRIBUTE_NORMAL|FILE_FLAG_SEQUENTIAL_SCAN, /* dwFlagsAndAttributes */ NULL); /* hTemplateFile */ return h; } _SOKOL_PRIVATE void _sfetch_file_close(_sfetch_file_handle_t h) { CloseHandle(h); } _SOKOL_PRIVATE bool _sfetch_file_handle_valid(_sfetch_file_handle_t h) { return h != _SFETCH_INVALID_FILE_HANDLE; } _SOKOL_PRIVATE uint32_t _sfetch_file_size(_sfetch_file_handle_t h) { return GetFileSize(h, NULL); } _SOKOL_PRIVATE bool _sfetch_file_read(_sfetch_file_handle_t h, uint32_t offset, uint32_t num_bytes, void* ptr) { LARGE_INTEGER offset_li; offset_li.QuadPart = offset; BOOL seek_res = SetFilePointerEx(h, offset_li, NULL, FILE_BEGIN); if (seek_res) { DWORD bytes_read = 0; BOOL read_res = ReadFile(h, ptr, (DWORD)num_bytes, &bytes_read, NULL); return read_res && (bytes_read == num_bytes); } else { return false; } } _SOKOL_PRIVATE bool _sfetch_thread_init(_sfetch_thread_t* thread, _sfetch_thread_func_t thread_func, void* thread_arg) { SOKOL_ASSERT(thread && !thread->valid && !thread->stop_requested); thread->incoming_event = CreateEventA(NULL, FALSE, FALSE, NULL); SOKOL_ASSERT(NULL != thread->incoming_event); InitializeCriticalSection(&thread->incoming_critsec); InitializeCriticalSection(&thread->outgoing_critsec); InitializeCriticalSection(&thread->running_critsec); InitializeCriticalSection(&thread->stop_critsec); EnterCriticalSection(&thread->running_critsec); const SIZE_T stack_size = 512 * 1024; thread->thread = CreateThread(NULL, 512*1024, thread_func, thread_arg, 0, NULL); thread->valid = (NULL != thread->thread); LeaveCriticalSection(&thread->running_critsec); return thread->valid; } _SOKOL_PRIVATE void _sfetch_thread_request_stop(_sfetch_thread_t* thread) { EnterCriticalSection(&thread->stop_critsec); thread->stop_requested = true; LeaveCriticalSection(&thread->stop_critsec); } _SOKOL_PRIVATE bool _sfetch_thread_stop_requested(_sfetch_thread_t* thread) { EnterCriticalSection(&thread->stop_critsec); bool stop_requested = thread->stop_requested; LeaveCriticalSection(&thread->stop_critsec); return stop_requested; } _SOKOL_PRIVATE void _sfetch_thread_join(_sfetch_thread_t* thread) { if (thread->valid) { EnterCriticalSection(&thread->incoming_critsec); _sfetch_thread_request_stop(thread); BOOL set_event_res = SetEvent(thread->incoming_event); SOKOL_ASSERT(set_event_res); LeaveCriticalSection(&thread->incoming_critsec); WaitForSingleObject(thread->thread, INFINITE); CloseHandle(thread->thread); thread->valid = false; } CloseHandle(thread->incoming_event); DeleteCriticalSection(&thread->stop_critsec); DeleteCriticalSection(&thread->running_critsec); DeleteCriticalSection(&thread->outgoing_critsec); DeleteCriticalSection(&thread->incoming_critsec); } _SOKOL_PRIVATE void _sfetch_thread_entered(_sfetch_thread_t* thread) { EnterCriticalSection(&thread->running_critsec); } /* called by the thread-func right before it is left */ _SOKOL_PRIVATE void _sfetch_thread_leaving(_sfetch_thread_t* thread) { LeaveCriticalSection(&thread->running_critsec); } _SOKOL_PRIVATE void _sfetch_thread_enqueue_incoming(_sfetch_thread_t* thread, _sfetch_ring_t* incoming, _sfetch_ring_t* src) { /* called from user thread */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(incoming && incoming->buf); SOKOL_ASSERT(src && src->buf); if (!_sfetch_ring_empty(src)) { EnterCriticalSection(&thread->incoming_critsec); while (!_sfetch_ring_full(incoming) && !_sfetch_ring_empty(src)) { _sfetch_ring_enqueue(incoming, _sfetch_ring_dequeue(src)); } LeaveCriticalSection(&thread->incoming_critsec); BOOL set_event_res = SetEvent(thread->incoming_event); SOKOL_ASSERT(set_event_res); } } _SOKOL_PRIVATE uint32_t _sfetch_thread_dequeue_incoming(_sfetch_thread_t* thread, _sfetch_ring_t* incoming) { /* called from thread function */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(incoming && incoming->buf); EnterCriticalSection(&thread->incoming_critsec); while (_sfetch_ring_empty(incoming) && !thread->stop_requested) { LeaveCriticalSection(&thread->incoming_critsec); WaitForSingleObject(&thread->incoming_event, INFINITE); EnterCriticalSection(&thread->incoming_critsec); } uint32_t item = 0; if (!thread->stop_requested) { item = _sfetch_ring_dequeue(incoming); } LeaveCriticalSection(&thread->incoming_critsec); return item; } _SOKOL_PRIVATE bool _sfetch_thread_enqueue_outgoing(_sfetch_thread_t* thread, _sfetch_ring_t* outgoing, uint32_t item) { /* called from thread function */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(outgoing && outgoing->buf); EnterCriticalSection(&thread->outgoing_critsec); bool result = false; if (!_sfetch_ring_full(outgoing)) { _sfetch_ring_enqueue(outgoing, item); } LeaveCriticalSection(&thread->outgoing_critsec); return result; } _SOKOL_PRIVATE void _sfetch_thread_dequeue_outgoing(_sfetch_thread_t* thread, _sfetch_ring_t* outgoing, _sfetch_ring_t* dst) { /* called from user thread */ SOKOL_ASSERT(thread && thread->valid); SOKOL_ASSERT(outgoing && outgoing->buf); SOKOL_ASSERT(dst && dst->buf); EnterCriticalSection(&thread->outgoing_critsec); while (!_sfetch_ring_full(dst) && !_sfetch_ring_empty(outgoing)) { _sfetch_ring_enqueue(dst, _sfetch_ring_dequeue(outgoing)); } LeaveCriticalSection(&thread->outgoing_critsec); } #endif /* _SFETCH_PLATFORM_WINDOWS */ /*=== IO CHANNEL implementation ==============================================*/ /* per-channel request handler for native platforms accessing the local filesystem */ #if _SFETCH_HAS_THREADS _SOKOL_PRIVATE void _sfetch_request_handler(_sfetch_t* ctx, uint32_t slot_id) { _sfetch_state_t state; _sfetch_path_t* path; _sfetch_item_thread_t* thread; _sfetch_buffer_t* buffer; uint32_t chunk_size; { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (!item) { return; } state = item->state; SOKOL_ASSERT((state == _SFETCH_STATE_FETCHING) || (state == _SFETCH_STATE_PAUSED) || (state == _SFETCH_STATE_FAILED)); path = &item->path; thread = &item->thread; buffer = &item->buffer; chunk_size = item->chunk_size; } if (thread->failed) { return; } if (state == _SFETCH_STATE_FETCHING) { if ((buffer->ptr == 0) || (buffer->size == 0)) { thread->error_code = SFETCH_ERROR_NO_BUFFER; thread->failed = true; } else { /* open file if not happened yet */ if (!_sfetch_file_handle_valid(thread->file_handle)) { SOKOL_ASSERT(path->buf[0]); SOKOL_ASSERT(thread->fetched_offset == 0); SOKOL_ASSERT(thread->fetched_size == 0); thread->file_handle = _sfetch_file_open(path); if (_sfetch_file_handle_valid(thread->file_handle)) { thread->content_size = _sfetch_file_size(thread->file_handle); } else { thread->error_code = SFETCH_ERROR_FILE_NOT_FOUND; thread->failed = true; } } if (!thread->failed) { uint32_t read_offset = 0; uint32_t bytes_to_read = 0; if (chunk_size == 0) { /* load entire file */ if (thread->content_size <= buffer->size) { bytes_to_read = thread->content_size; read_offset = 0; } else { /* provided buffer to small to fit entire file */ thread->error_code = SFETCH_ERROR_BUFFER_TOO_SMALL; thread->failed = true; } } else { if (chunk_size <= buffer->size) { bytes_to_read = chunk_size; read_offset = thread->fetched_offset; if ((read_offset + bytes_to_read) > thread->content_size) { bytes_to_read = thread->content_size - read_offset; } } else { /* provided buffer to small to fit next chunk */ thread->error_code = SFETCH_ERROR_BUFFER_TOO_SMALL; thread->failed = true; } } if (!thread->failed) { if (_sfetch_file_read(thread->file_handle, read_offset, bytes_to_read, buffer->ptr)) { thread->fetched_size = bytes_to_read; thread->fetched_offset += bytes_to_read; } else { thread->error_code = SFETCH_ERROR_UNEXPECTED_EOF; thread->failed = true; } } } } SOKOL_ASSERT(thread->fetched_offset <= thread->content_size); if (thread->failed || (thread->fetched_offset == thread->content_size)) { if (_sfetch_file_handle_valid(thread->file_handle)) { _sfetch_file_close(thread->file_handle); thread->file_handle = _SFETCH_INVALID_FILE_HANDLE; } thread->finished = true; } } /* ignore items in PAUSED or FAILED state */ } #if _SFETCH_PLATFORM_WINDOWS _SOKOL_PRIVATE DWORD WINAPI _sfetch_channel_thread_func(LPVOID arg) { #else _SOKOL_PRIVATE void* _sfetch_channel_thread_func(void* arg) { #endif _sfetch_channel_t* chn = (_sfetch_channel_t*) arg; _sfetch_thread_entered(&chn->thread); while (!_sfetch_thread_stop_requested(&chn->thread)) { /* block until work arrives */ uint32_t slot_id = _sfetch_thread_dequeue_incoming(&chn->thread, &chn->thread_incoming); /* slot_id will be invalid if the thread was woken up to join */ if (!_sfetch_thread_stop_requested(&chn->thread)) { SOKOL_ASSERT(0 != slot_id); chn->request_handler(chn->ctx, slot_id); SOKOL_ASSERT(!_sfetch_ring_full(&chn->thread_outgoing)); _sfetch_thread_enqueue_outgoing(&chn->thread, &chn->thread_outgoing, slot_id); } } _sfetch_thread_leaving(&chn->thread); return 0; } #endif /* _SFETCH_HAS_THREADS */ #if _SFETCH_PLATFORM_EMSCRIPTEN /*=== embedded Javascript helper functions ===================================*/ EM_JS(void, sfetch_js_send_head_request, (uint32_t slot_id, const char* path_cstr), { var path_str = UTF8ToString(path_cstr); var req = new XMLHttpRequest(); req.open('HEAD', path_str); req.onreadystatechange = function() { if (this.readyState == this.DONE) { if (this.status == 200) { var content_length = this.getResponseHeader('Content-Length'); __sfetch_emsc_head_response(slot_id, content_length); } else { __sfetch_emsc_failed_http_status(slot_id, this.status); } } }; req.send(); }); /* if bytes_to_read != 0, a range-request will be sent, otherwise a normal request */ EM_JS(void, sfetch_js_send_get_request, (uint32_t slot_id, const char* path_cstr, int offset, int bytes_to_read, void* buf_ptr, int buf_size), { var path_str = UTF8ToString(path_cstr); var req = new XMLHttpRequest(); req.open('GET', path_str); req.responseType = 'arraybuffer'; var need_range_request = (bytes_to_read > 0); if (need_range_request) { req.setRequestHeader('Range', 'bytes='+offset+'-'+(offset+bytes_to_read)); } req.onreadystatechange = function() { if (this.readyState == this.DONE) { if ((this.status == 206) || ((this.status == 200) && !need_range_request)) { var u8_array = new Uint8Array(req.response); var content_fetched_size = u8_array.length; if (content_fetched_size <= buf_size) { HEAPU8.set(u8_array, buf_ptr); __sfetch_emsc_get_response(slot_id, bytes_to_read, content_fetched_size); } else { __sfetch_emsc_failed_buffer_too_small(slot_id); } } else { __sfetch_emsc_failed_http_status(slot_id, this.status); } } }; req.send(); }); /*=== emscripten specific C helper functions =================================*/ #ifdef __cplusplus extern "C" { #endif void _sfetch_emsc_send_get_request(uint32_t slot_id, _sfetch_item_t* item) { if ((item->buffer.ptr == 0) || (item->buffer.size == 0)) { item->thread.error_code = SFETCH_ERROR_NO_BUFFER; item->thread.failed = true; } else { uint32_t offset = 0; uint32_t bytes_to_read = 0; if (item->chunk_size > 0) { /* send HTTP range request */ SOKOL_ASSERT(item->thread.content_size > 0); SOKOL_ASSERT(item->thread.http_range_offset < item->thread.content_size); bytes_to_read = item->thread.content_size - item->thread.http_range_offset; if (bytes_to_read > item->chunk_size) { bytes_to_read = item->chunk_size; } SOKOL_ASSERT(bytes_to_read > 0); offset = item->thread.http_range_offset; } sfetch_js_send_get_request(slot_id, item->path.buf, offset, bytes_to_read, item->buffer.ptr, item->buffer.size); } } /* called by JS when an initial HEAD request finished successfully (only when streaming chunks) */ EMSCRIPTEN_KEEPALIVE void _sfetch_emsc_head_response(uint32_t slot_id, uint32_t content_length) { _sfetch_t* ctx = _sfetch_ctx(); if (ctx && ctx->valid) { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (item) { SOKOL_ASSERT(item->buffer.ptr && (item->buffer.size > 0)); item->thread.content_size = content_length; _sfetch_emsc_send_get_request(slot_id, item); } } } /* called by JS when a followup GET request finished successfully */ EMSCRIPTEN_KEEPALIVE void _sfetch_emsc_get_response(uint32_t slot_id, uint32_t range_fetched_size, uint32_t content_fetched_size) { _sfetch_t* ctx = _sfetch_ctx(); if (ctx && ctx->valid) { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (item) { item->thread.fetched_size = content_fetched_size; item->thread.fetched_offset += content_fetched_size; item->thread.http_range_offset += range_fetched_size; if (item->chunk_size == 0) { item->thread.finished = true; } else if (item->thread.http_range_offset >= item->thread.content_size) { item->thread.finished = true; } _sfetch_ring_enqueue(&ctx->chn[item->channel].user_outgoing, slot_id); } } } /* called by JS when an error occurred */ EMSCRIPTEN_KEEPALIVE void _sfetch_emsc_failed_http_status(uint32_t slot_id, uint32_t http_status) { _sfetch_t* ctx = _sfetch_ctx(); if (ctx && ctx->valid) { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (item) { if (http_status == 404) { item->thread.error_code = SFETCH_ERROR_FILE_NOT_FOUND; } else { item->thread.error_code = SFETCH_ERROR_INVALID_HTTP_STATUS; } item->thread.failed = true; item->thread.finished = true; _sfetch_ring_enqueue(&ctx->chn[item->channel].user_outgoing, slot_id); } } } EMSCRIPTEN_KEEPALIVE void _sfetch_emsc_failed_buffer_too_small(uint32_t slot_id) { _sfetch_t* ctx = _sfetch_ctx(); if (ctx && ctx->valid) { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (item) { item->thread.error_code = SFETCH_ERROR_BUFFER_TOO_SMALL; item->thread.failed = true; item->thread.finished = true; _sfetch_ring_enqueue(&ctx->chn[item->channel].user_outgoing, slot_id); } } } #ifdef __cplusplus } /* extern "C" */ #endif _SOKOL_PRIVATE void _sfetch_request_handler(_sfetch_t* ctx, uint32_t slot_id) { _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, slot_id); if (!item) { return; } if (item->state == _SFETCH_STATE_FETCHING) { if ((item->chunk_size > 0) && (item->thread.content_size == 0)) { /* if streaming download is requested, and the content-length isn't known yet, need to send a HEAD request first */ sfetch_js_send_head_request(slot_id, item->path.buf); } else { /* otherwise, this is either a request to load the entire file, or to load the next streaming chunk */ _sfetch_emsc_send_get_request(slot_id, item); } } else { /* just move all other items (e.g. paused or cancelled) into the outgoing queue, so they wont get lost */ _sfetch_ring_enqueue(&ctx->chn[item->channel].user_outgoing, slot_id); } if (item->thread.failed) { item->thread.finished = true; } } #endif /* _SFETCH_PLATFORM_EMSCRIPTEN */ _SOKOL_PRIVATE void _sfetch_channel_discard(_sfetch_channel_t* chn) { SOKOL_ASSERT(chn); #if _SFETCH_HAS_THREADS if (chn->valid) { _sfetch_thread_join(&chn->thread); } _sfetch_ring_discard(&chn->thread_incoming); _sfetch_ring_discard(&chn->thread_outgoing); #endif _sfetch_ring_discard(&chn->free_lanes); _sfetch_ring_discard(&chn->user_sent); _sfetch_ring_discard(&chn->user_incoming); _sfetch_ring_discard(&chn->user_outgoing); _sfetch_ring_discard(&chn->free_lanes); chn->valid = false; } _SOKOL_PRIVATE bool _sfetch_channel_init(_sfetch_channel_t* chn, _sfetch_t* ctx, uint32_t num_items, uint32_t num_lanes, void (*request_handler)(_sfetch_t* ctx, uint32_t)) { SOKOL_ASSERT(chn && (num_items > 0) && request_handler); SOKOL_ASSERT(!chn->valid); bool valid = true; chn->request_handler = request_handler; chn->ctx = ctx; valid &= _sfetch_ring_init(&chn->free_lanes, num_lanes); for (uint32_t lane = 0; lane < num_lanes; lane++) { _sfetch_ring_enqueue(&chn->free_lanes, lane); } valid &= _sfetch_ring_init(&chn->user_sent, num_items); valid &= _sfetch_ring_init(&chn->user_incoming, num_lanes); valid &= _sfetch_ring_init(&chn->user_outgoing, num_lanes); #if _SFETCH_HAS_THREADS valid &= _sfetch_ring_init(&chn->thread_incoming, num_lanes); valid &= _sfetch_ring_init(&chn->thread_outgoing, num_lanes); #endif if (valid) { chn->valid = true; #if _SFETCH_HAS_THREADS _sfetch_thread_init(&chn->thread, _sfetch_channel_thread_func, chn); #endif return true; } else { _sfetch_channel_discard(chn); return false; } } /* put a request into the channels sent-queue, this is where all new requests are stored until a lane becomes free. */ _SOKOL_PRIVATE bool _sfetch_channel_send(_sfetch_channel_t* chn, uint32_t slot_id) { SOKOL_ASSERT(chn && chn->valid); if (!_sfetch_ring_full(&chn->user_sent)) { _sfetch_ring_enqueue(&chn->user_sent, slot_id); return true; } else { SOKOL_LOG("sfetch_send: user_sent queue is full)"); return false; } } _SOKOL_PRIVATE void _sfetch_invoke_response_callback(_sfetch_item_t* item) { sfetch_response_t response; memset(&response, 0, sizeof(response)); response.handle = item->handle; response.dispatched = (item->state == _SFETCH_STATE_DISPATCHED); response.fetched = (item->state == _SFETCH_STATE_FETCHED); response.paused = (item->state == _SFETCH_STATE_PAUSED); response.finished = item->user.finished; response.failed = (item->state == _SFETCH_STATE_FAILED); response.cancelled = item->user.cancel; response.error_code = item->user.error_code; response.channel = item->channel; response.lane = item->lane; response.path = item->path.buf; response.user_data = item->user.user_data; response.fetched_offset = item->user.fetched_offset - item->user.fetched_size; response.fetched_size = item->user.fetched_size; response.buffer_ptr = item->buffer.ptr; response.buffer_size = item->buffer.size; item->callback(&response); } /* per-frame channel stuff: move requests in and out of the IO threads, call response callbacks */ _SOKOL_PRIVATE void _sfetch_channel_dowork(_sfetch_channel_t* chn, _sfetch_pool_t* pool) { /* move items from sent- to incoming-queue permitting free lanes */ const uint32_t num_sent = _sfetch_ring_count(&chn->user_sent); const uint32_t avail_lanes = _sfetch_ring_count(&chn->free_lanes); const uint32_t num_move = (num_sent < avail_lanes) ? num_sent : avail_lanes; for (uint32_t i = 0; i < num_move; i++) { const uint32_t slot_id = _sfetch_ring_dequeue(&chn->user_sent); _sfetch_item_t* item = _sfetch_pool_item_lookup(pool, slot_id); SOKOL_ASSERT(item); SOKOL_ASSERT(item->state == _SFETCH_STATE_ALLOCATED); item->state = _SFETCH_STATE_DISPATCHED; item->lane = _sfetch_ring_dequeue(&chn->free_lanes); /* if no buffer provided yet, invoke response callback to do so */ if (0 == item->buffer.ptr) { _sfetch_invoke_response_callback(item); } _sfetch_ring_enqueue(&chn->user_incoming, slot_id); } /* prepare incoming items for being moved into the IO thread */ const uint32_t num_incoming = _sfetch_ring_count(&chn->user_incoming); for (uint32_t i = 0; i < num_incoming; i++) { const uint32_t slot_id = _sfetch_ring_peek(&chn->user_incoming, i); _sfetch_item_t* item = _sfetch_pool_item_lookup(pool, slot_id); SOKOL_ASSERT(item); SOKOL_ASSERT(item->state != _SFETCH_STATE_INITIAL); SOKOL_ASSERT(item->state != _SFETCH_STATE_FETCHING); /* transfer input params from user- to thread-data */ if (item->user.pause) { item->state = _SFETCH_STATE_PAUSED; item->user.pause = false; } if (item->user.cont) { if (item->state == _SFETCH_STATE_PAUSED) { item->state = _SFETCH_STATE_FETCHED; } item->user.cont = false; } if (item->user.cancel) { item->state = _SFETCH_STATE_FAILED; item->user.finished = true; } switch (item->state) { case _SFETCH_STATE_DISPATCHED: case _SFETCH_STATE_FETCHED: item->state = _SFETCH_STATE_FETCHING; break; default: break; } } #if _SFETCH_HAS_THREADS /* move new items into the IO threads and processed items out of IO threads */ _sfetch_thread_enqueue_incoming(&chn->thread, &chn->thread_incoming, &chn->user_incoming); _sfetch_thread_dequeue_outgoing(&chn->thread, &chn->thread_outgoing, &chn->user_outgoing); #else /* without threading just directly dequeue items from the user_incoming queue and call the request handler, the user_outgoing queue will be filled as the asynchronous HTTP requests sent by the request handler are completed */ while (!_sfetch_ring_empty(&chn->user_incoming)) { uint32_t slot_id = _sfetch_ring_dequeue(&chn->user_incoming); _sfetch_request_handler(chn->ctx, slot_id); } #endif /* drain the outgoing queue, prepare items for invoking the response callback, and finally call the response callback, free finished items */ while (!_sfetch_ring_empty(&chn->user_outgoing)) { const uint32_t slot_id = _sfetch_ring_dequeue(&chn->user_outgoing); SOKOL_ASSERT(slot_id); _sfetch_item_t* item = _sfetch_pool_item_lookup(pool, slot_id); SOKOL_ASSERT(item && item->callback); SOKOL_ASSERT(item->state != _SFETCH_STATE_INITIAL); SOKOL_ASSERT(item->state != _SFETCH_STATE_ALLOCATED); SOKOL_ASSERT(item->state != _SFETCH_STATE_DISPATCHED); SOKOL_ASSERT(item->state != _SFETCH_STATE_FETCHED); /* transfer output params from thread- to user-data */ item->user.fetched_offset = item->thread.fetched_offset; item->user.fetched_size = item->thread.fetched_size; if (item->user.cancel) { item->user.error_code = SFETCH_ERROR_CANCELLED; } else { item->user.error_code = item->thread.error_code; } if (item->thread.finished) { item->user.finished = true; } /* state transition */ if (item->thread.failed) { item->state = _SFETCH_STATE_FAILED; } else if (item->state == _SFETCH_STATE_FETCHING) { item->state = _SFETCH_STATE_FETCHED; } _sfetch_invoke_response_callback(item); /* when the request is finish, free the lane for another request, otherwise feed it back into the incoming queue */ if (item->user.finished) { _sfetch_ring_enqueue(&chn->free_lanes, item->lane); _sfetch_pool_item_free(pool, slot_id); } else { _sfetch_ring_enqueue(&chn->user_incoming, slot_id); } } } /*=== private high-level functions ===========================================*/ _SOKOL_PRIVATE bool _sfetch_validate_request(_sfetch_t* ctx, const sfetch_request_t* req) { #if defined(SOKOL_DEBUG) if (req->channel >= ctx->desc.num_channels) { SOKOL_LOG("_sfetch_validate_request: request.num_channels too big!"); return false; } if (!req->path) { SOKOL_LOG("_sfetch_validate_request: request.path is null!"); return false; } if (strlen(req->path) >= (SFETCH_MAX_PATH-1)) { SOKOL_LOG("_sfetch_validate_request: request.path is too long (must be < SFETCH_MAX_PATH-1)"); return false; } if (!req->callback) { SOKOL_LOG("_sfetch_validate_request: request.callback missing"); return false; } if (req->chunk_size > req->buffer_size) { SOKOL_LOG("_sfetch_validate_request: request.stream_size is greater request.buffer_size)"); return false; } if (req->user_data_ptr && (req->user_data_size == 0)) { SOKOL_LOG("_sfetch_validate_request: request.user_data_ptr is set, but req.user_data_size is null"); return false; } if (!req->user_data_ptr && (req->user_data_size > 0)) { SOKOL_LOG("_sfetch_validate_request: request.user_data_ptr is null, but req.user_data_size is not"); return false; } if (req->user_data_size > SFETCH_MAX_USERDATA_UINT64 * sizeof(uint64_t)) { SOKOL_LOG("_sfetch_validate_request: request.user_data_size is too big (see SFETCH_MAX_USERDATA_UINT64"); return false; } #endif return true; } /*=== PUBLIC API FUNCTIONS ===================================================*/ SOKOL_API_IMPL void sfetch_setup(const sfetch_desc_t* desc) { SOKOL_ASSERT(desc); SOKOL_ASSERT((desc->_start_canary == 0) && (desc->_end_canary == 0)); SOKOL_ASSERT(0 == _sfetch); _sfetch = (_sfetch_t*) SOKOL_MALLOC(sizeof(_sfetch_t)); SOKOL_ASSERT(_sfetch); memset(_sfetch, 0, sizeof(_sfetch_t)); _sfetch_t* ctx = _sfetch_ctx(); ctx->desc = *desc; ctx->setup = true; ctx->valid = true; /* replace zero-init items with default values */ ctx->desc.max_requests = _sfetch_def(ctx->desc.max_requests, 128); ctx->desc.num_channels = _sfetch_def(ctx->desc.num_channels, 1); ctx->desc.num_lanes = _sfetch_def(ctx->desc.num_lanes, 1); if (ctx->desc.num_channels > SFETCH_MAX_CHANNELS) { ctx->desc.num_channels = SFETCH_MAX_CHANNELS; SOKOL_LOG("sfetch_setup: clamping num_channels to SFETCH_MAX_CHANNELS"); } /* setup the global request item pool */ ctx->valid &= _sfetch_pool_init(&ctx->pool, ctx->desc.max_requests); /* setup IO channels (one thread per channel) */ for (uint32_t i = 0; i < ctx->desc.num_channels; i++) { ctx->valid &= _sfetch_channel_init(&ctx->chn[i], ctx, ctx->desc.max_requests, ctx->desc.num_lanes, _sfetch_request_handler); } } SOKOL_API_IMPL void sfetch_shutdown(void) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->setup); ctx->valid = false; /* IO threads must be shutdown first */ for (uint32_t i = 0; i < ctx->desc.num_channels; i++) { if (ctx->chn[i].valid) { _sfetch_channel_discard(&ctx->chn[i]); } } _sfetch_pool_discard(&ctx->pool); ctx->setup = false; SOKOL_FREE(ctx); _sfetch = 0; } SOKOL_API_IMPL bool sfetch_valid(void) { _sfetch_t* ctx = _sfetch_ctx(); return ctx && ctx->valid; } SOKOL_API_IMPL sfetch_desc_t sfetch_desc(void) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); return ctx->desc; } SOKOL_API_IMPL int sfetch_max_userdata_bytes(void) { return SFETCH_MAX_USERDATA_UINT64 * 8; } SOKOL_API_IMPL int sfetch_max_path(void) { return SFETCH_MAX_PATH; } SOKOL_API_IMPL bool sfetch_handle_valid(sfetch_handle_t h) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); /* shortcut invalid handle */ if (h.id == 0) { return false; } return 0 != _sfetch_pool_item_lookup(&ctx->pool, h.id); } SOKOL_API_IMPL sfetch_handle_t sfetch_send(const sfetch_request_t* request) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->setup); SOKOL_ASSERT(request && (request->_start_canary == 0) && (request->_end_canary == 0)); const sfetch_handle_t invalid_handle = _sfetch_make_handle(0); if (!ctx->valid) { return invalid_handle; } if (!_sfetch_validate_request(ctx, request)) { return invalid_handle; } SOKOL_ASSERT(request->channel < ctx->desc.num_channels); uint32_t slot_id = _sfetch_pool_item_alloc(&ctx->pool, request); if (0 == slot_id) { SOKOL_LOG("sfetch_send: request pool exhausted (too many active requests)"); return invalid_handle; } if (!_sfetch_channel_send(&ctx->chn[request->channel], slot_id)) { /* send failed because the channels sent-queue overflowed */ _sfetch_pool_item_free(&ctx->pool, slot_id); return invalid_handle; } return _sfetch_make_handle(slot_id); } SOKOL_API_IMPL void sfetch_dowork(void) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->setup); if (!ctx->valid) { return; } /* we're pumping each channel 2x so that unfinished request items coming out the IO threads can be moved back into the IO-thread immediately without having to wait a frame */ ctx->in_callback = true; for (int pass = 0; pass < 2; pass++) { for (uint32_t chn_index = 0; chn_index < ctx->desc.num_channels; chn_index++) { _sfetch_channel_dowork(&ctx->chn[chn_index], &ctx->pool); } } ctx->in_callback = false; } SOKOL_API_IMPL void sfetch_bind_buffer(sfetch_handle_t h, void* buffer_ptr, uint32_t buffer_size) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); SOKOL_ASSERT(ctx->in_callback); _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, h.id); if (item) { SOKOL_ASSERT((0 == item->buffer.ptr) && (0 == item->buffer.size)); item->buffer.ptr = (uint8_t*) buffer_ptr; item->buffer.size = buffer_size; } } SOKOL_API_IMPL void* sfetch_unbind_buffer(sfetch_handle_t h) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); SOKOL_ASSERT(ctx->in_callback); _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, h.id); if (item) { void* prev_buf_ptr = item->buffer.ptr; item->buffer.ptr = 0; item->buffer.size = 0; return prev_buf_ptr; } else { return 0; } } SOKOL_API_IMPL void sfetch_pause(sfetch_handle_t h) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, h.id); if (item) { item->user.pause = true; item->user.cont = false; } } SOKOL_API_IMPL void sfetch_continue(sfetch_handle_t h) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, h.id); if (item) { item->user.cont = true; item->user.pause = false; } } SOKOL_API_IMPL void sfetch_cancel(sfetch_handle_t h) { _sfetch_t* ctx = _sfetch_ctx(); SOKOL_ASSERT(ctx && ctx->valid); _sfetch_item_t* item = _sfetch_pool_item_lookup(&ctx->pool, h.id); if (item) { item->user.cont = false; item->user.pause = false; item->user.cancel = true; } } #endif /* SOKOL_IMPL */