test/drogon
1
0
Fork 0
mirror of https://gitee.com/an-tao/drogon.git synced 2024-12-02 03:38:03 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Support range for static file (#1060)

Browse Source
This commit is contained in:
NitroMelon 2021-10-28 21:53:58 +08:00 committed by GitHub
parent 7066b09edb
commit 0431f38a4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 435 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -263,6 +263,7 @@ set(DROGON_SOURCES
lib/src/MultiPart.cc
lib/src/NotFound.cc
lib/src/PluginsManager.cc
lib/src/RangeParser.cc
lib/src/SecureSSLRedirector.cc
lib/src/AccessLogger.cc
lib/src/SessionManager.cc

177
lib/src/RangeParser.cc Normal file
View File

@ -0,0 +1,177 @@
/**
*
* RangeParser.h
* He, Wanchen
*
* Copyright 2021, He,Wanchen. All rights reserved.
* https://github.com/drogonframework/drogon
* Use of this source code is governed by a MIT license
* that can be found in the License file.
*
* Drogon
*
*/
#include "RangeParser.h"
#include <limits>
using namespace drogon;
static constexpr size_t MAX_SIZE = std::numeric_limits<size_t>::max();
static constexpr size_t MAX_TEN = MAX_SIZE / 10;
static constexpr size_t MAX_DIGIT = MAX_SIZE % 10;
// clang-format off
#define DR_SKIP_WHITESPACE(p) while (*p == ' ') { ++(p); }
#define DR_ISDIGIT(p) ('0' <= *(p) && *(p) <= '9')
#define DR_WOULD_OVERFLOW(base, digit) \
((base) > MAX_TEN || (base >= MAX_TEN && (digit) - '0' > MAX_DIGIT))
// clang-format on
/** Following formats are valid range header according to rfc7233`
* Range: <unit>=<start>-
* Range: <unit>=<start>-<end>
* Range: <unit>=<start>-<end>, <start>-<end>
* Range: <unit>=<start>-<end>, <start>-<end>, <start>-<end>
* Range: <unit>=-<suffix-length>
*/
FileRangeParseResult drogon::parseRangeHeader(const std::string &rangeStr,
size_t contentLength,
std::vector<FileRange> &ranges)
{
if (rangeStr.size() < 7 || rangeStr.compare(0, 6, "bytes=") != 0)
{
return InvalidRange;
}
const char *iter = rangeStr.c_str() + 6;
size_t totalSize = 0;
while (true)
{
size_t start = 0;
size_t end = 0;
// If this is a suffix range: <unit>=-<suffix-length>
bool isSuffix = false;
DR_SKIP_WHITESPACE(iter);
if (*iter == '-')
{
isSuffix = true;
++iter;
}
// Parse start
else
{
if (!DR_ISDIGIT(iter))
{
return InvalidRange;
}
while (DR_ISDIGIT(iter))
{
// integer out of range
if (DR_WOULD_OVERFLOW(start, *iter))
{
return NotSatisfiable;
}
start = start * 10 + (*iter++ - '0');
}
DR_SKIP_WHITESPACE(iter);
// should be separator now
if (*iter++ != '-')
{
return InvalidRange;
}
DR_SKIP_WHITESPACE(iter);
// If this is a prefix range <unit>=<range-start>-
if (*iter == ',' || *iter == '\0')
{
end = contentLength;
// Handle found
if (start < end)
{
if (totalSize > MAX_SIZE - (end - start))
{
return NotSatisfiable;
}
totalSize += end - start;
ranges.push_back({start, end});
}
if (*iter++ != ',')
{
break;
}
continue;
}
}
// Parse end
if (!DR_ISDIGIT(iter))
{
return InvalidRange;
}
while (DR_ISDIGIT(iter))
{
if (DR_WOULD_OVERFLOW(end, *iter))
{
return NotSatisfiable;
}
end = end * 10 + (*iter++ - '0');
}
DR_SKIP_WHITESPACE(iter);
if (*iter != ',' && *iter != '\0')
{
return InvalidRange;
}
if (isSuffix)
{
start = (end < contentLength) ? contentLength - end : 0;
end = contentLength - 1;
}
// [start, end)
if (end >= contentLength)
{
end = contentLength;
}
else
{
++end;
}
// handle found
if (start < end)
{
ranges.push_back({start, end});
if (totalSize > MAX_SIZE - (end - start))
{
return NotSatisfiable;
}
totalSize += end - start;
// We restrict the number to be under 100, to avoid malicious
// requests.
// Though rfc does not say anything about max number of ranges,
// it does mention that server can ignore range header freely.
if (ranges.size() > 100)
{
return InvalidRange;
}
}
if (*iter++ != ',')
{
break;
}
}
if (ranges.size() == 0 || totalSize > contentLength)
{
return NotSatisfiable;
}
return ranges.size() == 1 ? SinglePart : MultiPart;
}
#undef DR_SKIP_WHITESPACE
#undef DR_ISDIGIT
#undef DR_WOULD_OVERFLOW

40
lib/src/RangeParser.h Normal file
View File

@ -0,0 +1,40 @@
/**
*
* RangeParser.h
* He, Wanchen
*
* Copyright 2021, He,Wanchen. All rights reserved.
* https://github.com/drogonframework/drogon
* Use of this source code is governed by a MIT license
* that can be found in the License file.
*
* Drogon
*
*/
#include <string>
#include <vector>
#include <sys/types.h>
namespace drogon
{
// [start, end)
struct FileRange
{
size_t start;
size_t end;
};
enum FileRangeParseResult
{
InvalidRange = -1,
NotSatisfiable = 0,
SinglePart = 1,
MultiPart = 2
};
FileRangeParseResult parseRangeHeader(const std::string &rangeStr,
size_t contentLength,
std::vector<FileRange> &ranges);
} // namespace drogon

217
lib/src/StaticFileRouter.cc
View File

@ -16,6 +16,7 @@
#include "HttpAppFrameworkImpl.h"
#include "HttpRequestImpl.h"
#include "HttpResponseImpl.h"
#include "RangeParser.h"
#include <fstream>
#include <iostream>
#include <algorithm>
@ -259,12 +260,153 @@ void StaticFileRouter::route(
defaultHandler_(req, std::move(callback));
}
// Expand this struct as you need, nothing to worry about
struct FileStat
{
size_t fileSize_;
struct tm modifiedTime_;
std::string modifiedTimeStr_;
};
// A wrapper to call stat()
// std::filesystem::file_time_type::clock::to_time_t still not
// implemented by M$, even in c++20, so keep calls to stat()
static bool getFileStat(const std::string &filePath, FileStat &myStat)
{
#if defined(_WIN32) && !defined(__MINGW32__)
struct _stati64 fileStat;
#else // _WIN32
struct stat fileStat;
#endif // _WIN32
if (stat(utils::toNativePath(filePath).c_str(), &fileStat) == 0 &&
S_ISREG(fileStat.st_mode))
{
LOG_TRACE << "last modify time:" << fileStat.st_mtime;
#ifdef _WIN32
gmtime_s(&myStat.modifiedTime_, &fileStat.st_mtime);
#else
gmtime_r(&fileStat.st_mtime, &myStat.modifiedTime_);
#endif
std::string &timeStr = myStat.modifiedTimeStr_;
timeStr.resize(64);
size_t len = strftime(timeStr.data(),
timeStr.size(),
"%a, %d %b %Y %H:%M:%S GMT",
&myStat.modifiedTime_);
timeStr.resize(len);
myStat.fileSize_ = fileStat.st_size;
return true;
}
return false;
}
void StaticFileRouter::sendStaticFileResponse(
const std::string &filePath,
const HttpRequestImplPtr &req,
std::function<void(const HttpResponsePtr &)> &&callback,
const string_view &defaultContentType)
{ // find cached response
{
if (req->method() != Get)
{
callback(app().getCustomErrorHandler()(k405MethodNotAllowed));
return;
}
FileStat fileStat;
bool fileExists = false;
const std::string &rangeStr = req->getHeaderBy("range");
if (enableRange_ && !rangeStr.empty())
{
if (!getFileStat(filePath, fileStat))
{
defaultHandler_(req, std::move(callback));
return;
}
fileExists = true;
// Check last modified time, rfc2616-14.25
// If-Modified-Since: Mon, 15 Oct 2018 06:26:33 GMT
// According to rfc 7233-3.1, preconditions must be evaluated before
const std::string &modiStr = req->getHeaderBy("if-modified-since");
if (enableLastModify_ && modiStr == fileStat.modifiedTimeStr_)
{
LOG_TRACE << "Not modified!";
std::shared_ptr<HttpResponseImpl> resp =
std::make_shared<HttpResponseImpl>();
resp->setStatusCode(k304NotModified);
resp->setContentTypeCode(CT_NONE);
HttpAppFrameworkImpl::instance().callCallback(req, resp, callback);
return;
}
// Check If-Range precondition
const std::string &ifRange = req->getHeaderBy("if-range");
if (ifRange.empty() || ifRange == fileStat.modifiedTimeStr_)
{
std::vector<FileRange> ranges;
switch (parseRangeHeader(rangeStr, fileStat.fileSize_, ranges))
{
// TODO: support only single range now
// Contributions are welcomed.
case FileRangeParseResult::SinglePart:
case FileRangeParseResult::MultiPart:
{
auto firstRange = ranges.front();
auto ct = fileNameToContentTypeAndMime(filePath);
auto resp =
HttpResponse::newFileResponse(filePath,
firstRange.start,
firstRange.end -
firstRange.start,
true,
"",
ct.first,
std::string(ct.second));
if (!fileStat.modifiedTimeStr_.empty())
{
resp->addHeader("Last-Modified",
fileStat.modifiedTimeStr_);
resp->addHeader("Expires",
"Thu, 01 Jan 1970 00:00:00 GMT");
}
HttpAppFrameworkImpl::instance().callCallback(req,
resp,
callback);
return;
}
case FileRangeParseResult::NotSatisfiable:
{
auto resp = HttpResponse::newHttpResponse();
resp->setStatusCode(k416RequestedRangeNotSatisfiable);
char buf[64];
snprintf(buf,
sizeof(buf),
"bytes */%zu",
fileStat.fileSize_);
resp->addHeader("Content-Range", std::string(buf));
HttpAppFrameworkImpl::instance().callCallback(req,
resp,
callback);
return;
}
/** rfc7233 4.4.
* > Note: Because servers are free to ignore Range, many
* implementations will simply respond with the entire selected
* representation in a 200 (OK) response. That is partly
* because most clients are prepared to receive a 200 (OK) to
* complete the task (albeit less efficiently) and partly
* because clients might not stop making an invalid partial
* request until they have received a complete representation.
* Thus, clients cannot depend on receiving a 416 (Range Not
* Satisfiable) response even when it is most appropriate.
*/
default:
break;
}
}
}
// find cached response
HttpResponsePtr cachedResp;
auto &cacheMap = staticFilesCache_->getThreadData();
auto iter = cacheMap.find(filePath);
@ -273,20 +415,10 @@ void StaticFileRouter::sendStaticFileResponse(
cachedResp = iter->second;
}
// check last modified time,rfc2616-14.25
// If-Modified-Since: Mon, 15 Oct 2018 06:26:33 GMT
std::string timeStr;
bool fileExists{false};
if (enableLastModify_)
{
if (cachedResp)
{
if (req->method() != Get)
{
callback(app().getCustomErrorHandler()(k405MethodNotAllowed));
return;
}
if (static_cast<HttpResponseImpl *>(cachedResp.get())
->getHeaderBy("last-modified") ==
req->getHeaderBy("if-modified-since"))
@ -304,39 +436,14 @@ void StaticFileRouter::sendStaticFileResponse(
else
{
LOG_TRACE << "enabled LastModify";
// std::filesystem::file_time_type::clock::to_time_t still not
// implemented by M$, even in c++20, so keep calls to stat()
#if defined(_WIN32) && !defined(__MINGW32__)
struct _stati64 fileStat;
#else // _WIN32
struct stat fileStat;
#endif // _WIN32
if (stat(utils::toNativePath(filePath).c_str(), &fileStat) == 0 &&
S_ISREG(fileStat.st_mode))
if (!fileExists && !getFileStat(filePath, fileStat))
{
fileExists = true;
LOG_TRACE << "last modify time:" << fileStat.st_mtime;
if (req->method() != Get)
{
callback(
app().getCustomErrorHandler()(k405MethodNotAllowed));
defaultHandler_(req, std::move(callback));
return;
}
struct tm tm1;
#ifdef _WIN32
gmtime_s(&tm1, &fileStat.st_mtime);
#else
gmtime_r(&fileStat.st_mtime, &tm1);
#endif
timeStr.resize(64);
auto len = strftime((char *)timeStr.data(),
timeStr.size(),
"%a, %d %b %Y %H:%M:%S GMT",
&tm1);
timeStr.resize(len);
const std::string &modiStr =
req->getHeaderBy("if-modified-since");
if (modiStr == timeStr && !modiStr.empty())
fileExists = true;
const std::string &modiStr = req->getHeaderBy("if-modified-since");
if (modiStr == fileStat.modifiedTimeStr_)
{
LOG_TRACE << "not Modified!";
std::shared_ptr<HttpResponseImpl> resp =
@ -349,26 +456,16 @@ void StaticFileRouter::sendStaticFileResponse(
return;
}
}
else
{
defaultHandler_(req, std::move(callback));
return;
}
}
}
if (cachedResp)
{
if (req->method() != Get)
{
callback(app().getCustomErrorHandler()(k405MethodNotAllowed));
return;
}
LOG_TRACE << "Using file cache";
HttpAppFrameworkImpl::instance().callCallback(req,
cachedResp,
callback);
return;
}
// Check existence
if (!fileExists)
{
filesystem::path fsFilePath(utils::toNativePath(filePath));
@ -381,12 +478,6 @@ void StaticFileRouter::sendStaticFileResponse(
}
}
if (req->method() != Get)
{
callback(app().getCustomErrorHandler()(k405MethodNotAllowed));
return;
}
HttpResponsePtr resp;
auto &acceptEncoding = req->getHeaderBy("accept-encoding");
@ -441,11 +532,15 @@ void StaticFileRouter::sendStaticFileResponse(
resp->setContentTypeCodeAndCustomString(CT_CUSTOM,
defaultContentType);
}
if (!timeStr.empty())
if (!fileStat.modifiedTimeStr_.empty())
{
resp->addHeader("Last-Modified", timeStr);
resp->addHeader("Last-Modified", fileStat.modifiedTimeStr_);
resp->addHeader("Expires", "Thu, 01 Jan 1970 00:00:00 GMT");
}
if (enableRange_)
{
resp->addHeader("accept-range", "bytes");
}
if (!headers_.empty())
{
for (auto &header : headers_)
@ -474,6 +569,7 @@ void StaticFileRouter::sendStaticFileResponse(
callback(resp);
return;
}
void StaticFileRouter::setFileTypes(const std::vector<std::string> &types)
{
fileTypeSet_.clear();
@ -482,6 +578,7 @@ void StaticFileRouter::setFileTypes(const std::vector<std::string> &types)
fileTypeSet_.insert(type);
}
}
void StaticFileRouter::defaultHandler(
const HttpRequestPtr & /*req*/,
std::function<void(const HttpResponsePtr &)> &&callback)

1
lib/src/StaticFileRouter.h
View File

@ -125,6 +125,7 @@ class StaticFileRouter
int staticFilesCacheTime_{5};
bool enableLastModify_{true};
bool enableRange_{true};
bool gzipStaticFlag_{true};
bool brStaticFlag_{true};
std::unique_ptr<

49
lib/tests/integration_test/client/main.cc
View File

@ -712,7 +712,7 @@ void doTest(const HttpClientPtr &client, std::shared_ptr<test::Case> TEST_CTX)
CHECK((*json)["P2"] == "test");
});
// Test send file by range
// Test newFileResponse
req = HttpRequest::newHttpRequest();
req->setPath("/RangeTestController/");
client->sendRequest(
@ -766,6 +766,53 @@ void doTest(const HttpClientPtr &client, std::shared_ptr<test::Case> TEST_CTX)
CHECK(resp->getStatusCode() == k416RequestedRangeNotSatisfiable);
});
//
// Test StaticFileRouter with range header
//
req = HttpRequest::newHttpRequest();
req->setPath("/range-test.txt");
req->setMethod(drogon::Head);
client->sendRequest(
req, [req, TEST_CTX](ReqResult result, const HttpResponsePtr &resp) {
REQUIRE(result == ReqResult::Ok);
CHECK(resp->getStatusCode() == k200OK);
CHECK(resp->getHeader("content-length") == "1000000");
CHECK(resp->getHeader("accept-range") == "bytes");
});
req = HttpRequest::newHttpRequest();
req->setPath("/range-test.txt");
req->addHeader("range", "bytes=0-19");
client->sendRequest(req,
[req, TEST_CTX](ReqResult result,
const HttpResponsePtr &resp) {
REQUIRE(result == ReqResult::Ok);
CHECK(resp->getStatusCode() == k206PartialContent);
CHECK(resp->getBody() == "01234567890123456789");
});
req = HttpRequest::newHttpRequest();
req->setPath("/range-test.txt");
req->addHeader("range", "bytes=-20");
client->sendRequest(req,
[req, TEST_CTX](ReqResult result,
const HttpResponsePtr &resp) {
REQUIRE(result == ReqResult::Ok);
CHECK(resp->getStatusCode() == k206PartialContent);
CHECK(resp->getBody() == "01234567890123456789");
});
req = HttpRequest::newHttpRequest();
req->setPath("/range-test.txt");
req->addHeader("range", "bytes=999980-");
client->sendRequest(req,
[req, TEST_CTX](ReqResult result,
const HttpResponsePtr &resp) {
REQUIRE(result == ReqResult::Ok);
CHECK(resp->getStatusCode() == k206PartialContent);
CHECK(resp->getBody() == "01234567890123456789");
});
// Using .. to access a upper directory should be permitted as long as
// it never leaves the document root
req = HttpRequest::newHttpRequest();

2
lib/tests/integration_test/server/main.cc
View File

@ -362,7 +362,7 @@ int main()
}
});
app().registerCustomExtensionMime("md", "text/markdown");
app().setFileTypes({"md", "html", "jpg", "cc"});
app().setFileTypes({"md", "html", "jpg", "cc", "txt"});
std::cout << "Date: "
<< std::string{drogon::utils::getHttpFullDate(
trantor::Date::now())}