From d16287c9ddab4fd37849b5e0ea67d3098765a5e5 Mon Sep 17 00:00:00 2001 From: antao Date: Fri, 1 Feb 2019 18:00:34 +0800 Subject: [PATCH] Remove spin-locks --- lib/src/HttpAppFrameworkImpl.cc | 21 ++++++++++++++++--- lib/src/HttpControllersRouter.cc | 29 +++++++++++++++----------- lib/src/HttpControllersRouter.h | 6 ++---- lib/src/HttpFileUploadRequest.cc | 3 ++- lib/src/HttpRequestImpl.cc | 6 +++--- lib/src/HttpRequestImpl.h | 12 ++++++++--- lib/src/HttpRequestParser.cc | 3 ++- lib/src/HttpRequestParser.h | 4 +++- lib/src/HttpResponseImpl.cc | 16 +++++--------- lib/src/HttpResponseImpl.h | 5 +---- lib/src/HttpServer.h | 8 +++++++ lib/src/HttpSimpleControllersRouter.cc | 25 +++++++++++++--------- lib/src/HttpSimpleControllersRouter.h | 6 +++--- 13 files changed, 88 insertions(+), 56 deletions(-) diff --git a/lib/src/HttpAppFrameworkImpl.cc b/lib/src/HttpAppFrameworkImpl.cc index 0ffad4fa..a7c2b225 100755 --- a/lib/src/HttpAppFrameworkImpl.cc +++ b/lib/src/HttpAppFrameworkImpl.cc @@ -279,9 +279,8 @@ void HttpAppFrameworkImpl::run() } std::vector> servers; std::vector> loopThreads; - _httpCtrlsRouter.init(); - _httpSimpleCtrlsRouter.init(); - _websockCtrlsRouter.init(); + + std::vector ioLoops; for (auto const &listener : _listeners) { LOG_TRACE << "thread num=" << _threadNum; @@ -316,6 +315,7 @@ void HttpAppFrameworkImpl::run() serverPtr->kickoffIdleConnections(_idleConnectionTimeout); serverPtr->start(); servers.push_back(serverPtr); + ioLoops.push_back(serverPtr->getLoop()); } #else auto loopThreadPtr = std::make_shared("DrogonIoLoop"); @@ -349,9 +349,24 @@ void HttpAppFrameworkImpl::run() serverPtr->setConnectionCallback(std::bind(&HttpAppFrameworkImpl::onConnection, this, _1)); serverPtr->kickoffIdleConnections(_idleConnectionTimeout); serverPtr->start(); + std::promise pro; + auto f = pro.get_future(); + serverPtr->getLoop()->runInLoop([&pro]() { + pro.set_value(1); + }); + f.get(); + auto serverIoLoops = serverPtr->getIoLoops(); + for (auto serverIoLoop : serverIoLoops) + { + ioLoops.push_back(serverIoLoop); + } servers.push_back(serverPtr); #endif } + _httpCtrlsRouter.init(ioLoops); + _httpSimpleCtrlsRouter.init(ioLoops); + _websockCtrlsRouter.init(); + if (_useSession) { if (_sessionTimeout > 0) diff --git a/lib/src/HttpControllersRouter.cc b/lib/src/HttpControllersRouter.cc index bc60377a..38284c01 100644 --- a/lib/src/HttpControllersRouter.cc +++ b/lib/src/HttpControllersRouter.cc @@ -17,11 +17,10 @@ #include "HttpRequestImpl.h" #include "HttpResponseImpl.h" #include "HttpAppFrameworkImpl.h" -#include "SpinLock.h" using namespace drogon; -void HttpControllersRouter::init() +void HttpControllersRouter::init(const std::vector &ioLoops) { std::string regString; for (auto &router : _ctrlVector) @@ -30,11 +29,15 @@ void HttpControllersRouter::init() std::string tmp = std::regex_replace(router._pathParameterPattern, reg, "[^/]*"); router._regex = std::regex(router._pathParameterPattern, std::regex_constants::icase); regString.append("(").append(tmp).append(")|"); - for(auto &binder:router._binders) + for (auto &binder : router._binders) { - if(binder) + if (binder) { binder->_filters = FiltersFunction::createFilters(binder->_filterNames); + for(auto ioloop:ioLoops) + { + binder->_responsePtrMap[ioloop] = std::shared_ptr(); + } } } } @@ -218,12 +221,7 @@ void HttpControllersRouter::doControllerHandler(const CtrlBinderPtr &ctrlBinderP bool needSetJsessionid, std::string &&sessionId) { - HttpResponsePtr responsePtr; - { - SimpleSpinLock guard(ctrlBinderPtr->_binderMtx); - responsePtr = ctrlBinderPtr->_responsePtr; - } - + HttpResponsePtr &responsePtr = ctrlBinderPtr->_responsePtrMap[req->getLoop()]; if (responsePtr && (responsePtr->expiredTime() == 0 || (trantor::Date::now() < responsePtr->creationDate().after(responsePtr->expiredTime())))) { //use cached response! @@ -282,9 +280,16 @@ void HttpControllersRouter::doControllerHandler(const CtrlBinderPtr &ctrlBinderP { //cache the response; std::dynamic_pointer_cast(resp)->makeHeaderString(); + auto loop = req->getLoop(); + if (loop->isInLoopThread()) { - SimpleSpinLock guard(ctrlBinderPtr->_binderMtx); - ctrlBinderPtr->_responsePtr = resp; + ctrlBinderPtr->_responsePtrMap[loop] = resp; + } + else + { + req->getLoop()->queueInLoop([loop, resp, ctrlBinderPtr]() { + ctrlBinderPtr->_responsePtrMap[loop] = resp; + }); } } if (needSetJsessionid) diff --git a/lib/src/HttpControllersRouter.h b/lib/src/HttpControllersRouter.h index 20280b7d..434ae604 100644 --- a/lib/src/HttpControllersRouter.h +++ b/lib/src/HttpControllersRouter.h @@ -32,7 +32,7 @@ class HttpControllersRouter : public trantor::NonCopyable { public: HttpControllersRouter() {} - void init(); + void init(const std::vector &ioLoops); void addHttpPath(const std::string &path, const internal::HttpBinderBasePtr &binder, const std::vector &validMethods, @@ -50,9 +50,7 @@ class HttpControllersRouter : public trantor::NonCopyable std::vector> _filters; std::vector _parameterPlaces; std::map _queryParametersPlaces; - //std::atomic _binderMtx = ATOMIC_VAR_INIT(false); - std::atomic_flag _binderMtx = ATOMIC_FLAG_INIT; - std::shared_ptr _responsePtr; + std::map> _responsePtrMap; }; typedef std::shared_ptr CtrlBinderPtr; struct HttpControllerRouterItem diff --git a/lib/src/HttpFileUploadRequest.cc b/lib/src/HttpFileUploadRequest.cc index 8df9dd6e..b0e15399 100644 --- a/lib/src/HttpFileUploadRequest.cc +++ b/lib/src/HttpFileUploadRequest.cc @@ -18,7 +18,8 @@ using namespace drogon; HttpFileUploadRequest::HttpFileUploadRequest(const std::vector &files) - : _files(files) + : HttpRequestImpl(nullptr), + _files(files) { _boundary = genRandomString(32); setMethod(drogon::Post); diff --git a/lib/src/HttpRequestImpl.cc b/lib/src/HttpRequestImpl.cc index 71bd9831..0cfc622e 100755 --- a/lib/src/HttpRequestImpl.cc +++ b/lib/src/HttpRequestImpl.cc @@ -317,7 +317,7 @@ void HttpRequestImpl::addHeader(const char *start, const char *colon, const char HttpRequestPtr HttpRequest::newHttpRequest() { - auto req = std::make_shared(); + auto req = std::make_shared(nullptr); req->setMethod(drogon::Get); req->setVersion(drogon::HttpRequest::kHttp11); return req; @@ -325,7 +325,7 @@ HttpRequestPtr HttpRequest::newHttpRequest() HttpRequestPtr HttpRequest::newHttpFormPostRequest() { - auto req = std::make_shared(); + auto req = std::make_shared(nullptr); req->setMethod(drogon::Post); req->setVersion(drogon::HttpRequest::kHttp11); req->_contentType = CT_APPLICATION_X_FORM; @@ -334,7 +334,7 @@ HttpRequestPtr HttpRequest::newHttpFormPostRequest() HttpRequestPtr HttpRequest::newHttpJsonRequest(const Json::Value &data) { - auto req = std::make_shared(); + auto req = std::make_shared(nullptr); req->setMethod(drogon::Get); req->setVersion(drogon::HttpRequest::kHttp11); req->_contentType = CT_APPLICATION_JSON; diff --git a/lib/src/HttpRequestImpl.h b/lib/src/HttpRequestImpl.h index 5c72a788..66e3f533 100755 --- a/lib/src/HttpRequestImpl.h +++ b/lib/src/HttpRequestImpl.h @@ -23,6 +23,8 @@ #include #include #include +#include + #include #include #include @@ -40,14 +42,17 @@ class HttpRequestImpl : public HttpRequest public: friend class HttpRequestParser; - HttpRequestImpl() + HttpRequestImpl(trantor::EventLoop *loop) : _method(Invalid), _version(kUnknown), _date(trantor::Date::now()), - _contentLen(0) + _contentLen(0), + _loop(loop) { } + trantor::EventLoop *getLoop() { return _loop; } + void setVersion(Version v) { _version = v; @@ -368,10 +373,11 @@ class HttpRequestImpl : public HttpRequest trantor::InetAddress _peer; trantor::InetAddress _local; trantor::Date _date; - + protected: std::string _content; size_t _contentLen; + trantor::EventLoop *_loop; ContentType _contentType = CT_TEXT_PLAIN; }; diff --git a/lib/src/HttpRequestParser.cc b/lib/src/HttpRequestParser.cc index c1fd19cd..55d80e0e 100755 --- a/lib/src/HttpRequestParser.cc +++ b/lib/src/HttpRequestParser.cc @@ -24,7 +24,8 @@ using namespace drogon; HttpRequestParser::HttpRequestParser(const trantor::TcpConnectionPtr &connPtr) : _state(kExpectRequestLine), - _request(new HttpRequestImpl), + _loop(connPtr->getLoop()), + _request(new HttpRequestImpl(connPtr->getLoop())), _conn(connPtr) { } diff --git a/lib/src/HttpRequestParser.h b/lib/src/HttpRequestParser.h index 0cf9ea80..4c50a45a 100755 --- a/lib/src/HttpRequestParser.h +++ b/lib/src/HttpRequestParser.h @@ -49,7 +49,7 @@ class HttpRequestParser void reset() { _state = kExpectRequestLine; - _request.reset(new HttpRequestImpl); + _request.reset(new HttpRequestImpl(_loop)); } const HttpRequestPtr request() const @@ -96,6 +96,8 @@ class HttpRequestParser bool processRequestLine(const char *begin, const char *end); HttpRequestParseState _state; + + trantor::EventLoop *_loop; HttpRequestImplPtr _request; bool _firstRequest = true; diff --git a/lib/src/HttpResponseImpl.cc b/lib/src/HttpResponseImpl.cc index 27ccb899..7346fecd 100755 --- a/lib/src/HttpResponseImpl.cc +++ b/lib/src/HttpResponseImpl.cc @@ -14,7 +14,6 @@ #include "HttpAppFrameworkImpl.h" #include "HttpResponseImpl.h" -#include "SpinLock.h" #include #include #include @@ -350,17 +349,13 @@ std::shared_ptr HttpResponseImpl::renderToString() const { _httpStringDate = now.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC; auto newDate = getHttpFullDate(now); - { - SimpleSpinLock lock(*_httpStringMutex); - _httpString = std::make_shared(*_httpString); - memcpy((void *)&(*_httpString)[_datePos], newDate, strlen(newDate)); - return _httpString; - } - } - { - SimpleSpinLock lock(*_httpStringMutex); + + _httpString = std::make_shared(*_httpString); + memcpy((void *)&(*_httpString)[_datePos], newDate, strlen(newDate)); return _httpString; } + + return _httpString; } } auto httpString = std::make_shared(); @@ -394,7 +389,6 @@ std::shared_ptr HttpResponseImpl::renderToString() const httpString->append(*_bodyPtr); if (_expriedTime >= 0) { - SimpleSpinLock lock(*_httpStringMutex); _datePos = datePos; _httpString = httpString; } diff --git a/lib/src/HttpResponseImpl.h b/lib/src/HttpResponseImpl.h index 44853d53..1263d66a 100755 --- a/lib/src/HttpResponseImpl.h +++ b/lib/src/HttpResponseImpl.h @@ -40,10 +40,8 @@ class HttpResponseImpl : public HttpResponse _closeConnection(false), _leftBodyLength(0), _currentChunkLength(0), - _bodyPtr(new std::string()), - _httpStringMutex(new std::atomic_flag) + _bodyPtr(new std::string()) { - _httpStringMutex->clear(); } virtual HttpStatusCode statusCode() override { @@ -388,7 +386,6 @@ class HttpResponseImpl : public HttpResponse std::shared_ptr _fullHeaderString; mutable std::shared_ptr _httpString; - mutable std::shared_ptr _httpStringMutex; mutable std::string::size_type _datePos = std::string::npos; mutable int64_t _httpStringDate = -1; diff --git a/lib/src/HttpServer.h b/lib/src/HttpServer.h index 89d6dc71..b0cba369 100755 --- a/lib/src/HttpServer.h +++ b/lib/src/HttpServer.h @@ -78,6 +78,14 @@ class HttpServer : trantor::NonCopyable { _server.kickoffIdleConnections(timeout); } + trantor::EventLoop *getLoop() + { + return _server.getLoop(); + } + std::vector getIoLoops() + { + return _server.getIoLoops(); + } void start(); #ifdef USE_OPENSSL diff --git a/lib/src/HttpSimpleControllersRouter.cc b/lib/src/HttpSimpleControllersRouter.cc index 8e14f21c..bf740f0e 100644 --- a/lib/src/HttpSimpleControllersRouter.cc +++ b/lib/src/HttpSimpleControllersRouter.cc @@ -15,7 +15,6 @@ #include "FiltersFunction.h" #include "HttpSimpleControllersRouter.h" #include "HttpAppFrameworkImpl.h" -#include "SpinLock.h" using namespace drogon; @@ -124,12 +123,7 @@ void HttpSimpleControllersRouter::doControllerHandler(SimpleControllerRouterItem auto &controller = item._controller; if (controller) { - HttpResponsePtr responsePtr; - { - //Maybe update the _responsePtr, so we use shared_lock to protect; - SimpleSpinLock guard(item._mutex); - responsePtr = item._responsePtr; - } + HttpResponsePtr &responsePtr = item._responsePtrMap[req->getLoop()]; if (responsePtr && (responsePtr->expiredTime() == 0 || (trantor::Date::now() < responsePtr->creationDate().after(responsePtr->expiredTime())))) { //use cached response! @@ -154,9 +148,16 @@ void HttpSimpleControllersRouter::doControllerHandler(SimpleControllerRouterItem { //cache the response; std::dynamic_pointer_cast(resp)->makeHeaderString(); + auto loop = req->getLoop(); + if (loop->isInLoopThread()) { - SimpleSpinLock guard(item._mutex); - item._responsePtr = resp; + item._responsePtrMap[loop] = resp; + } + else + { + loop->queueInLoop([loop, resp, &item]() { + item._responsePtrMap[loop] = resp; + }); } } if (needSetJsessionid) @@ -186,11 +187,15 @@ void HttpSimpleControllersRouter::doControllerHandler(SimpleControllerRouterItem } } -void HttpSimpleControllersRouter::init() +void HttpSimpleControllersRouter::init(const std::vector &ioLoops) { for (auto &iter : _simpCtrlMap) { auto &item = iter.second; item._filters = FiltersFunction::createFilters(item._filterNames); + for (auto ioloop : ioLoops) + { + item._responsePtrMap[ioloop] = std::shared_ptr(); + } } } \ No newline at end of file diff --git a/lib/src/HttpSimpleControllersRouter.h b/lib/src/HttpSimpleControllersRouter.h index 72252dd5..3956c9a7 100644 --- a/lib/src/HttpSimpleControllersRouter.h +++ b/lib/src/HttpSimpleControllersRouter.h @@ -44,7 +44,7 @@ class HttpSimpleControllersRouter : public trantor::NonCopyable std::function &&callback, bool needSetJsessionid, std::string &&sessionId); - void init(); + void init(const std::vector &ioLoops); private: HttpControllersRouter &_httpCtrlsRouter; @@ -55,9 +55,9 @@ class HttpSimpleControllersRouter : public trantor::NonCopyable std::vector> _filters; std::vector _validMethodsFlags; std::shared_ptr _controller; - std::shared_ptr _responsePtr; + std::map> _responsePtrMap; //std::atomic _mutex = ATOMIC_VAR_INIT(false); - std::atomic_flag _mutex = ATOMIC_FLAG_INIT; + //std::atomic_flag _mutex = ATOMIC_FLAG_INIT; }; std::unordered_map _simpCtrlMap; std::mutex _simpCtrlMutex;