-
Notifications
You must be signed in to change notification settings - Fork 62
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
MB-48816: Avoid unsafe use of cookie from background tasks
Previously, StatCheckpointTask and StatDCPTask immediately wrote
responses when collecting stats while on a background thread.
TSAN reported this as unsafe; no locks prevent potential racing with a
frontend thread manipulating the cookie.
Change both tasks to accumulate task values, but leave the frontend
thread to actually write the responses when it resumes the
ewouldblock'ed operation.
TSAN Report:
WARNING: ThreadSanitizer: data race (pid=24371)
Read of size 8 at 0x7b54000a2df0 by thread T62:
#0 Cookie::getHeader() const kv_engine/daemon/cookie.cc:201 (memcached+0x6508ac)
#1 append_stats kv_engine/daemon/protocol/mcbp/stats_context.cc:95 (memcached+0x71fd6c)
....
#19 void StatCollector::addStat<cb::stats::Key, unsigned long const&>(cb::stats::Key&&, unsigned long const&) const ../kv_engine/include/statistics/collector.h:336 (memcached+0x7e50e5)
#20 EventuallyPersistentEngine::addAggregatedProducerStats(BucketStatCollector const&, ConnCounter const&) kv_engine/engines/ep/src/ep_engine.cc:4038 (memcached+0x7e50e5)
#21 EventuallyPersistentEngine::doDcpStatsInner(CookieIface const*, std::function<void (std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, void const*)> const&, std::basic_string_view<char, std::char_traits<char> >) kv_engine/engines/ep/src/ep_engine.cc:4030 (memcached+0x81bd05)
Previous write of size 8 at 0x7b54000a2df0 by thread T21 (mutexes: write M3843):
#0 Cookie::setPacket(cb::mcbp::Header const&, bool) kv_engine/daemon/cookie.cc:186 (memcached+0x65080e)
#1 Cookie::preserveRequest() kv_engine/daemon/cookie.h:225 (memcached+0x696aa7)
#2 Connection::executeCommandPipeline() kv_engine/daemon/connection.cc:581 (memcached+0x696aa7)
#3 Connection::executeCommandsCallback() kv_engine/daemon/connection.cc:793 (memcached+0x696be8)
#4 Connection::rw_callback(bufferevent*, void*) kv_engine/daemon/connection.cc:942 (memcached+0x697851)
#5 bufferevent_run_deferred_callbacks_unlocked /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 (libevent_core-2.1.so.7+0xf71d)
#6 folly::EventBase::loopBody(int, bool) folly/io/async/EventBase.cpp:397 (memcached+0xfc9b52)
#7 folly::EventBase::loop() folly/io/async/EventBase.cpp:315 (memcached+0xfcb06b)
#8 folly::EventBase::loopForever() folly/io/async/EventBase.cpp:538 (memcached+0xfcb06b)
#9 worker_libevent kv_engine/daemon/thread.cc:115 (memcached+0x6c16af)
#10 CouchbaseThread::run() platform/src/cb_pthreads.cc:51 (memcached+0xf217d5)
#11 platform_thread_wrap platform/src/cb_pthreads.cc:64 (memcached+0xf217d5)
Change-Id: I3fbd8d51e174a7d19c5cb608a969795e445b8e86
Reviewed-on: http://review.couchbase.org/c/kv_engine/+/163709
Tested-by: Build Bot <[email protected]>
Reviewed-by: Dave Rigby <[email protected]>- Loading branch information
Showing
5 changed files
with
220 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ | ||
| /* | ||
| * Copyright 2021-Present Couchbase, Inc. | ||
| * | ||
| * Use of this software is governed by the Business Source License included | ||
| * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified | ||
| * in that file, in accordance with the Business Source License, use of this | ||
| * software will be governed by the Apache License, Version 2.0, included in | ||
| * the file licenses/APL2.txt. | ||
| */ | ||
|
|
||
| #include "background_stat_task.h" | ||
|
|
||
| #include "bucket_logger.h" | ||
| #include "ep_engine.h" | ||
| #include <phosphor/phosphor.h> | ||
| #include <string_view> | ||
|
|
||
| BackgroundStatTask::BackgroundStatTask(EventuallyPersistentEngine* e, | ||
| const CookieIface* cookie, | ||
| TaskId taskId) | ||
| : GlobalTask(e, taskId, 0, false), e(e), cookie(cookie) { | ||
| } | ||
|
|
||
| bool BackgroundStatTask::run() { | ||
| TRACE_EVENT0("ep-engine/task", "BackgroundStatTask"); | ||
| try { | ||
| status = collectStats(); | ||
| } catch (const std::exception& e) { | ||
| EP_LOG_WARN( | ||
| "BackgroundStatTask: callback threw exception: \"{}\" task " | ||
| "desc:\"{}\"", | ||
| e.what(), | ||
| getDescription()); | ||
| } | ||
| // _must_ notify success to ensure the frontend calls back into | ||
| // getStats - the second call is required to avoid leaking data | ||
| // allocated to store in the engine specific. | ||
| // The real status is stored and shall be retrieved later. | ||
| e->notifyIOComplete(cookie, cb::engine_errc::success); | ||
| return false; | ||
| } | ||
|
|
||
| cb::engine_errc BackgroundStatTask::maybeWriteResponse( | ||
| const AddStatFn& addStat) const { | ||
| if (status == cb::engine_errc::success) { | ||
| for (const auto& [key, value] : stats) { | ||
| addStat(key, value, cookie); | ||
| } | ||
| } | ||
| return status; | ||
| } | ||
|
|
||
| AddStatFn BackgroundStatTask::getDeferredAddStat() { | ||
| return [this](std::string_view key, std::string_view value, const void*) { | ||
| this->stats.emplace_back(key, value); | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ | ||
| /* | ||
| * Copyright 2021-Present Couchbase, Inc. | ||
| * | ||
| * Use of this software is governed by the Business Source License included | ||
| * in the file licenses/BSL-Couchbase.txt. As of the Change Date specified | ||
| * in that file, in accordance with the Business Source License, use of this | ||
| * software will be governed by the Apache License, Version 2.0, included in | ||
| * the file licenses/APL2.txt. | ||
| */ | ||
| #pragma once | ||
|
|
||
| #include <executor/globaltask.h> | ||
| #include <memcached/cookie_iface.h> | ||
| #include <memcached/engine_common.h> | ||
| #include <memcached/engine_error.h> | ||
|
|
||
| #include <functional> | ||
| #include <string> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| // forward decl | ||
| enum class TaskId : int; | ||
|
|
||
| /** | ||
| * Base type for tasks which gather stats on a background task. | ||
| * | ||
| * For use when generating stats is likely to be expensive, to avoid | ||
| * taking up frontend thread time. | ||
| * Users should construct and schedule the task, then store it in the | ||
| * cookie engine specific with | ||
| * EventuallyPersistentEngine::storeStatTask(...) | ||
| * Once the task has notified the frontend, the task should be retrieved with | ||
| * EventuallyPersistentEngine::retrieveStatTask(...) | ||
| */ | ||
| class BackgroundStatTask : public GlobalTask { | ||
| public: | ||
| using Callback = std::function<cb::engine_errc(EventuallyPersistentEngine*, | ||
| const AddStatFn&)>; | ||
| BackgroundStatTask(EventuallyPersistentEngine* e, | ||
| const CookieIface* cookie, | ||
| TaskId taskId); | ||
|
|
||
| bool run() override; | ||
|
|
||
| /** | ||
| * If the task was successful, write all gathered stats as responses. | ||
| * | ||
| * Should only be called from a frontend thread. | ||
| * @param addStat frontend provided callback | ||
| * @return errc reflecting status of the operation | ||
| */ | ||
| cb::engine_errc maybeWriteResponse(const AddStatFn& addStat) const; | ||
|
|
||
| protected: | ||
| /** | ||
| * Do potentially expensive work to collect stats in a background task. | ||
| * @return status of operation | ||
| */ | ||
| virtual cb::engine_errc collectStats() = 0; | ||
|
|
||
| /** | ||
| * Get a callback used to store stats which have been computed by the | ||
| * background task. | ||
| * | ||
| * Stats cannot be immediately written as responses from the background | ||
| * task as doing so could be racy. Instead, store them for when the | ||
| * frontend thread revisits this operation. | ||
| */ | ||
| AddStatFn getDeferredAddStat(); | ||
|
|
||
| EventuallyPersistentEngine* e; | ||
| const CookieIface* cookie; | ||
|
|
||
| // stats which have been collected while running in a background task | ||
| std::vector<std::pair<std::string, std::string>> stats; | ||
| cb::engine_errc status = cb::engine_errc::success; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters