220 lines
8.0 KiB
C++
220 lines
8.0 KiB
C++
// Copyright 2017 The Abseil Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// https://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
// -----------------------------------------------------------------------------
|
|
// File: call_once.h
|
|
// -----------------------------------------------------------------------------
|
|
//
|
|
// This header file provides an Abseil version of `std::call_once` for invoking
|
|
// a given function at most once, across all threads. This Abseil version is
|
|
// faster than the C++11 version and incorporates the C++17 argument-passing
|
|
// fix, so that (for example) non-const references may be passed to the invoked
|
|
// function.
|
|
|
|
#ifndef ABSL_BASE_CALL_ONCE_H_
|
|
#define ABSL_BASE_CALL_ONCE_H_
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <cstdint>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
|
|
#include "absl/base/internal/invoke.h"
|
|
#include "absl/base/internal/low_level_scheduling.h"
|
|
#include "absl/base/internal/raw_logging.h"
|
|
#include "absl/base/internal/scheduling_mode.h"
|
|
#include "absl/base/internal/spinlock_wait.h"
|
|
#include "absl/base/macros.h"
|
|
#include "absl/base/optimization.h"
|
|
#include "absl/base/port.h"
|
|
|
|
namespace absl {
|
|
ABSL_NAMESPACE_BEGIN
|
|
|
|
class once_flag;
|
|
|
|
namespace base_internal {
|
|
std::atomic<uint32_t>* ControlWord(absl::once_flag* flag);
|
|
} // namespace base_internal
|
|
|
|
// call_once()
|
|
//
|
|
// For all invocations using a given `once_flag`, invokes a given `fn` exactly
|
|
// once across all threads. The first call to `call_once()` with a particular
|
|
// `once_flag` argument (that does not throw an exception) will run the
|
|
// specified function with the provided `args`; other calls with the same
|
|
// `once_flag` argument will not run the function, but will wait
|
|
// for the provided function to finish running (if it is still running).
|
|
//
|
|
// This mechanism provides a safe, simple, and fast mechanism for one-time
|
|
// initialization in a multi-threaded process.
|
|
//
|
|
// Example:
|
|
//
|
|
// class MyInitClass {
|
|
// public:
|
|
// ...
|
|
// mutable absl::once_flag once_;
|
|
//
|
|
// MyInitClass* init() const {
|
|
// absl::call_once(once_, &MyInitClass::Init, this);
|
|
// return ptr_;
|
|
// }
|
|
//
|
|
template <typename Callable, typename... Args>
|
|
void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
|
|
|
|
// once_flag
|
|
//
|
|
// Objects of this type are used to distinguish calls to `call_once()` and
|
|
// ensure the provided function is only invoked once across all threads. This
|
|
// type is not copyable or movable. However, it has a `constexpr`
|
|
// constructor, and is safe to use as a namespace-scoped global variable.
|
|
class once_flag {
|
|
public:
|
|
constexpr once_flag() : control_(0) {}
|
|
once_flag(const once_flag&) = delete;
|
|
once_flag& operator=(const once_flag&) = delete;
|
|
|
|
private:
|
|
friend std::atomic<uint32_t>* base_internal::ControlWord(once_flag* flag);
|
|
std::atomic<uint32_t> control_;
|
|
};
|
|
|
|
//------------------------------------------------------------------------------
|
|
// End of public interfaces.
|
|
// Implementation details follow.
|
|
//------------------------------------------------------------------------------
|
|
|
|
namespace base_internal {
|
|
|
|
// Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
|
|
// initialize entities used by the scheduler implementation.
|
|
template <typename Callable, typename... Args>
|
|
void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args);
|
|
|
|
// Disables scheduling while on stack when scheduling mode is non-cooperative.
|
|
// No effect for cooperative scheduling modes.
|
|
class SchedulingHelper {
|
|
public:
|
|
explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
|
|
if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
|
|
guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
|
|
}
|
|
}
|
|
|
|
~SchedulingHelper() {
|
|
if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
|
|
base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
|
|
}
|
|
}
|
|
|
|
private:
|
|
base_internal::SchedulingMode mode_;
|
|
bool guard_result_;
|
|
};
|
|
|
|
// Bit patterns for call_once state machine values. Internal implementation
|
|
// detail, not for use by clients.
|
|
//
|
|
// The bit patterns are arbitrarily chosen from unlikely values, to aid in
|
|
// debugging. However, kOnceInit must be 0, so that a zero-initialized
|
|
// once_flag will be valid for immediate use.
|
|
enum {
|
|
kOnceInit = 0,
|
|
kOnceRunning = 0x65C2937B,
|
|
kOnceWaiter = 0x05A308D2,
|
|
// A very small constant is chosen for kOnceDone so that it fit in a single
|
|
// compare with immediate instruction for most common ISAs. This is verified
|
|
// for x86, POWER and ARM.
|
|
kOnceDone = 221, // Random Number
|
|
};
|
|
|
|
template <typename Callable, typename... Args>
|
|
ABSL_ATTRIBUTE_NOINLINE
|
|
void CallOnceImpl(std::atomic<uint32_t>* control,
|
|
base_internal::SchedulingMode scheduling_mode, Callable&& fn,
|
|
Args&&... args) {
|
|
#ifndef NDEBUG
|
|
{
|
|
uint32_t old_control = control->load(std::memory_order_relaxed);
|
|
if (old_control != kOnceInit &&
|
|
old_control != kOnceRunning &&
|
|
old_control != kOnceWaiter &&
|
|
old_control != kOnceDone) {
|
|
ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx",
|
|
static_cast<unsigned long>(old_control)); // NOLINT
|
|
}
|
|
}
|
|
#endif // NDEBUG
|
|
static const base_internal::SpinLockWaitTransition trans[] = {
|
|
{kOnceInit, kOnceRunning, true},
|
|
{kOnceRunning, kOnceWaiter, false},
|
|
{kOnceDone, kOnceDone, true}};
|
|
|
|
// Must do this before potentially modifying control word's state.
|
|
base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
|
|
// Short circuit the simplest case to avoid procedure call overhead.
|
|
// The base_internal::SpinLockWait() call returns either kOnceInit or
|
|
// kOnceDone. If it returns kOnceDone, it must have loaded the control word
|
|
// with std::memory_order_acquire and seen a value of kOnceDone.
|
|
uint32_t old_control = kOnceInit;
|
|
if (control->compare_exchange_strong(old_control, kOnceRunning,
|
|
std::memory_order_relaxed) ||
|
|
base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans,
|
|
scheduling_mode) == kOnceInit) {
|
|
base_internal::invoke(std::forward<Callable>(fn),
|
|
std::forward<Args>(args)...);
|
|
old_control =
|
|
control->exchange(base_internal::kOnceDone, std::memory_order_release);
|
|
if (old_control == base_internal::kOnceWaiter) {
|
|
base_internal::SpinLockWake(control, true);
|
|
}
|
|
} // else *control is already kOnceDone
|
|
}
|
|
|
|
inline std::atomic<uint32_t>* ControlWord(once_flag* flag) {
|
|
return &flag->control_;
|
|
}
|
|
|
|
template <typename Callable, typename... Args>
|
|
void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args) {
|
|
std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
|
|
uint32_t s = once->load(std::memory_order_acquire);
|
|
if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
|
|
base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
|
|
std::forward<Callable>(fn),
|
|
std::forward<Args>(args)...);
|
|
}
|
|
}
|
|
|
|
} // namespace base_internal
|
|
|
|
template <typename Callable, typename... Args>
|
|
void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
|
|
std::atomic<uint32_t>* once = base_internal::ControlWord(&flag);
|
|
uint32_t s = once->load(std::memory_order_acquire);
|
|
if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
|
|
base_internal::CallOnceImpl(
|
|
once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
|
|
std::forward<Callable>(fn), std::forward<Args>(args)...);
|
|
}
|
|
}
|
|
|
|
ABSL_NAMESPACE_END
|
|
} // namespace absl
|
|
|
|
#endif // ABSL_BASE_CALL_ONCE_H_
|