4#include "envelope_reader.hpp"
8#include <coroio/accept_errors.hpp>
9#include <coroio/arena.hpp>
10#include <coroio/sockutils.hpp>
22enum class ESystemMessages : TMessageId {
38 static constexpr TMessageId MessageId =
static_cast<TMessageId
>(ESystemMessages::PoisonPill);
45 THandle Handle =
nullptr;
46 TMessageId MessageId = 0;
53 std::unique_ptr<TUnboundedVectorQueue<TEnvelope>> Mailbox;
58 uint32_t IsReady : 1 = 0;
63class TAsk final :
public IActor {
69 void Receive(TMessageId messageId,
TBlob blob, TActorContext::TPtr ctx)
override;
72 std::shared_ptr<TAskState<T>> State;
113 , Logger_(std::move(logger))
133 TActorId
Register(IActor::TPtr actor);
150 return Poller->Sleep(until);
154 template<
typename Rep,
typename Period>
155 auto Sleep(std::chrono::duration<Rep,Period> duration) {
156 return Poller->Sleep(duration);
176 template<
typename T,
typename... Args>
178 if (recipient.
NodeId() == NodeId_) {
179 auto blob = SerializeNear<T>(GetPodAllocator(), std::forward<Args>(args)...);
180 Send(sender, recipient, T::MessageId, std::move(blob));
181 }
else if constexpr (is_local_message_v<T>) {
182 if (Logger_) Logger_(
"Local message cannot be sent to a remote actor: " + recipient.
ToString());
184 auto& maybeRemote = Nodes[recipient.
NodeId()];
185 if (!maybeRemote.Node) {
186 if (Logger_) Logger_(
"Cannot send message to actor on different node: " + recipient.
ToString());
189 SerializeFarInplace<T>(*maybeRemote.Node, sender, recipient, std::forward<Args>(args)...);
190 if (maybeRemote.Flags.IsReady == 0) {
191 maybeRemote.Flags.IsReady = 1;
192 ReadyNodes.Push(recipient.
NodeId());
211 template<
typename T,
typename... Args>
213 auto blob = SerializeNear<T>(GetPodAllocator(), std::forward<Args>(args)...);
214 return Schedule(when, sender, recipient, T::MessageId, std::move(blob));
223 void Cancel(TEvent event);
242 template<
typename T,
typename TQuestion>
251 bool await_ready()
const noexcept {
255 void await_suspend(THandle h) {
260 if (T::MessageId != State->MessageId) {
261 throw std::runtime_error(
"MessageId mismatch in Ask awaiter");
264 return DeserializeNear<T>(State->Blob);
268 std::shared_ptr<TAskState<T>> State;
271 auto state = std::make_shared<TAskState<T>>();
272 auto askActor = std::make_unique<TAsk<T>>(state);
273 auto actorId =
Register(std::move(askActor));
274 Send(actorId, recepient, TQuestion::MessageId, SerializeNear<TQuestion>(GetPodAllocator(), std::forward<TQuestion>(message)));
275 return TAskAwaiter{state};
299 void AddNode(
int id, std::unique_ptr<INode> node);
320 template<
typename TSocket,
typename TEnvelopeReader = TZeroCopyEnvelopeReader>
323 Handles.emplace_back(InboundServe<TSocket, TEnvelopeReader>(std::move(socket)));
324 for (
int i = 0; i < static_cast<int>(Nodes.size()); ++i) {
326 Handles.emplace_back(OutboundServe(i));
332 void GcIterationSync();
334 void DrainReadyNodes();
335 void AddPendingFuture(TLocalActorId
id,
TFuture<void>&& future);
338 Nodes[id].Node->StartConnect();
340 co_await SuspendExecution(
id);
341 auto& node = Nodes[id].Node;
345 if (Logger_) Logger_(
"Node with id: " + std::to_string(
id) +
" is not registered");
351 template<
typename TSocket,
typename TEnvelopeReader>
352 TVoidSuspendedTask InboundServe(TSocket socket) {
354 auto client =
co_await socket.Accept();
356 client->SetNoDelay();
357 InboundConnection<TSocket, TEnvelopeReader>(std::move(*client));
358 }
else if (!IsTransientAcceptError(client.error())) {
359 if (Logger_) Logger_(
"Accept failed: " + std::system_category().message(client.error()));
360 co_await Poller->Sleep(std::chrono::milliseconds(100));
365 template<
typename TSocket,
typename TEnvelopeReader>
366 TVoidTask InboundConnection(TSocket socket) {
367 static constexpr size_t ReadSize = 512 * 1024;
368 static constexpr size_t InflightBytes = 16 * 1024 * 1024;
369 static constexpr size_t MaxBytesBeforeYield = 2 * 1024 * 1024;
370 TEnvelopeReader envelopeReader(InflightBytes, 1024);
371 uint64_t message = 0;
375 bool mayHaveEnvelopes =
false;
379 if (!mayHaveEnvelopes) {
380 auto buffer = envelopeReader.Acquire(ReadSize);
381 auto size =
co_await socket.ReadSome(buffer.data(), buffer.size());
386 throw std::runtime_error(
"Socket closed");
388 envelopeReader.Commit(size);
393 size_t bytesProcessed = 0;
394 mayHaveEnvelopes =
false;
395 while (
auto envelope = envelopeReader.Pop()) {
396 if (envelope->Recipient.NodeId() != NodeId_) [[unlikely]] {
397 if (Logger_) Logger_(
"Received message for different node: " + envelope->Recipient.ToString());
401 bytesProcessed += envelope->Blob.Size +
sizeof(THeader);
402 Send(envelope->Sender, envelope->Recipient, envelope->MessageId, std::move(envelope->Blob));
403 if (bytesProcessed >= MaxBytesBeforeYield) {
404 mayHaveEnvelopes =
true;
409 co_await Poller->Yield();
411 }
catch (
const std::exception& e) {
412 if (Logger_) Logger_(std::string(
"Error in InboundConnection: ") + e.what());
416 void ShutdownActor(TLocalActorId actorId) {
417 if (actorId < Actors.size()) {
418 if (
auto* actor = Actors[actorId].Actor.get()) {
419 TActorId self{NodeId_, actorId, Actors[actorId].Cookie};
420 actor->BeforeUnregister(std::unique_ptr<TActorContext>(
421 new (
this) TActorContext(self, self,
this)
425 Actors[actorId] = {};
427 FreeActorIds.push(actorId);
432 void* AllocateActorContext() {
433 return ContextAllocator.Allocate();
436 void DeallocateActorContext(TActorContext* ptr) {
437 ContextAllocator.Deallocate(ptr);
440 TFuture<void> SuspendExecution(
int nodeId) {
441 Nodes[nodeId].Pending =
co_await Self();
442 co_await std::suspend_always();
443 Nodes[nodeId].Pending = {};
448 struct TPodAllocator {
449 void* Acquire(
size_t size) {
450 return ::operator
new(size);
453 void Release(
void* ptr) {
454 ::operator
delete(ptr);
458 TPodAllocator& GetPodAllocator() {
464 TUnboundedVectorQueue<TLocalActorId> ReadyActors;
465 std::vector<TActorInternalState> Actors;
468 std::vector<TFuture<void>> CleanupMessages;
469 std::stack<TLocalActorId, std::vector<TLocalActorId>> FreeActorIds;
471 TActorId Emplace(TLocalActorId
id, TCookie cookie, IActor::TPtr actor);
473 TArenaAllocator<TActorContext> ContextAllocator;
475 TPodAllocator PodAllocator;
478 TCookie NextCookie_ = 1;
480 THandle YieldCoroutine_{};
481 THandle ScheduleCoroutine_{};
482 bool IsYielding_ =
true;
483 std::function<void(
const std::string&)> Logger_;
486 std::unique_ptr<INode> Node;
489 uint32_t IsReady : 1 = 0;
492 std::vector<TNodeState> Nodes;
497 unsigned TimerId = 0;
501 TMessageId MessageId;
504 bool operator<(
const TDelayed& other)
const {
505 return std::tie(When, TimerId, valid) > std::tie(other.When, other.TimerId, other.valid);
509 std::priority_queue<TDelayed> DelayedMessages;
511 std::vector<THandle> Handles;
513 friend class TActorContext;
518 State->MessageId = messageId;
519 State->Blob = std::move(blob);
520 State->Handle.resume();
521 ctx->Send<
TPoison>(ctx->Self());
525 co_return co_await ActorSystem->Sleep(until);
528template<
typename Rep,
typename Period>
530 co_return co_await ActorSystem->Sleep(duration);
533template<
typename T,
typename TQuestion>
535 co_return co_await ActorSystem->Ask<T>(recipient, std::forward<TQuestion>(question));
538template<
typename T,
typename... Args>
540 ActorSystem->Send<T>(
Self(), to, std::forward<Args>(args)...);
542template<
typename T,
typename... Args>
544 ActorSystem->Send<T>(
Sender(), to, std::forward<Args>(args)...);
547template<
typename T,
typename... Args>
549 ActorSystem->Send<T>(from, to, std::forward<Args>(args)...);
552template<
typename T,
typename... Args>
554 auto blob = SerializeNear<T>(ActorSystem->GetPodAllocator(), std::forward<Args>(args)...);
555 return Schedule(when, sender, recipient, T::MessageId, std::move(blob));
558inline void* TActorContext::operator
new(
size_t size,
TActorSystem* actorSystem) {
559 return actorSystem->AllocateActorContext();
562inline void TActorContext::operator
delete(
void* ptr) {
564 auto* self =
static_cast<TActorContext*
>(ptr);
565 auto* actorSystem = self->ActorSystem;
566 actorSystem->DeallocateActorContext(self);
Actor system implementation with message passing and behavior support.
TActorId Self() const
Get this actor's ID.
Definition actor.hpp:158
void SendAs(TActorId from, TActorId to, TMessageId messageId, TBlob blob)
Send a message with an explicitly specified sender.
Definition actorsystem.cpp:41
void Forward(TActorId to, TMessageId messageId, TBlob blob)
Forward a message to another actor (preserves original sender).
Definition actorsystem.cpp:36
TFuture< void > Sleep(TTime until)
Suspend the current handler until a specific time.
Definition actorsystem.hpp:524
TFuture< T > Ask(TActorId recipient, TQuestion &&question)
Send a request and suspend until a reply of type T arrives.
Definition actorsystem.hpp:534
void Send(TActorId to, TMessageId messageId, TBlob blob)
Send a message to another actor.
Definition actorsystem.cpp:31
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Schedule a message to be delivered at a specific time.
Definition actorsystem.cpp:46
TActorId Sender() const
Get the sender of the current message.
Definition actor.hpp:153
Globally unique identifier for actors across a distributed system.
Definition actorid.hpp:34
TNodeId NodeId() const
Get the node ID component.
Definition actorid.hpp:51
std::string ToString() const
Convert actor ID to a human-readable string.
Definition actorid.hpp:69
Single-threaded actor runtime.
Definition actorsystem.hpp:101
auto Ask(TActorId recepient, TQuestion &&message)
Sends a request and returns an awaitable for the reply of type T.
Definition actorsystem.hpp:243
size_t ActorsSize() const
Returns the number of currently registered (alive) actors.
Definition actorsystem.cpp:279
TActorSystem(TPollerBase *poller, TNodeId nodeId=1, std::function< void(const std::string &)> logger={})
Constructs the actor system.
Definition actorsystem.hpp:110
void Serve(TSocket socket)
Starts the actor system with inbound and outbound network serving.
Definition actorsystem.hpp:321
auto Sleep(std::chrono::duration< Rep, Period > duration)
Suspends the current coroutine for duration. Delegates to the poller.
Definition actorsystem.hpp:155
static constexpr TCookie WellKnownCookie
Fixed cookie of well-known actors, so any node can build their TActorId.
Definition actorsystem.hpp:138
void Send(TActorId sender, TActorId recipient, Args &&... args)
Sends a typed message from sender to recipient (non-blocking).
Definition actorsystem.hpp:177
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Low-level schedule with a pre-serialized blob. Prefer the typed Schedule<T> overload.
Definition actorsystem.cpp:289
void Serve()
Starts the actor system event loop for local actors only.
Definition actorsystem.cpp:215
TActorId RegisterWellKnown(TLocalActorId id, IActor::TPtr actor)
Registers an actor under a fixed local id in [1, FirstDynamicActorId).
Definition actorsystem.cpp:72
void Cancel(TEvent event)
Cancels a previously scheduled message.
Definition actorsystem.cpp:304
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, Args &&... args)
Schedules a typed message to be delivered at when.
Definition actorsystem.hpp:212
void YieldNotify()
Wakes up the internal scheduler to process newly ready nodes.
Definition actorsystem.cpp:283
void Send(TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Low-level send with a pre-serialized blob. Prefer the typed Send<T> overload.
Definition actorsystem.cpp:100
auto Sleep(TTime until)
Suspends the current coroutine until until. Delegates to the poller.
Definition actorsystem.hpp:149
TActorId Register(IActor::TPtr actor)
Registers an actor and returns its system-wide ID.
Definition actorsystem.cpp:61
std::unique_ptr< INotifier > CreateNotifier()
Creates a notifier on the poller thread. The poller must outlive it.
Definition actorsystem.cpp:19
static constexpr TLocalActorId FirstDynamicActorId
Local ids in [1, FirstDynamicActorId) are reserved for well-known actors.
Definition actorsystem.hpp:136
void AddNode(int id, std::unique_ptr< INode > node)
Registers a remote node for distributed message routing.
Definition actorsystem.cpp:267
Definition actorsystem.hpp:43
void Receive(TMessageId messageId, TBlob blob, TActorContext::TPtr ctx) override
Process an incoming message.
Definition actorsystem.hpp:517
Backend-independent base for I/O pollers.
Definition poller.hpp:49
High-level asynchronous socket for network communication.
Definition socket.hpp:404
Memory-efficient unbounded queue implementation for actor message passing.
Definition actorsystem.hpp:57
Definition actorsystem.hpp:51
Definition actorsystem.hpp:488
Opaque message payload with Near/Far duality.
Definition messages.hpp:101
System message that terminates the receiving actor.
Definition actorsystem.hpp:37
Unbounded queue with automatic capacity growth.
Definition queue.hpp:63
Owned coroutine handle that carries a result of type T.
Definition corochain.hpp:185
Fire-and-forget coroutine handle.
Definition promises.hpp:26