COROIO: coroio/actors/actorsystem.hpp Source File
COROIO
Loading...
Searching...
No Matches
actorsystem.hpp
1#pragma once
2
3#include "actor.hpp"
4#include "envelope_reader.hpp"
5#include "node.hpp"
6#include "queue.hpp"
7
8#include <coroio/accept_errors.hpp>
9#include <coroio/arena.hpp>
10#include <coroio/sockutils.hpp>
11
12#include <functional>
13#include <stack>
14
15#ifdef Yield
16#undef Yield
17#endif
18
19namespace NNet {
20namespace NActors {
21
22enum class ESystemMessages : TMessageId {
23 PoisonPill = 1
24};
25
37struct TPoison {
38 static constexpr TMessageId MessageId = static_cast<TMessageId>(ESystemMessages::PoisonPill);
39};
40
41template<typename T>
43{
44public:
45 THandle Handle = nullptr;
46 TMessageId MessageId = 0;
47 TBlob Blob;
48};
49
51{
52 TCookie Cookie = 0;
53 std::unique_ptr<TUnboundedVectorQueue<TEnvelope>> Mailbox;
54 TFuture<void> Pending;
55 IActor::TPtr Actor;
56
57 struct TFlags {
58 uint32_t IsReady : 1 = 0; // Is the actor exists in ReadyActors queue
59 } Flags = {};
60};
61
62template<typename T>
63class TAsk final : public IActor {
64public:
65 TAsk(const std::shared_ptr<TAskState<T>>& state)
66 : State(state)
67 { }
68
69 void Receive(TMessageId messageId, TBlob blob, TActorContext::TPtr ctx) override;
70
71private:
72 std::shared_ptr<TAskState<T>> State;
73};
74
101{
102public:
110 TActorSystem(TPollerBase* poller, TNodeId nodeId = 1, std::function<void(const std::string&)> logger = {})
111 : Poller(poller)
112 , NodeId_(nodeId)
113 , Logger_(std::move(logger))
114 { }
115
116 ~TActorSystem();
117
118#ifndef _WIN32
120 std::unique_ptr<INotifier> CreateNotifier();
121#endif
122
133 TActorId Register(IActor::TPtr actor);
134
136 static constexpr TLocalActorId FirstDynamicActorId = 128;
138 static constexpr TCookie WellKnownCookie = 0;
139
146 TActorId RegisterWellKnown(TLocalActorId id, IActor::TPtr actor);
147
149 auto Sleep(TTime until) {
150 return Poller->Sleep(until);
151 }
152
154 template<typename Rep, typename Period>
155 auto Sleep(std::chrono::duration<Rep,Period> duration) {
156 return Poller->Sleep(duration);
157 }
158
160 void Send(TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob);
161
176 template<typename T, typename... Args>
177 void Send(TActorId sender, TActorId recipient, Args&&... args) {
178 if (recipient.NodeId() == NodeId_) {
179 auto blob = SerializeNear<T>(GetPodAllocator(), std::forward<Args>(args)...);
180 Send(sender, recipient, T::MessageId, std::move(blob));
181 } else if constexpr (is_local_message_v<T>) {
182 if (Logger_) Logger_("Local message cannot be sent to a remote actor: " + recipient.ToString());
183 } else {
184 auto& maybeRemote = Nodes[recipient.NodeId()];
185 if (!maybeRemote.Node) {
186 if (Logger_) Logger_("Cannot send message to actor on different node: " + recipient.ToString());
187 return;
188 }
189 SerializeFarInplace<T>(*maybeRemote.Node, sender, recipient, std::forward<Args>(args)...);
190 if (maybeRemote.Flags.IsReady == 0) {
191 maybeRemote.Flags.IsReady = 1;
192 ReadyNodes.Push(recipient.NodeId());
193 }
194
195 YieldNotify();
196 }
197 }
198
200 TEvent Schedule(TTime when, TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob);
201
211 template<typename T, typename... Args>
212 TEvent Schedule(TTime when, TActorId sender, TActorId recipient, Args&&... args) {
213 auto blob = SerializeNear<T>(GetPodAllocator(), std::forward<Args>(args)...);
214 return Schedule(when, sender, recipient, T::MessageId, std::move(blob));
215 }
216
223 void Cancel(TEvent event);
224
242 template<typename T, typename TQuestion>
243 auto Ask(TActorId recepient, TQuestion&& message) {
244 class TAskAwaiter
245 {
246 public:
247 TAskAwaiter(const std::shared_ptr<TAskState<T>>& state)
248 : State(state)
249 { }
250
251 bool await_ready() const noexcept {
252 return false;
253 }
254
255 void await_suspend(THandle h) {
256 State->Handle = h;
257 }
258
259 T await_resume() {
260 if (T::MessageId != State->MessageId) {
261 throw std::runtime_error("MessageId mismatch in Ask awaiter");
262 }
263
264 return DeserializeNear<T>(State->Blob);
265 }
266
267 private:
268 std::shared_ptr<TAskState<T>> State;
269 };
270
271 auto state = std::make_shared<TAskState<T>>();
272 auto askActor = std::make_unique<TAsk<T>>(state);
273 auto actorId = Register(std::move(askActor));
274 Send(actorId, recepient, TQuestion::MessageId, SerializeNear<TQuestion>(GetPodAllocator(), std::forward<TQuestion>(message)));
275 return TAskAwaiter{state};
276 }
277
284 void YieldNotify();
285
287 size_t ActorsSize() const;
288
299 void AddNode(int id, std::unique_ptr<INode> node);
300
308 void Serve();
309
320 template<typename TSocket, typename TEnvelopeReader = TZeroCopyEnvelopeReader>
321 void Serve(TSocket socket) {
322 Serve();
323 Handles.emplace_back(InboundServe<TSocket, TEnvelopeReader>(std::move(socket)));
324 for (int i = 0; i < static_cast<int>(Nodes.size()); ++i) {
325 if (Nodes[i].Node) {
326 Handles.emplace_back(OutboundServe(i));
327 }
328 }
329 }
330
331private:
332 void GcIterationSync();
333 void ExecuteSync();
334 void DrainReadyNodes();
335 void AddPendingFuture(TLocalActorId id, TFuture<void>&& future);
336
337 TVoidTask OutboundServe(int id) {
338 Nodes[id].Node->StartConnect();
339 while (true) {
340 co_await SuspendExecution(id);
341 auto& node = Nodes[id].Node;
342 if (node) {
343 node->Drain();
344 } else {
345 if (Logger_) Logger_("Node with id: " + std::to_string(id) + " is not registered");
346 }
347 }
348 }
349
350 // Suspends at the end instead of freeing itself: the destructor destroys Handles.
351 template<typename TSocket, typename TEnvelopeReader>
352 TVoidSuspendedTask InboundServe(TSocket socket) {
353 while (true) {
354 auto client = co_await socket.Accept();
355 if (client) {
356 client->SetNoDelay();
357 InboundConnection<TSocket, TEnvelopeReader>(std::move(*client));
358 } else if (!IsTransientAcceptError(client.error())) {
359 if (Logger_) Logger_("Accept failed: " + std::system_category().message(client.error()));
360 co_await Poller->Sleep(std::chrono::milliseconds(100));
361 }
362 }
363 }
364
365 template<typename TSocket, typename TEnvelopeReader>
366 TVoidTask InboundConnection(TSocket socket) {
367 static constexpr size_t ReadSize = 512 * 1024;
368 static constexpr size_t InflightBytes = 16 * 1024 * 1024;
369 static constexpr size_t MaxBytesBeforeYield = 2 * 1024 * 1024;
370 TEnvelopeReader envelopeReader(InflightBytes, /*lowWatermark = */ 1024);
371 uint64_t message = 0;
372
373 // Set when the last batch stopped at MaxBytesBeforeYield: complete envelopes
374 // may still be buffered, and a read would wait for bytes that may never come.
375 bool mayHaveEnvelopes = false;
376
377 try {
378 while (true) {
379 if (!mayHaveEnvelopes) {
380 auto buffer = envelopeReader.Acquire(ReadSize);
381 auto size = co_await socket.ReadSome(buffer.data(), buffer.size());
382 if (size < 0) {
383 continue;
384 }
385 if (size == 0) {
386 throw std::runtime_error("Socket closed");
387 }
388 envelopeReader.Commit(size);
389 }
390
391 //envelopeReader.PrintDebugInfo();
392
393 size_t bytesProcessed = 0;
394 mayHaveEnvelopes = false;
395 while (auto envelope = envelopeReader.Pop()) {
396 if (envelope->Recipient.NodeId() != NodeId_) [[unlikely]] {
397 if (Logger_) Logger_("Received message for different node: " + envelope->Recipient.ToString());
398 continue;
399 }
400
401 bytesProcessed += envelope->Blob.Size + sizeof(THeader);
402 Send(envelope->Sender, envelope->Recipient, envelope->MessageId, std::move(envelope->Blob));
403 if (bytesProcessed >= MaxBytesBeforeYield) {
404 mayHaveEnvelopes = true;
405 break;
406 }
407 }
408
409 co_await Poller->Yield();
410 }
411 } catch (const std::exception& e) {
412 if (Logger_) Logger_(std::string("Error in InboundConnection: ") + e.what());
413 }
414 }
415
416 void ShutdownActor(TLocalActorId actorId) {
417 if (actorId < Actors.size()) {
418 if (auto* actor = Actors[actorId].Actor.get()) {
419 TActorId self{NodeId_, actorId, Actors[actorId].Cookie};
420 actor->BeforeUnregister(std::unique_ptr<TActorContext>(
421 new (this) TActorContext(self, self, this)
422 ));
423 }
424 AliveActors--;
425 Actors[actorId] = {};
426 if (actorId >= FirstDynamicActorId) {
427 FreeActorIds.push(actorId);
428 }
429 }
430 }
431
432 void* AllocateActorContext() {
433 return ContextAllocator.Allocate();
434 }
435
436 void DeallocateActorContext(TActorContext* ptr) {
437 ContextAllocator.Deallocate(ptr);
438 }
439
440 TFuture<void> SuspendExecution(int nodeId) {
441 Nodes[nodeId].Pending = co_await Self();
442 co_await std::suspend_always();
443 Nodes[nodeId].Pending = {};
444 co_return;
445 }
446
447 // TODO: rewrite
448 struct TPodAllocator {
449 void* Acquire(size_t size) {
450 return ::operator new(size);
451 }
452
453 void Release(void* ptr) {
454 ::operator delete(ptr);
455 }
456 };
457
458 TPodAllocator& GetPodAllocator() {
459 return PodAllocator;
460 }
461
462 TPollerBase* Poller;
463
464 TUnboundedVectorQueue<TLocalActorId> ReadyActors;
465 std::vector<TActorInternalState> Actors;
466 int AliveActors = 0;
467
468 std::vector<TFuture<void>> CleanupMessages;
469 std::stack<TLocalActorId, std::vector<TLocalActorId>> FreeActorIds;
470
471 TActorId Emplace(TLocalActorId id, TCookie cookie, IActor::TPtr actor);
472
473 TArenaAllocator<TActorContext> ContextAllocator;
474
475 TPodAllocator PodAllocator;
476
477 TLocalActorId NextActorId_ = FirstDynamicActorId;
478 TCookie NextCookie_ = 1;
479 TNodeId NodeId_ = 1;
480 THandle YieldCoroutine_{};
481 THandle ScheduleCoroutine_{};
482 bool IsYielding_ = true;
483 std::function<void(const std::string&)> Logger_;
484
485 struct TNodeState {
486 std::unique_ptr<INode> Node;
487 THandle Pending;
488 struct TFlags {
489 uint32_t IsReady : 1 = 0;
490 } Flags = {};
491 };
492 std::vector<TNodeState> Nodes;
494
495 struct TDelayed {
496 TTime When;
497 unsigned TimerId = 0;
498 bool valid = true;
499 TActorId Sender;
500 TActorId Recipient;
501 TMessageId MessageId;
502 TBlob Blob;
503
504 bool operator<(const TDelayed& other) const {
505 return std::tie(When, TimerId, valid) > std::tie(other.When, other.TimerId, other.valid);
506 }
507 };
508
509 std::priority_queue<TDelayed> DelayedMessages;
510
511 std::vector<THandle> Handles;
512
513 friend class TActorContext;
514};
515
516template<typename T>
517void TAsk<T>::Receive(TMessageId messageId, TBlob blob, TActorContext::TPtr ctx) {
518 State->MessageId = messageId;
519 State->Blob = std::move(blob);
520 State->Handle.resume();
521 ctx->Send<TPoison>(ctx->Self());
522}
523
525 co_return co_await ActorSystem->Sleep(until);
526}
527
528template<typename Rep, typename Period>
529inline TFuture<void> TActorContext::Sleep(std::chrono::duration<Rep,Period> duration) {
530 co_return co_await ActorSystem->Sleep(duration);
531}
532
533template<typename T, typename TQuestion>
534inline TFuture<T> TActorContext::Ask(TActorId recipient, TQuestion&& question) {
535 co_return co_await ActorSystem->Ask<T>(recipient, std::forward<TQuestion>(question));
536}
537
538template<typename T, typename... Args>
539inline void TActorContext::Send(TActorId to, Args&&... args) {
540 ActorSystem->Send<T>(Self(), to, std::forward<Args>(args)...);
541}
542template<typename T, typename... Args>
543inline void TActorContext::Forward(TActorId to, Args&&... args) {
544 ActorSystem->Send<T>(Sender(), to, std::forward<Args>(args)...);
545}
546
547template<typename T, typename... Args>
548inline void TActorContext::SendAs(TActorId from, TActorId to, Args&&... args) {
549 ActorSystem->Send<T>(from, to, std::forward<Args>(args)...);
550}
551
552template<typename T, typename... Args>
553inline TEvent TActorContext::Schedule(TTime when, TActorId sender, TActorId recipient, Args&&... args) {
554 auto blob = SerializeNear<T>(ActorSystem->GetPodAllocator(), std::forward<Args>(args)...);
555 return Schedule(when, sender, recipient, T::MessageId, std::move(blob));
556}
557
558inline void* TActorContext::operator new(size_t size, TActorSystem* actorSystem) {
559 return actorSystem->AllocateActorContext();
560}
561
562inline void TActorContext::operator delete(void* ptr) {
563 if (ptr) {
564 auto* self = static_cast<TActorContext*>(ptr);
565 auto* actorSystem = self->ActorSystem;
566 actorSystem->DeallocateActorContext(self);
567 }
568}
569
570} // namespace NActors
571} // namespace NNet
572
Actor system implementation with message passing and behavior support.
TActorId Self() const
Get this actor's ID.
Definition actor.hpp:158
void SendAs(TActorId from, TActorId to, TMessageId messageId, TBlob blob)
Send a message with an explicitly specified sender.
Definition actorsystem.cpp:41
void Forward(TActorId to, TMessageId messageId, TBlob blob)
Forward a message to another actor (preserves original sender).
Definition actorsystem.cpp:36
TFuture< void > Sleep(TTime until)
Suspend the current handler until a specific time.
Definition actorsystem.hpp:524
TFuture< T > Ask(TActorId recipient, TQuestion &&question)
Send a request and suspend until a reply of type T arrives.
Definition actorsystem.hpp:534
void Send(TActorId to, TMessageId messageId, TBlob blob)
Send a message to another actor.
Definition actorsystem.cpp:31
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Schedule a message to be delivered at a specific time.
Definition actorsystem.cpp:46
TActorId Sender() const
Get the sender of the current message.
Definition actor.hpp:153
Globally unique identifier for actors across a distributed system.
Definition actorid.hpp:34
TNodeId NodeId() const
Get the node ID component.
Definition actorid.hpp:51
std::string ToString() const
Convert actor ID to a human-readable string.
Definition actorid.hpp:69
Single-threaded actor runtime.
Definition actorsystem.hpp:101
auto Ask(TActorId recepient, TQuestion &&message)
Sends a request and returns an awaitable for the reply of type T.
Definition actorsystem.hpp:243
size_t ActorsSize() const
Returns the number of currently registered (alive) actors.
Definition actorsystem.cpp:279
TActorSystem(TPollerBase *poller, TNodeId nodeId=1, std::function< void(const std::string &)> logger={})
Constructs the actor system.
Definition actorsystem.hpp:110
void Serve(TSocket socket)
Starts the actor system with inbound and outbound network serving.
Definition actorsystem.hpp:321
auto Sleep(std::chrono::duration< Rep, Period > duration)
Suspends the current coroutine for duration. Delegates to the poller.
Definition actorsystem.hpp:155
static constexpr TCookie WellKnownCookie
Fixed cookie of well-known actors, so any node can build their TActorId.
Definition actorsystem.hpp:138
void Send(TActorId sender, TActorId recipient, Args &&... args)
Sends a typed message from sender to recipient (non-blocking).
Definition actorsystem.hpp:177
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Low-level schedule with a pre-serialized blob. Prefer the typed Schedule<T> overload.
Definition actorsystem.cpp:289
void Serve()
Starts the actor system event loop for local actors only.
Definition actorsystem.cpp:215
TActorId RegisterWellKnown(TLocalActorId id, IActor::TPtr actor)
Registers an actor under a fixed local id in [1, FirstDynamicActorId).
Definition actorsystem.cpp:72
void Cancel(TEvent event)
Cancels a previously scheduled message.
Definition actorsystem.cpp:304
TEvent Schedule(TTime when, TActorId sender, TActorId recipient, Args &&... args)
Schedules a typed message to be delivered at when.
Definition actorsystem.hpp:212
void YieldNotify()
Wakes up the internal scheduler to process newly ready nodes.
Definition actorsystem.cpp:283
void Send(TActorId sender, TActorId recipient, TMessageId messageId, TBlob blob)
Low-level send with a pre-serialized blob. Prefer the typed Send<T> overload.
Definition actorsystem.cpp:100
auto Sleep(TTime until)
Suspends the current coroutine until until. Delegates to the poller.
Definition actorsystem.hpp:149
TActorId Register(IActor::TPtr actor)
Registers an actor and returns its system-wide ID.
Definition actorsystem.cpp:61
std::unique_ptr< INotifier > CreateNotifier()
Creates a notifier on the poller thread. The poller must outlive it.
Definition actorsystem.cpp:19
static constexpr TLocalActorId FirstDynamicActorId
Local ids in [1, FirstDynamicActorId) are reserved for well-known actors.
Definition actorsystem.hpp:136
void AddNode(int id, std::unique_ptr< INode > node)
Registers a remote node for distributed message routing.
Definition actorsystem.cpp:267
Definition actorsystem.hpp:43
void Receive(TMessageId messageId, TBlob blob, TActorContext::TPtr ctx) override
Process an incoming message.
Definition actorsystem.hpp:517
Backend-independent base for I/O pollers.
Definition poller.hpp:49
High-level asynchronous socket for network communication.
Definition socket.hpp:404
Memory-efficient unbounded queue implementation for actor message passing.
Definition actorsystem.hpp:57
Definition actorsystem.hpp:51
Opaque message payload with Near/Far duality.
Definition messages.hpp:101
System message that terminates the receiving actor.
Definition actorsystem.hpp:37
Unbounded queue with automatic capacity growth.
Definition queue.hpp:63
Owned coroutine handle that carries a result of type T.
Definition corochain.hpp:185
Fire-and-forget coroutine handle.
Definition promises.hpp:26