Pipecat C++ Client SDK 1.3.0
Connect native apps to Pipecat bots
Loading...
Searching...
No Matches
messages.h
Go to the documentation of this file.
1//
2// Copyright (c) 2024-2026, Daily
3//
4// SPDX-License-Identifier: BSD-2-Clause
5//
6
9
10#ifndef PIPECAT_RTVI_MESSAGES_H
11#define PIPECAT_RTVI_MESSAGES_H
12
13#include <nlohmann/json.hpp>
14
15#include <cstdint>
16#include <optional>
17#include <string>
18#include <vector>
19
29namespace pipecat::rtvi {
30
32inline constexpr const char* PROTOCOL_VERSION = "2.3.0";
33
35inline constexpr const char* MESSAGE_LABEL = "rtvi-ai";
36
76
78const char* to_string(MessageType type);
79
81std::optional<MessageType> parse_message_type(const std::string& type);
82
86 std::string library;
88 std::optional<std::string> library_version;
90 std::optional<std::string> platform;
92 std::optional<std::string> platform_version;
94 nlohmann::json platform_details;
95};
96
99
103 std::string type;
105 nlohmann::json data;
106};
107
111 std::optional<bool> run_immediately;
113 std::optional<bool> audio_response;
114};
115
119 std::string function_name;
121 std::string tool_call_id;
123 nlohmann::json arguments;
125 nlohmann::json result;
126};
127
134 std::optional<bool> audio_in;
136 std::optional<bool> audio_out;
138 std::optional<bool> video_in;
140 std::optional<bool> video_out;
142 std::optional<bool> screen_in;
144 std::optional<bool> metrics;
145};
146
150 std::string version;
152 nlohmann::json about;
156};
157
159struct ErrorData {
161 std::string error;
163 bool fatal = false;
164};
165
169 std::string label;
171 double confidence = 0;
172};
173
178 std::string name;
180 std::vector<UserPerceptionLabel> labels;
182 std::optional<double> value;
183};
184
188 std::string label;
191 int number = 0;
192};
193
197 std::string text;
202 std::string input_type;
205 bool final = false;
207 std::string timestamp;
210 std::optional<std::string> user_id;
212 std::optional<std::string> msg_id;
216 std::vector<UserPerceptionSignal> perception;
220 std::optional<TranscriptionSpeaker> speaker;
221};
222
226 std::string text;
228 bool final = false;
230 std::string timestamp;
232 std::string user_id;
235 std::optional<TranscriptionSpeaker> speaker;
236};
237
239struct TextData {
241 std::string text;
242};
243
246
249
252
256 std::string accumulated_text;
258 std::string remaining_text;
259};
260
267 std::string text;
276 std::string output_type;
278 std::optional<int64_t> segment_id;
280 std::optional<bool> will_be_spoken;
286 std::optional<std::string> spoken_status;
288 std::optional<SpokenProgressData> spoken_progress;
293 std::optional<std::vector<std::string>> delivery;
294};
295
299 std::string processor;
301 std::optional<std::string> model;
303 double value = 0;
304};
305
309 std::vector<MetricData> ttfb;
311 std::vector<MetricData> processing;
313 std::vector<MetricData> characters;
315 nlohmann::json raw;
316};
317
321 std::optional<std::string> function_name;
322};
323
327 std::optional<std::string> function_name;
329 std::string tool_call_id;
332 nlohmann::json arguments;
333};
334
338 std::optional<std::string> function_name;
340 std::string tool_call_id;
342 bool cancelled = false;
344 nlohmann::json result;
345};
346
350 std::string text;
352 std::vector<double> confidence;
353};
354
358 std::optional<std::string> site_uri;
360 std::optional<std::string> site_title;
362 std::vector<LLMSearchResult> results;
363};
364
368 std::optional<std::string> search_result;
370 std::optional<std::string> rendered_content;
372 std::vector<LLMSearchOrigin> origins;
373};
374
379struct Message {
381 std::string id;
383 std::string label = MESSAGE_LABEL;
385 std::string type;
387 nlohmann::json data;
388
390 Message() = default;
391
393 explicit Message(MessageType type, nlohmann::json data = nullptr);
394
397
400
402 static Message
403 client_message(const std::string& type, const nlohmann::json& data);
404
406 static Message
407 send_text(const std::string& content, const SendTextOptions& options = {});
408
410 static Message dtmf(const std::string& buttons);
411
415 );
416};
417
418//
419// JSON conversions, used by nlohmann::json (e.g. `data.get<BotOutputData>()`).
420//
421
423
424void to_json(nlohmann::json& j, const Message& message);
425void from_json(const nlohmann::json& j, Message& message);
426
427void to_json(nlohmann::json& j, const AboutClientData& data);
428void to_json(nlohmann::json& j, const SendTextOptions& options);
429void to_json(nlohmann::json& j, const LLMFunctionCallResultData& data);
430
431void to_json(nlohmann::json& j, const ClientMessageData& data);
432void from_json(const nlohmann::json& j, ClientMessageData& data);
433
434void from_json(const nlohmann::json& j, BotCapabilities& data);
435void from_json(const nlohmann::json& j, BotReadyData& data);
436void from_json(const nlohmann::json& j, ErrorData& data);
437void from_json(const nlohmann::json& j, UserPerceptionLabel& data);
438void from_json(const nlohmann::json& j, UserPerceptionSignal& data);
439void from_json(const nlohmann::json& j, TranscriptionSpeaker& data);
440void from_json(const nlohmann::json& j, UserInputData& data);
441void from_json(const nlohmann::json& j, STTRawTextData& data);
442void from_json(const nlohmann::json& j, TextData& data);
443void from_json(const nlohmann::json& j, SpokenProgressData& data);
444void from_json(const nlohmann::json& j, BotOutputData& data);
445void from_json(const nlohmann::json& j, MetricData& data);
446void from_json(const nlohmann::json& j, MetricsData& data);
447void from_json(const nlohmann::json& j, LLMFunctionCallStartedData& data);
448void from_json(const nlohmann::json& j, LLMFunctionCallInProgressData& data);
449void from_json(const nlohmann::json& j, LLMFunctionCallStoppedData& data);
450void from_json(const nlohmann::json& j, LLMSearchResult& data);
451void from_json(const nlohmann::json& j, LLMSearchOrigin& data);
452void from_json(const nlohmann::json& j, BotLLMSearchResponseData& data);
453
455
456} // namespace pipecat::rtvi
457
458#endif
Definition messages.h:29
constexpr const char * PROTOCOL_VERSION
The RTVI protocol version this library speaks.
Definition messages.h:32
std::optional< MessageType > parse_message_type(const std::string &type)
The message type with an RTVI name, or std::nullopt if it's unknown.
constexpr const char * MESSAGE_LABEL
The label of every RTVI message.
Definition messages.h:35
const char * to_string(MessageType type)
The RTVI name of a message type, e.g. "bot-ready".
AboutClientData default_about_client()
Information about this library and the platform it runs on.
MessageType
Types of RTVI messages.
Definition messages.h:38
@ STTRawText
A transcription, as the bot made it.
@ LLMFunctionCallInProgress
A function call is running.
@ ServerMessage
An app-defined message from the bot.
@ BotInterrupted
The bot was interrupted.
@ BotTTSStarted
The bot's text-to-speech started.
@ UserLLMText
User text sent to the bot's LLM.
@ LLMRawText
Text from the bot's LLM, as it streams.
@ ClientMessage
An app-defined message to the bot.
@ TTSRawText
Text the bot speaks, as it speaks it.
@ UserStartedSpeaking
The user started speaking.
@ BotOutput
What the bot says or outputs.
@ LLMFunctionCallStarted
The bot's LLM started a function call.
@ DisconnectBot
Asks the bot to leave.
@ Error
An error in the bot.
@ BotStoppedSpeaking
The bot stopped speaking.
@ UserMuteStopped
The bot stopped ignoring user audio.
@ ErrorResponse
The bot couldn't handle a message.
@ LLMFunctionCallResult
The result of a function call.
@ UserMuteStarted
The bot started ignoring user audio.
@ UserStoppedSpeaking
The user stopped speaking.
@ UserInput
What the user said or typed.
@ LLMFunctionCallStopped
A function call finished.
@ ServerResponse
The bot's answer to a request.
@ BotStartedSpeaking
The bot started speaking.
@ SendText
Text for the bot's LLM.
@ BotTTSStopped
The bot's text-to-speech stopped.
@ ClientReady
The client is ready.
@ BotLLMStopped
The bot's LLM finished responding.
@ BotLLMStarted
The bot's LLM started responding.
@ BotLLMSearchResponse
Search results used by the bot's LLM.
@ Metrics
Performance metrics.
@ BotReady
The bot is ready.
Information about the client, sent to the bot when connecting.
Definition messages.h:84
std::optional< std::string > platform
Operating system, e.g. "Linux".
Definition messages.h:90
std::string library
Name of the library, e.g. "pipecat-client-cxx".
Definition messages.h:86
nlohmann::json platform_details
Any other details, as a JSON object. Null to leave them out.
Definition messages.h:94
std::optional< std::string > platform_version
Version of the OS.
Definition messages.h:92
std::optional< std::string > library_version
Version of the library.
Definition messages.h:88
Definition messages.h:132
std::optional< bool > metrics
Whether the bot sends metrics.
Definition messages.h:144
std::optional< bool > audio_in
Whether the bot receives the user's audio.
Definition messages.h:134
std::optional< bool > audio_out
Whether the bot sends audio to the user.
Definition messages.h:136
std::optional< bool > screen_in
Whether the bot receives the user's screen share.
Definition messages.h:142
std::optional< bool > video_in
Whether the bot receives the user's video.
Definition messages.h:138
std::optional< bool > video_out
Whether the bot sends video to the user.
Definition messages.h:140
Search results used by the bot's LLM.
Definition messages.h:366
std::vector< LLMSearchOrigin > origins
Where the results come from.
Definition messages.h:372
std::optional< std::string > search_result
Summary of the results.
Definition messages.h:368
std::optional< std::string > rendered_content
Results ready to show.
Definition messages.h:370
Definition messages.h:265
std::string text
The text.
Definition messages.h:267
std::optional< bool > will_be_spoken
Whether the bot will speak it.
Definition messages.h:280
std::optional< std::string > spoken_status
Definition messages.h:286
std::string output_type
Definition messages.h:276
std::optional< SpokenProgressData > spoken_progress
How much of the text the bot has spoken. Only for text it speaks.
Definition messages.h:288
std::optional< int64_t > segment_id
ID shared by updates of a text.
Definition messages.h:278
std::optional< std::vector< std::string > > delivery
Definition messages.h:293
Sent by the bot when it's ready.
Definition messages.h:148
nlohmann::json about
Information about the bot. Can be null.
Definition messages.h:152
std::string version
RTVI version of the bot.
Definition messages.h:150
BotCapabilities capabilities
Definition messages.h:155
An app-defined message between the client and the bot.
Definition messages.h:101
nlohmann::json data
Data of the message. Can be null.
Definition messages.h:105
std::string type
Type of the message, defined by your app.
Definition messages.h:103
An error.
Definition messages.h:159
bool fatal
Whether the error is fatal, e.g. the bot stopped.
Definition messages.h:163
std::string error
What went wrong.
Definition messages.h:161
A function call is running.
Definition messages.h:325
nlohmann::json arguments
Definition messages.h:332
std::string tool_call_id
ID of the function call.
Definition messages.h:329
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:327
The result of a function call, sent to the bot.
Definition messages.h:117
std::string tool_call_id
ID of the function call.
Definition messages.h:121
nlohmann::json result
The result.
Definition messages.h:125
std::string function_name
Name of the function.
Definition messages.h:119
nlohmann::json arguments
Arguments the function was called with.
Definition messages.h:123
The bot's LLM started a function call.
Definition messages.h:319
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:321
A function call finished or was cancelled.
Definition messages.h:336
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:338
nlohmann::json result
The result of the function, if the bot shares it. Can be null.
Definition messages.h:344
std::string tool_call_id
ID of the function call.
Definition messages.h:340
bool cancelled
Whether the call was cancelled.
Definition messages.h:342
Where search results come from, e.g. a website.
Definition messages.h:356
std::optional< std::string > site_title
Title of the site.
Definition messages.h:360
std::optional< std::string > site_uri
Address of the site.
Definition messages.h:358
std::vector< LLMSearchResult > results
Results from this site.
Definition messages.h:362
A search result.
Definition messages.h:348
std::string text
Text of the result.
Definition messages.h:350
std::vector< double > confidence
Confidence scores, from 0 to 1.
Definition messages.h:352
Definition messages.h:379
static Message disconnect_bot()
A message that asks the bot to leave.
std::string label
Always "rtvi-ai".
Definition messages.h:383
std::string type
Type of the message, e.g. "bot-ready".
Definition messages.h:385
static Message dtmf(const std::string &buttons)
One or more DTMF keys (0-9, * and #), for bots on RTVI 2.1 or newer.
static Message client_ready(const AboutClientData &about)
A message that says the client is ready, with information about it.
static Message llm_function_call_result(const LLMFunctionCallResultData &data)
The result of a function call.
std::string id
ID of the message. Answers have the request's ID.
Definition messages.h:381
Message(MessageType type, nlohmann::json data=nullptr)
A message of type with data, and a new random ID.
static Message client_message(const std::string &type, const nlohmann::json &data)
An app-defined message to the bot.
Message()=default
An empty message.
nlohmann::json data
Data of the message. Null if it has none.
Definition messages.h:387
static Message send_text(const std::string &content, const SendTextOptions &options={})
Text for the bot's LLM.
A single measurement.
Definition messages.h:297
std::string processor
What was measured, e.g. a service in the bot.
Definition messages.h:299
std::optional< std::string > model
The AI model used, if any.
Definition messages.h:301
double value
The measured value, e.g. a time in seconds.
Definition messages.h:303
Performance metrics from the bot.
Definition messages.h:307
std::vector< MetricData > characters
Characters sent to be spoken.
Definition messages.h:313
nlohmann::json raw
All the metrics, including those without a field above.
Definition messages.h:315
std::vector< MetricData > ttfb
Times to first byte, in seconds.
Definition messages.h:309
std::vector< MetricData > processing
Processing times, in seconds.
Definition messages.h:311
A transcription, as the bot's speech-to-text made it.
Definition messages.h:224
std::string user_id
ID of the user who spoke.
Definition messages.h:232
std::optional< TranscriptionSpeaker > speaker
Definition messages.h:235
std::string text
The words.
Definition messages.h:226
std::string timestamp
When it was said, as an ISO 8601 date.
Definition messages.h:230
Options for sending text to the bot.
Definition messages.h:109
std::optional< bool > run_immediately
Whether the bot responds right away. Defaults to true.
Definition messages.h:111
std::optional< bool > audio_response
Whether the bot responds with audio. Defaults to true.
Definition messages.h:113
How much of a text the bot has spoken.
Definition messages.h:254
std::string remaining_text
Text not spoken yet.
Definition messages.h:258
std::string accumulated_text
Text spoken so far, including the current word.
Definition messages.h:256
Some text.
Definition messages.h:239
std::string text
The text.
Definition messages.h:241
Who said a transcription, if the bot's speech-to-text tells speakers apart.
Definition messages.h:186
std::string label
The speech-to-text service's name for the speaker, e.g. "A" or "S1".
Definition messages.h:188
int number
Definition messages.h:191
What the user said or typed.
Definition messages.h:195
std::vector< UserPerceptionSignal > perception
Definition messages.h:216
std::optional< TranscriptionSpeaker > speaker
Definition messages.h:220
std::optional< std::string > user_id
Definition messages.h:210
std::string input_type
Definition messages.h:202
std::optional< std::string > msg_id
For chat, the ID PipecatClient::send_text() returned.
Definition messages.h:212
std::string text
The words.
Definition messages.h:197
std::string timestamp
When it was said or sent, as an ISO 8601 date.
Definition messages.h:207
A label for how the user spoke, e.g. "frustrated", with how sure the bot is.
Definition messages.h:167
double confidence
How sure the bot is of the label, from 0 to 1.
Definition messages.h:171
std::string label
The label, e.g. "frustrated".
Definition messages.h:169
Definition messages.h:176
std::optional< double > value
The value, if the bot reads it as a value.
Definition messages.h:182
std::vector< UserPerceptionLabel > labels
The likeliest labels, likeliest first, if the bot reads it as labels.
Definition messages.h:180
std::string name
What the bot read, e.g. "tone" or "speech_rate".
Definition messages.h:178