10#ifndef PIPECAT_RTVI_MESSAGES_H
11#define PIPECAT_RTVI_MESSAGES_H
13#include <nlohmann/json.hpp>
424void to_json(nlohmann::json& j,
const Message& message);
425void from_json(
const nlohmann::json& j,
Message& message);
435void from_json(
const nlohmann::json& j,
BotReadyData& data);
436void from_json(
const nlohmann::json& j,
ErrorData& data);
442void from_json(
const nlohmann::json& j,
TextData& data);
445void from_json(
const nlohmann::json& j,
MetricData& data);
446void from_json(
const nlohmann::json& j,
MetricsData& data);
constexpr const char * PROTOCOL_VERSION
The RTVI protocol version this library speaks.
Definition messages.h:32
std::optional< MessageType > parse_message_type(const std::string &type)
The message type with an RTVI name, or std::nullopt if it's unknown.
constexpr const char * MESSAGE_LABEL
The label of every RTVI message.
Definition messages.h:35
const char * to_string(MessageType type)
The RTVI name of a message type, e.g. "bot-ready".
AboutClientData default_about_client()
Information about this library and the platform it runs on.
MessageType
Types of RTVI messages.
Definition messages.h:38
@ STTRawText
A transcription, as the bot made it.
@ LLMFunctionCallInProgress
A function call is running.
@ ServerMessage
An app-defined message from the bot.
@ BotInterrupted
The bot was interrupted.
@ BotTTSStarted
The bot's text-to-speech started.
@ UserLLMText
User text sent to the bot's LLM.
@ LLMRawText
Text from the bot's LLM, as it streams.
@ ClientMessage
An app-defined message to the bot.
@ TTSRawText
Text the bot speaks, as it speaks it.
@ UserStartedSpeaking
The user started speaking.
@ BotOutput
What the bot says or outputs.
@ LLMFunctionCallStarted
The bot's LLM started a function call.
@ DisconnectBot
Asks the bot to leave.
@ Error
An error in the bot.
@ BotStoppedSpeaking
The bot stopped speaking.
@ UserMuteStopped
The bot stopped ignoring user audio.
@ ErrorResponse
The bot couldn't handle a message.
@ LLMFunctionCallResult
The result of a function call.
@ UserMuteStarted
The bot started ignoring user audio.
@ UserStoppedSpeaking
The user stopped speaking.
@ UserInput
What the user said or typed.
@ LLMFunctionCallStopped
A function call finished.
@ ServerResponse
The bot's answer to a request.
@ BotStartedSpeaking
The bot started speaking.
@ SendText
Text for the bot's LLM.
@ BotTTSStopped
The bot's text-to-speech stopped.
@ ClientReady
The client is ready.
@ BotLLMStopped
The bot's LLM finished responding.
@ BotLLMStarted
The bot's LLM started responding.
@ BotLLMSearchResponse
Search results used by the bot's LLM.
@ Metrics
Performance metrics.
@ BotReady
The bot is ready.
Information about the client, sent to the bot when connecting.
Definition messages.h:84
std::optional< std::string > platform
Operating system, e.g. "Linux".
Definition messages.h:90
std::string library
Name of the library, e.g. "pipecat-client-cxx".
Definition messages.h:86
nlohmann::json platform_details
Any other details, as a JSON object. Null to leave them out.
Definition messages.h:94
std::optional< std::string > platform_version
Version of the OS.
Definition messages.h:92
std::optional< std::string > library_version
Version of the library.
Definition messages.h:88
Definition messages.h:132
std::optional< bool > metrics
Whether the bot sends metrics.
Definition messages.h:144
std::optional< bool > audio_in
Whether the bot receives the user's audio.
Definition messages.h:134
std::optional< bool > audio_out
Whether the bot sends audio to the user.
Definition messages.h:136
std::optional< bool > screen_in
Whether the bot receives the user's screen share.
Definition messages.h:142
std::optional< bool > video_in
Whether the bot receives the user's video.
Definition messages.h:138
std::optional< bool > video_out
Whether the bot sends video to the user.
Definition messages.h:140
Search results used by the bot's LLM.
Definition messages.h:366
std::vector< LLMSearchOrigin > origins
Where the results come from.
Definition messages.h:372
std::optional< std::string > search_result
Summary of the results.
Definition messages.h:368
std::optional< std::string > rendered_content
Results ready to show.
Definition messages.h:370
Definition messages.h:265
std::string text
The text.
Definition messages.h:267
std::optional< bool > will_be_spoken
Whether the bot will speak it.
Definition messages.h:280
std::optional< std::string > spoken_status
Definition messages.h:286
std::string output_type
Definition messages.h:276
std::optional< SpokenProgressData > spoken_progress
How much of the text the bot has spoken. Only for text it speaks.
Definition messages.h:288
std::optional< int64_t > segment_id
ID shared by updates of a text.
Definition messages.h:278
std::optional< std::vector< std::string > > delivery
Definition messages.h:293
Sent by the bot when it's ready.
Definition messages.h:148
nlohmann::json about
Information about the bot. Can be null.
Definition messages.h:152
std::string version
RTVI version of the bot.
Definition messages.h:150
BotCapabilities capabilities
Definition messages.h:155
An app-defined message between the client and the bot.
Definition messages.h:101
nlohmann::json data
Data of the message. Can be null.
Definition messages.h:105
std::string type
Type of the message, defined by your app.
Definition messages.h:103
An error.
Definition messages.h:159
bool fatal
Whether the error is fatal, e.g. the bot stopped.
Definition messages.h:163
std::string error
What went wrong.
Definition messages.h:161
A function call is running.
Definition messages.h:325
nlohmann::json arguments
Definition messages.h:332
std::string tool_call_id
ID of the function call.
Definition messages.h:329
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:327
The result of a function call, sent to the bot.
Definition messages.h:117
std::string tool_call_id
ID of the function call.
Definition messages.h:121
nlohmann::json result
The result.
Definition messages.h:125
std::string function_name
Name of the function.
Definition messages.h:119
nlohmann::json arguments
Arguments the function was called with.
Definition messages.h:123
The bot's LLM started a function call.
Definition messages.h:319
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:321
A function call finished or was cancelled.
Definition messages.h:336
std::optional< std::string > function_name
Name of the function, if the bot shares it.
Definition messages.h:338
nlohmann::json result
The result of the function, if the bot shares it. Can be null.
Definition messages.h:344
std::string tool_call_id
ID of the function call.
Definition messages.h:340
bool cancelled
Whether the call was cancelled.
Definition messages.h:342
Where search results come from, e.g. a website.
Definition messages.h:356
std::optional< std::string > site_title
Title of the site.
Definition messages.h:360
std::optional< std::string > site_uri
Address of the site.
Definition messages.h:358
std::vector< LLMSearchResult > results
Results from this site.
Definition messages.h:362
A search result.
Definition messages.h:348
std::string text
Text of the result.
Definition messages.h:350
std::vector< double > confidence
Confidence scores, from 0 to 1.
Definition messages.h:352
Definition messages.h:379
static Message disconnect_bot()
A message that asks the bot to leave.
std::string label
Always "rtvi-ai".
Definition messages.h:383
std::string type
Type of the message, e.g. "bot-ready".
Definition messages.h:385
static Message dtmf(const std::string &buttons)
One or more DTMF keys (0-9, * and #), for bots on RTVI 2.1 or newer.
static Message client_ready(const AboutClientData &about)
A message that says the client is ready, with information about it.
static Message llm_function_call_result(const LLMFunctionCallResultData &data)
The result of a function call.
std::string id
ID of the message. Answers have the request's ID.
Definition messages.h:381
Message(MessageType type, nlohmann::json data=nullptr)
A message of type with data, and a new random ID.
static Message client_message(const std::string &type, const nlohmann::json &data)
An app-defined message to the bot.
Message()=default
An empty message.
nlohmann::json data
Data of the message. Null if it has none.
Definition messages.h:387
static Message send_text(const std::string &content, const SendTextOptions &options={})
Text for the bot's LLM.
A single measurement.
Definition messages.h:297
std::string processor
What was measured, e.g. a service in the bot.
Definition messages.h:299
std::optional< std::string > model
The AI model used, if any.
Definition messages.h:301
double value
The measured value, e.g. a time in seconds.
Definition messages.h:303
Performance metrics from the bot.
Definition messages.h:307
std::vector< MetricData > characters
Characters sent to be spoken.
Definition messages.h:313
nlohmann::json raw
All the metrics, including those without a field above.
Definition messages.h:315
std::vector< MetricData > ttfb
Times to first byte, in seconds.
Definition messages.h:309
std::vector< MetricData > processing
Processing times, in seconds.
Definition messages.h:311
A transcription, as the bot's speech-to-text made it.
Definition messages.h:224
std::string user_id
ID of the user who spoke.
Definition messages.h:232
std::optional< TranscriptionSpeaker > speaker
Definition messages.h:235
std::string text
The words.
Definition messages.h:226
std::string timestamp
When it was said, as an ISO 8601 date.
Definition messages.h:230
Options for sending text to the bot.
Definition messages.h:109
std::optional< bool > run_immediately
Whether the bot responds right away. Defaults to true.
Definition messages.h:111
std::optional< bool > audio_response
Whether the bot responds with audio. Defaults to true.
Definition messages.h:113
How much of a text the bot has spoken.
Definition messages.h:254
std::string remaining_text
Text not spoken yet.
Definition messages.h:258
std::string accumulated_text
Text spoken so far, including the current word.
Definition messages.h:256
Some text.
Definition messages.h:239
std::string text
The text.
Definition messages.h:241
Who said a transcription, if the bot's speech-to-text tells speakers apart.
Definition messages.h:186
std::string label
The speech-to-text service's name for the speaker, e.g. "A" or "S1".
Definition messages.h:188
int number
Definition messages.h:191
A label for how the user spoke, e.g. "frustrated", with how sure the bot is.
Definition messages.h:167
double confidence
How sure the bot is of the label, from 0 to 1.
Definition messages.h:171
std::string label
The label, e.g. "frustrated".
Definition messages.h:169
Definition messages.h:176
std::optional< double > value
The value, if the bot reads it as a value.
Definition messages.h:182
std::vector< UserPerceptionLabel > labels
The likeliest labels, likeliest first, if the bot reads it as labels.
Definition messages.h:180
std::string name
What the bot read, e.g. "tone" or "speech_rate".
Definition messages.h:178