LiveSessionResponse.cs

/*
 * Copyright 2025 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

usingSystem.Collections.Generic;
usingSystem.Collections.ObjectModel;
usingGoogle.MiniJSON;
usingFirebase.VertexAI.Internal;
usingSystem.Linq;
usingSystem;
usingSystem.Text;

namespaceFirebase.VertexAI{

/// <summary>
/// Represents the response from the model for live content updates.
/// </summary>
publicreadonlystructLiveSessionResponse{

/// <summary>
/// The detailed message from the live session.
/// </summary>
publicreadonlyILiveSessionMessageMessage{get;}

/// <summary>
/// The response's content as text, if it exists.
/// </summary>
publicstringText{
get{
StringBuilderstringBuilder=new();
if(MessageisLiveSessionContentcontent){
foreach(varpartincontent.Content?.Parts){
if(partisModelContent.TextParttextPart){
stringBuilder.Append(textPart.Text);
}
}
}
returnstringBuilder.ToString();
}
}

/// <summary>
/// The response's content that was audio, if it exists.
/// </summary>
publicIEnumerable<byte[]>Audio{
get{
if(MessageisLiveSessionContentcontent){
returncontent.Content?.Parts
.OfType<ModelContent.InlineDataPart>()
.Where(part =>part.MimeType=="audio/pcm")
.Select(part =>part.Data.ToArray());
}
returnnull;
}
}

/// <summary>
/// The response's content that was audio, if it exists, converted into floats.
/// </summary>
publicIEnumerable<float[]>AudioAsFloat{
get{
returnAudio?.Select(ConvertBytesToFloat);
}
}

// Helper function to convert a byte array representing a 16-bit encoded
// Audio snippit into a float array, which Unity's built in libraries supports.
privatefloat[]ConvertBytesToFloat(byte[]byteArray){
// Assumes 16 bit encoding, which would be two bytes per sample.
intsampleCount=byteArray.Length/2;
float[]floatArray=newfloat[sampleCount];

for(inti=0;i<sampleCount;i++){
floatsample=(short)(byteArray[i*2]|(byteArray[i*2+1]<<8))/32768f;
floatArray[i]=Math.Clamp(sample,-1f,1f);// Ensure values are within the valid range
}

returnfloatArray;
}

privateLiveSessionResponse(ILiveSessionMessageliveSessionMessage){
Message=liveSessionMessage;
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticLiveSessionResponse?FromJson(stringjsonString){
returnFromJson(Json.Deserialize(jsonString)asDictionary<string,object>);
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticLiveSessionResponse?FromJson(Dictionary<string,object>jsonDict){
if(jsonDict.ContainsKey("setupComplete")){
// We don't want to pass this along to the user, so return null instead.
returnnull;
}elseif(jsonDict.TryParseValue("serverContent",outDictionary<string,object>serverContent)){
// TODO: Other fields
returnnewLiveSessionResponse(LiveSessionContent.FromJson(serverContent));
}elseif(jsonDict.TryParseValue("toolCall",outDictionary<string,object>toolCall)){
returnnewLiveSessionResponse(LiveSessionToolCall.FromJson(toolCall));
}elseif(jsonDict.TryParseValue("toolCallCancellation",outDictionary<string,object>toolCallCancellation)){
returnnewLiveSessionResponse(LiveSessionToolCallCancellation.FromJson(toolCallCancellation));
}else{
// TODO: Determine if we want to log this, or just ignore it?
#if FIREBASE_LOG_REST_CALLS
UnityEngine.Debug.Log($"Failed to parse LiveSessionResponse from JSON, with keys: {string.Join(',',jsonDict.Keys)}");
#endif
returnnull;
}
}
}

/// <summary>
/// Represents a message received from a live session.
/// </summary>
publicinterfaceILiveSessionMessage{}

/// <summary>
/// Content generated by the model in a live session.
/// </summary>
publicreadonlystructLiveSessionContent:ILiveSessionMessage{
/// <summary>
/// The main content data of the response. This can be `null` if there was no content.
/// </summary>
publicreadonlyModelContent?Content{get;}

/// <summary>
/// Whether the turn is complete. If true, indicates that the model is done
/// generating.
/// </summary>
publicreadonlyboolTurnComplete{get;}

/// <summary>
/// Whether generation was interrupted. If true, indicates that a
/// client message has interrupted current model.
/// </summary>
publicreadonlyboolInterrupted{get;}

privateLiveSessionContent(ModelContent?content,boolturnComplete,boolinterrupted){
Content=content;
TurnComplete=turnComplete;
Interrupted=interrupted;
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticLiveSessionContentFromJson(Dictionary<string,object>jsonDict){
returnnewLiveSessionContent(
jsonDict.ParseNullableObject("modelTurn",ModelContent.FromJson),
jsonDict.ParseValue<bool>("turnComplete"),
jsonDict.ParseValue<bool>("interrupted")
);
}
}

/// <summary>
/// A request to use a tool from the live session.
/// </summary>
publicreadonlystructLiveSessionToolCall:ILiveSessionMessage{
privatereadonlyReadOnlyCollection<ModelContent.FunctionCallPart>_functionCalls;

/// <summary>
/// A list of `ModelContent.FunctionCallPart` included in the response, if any.
///
/// This will be empty if no function calls are present.
/// </summary>
publicIEnumerable<ModelContent.FunctionCallPart>FunctionCalls=>
_functionCalls??newReadOnlyCollection<ModelContent.FunctionCallPart>(
newList<ModelContent.FunctionCallPart>());

privateLiveSessionToolCall(List<ModelContent.FunctionCallPart>functionCalls){
_functionCalls=newReadOnlyCollection<ModelContent.FunctionCallPart>(
functionCalls??newList<ModelContent.FunctionCallPart>());
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticLiveSessionToolCallFromJson(Dictionary<string,object>jsonDict){
returnnewLiveSessionToolCall(
jsonDict.ParseObjectList("functionCalls",ModelContentJsonParsers.FunctionCallPartFromJson));
}
}

/// <summary>
/// A request to cancel using a tool from the live session.
/// </summary>
publicreadonlystructLiveSessionToolCallCancellation:ILiveSessionMessage{
privatereadonlyReadOnlyCollection<string>_functionIds;

/// <summary>
/// The list of Function IDs to cancel.
/// </summary>
publicIEnumerable<string>FunctionIds=>
_functionIds??newReadOnlyCollection<string>(newList<string>());

privateLiveSessionToolCallCancellation(List<string>functionIds){
_functionIds=newReadOnlyCollection<string>(
functionIds??newList<string>());
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticLiveSessionToolCallCancellationFromJson(Dictionary<string,object>jsonDict){
returnnewLiveSessionToolCallCancellation(
jsonDict.ParseStringList("ids"));
}
}

}