ModelContent.cs

/*
 * Copyright 2025 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

usingSystem;
usingSystem.Collections.Generic;
usingSystem.Collections.ObjectModel;
usingSystem.Linq;
usingFirebase.VertexAI.Internal;

namespaceFirebase.VertexAI{

/// <summary>
/// A type describing data in media formats interpretable by an AI model. Each generative AI
/// request or response contains a list of `ModelContent`s, and each `ModelContent` value
/// may comprise multiple heterogeneous `ModelContent.Part`s.
/// </summary>
publicreadonlystructModelContent{
privatereadonlystring_role;
privatereadonlyReadOnlyCollection<Part>_parts;

/// <summary>
/// The role of the entity creating the `ModelContent`. For user-generated client requests,
/// for example, the role is `user`.
/// </summary>
publicstringRole=>string.IsNullOrWhiteSpace(_role)?"user":_role;

/// <summary>
/// The data parts comprising this `ModelContent` value.
/// </summary>
publicIEnumerable<Part>Parts=>_parts??newReadOnlyCollection<Part>(newList<Part>());

/// <summary>
/// Creates a `ModelContent` with the given `Part`s, using the default `user` role.
/// </summary>
publicModelContent(paramsPart[]parts):this("user",parts){}

/// <summary>
/// Creates a `ModelContent` with the given `Part`s, using the default `user` role.
/// </summary>
publicModelContent(IEnumerable<Part>parts):this("user",parts){}

/// <summary>
/// Creates a `ModelContent` with the given role and `Part`s.
/// </summary>
publicModelContent(stringrole,paramsPart[]parts):this(role,(IEnumerable<Part>)parts){}

/// <summary>
/// Creates a `ModelContent` with the given role and `Part`s.
/// </summary>
publicModelContent(stringrole,IEnumerable<Part>parts){
_role=role;
_parts=newReadOnlyCollection<Part>(parts==null?newList<Part>():parts.ToList());
}

#region Helper Factories

/// <summary>
/// Creates a new `ModelContent` with the default `user` role, and a
/// `TextPart` containing the given text.
/// </summary>
publicstaticModelContentText(stringtext){
returnnewModelContent(newTextPart(text));
}

/// <summary>
/// Creates a new `ModelContent` with the default `user` role, and an
/// `InlineDataPart` containing the given mimeType and data.
/// </summary>
publicstaticModelContentInlineData(stringmimeType,byte[]data){
returnnewModelContent(newInlineDataPart(mimeType,data));
}

/// <summary>
/// Creates a new `ModelContent` with the default `user` role, and a
/// `FileDataPart` containing the given mimeType and data.
/// </summary>
publicstaticModelContentFileData(stringmimeType,System.Uriuri){
returnnewModelContent(newFileDataPart(mimeType,uri));
}

/// <summary>
/// Creates a new `ModelContent` with the default `user` role, and a
/// `FunctionResponsePart` containing the given name and args.
/// </summary>
publicstaticModelContentFunctionResponse(
stringname,IDictionary<string,object>response){
returnnewModelContent(newFunctionResponsePart(name,response));
}

// TODO: Possibly more, like Multi, Model, FunctionResponses, System (only on Dart?)

// TODO: Do we want to include helper factories for common C# or Unity types?
#endregion

#region Parts

/// <summary>
/// A discrete piece of data in a media format interpretable by an AI model. Within a
/// single value of `Part`, different data types may not mix.
/// </summary>
publicinterfacePart{
#if !DOXYGEN
/// <summary>
/// Intended for internal use only.
/// This method is used for serializing the object to JSON for the API request.
/// </summary>
Dictionary<string,object>ToJson();
#endif
}

/// <summary>
/// A text part containing a string value.
/// </summary>
publicreadonlystructTextPart:Part{
/// <summary>
/// Text value.
/// </summary>
publicstringText{get;}

/// <summary>
/// Creates a `TextPart` with the given text.
/// </summary>
/// <param name="text">The text value to use.</param>
publicTextPart(stringtext){Text=text;}

Dictionary<string,object>Part.ToJson(){
returnnewDictionary<string,object>(){{"text",Text}};
}
}

/// <summary>
/// Data with a specified media type.
/// Note: Not all media types may be supported by the AI model.
/// </summary>
publicreadonlystructInlineDataPart:Part{
/// <summary>
/// The IANA standard MIME type of the data.
/// </summary>
publicstringMimeType{get;}
/// <summary>
/// The data provided in the inline data part.
/// </summary>
publicReadOnlyMemory<byte>Data{get;}

/// <summary>
/// Creates an `InlineDataPart` from data and a MIME type.
///
/// > Important: Supported input types depend on the model on the model being used; see [input
/// files and requirements](https://firebase.google.com/docs/vertex-ai/input-file-requirements)
/// for more details.
/// </summary>
/// <param name="mimeType">The IANA standard MIME type of the data, for example, `"image/jpeg"` or
/// `"video/mp4"`; see [input files and
/// requirements](https://firebase.google.com/docs/vertex-ai/input-file-requirements) for
/// supported values.</param>
/// <param name="data">The data representation of an image, video, audio or document; see [input files and
/// requirements](https://firebase.google.com/docs/vertex-ai/input-file-requirements) for
/// supported media types.</param>
publicInlineDataPart(stringmimeType,byte[]data){MimeType=mimeType;Data=data;}

Dictionary<string,object>Part.ToJson(){
returnnewDictionary<string,object>(){
{"inlineData",newDictionary<string,object>(){
{"mimeType",MimeType},
{"data",Convert.ToBase64String(Data.ToArray())}
}
}
};
}
}

/// <summary>
/// File data stored in Cloud Storage for Firebase, referenced by a URI.
/// </summary>
publicreadonlystructFileDataPart:Part{
/// <summary>
/// The IANA standard MIME type of the data.
/// </summary>
publicstringMimeType{get;}
/// <summary>
/// The URI of the file.
/// </summary>
publicSystem.UriUri{get;}

/// <summary>
/// Constructs a new file data part.
/// </summary>
/// <param name="mimeType">The IANA standard MIME type of the uploaded file, for example, `"image/jpeg"`
/// or `"video/mp4"`; see [media requirements
/// ](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#media_requirements)
/// for supported values.</param>
/// <param name="uri">The `"gs://"`-prefixed URI of the file in Cloud Storage for Firebase, for example,
/// `"gs://bucket-name/path/image.jpg"`</param>
publicFileDataPart(stringmimeType,System.Uriuri){MimeType=mimeType;Uri=uri;}

Dictionary<string,object>Part.ToJson(){
returnnewDictionary<string,object>(){
{"fileData",newDictionary<string,object>(){
{"mimeType",MimeType},
{"fileUri",Uri.AbsoluteUri}
}
}
};
}
}

/// <summary>
/// A predicted function call returned from the model.
/// </summary>
publicreadonlystructFunctionCallPart:Part{
/// <summary>
/// The name of the registered function to call.
/// </summary>
publicstringName{get;}
/// <summary>
/// The function parameters and values, matching the registered schema.
/// </summary>
publicIReadOnlyDictionary<string,object>Args{get;}

/// <summary>
/// Intended for internal use only.
/// </summary>
internalFunctionCallPart(stringname,IDictionary<string,object>args){
Name=name;
Args=newDictionary<string,object>(args);
}

Dictionary<string,object>Part.ToJson(){
returnnewDictionary<string,object>(){
{"functionCall",newDictionary<string,object>(){
{"name",Name},
{"args",Args}
}
}
};
}
}

/// <summary>
/// Result output from a function call.
///
/// Contains a string representing the `FunctionDeclaration.name` and a structured JSON object
/// containing any output from the function is used as context to the model. This should contain the
/// result of a `FunctionCallPart` made based on model prediction.
/// </summary>
publicreadonlystructFunctionResponsePart:Part{
/// <summary>
/// The name of the function that was called.
/// </summary>
publicstringName{get;}
/// <summary>
/// The function's response or return value.
/// </summary>
publicIReadOnlyDictionary<string,object>Response{get;}

/// <summary>
/// Constructs a new `FunctionResponsePart`.
/// </summary>
/// <param name="name">The name of the function that was called.</param>
/// <param name="response">The function's response.</param>
publicFunctionResponsePart(stringname,IDictionary<string,object>response){
Name=name;
Response=newDictionary<string,object>(response);
}

Dictionary<string,object>Part.ToJson(){
returnnewDictionary<string,object>(){
{"functionResponse",newDictionary<string,object>(){
{"name",Name},
{"response",Response}
}
}
};
}
}

#endregion

/// <summary>
/// Intended for internal use only.
/// This method is used for serializing the object to JSON for the API request.
/// </summary>
internalDictionary<string,object>ToJson(){
returnnewDictionary<string,object>(){
["role"]=Role,
["parts"]=Parts.Select(p =>p.ToJson()).ToList()
};
}

/// <summary>
/// Intended for internal use only.
/// This method is used for deserializing JSON responses and should not be called directly.
/// </summary>
internalstaticModelContentFromJson(Dictionary<string,object>jsonDict){
// Both role and parts are required keys
returnnewModelContent(
jsonDict.ParseValue<string>("role",JsonParseOptions.ThrowEverything),
jsonDict.ParseObjectList("parts",PartFromJson,JsonParseOptions.ThrowEverything));
}

privatestaticInlineDataPartInlineDataPartFromJson(Dictionary<string,object>jsonDict){
returnnewInlineDataPart(
jsonDict.ParseValue<string>("mimeType",JsonParseOptions.ThrowEverything),
Convert.FromBase64String(jsonDict.ParseValue<string>("data",JsonParseOptions.ThrowEverything)));
}

privatestaticPartPartFromJson(Dictionary<string,object>jsonDict){
if(jsonDict.TryParseValue("text",outstringtext)){
returnnewTextPart(text);
}elseif(jsonDict.TryParseObject("functionCall",ModelContentJsonParsers.FunctionCallPartFromJson,
outvarfcPart)){
returnfcPart;
}elseif(jsonDict.TryParseObject("inlineData",InlineDataPartFromJson,
outvarinlineDataPart)){
returninlineDataPart;
}else{
thrownewVertexAISerializationException("Unable to parse given 'part' into a known Part.");
}
}
}

namespaceInternal{

// Class for parsing Parts that need to be called from other files as well.
internalstaticclassModelContentJsonParsers{
internalstaticModelContent.FunctionCallPartFunctionCallPartFromJson(Dictionary<string,object>jsonDict){
returnnewModelContent.FunctionCallPart(
jsonDict.ParseValue<string>("name",JsonParseOptions.ThrowEverything),
jsonDict.ParseValue<Dictionary<string,object>>("args",JsonParseOptions.ThrowEverything));
}
}

}

}