google/ai/generativelanguage/v1beta/content.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.ai.generativelanguage.v1beta;

import "google/api/field_behavior.proto";
import "google/protobuf/struct.proto";

option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1beta/generativelanguagepb;generativelanguagepb";
option java_multiple_files = true;
option java_outer_classname = "ContentProto";
option java_package = "com.google.ai.generativelanguage.v1beta";

// Type contains the list of OpenAPI data types as defined by
// https://spec.openapis.org/oas/v3.0.3#data-types
enum Type {
  // Not specified, should not be used.
  TYPE_UNSPECIFIED = 0;

  // String type.
  STRING = 1;

  // Number type.
  NUMBER = 2;

  // Integer type.
  INTEGER = 3;

  // Boolean type.
  BOOLEAN = 4;

  // Array type.
  ARRAY = 5;

  // Object type.
  OBJECT = 6;
}

// The base structured datatype containing multi-part content of a message.
//
// A `Content` includes a `role` field designating the producer of the `Content`
// and a `parts` field containing multi-part data that contains the content of
// the message turn.
message Content {
  // Ordered `Parts` that constitute a single message. Parts may have different
  // MIME types.
  repeated Part parts = 1;

  // Optional. The producer of the content. Must be either 'user' or 'model'.
  //
  // Useful to set for multi-turn conversations, otherwise can be left blank
  // or unset.
  string role = 2 [(google.api.field_behavior) = OPTIONAL];
}

// A datatype containing media that is part of a multi-part `Content` message.
//
// A `Part` consists of data which has an associated datatype. A `Part` can only
// contain one of the accepted types in `Part.data`.
//
// A `Part` must have a fixed IANA MIME type identifying the type and subtype
// of the media if the `inline_data` field is filled with raw bytes.
message Part {
  oneof data {
    // Inline text.
    string text = 2;

    // Inline media bytes.
    Blob inline_data = 3;

    // A predicted `FunctionCall` returned from the model that contains
    // a string representing the `FunctionDeclaration.name` with the
    // arguments and their values.
    FunctionCall function_call = 4;

    // The result output of a `FunctionCall` that contains a string
    // representing the `FunctionDeclaration.name` and a structured JSON
    // object containing any output from the function is used as context to
    // the model.
    FunctionResponse function_response = 5;

    // URI based data.
    FileData file_data = 6;
  }
}

// Raw media bytes.
//
// Text should not be sent as raw bytes, use the 'text' field.
message Blob {
  // The IANA standard MIME type of the source data.
  // Accepted types include: "image/png", "image/jpeg", "image/heic",
  // "image/heif", "image/webp".
  string mime_type = 1;

  // Raw bytes for media formats.
  bytes data = 2;
}

// URI based data.
message FileData {
  // Optional. The IANA standard MIME type of the source data.
  string mime_type = 1 [(google.api.field_behavior) = OPTIONAL];

  // Required. URI.
  string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
}

// Tool details that the model may use to generate response.
//
// A `Tool` is a piece of code that enables the system to interact with
// external systems to perform an action, or set of actions, outside of
// knowledge and scope of the model.
message Tool {
  // Optional. A list of `FunctionDeclarations` available to the model that can
  // be used for function calling.
  //
  // The model or system does not execute the function. Instead the defined
  // function may be returned as a [FunctionCall][content.part.function_call]
  // with arguments to the client side for execution. The model may decide to
  // call a subset of these functions by populating
  // [FunctionCall][content.part.function_call] in the response. The next
  // conversation turn may contain a
  // [FunctionResponse][content.part.function_response]
  // with the [content.role] "function" generation context for the next model
  // turn.
  repeated FunctionDeclaration function_declarations = 1
      [(google.api.field_behavior) = OPTIONAL];
}

// The Tool configuration containing parameters for specifying `Tool` use
// in the request.
message ToolConfig {
  // Optional. Function calling config.
  FunctionCallingConfig function_calling_config = 1
      [(google.api.field_behavior) = OPTIONAL];
}

// Configuration for specifying function calling behavior.
message FunctionCallingConfig {
  // Defines the execution behavior for function calling by defining the
  // execution mode.
  enum Mode {
    // Unspecified function calling mode. This value should not be used.
    MODE_UNSPECIFIED = 0;

    // Default model behavior, model decides to predict either a function call
    // or a natural language repspose.
    AUTO = 1;

    // Model is constrained to always predicting a function call only.
    // If "allowed_function_names" are set, the predicted function call will be
    // limited to any one of "allowed_function_names", else the predicted
    // function call will be any one of the provided "function_declarations".
    ANY = 2;

    // Model will not predict any function call. Model behavior is same as when
    // not passing any function declarations.
    NONE = 3;
  }

  // Optional. Specifies the mode in which function calling should execute. If
  // unspecified, the default value will be set to AUTO.
  Mode mode = 1 [(google.api.field_behavior) = OPTIONAL];

  // Optional. A set of function names that, when provided, limits the functions
  // the model will call.
  //
  // This should only be set when the Mode is ANY. Function names
  // should match [FunctionDeclaration.name]. With mode set to ANY, model will
  // predict a function call from the set of function names provided.
  repeated string allowed_function_names = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// Structured representation of a function declaration as defined by the
// [OpenAPI 3.03 specification](https://spec.openapis.org/oas/v3.0.3). Included
// in this declaration are the function name and parameters. This
// FunctionDeclaration is a representation of a block of code that can be used
// as a `Tool` by the model and executed by the client.
message FunctionDeclaration {
  // Required. The name of the function.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. A brief description of the function.
  string description = 2 [(google.api.field_behavior) = REQUIRED];

  // Optional. Describes the parameters to this function. Reflects the Open
  // API 3.03 Parameter Object string Key: the name of the parameter. Parameter
  // names are case sensitive. Schema Value: the Schema defining the type used
  // for the parameter.
  optional Schema parameters = 3 [(google.api.field_behavior) = OPTIONAL];
}

// A predicted `FunctionCall` returned from the model that contains
// a string representing the `FunctionDeclaration.name` with the
// arguments and their values.
message FunctionCall {
  // Required. The name of the function to call.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Optional. The function parameters and values in JSON object format.
  optional google.protobuf.Struct args = 2
      [(google.api.field_behavior) = OPTIONAL];
}

// The result output from a `FunctionCall` that contains a string
// representing the `FunctionDeclaration.name` and a structured JSON
// object containing any output from the function is used as context to
// the model. This should contain the result of a`FunctionCall` made
// based on model prediction.
message FunctionResponse {
  // Required. The name of the function to call.
  // Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
  // length of 63.
  string name = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The function response in JSON object format.
  google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
}

// The `Schema` object allows the definition of input and output data types.
// These types can be objects, but also primitives and arrays.
// Represents a select subset of an [OpenAPI 3.0 schema
// object](https://spec.openapis.org/oas/v3.0.3#schema).
message Schema {
  // Required. Data type.
  Type type = 1 [(google.api.field_behavior) = REQUIRED];

  // Optional. The format of the data. This is used only for primitive
  // datatypes. Supported formats:
  //  for NUMBER type: float, double
  //  for INTEGER type: int32, int64
  string format = 2 [(google.api.field_behavior) = OPTIONAL];

  // Optional. A brief description of the parameter. This could contain examples
  // of use. Parameter description may be formatted as Markdown.
  string description = 3 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Indicates if the value may be null.
  bool nullable = 4 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Possible values of the element of Type.STRING with enum format.
  // For example we can define an Enum Direction as :
  // {type:STRING, format:enum, enum:["EAST", NORTH", "SOUTH", "WEST"]}
  repeated string enum = 5 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Schema of the elements of Type.ARRAY.
  optional Schema items = 6 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Properties of Type.OBJECT.
  map<string, Schema> properties = 7 [(google.api.field_behavior) = OPTIONAL];

  // Optional. Required properties of Type.OBJECT.
  repeated string required = 8 [(google.api.field_behavior) = OPTIONAL];
}

// Passage included inline with a grounding configuration.
message GroundingPassage {
  // Identifier for the passage for attributing this passage in grounded
  // answers.
  string id = 1;

  // Content of the passage.
  Content content = 2;
}

// A repeated list of passages.
message GroundingPassages {
  // List of passages.
  repeated GroundingPassage passages = 1;
}