evaluators.go

// Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0

// Package evaluators defines a set of Genkit Evaluators for popular use-cases
package evaluators

import (
"context"
"errors"
"fmt"
"reflect"
"regexp"
"sync"

 jsonata "github.com/blues/jsonata-go"
"github.com/firebase/genkit/go/ai"
"github.com/firebase/genkit/go/core/logger"
"github.com/firebase/genkit/go/genkit"
)

constprovider="genkitEval"

// EvaluatorType is an enum used to indicate the type of evaluator being
// configured for use
typeEvaluatorTypeint

const (
EvaluatorDeepEqualEvaluatorType=iota
EvaluatorRegex
EvaluatorJsonata
)

varevaluatorTypeName=map[EvaluatorType]string{
EvaluatorDeepEqual: "DEEP_EQUAL",
EvaluatorRegex: "REGEX",
EvaluatorJsonata: "JSONATA",
}

func (ssEvaluatorType) String() string {
returnevaluatorTypeName[ss]
}

// MetricConfig provides configuration options for a specific metric. More
// Params (judge LLMs, etc.) could be configured by extending this struct
typeMetricConfigstruct {
MetricTypeEvaluatorType
}

// GenkitEval is a Genkit plugin that provides evaluators
typeGenkitEvalstruct {
Metrics []MetricConfig// Configs for individual metrics
inittedbool// Whether the plugin has been initialized
mu sync.Mutex// Mutex to manage locks
}

func (ge*GenkitEval) Name() string {
returnprovider
}

// Init initializes the plugin.
func (ge*GenkitEval) Init(ctx context.Context, g*genkit.Genkit) (errerror) {
ifge==nil {
ge=&GenkitEval{}
 }
ge.mu.Lock()
deferge.mu.Unlock()
ifge.initted {
panic("genkitEval.Init already called")
 }
ifge==nil||len(ge.Metrics) ==0 {
returnerrors.New("genkitEval: need to configure at least one metric")
 }
ge.initted=true

for_, metric:=rangege.Metrics {
ConfigureMetric(g, metric)
 }
returnnil
}

funcConfigureMetric(g*genkit.Genkit, metricMetricConfig) (ai.Evaluator, error) {
switchmetric.MetricType {
caseEvaluatorDeepEqual:
returnconfigureDeepEqualEvaluator(g)
caseEvaluatorJsonata:
returnconfigureJsonataEvaluator(g)
caseEvaluatorRegex:
returnconfigureRegexEvaluator(g)
default:
panic(fmt.Sprintf("Unsupported genkitEval metric type: %s", metric.MetricType.String()))
 }
}

funcconfigureRegexEvaluator(g*genkit.Genkit) (ai.Evaluator, error) {
evalOptions:= ai.EvaluatorOptions{
DisplayName: "RegExp",
Definition: "Tests output against the regexp provided as reference",
IsBilled: false,
 }
evaluator, err:=genkit.DefineEvaluator(g, provider, "regex", &evalOptions, func(ctx context.Context, req*ai.EvaluatorCallbackRequest) (*ai.EvaluatorCallbackResponse, error) {
dataPoint:=req.Input
varscore ai.Score
ifdataPoint.Output==nil {
returnnil, errors.New("output was not provided")
 }
ifdataPoint.Reference==nil {
returnnil, errors.New("reference was not provided")
 }
ifreflect.TypeOf(dataPoint.Reference).String() !="string" {
returnnil, errors.New("reference must be a string (regex)")
 }
ifreflect.TypeOf(dataPoint.Output).String() =="string" {
// Test against provided regexp
match, _:=regexp.MatchString((dataPoint.Reference).(string), (dataPoint.Output).(string))
status:=ai.ScoreStatusUnknown
ifmatch {
status=ai.ScoreStatusPass
 } else {
status=ai.ScoreStatusFail
 }
score= ai.Score{
Score: match,
Status: status.String(),
 }
 } else {
// Mark as failed if output is not string type
logger.FromContext(ctx).Debug("genkitEval",
"regex", fmt.Sprintf("Failed regex evaluation, as output is not string type. TestCaseId: %s", dataPoint.TestCaseId))
score= ai.Score{
Score: false,
Status: ai.ScoreStatusFail.String(),
 }
 }
callbackResponse:= ai.EvaluatorCallbackResponse{
TestCaseId: req.Input.TestCaseId,
Evaluation: []ai.Score{score},
 }
return&callbackResponse, nil
 })
iferr!=nil {
returnnil, err
 }
returnevaluator, nil
}

funcconfigureDeepEqualEvaluator(g*genkit.Genkit) (ai.Evaluator, error) {
evalOptions:= ai.EvaluatorOptions{
DisplayName: "Deep Equal",
Definition: "Tests equality of output against the provided reference",
IsBilled: false,
 }
evaluator, err:=genkit.DefineEvaluator(g, provider, "deep_equal", &evalOptions, func(ctx context.Context, req*ai.EvaluatorCallbackRequest) (*ai.EvaluatorCallbackResponse, error) {
dataPoint:=req.Input
varscore ai.Score
ifdataPoint.Output==nil {
returnnil, errors.New("output was not provided")
 }
ifdataPoint.Reference==nil {
returnnil, errors.New("reference was not provided")
 }
deepEqual:=reflect.DeepEqual(dataPoint.Reference, dataPoint.Output)
status:=ai.ScoreStatusUnknown
ifdeepEqual {
status=ai.ScoreStatusPass
 } else {
status=ai.ScoreStatusFail
 }
score= ai.Score{
Score: deepEqual,
Status: status.String(),
 }

callbackResponse:= ai.EvaluatorCallbackResponse{
TestCaseId: req.Input.TestCaseId,
Evaluation: []ai.Score{score},
 }
return&callbackResponse, nil
 })
iferr!=nil {
returnnil, err
 }
returnevaluator, nil
}

funcconfigureJsonataEvaluator(g*genkit.Genkit) (ai.Evaluator, error) {
evalOptions:= ai.EvaluatorOptions{
DisplayName: "JSONata",
Definition: "Tests JSONata expression (provided in reference) against output",
IsBilled: false,
 }
evaluator, err:=genkit.DefineEvaluator(g, provider, "jsonata", &evalOptions, func(ctx context.Context, req*ai.EvaluatorCallbackRequest) (*ai.EvaluatorCallbackResponse, error) {
dataPoint:=req.Input
varscore ai.Score
ifdataPoint.Output==nil {
returnnil, errors.New("output was not provided")
 }
ifdataPoint.Reference==nil {
returnnil, errors.New("reference was not provided")
 }
ifreflect.TypeOf(dataPoint.Reference).String() !="string" {
returnnil, errors.New("reference must be a string (jsonata)")
 }
// Test against provided jsonata
exp:=jsonata.MustCompile((dataPoint.Reference).(string))
res, err:=exp.Eval(dataPoint.Output)
iferr!=nil {
returnnil, err
 }
status:=ai.ScoreStatusUnknown
ifres==false||res==""||res==nil {
status=ai.ScoreStatusFail
 } else {
status=ai.ScoreStatusPass
 }
score= ai.Score{
Score: res,
Status: status.String(),
 }

callbackResponse:= ai.EvaluatorCallbackResponse{
TestCaseId: req.Input.TestCaseId,
Evaluation: []ai.Score{score},
 }
return&callbackResponse, nil
 })
iferr!=nil {
returnnil, err
 }
returnevaluator, nil
}