Evaluation<R, E, M>

​

Example

// Collect your examples into a dataset
const dataset = new weave.Dataset({
  id: 'my-dataset',
  rows: [
    { question: 'What is the capital of France?', expected: 'Paris' },
    { question: 'Who wrote "To Kill a Mockingbird"?', expected: 'Harper Lee' },
    { question: 'What is the square root of 64?', expected: '8' },
  ],
});

// Define any custom scoring function
const scoringFunction = weave.op(function isEqual({ modelOutput, datasetRow }) {
  return modelOutput == datasetRow.expected;
});

// Define the function to evaluate
const model = weave.op(async function alwaysParisModel({ question }) {
  return 'Paris';
});

// Start evaluating
const evaluation = new weave.Evaluation({
  id: 'my-evaluation',
  dataset: dataset,
  scorers: [scoringFunction],
});

const results = await evaluation.evaluate({ model });

​

Extends

• WeaveObject

​

Type Parameters

​

R

​

E

​

M

​

Constructors

​

Constructor

new Evaluation<R, E, M>(parameters): Evaluation<R, E, M>

​

Parameters

parameters

columnMapping?

columnMapping.__savedRef?

dataset

description?

maxConcurrency?

name?

scorers

​

Returns

​

Overrides

​

Properties

​

__savedRef

optional __savedRef: ObjectRef | Promise<ObjectRef>

​

Inherited from

​

Accessors

​

description

​

Get Signature

get description(): string | undefined

Returns

​

Inherited from

​

name

​

Get Signature

get name(): string

Returns

​

Inherited from

​

Methods

​

evaluate()

evaluate(__namedParameters): Promise<Record<string, any>>

​

Parameters

__namedParameters

maxConcurrency?

model

nTrials?

​

Returns

​

predictAndScore()

predictAndScore(__namedParameters): Promise<{ model_latency: number; model_output: any; model_success: boolean; scores: {[key: string]: any; }; }>

​

Parameters

__namedParameters

columnMapping?

columnMapping.__savedRef?

example

model

​

Returns

​

saveAttrs()

saveAttrs(): object

​

Returns

​

Inherited from