Experiments

inputs = [{"question": "question"}]

experiment = literalai_client.api.create_experiment(
    name="Foo", params=[{"foo": "bar"}]  # optional
)

@literalai_client.run
def my_agent(input):
    # Faking the agent response
    return {"content": "answer"}

def score_output(output):
    # Faking the scoring
    return [{"name": "context_relevancy", "type": "AI", "value": 0.6}]

@literalai_client.experiment_item_run
def run_and_eval(input):
    output = my_agent(input)
    experiment_item = {
      "scores": score_output(output),
      "input": input,
      "output": output
    }
    experiment.log(experiment_item)

def run_experiment(inputs):
    for input in inputs:
        run_and_eval(input)

run_experiment(inputs)

async function myAgent(input: Record<string, unknown>) {
  // This is a fake agent
  return literalaiClient.step({ name: 'agent', type: 'run' }).wrap(async () => {
    return { answer: 'Fake Answer' };
  });
}

function scoreOutput(output: Record<string, unknown>) {
  // This is a fake scoring function
  return [
    {
      name: 'context_relevancy',
      type: 'AI' as const,
      value: 0.6
    }
  ];
}

async function main() {
  const experiment = await literalaiClient.api.createExperiment({
    name: 'Foo',
    params: { foo: 'bar' } // optional
  });

  const inputs = [{ question: 'Fake question' }];

  for (const input of inputs) {
    await literalaiClient.experimentItemRun().wrap(async () => {
      const agentOutput = await myAgent(input);
      const scores = scoreOutput(agentOutput);

      const experimentItem = {
        scores: scores,
        input: input,
        output: agentOutput
      };

      await experiment.log(experimentItem);
    });
  }
}

main();

dataset_id = "MY_DATASET_ID"

dataset = literalai_client.api.get_dataset(dataset_id)

experiment = literalai_client.api.create_experiment(
    dataset_id=dataset_id,
    name="Foo",
    params=[{"foo": "bar"}]  # optional
)

@literalai_client.experiment_item_run
def run_and_eval(item):
    output = my_agent(item.input)
    experiment_item = {
      # Notice that the experiment item is now linked to the dataset item
      "datasetItemId": item.id,
      "scores": score_output(output),
      "input": item.input,
      "output": output
    }
    experiment.log(experiment_item)

def run_experiment():
    for item in dataset.items:
        run_and_eval(item)

const datasetId = 'MY_DATASET_ID';
const dataset = await literalAiClient.api.getDataset({id: datasetId});

async function main() {
    const experiment = await literalaiClient.api.createExperiment({
        datasetId,
        name: 'Foo',
        params: { foo: 'bar' } // optional
    });

    for (const item of dataset.items) {
        await literalaiClient.experimentItemRun().wrap(async () => {
            const agentOutput = await myAgent(item.input);
            const scores = scoreOutput(agentOutput);

            const experimentItem = {
                // Notice that the experiment item is now linked to the dataset item
                datasetItemId: item.id,
                scores: scores,
                input: item.input,
                output: agentOutput
            };

            await experiment.log(experimentItem);
        });
    }
}

prompt = literalai_client.api.get_prompt(name="MY_PROMPT", version=0)

experiment = literalai_client.api.create_experiment(
    prompt_id=prompt.id,
    name="Foo",
    params=[{"foo": "bar"}]  # optional
)

# Run the experiment the same way as before

const promptName = 'MY_PROMPT';
const promptVersion = 0
prompt = literalAiClient.api.getPrompt(promptName, promptVersion);

async function main() {
    const experiment = await literalaiClient.api.createExperiment({
        promptId: prompt.id,
        name: 'Foo',
        params: { foo: 'bar' } // optional
    });
    // Run the experiment the same way as before
}