Instrument AI Agents

// Example agent implementation for demonstration
const myAgent = {
  name: "Weather Agent",
  modelProvider: "openai",
  model: "gpt-4o-mini",
  async run() {
    // Agent implementation
    return {
      output: "The weather in Paris is sunny",
      usage: {
        inputTokens: 15,
        outputTokens: 8,
      },
    };
  },
};

Sentry.startSpan(
  {
    op: "gen_ai.invoke_agent",
    name: `invoke_agent ${myAgent.name}`,
    attributes: {
      "gen_ai.operation.name": "invoke_agent",
      "gen_ai.request.model": myAgent.model,
      "gen_ai.agent.name": myAgent.name,
    },
  },
  async (span) => {
    // run the agent
    const result = await myAgent.run();

    // set agent response
    // we assume result.output is a string
    // type of `gen_ai.response.text` needs to be a string
    span.setAttribute(
      "gen_ai.response.text",
      JSON.stringify([result.output]),
    );

    // set token usage
    // we assume the result includes the tokens used
    span.setAttribute(
      "gen_ai.usage.input_tokens",
      result.usage.inputTokens,
    );
    span.setAttribute(
      "gen_ai.usage.output_tokens",
      result.usage.outputTokens,
    );

    return result;
  },
);

// Example implementation for demonstration
const myAi = {
  modelProvider: "openai",
  model: "gpt-4o-mini",
  modelConfig: {
    temperature: 0.1,
    presencePenalty: 0.5,
  },
  async createMessage(messages, maxTokens) {
    // AI implementation
    return {
      output:
        "Here's a joke: Why don't scientists trust atoms? Because they make up everything!",
      usage: {
        inputTokens: 12,
        outputTokens: 24,
      },
    };
  },
};

Sentry.startSpan(
  {
    op: "gen_ai.chat",
    name: `chat ${myAi.model}`,
    attributes: {
      "gen_ai.operation.name": "chat",
      "gen_ai.request.model": myAi.model,
    },
  },
  async (span) => {
    // set up messages for LLM
    const maxTokens = 1024;
    const prompt = "Tell me a joke";
    const messages = [{ role: "user", content: prompt }];

    // set chat request data
    span.setAttribute("gen_ai.request.messages", JSON.stringify(messages));
    span.setAttribute("gen_ai.request.max_tokens", maxTokens);
    span.setAttribute(
      "gen_ai.request.temperature",
      myAi.modelConfig.temperature,
    );
    span.setAttribute(
      "gen_ai.request.presence_penalty",
      myAi.modelConfig.presencePenalty,
    );

    // ask the LLM
    const result = await myAi.createMessage(messages, maxTokens);

    // set response
    // we assume result.output is a string
    // type of `gen_ai.response.text` needs to be a string
    span.setAttribute(
      "gen_ai.response.text",
      JSON.stringify([result.output]),
    );

    // set token usage
    // we assume the result includes the tokens used
    span.setAttribute(
      "gen_ai.usage.input_tokens",
      result.usage.inputTokens,
    );
    span.setAttribute(
      "gen_ai.usage.output_tokens",
      result.usage.outputTokens,
    );

    return result;
  },
);

// Example implementation for demonstration
const myAi = {
  modelProvider: "openai",
  model: "gpt-4o-mini",
  async createMessage(messages, maxTokens) {
    // AI implementation that returns tool calls
    return {
      toolCalls: [
        {
          name: "random_number",
          description: "Generate a random number",
          arguments: { max: 10 },
        },
      ],
    };
  },
};

const prompt = "Generate a random number between 0 and 10";
const messages = [{ role: "user", content: prompt }];

// First, make the AI call
const result = await Sentry.startSpan(
  { op: "gen_ai.chat", name: `chat ${myAi.model}` },
  () => myAi.createMessage(messages, 1024),
);

// Check if we should call a tool
if (result.toolCalls && result.toolCalls.length > 0) {
  const tool = result.toolCalls[0];

  await Sentry.startSpan(
    {
      op: "gen_ai.execute_tool",
      name: `execute_tool ${tool.name}`,
      attributes: {
        "gen_ai.request.model": myAi.model,
        "gen_ai.tool.type": "function",
        "gen_ai.tool.name": tool.name,
        "gen_ai.tool.description": tool.description,
        "gen_ai.tool.input": JSON.stringify(tool.arguments),
      },
    },
    async (span) => {
      // run tool (example implementation)
      const toolResult = Math.floor(Math.random() * tool.arguments.max);

      // set tool result
      span.setAttribute("gen_ai.tool.output", String(toolResult));

      return toolResult;
    },
  );
}

// Example agent implementation for demonstration
const myAgent = {
  name: "Weather Agent",
  modelProvider: "openai",
  model: "gpt-4o-mini",
  async run() {
    // Agent implementation
    return {
      handoffTo: "Travel Agent",
      output:
        "I need to handoff to the travel agent for booking recommendations",
    };
  },
};

const otherAgent = {
  name: "Travel Agent",
  modelProvider: "openai",
  model: "gpt-4o-mini",
  async run() {
    // Other agent implementation
    return { output: "Here are some travel recommendations..." };
  },
};

// First agent execution
const result = await Sentry.startSpan(
  { op: "gen_ai.invoke_agent", name: `invoke_agent ${myAgent.name}` },
  () => myAgent.run(),
);

// Check if we should handoff to another agent
if (result.handoffTo) {
  // Create handoff span
  await Sentry.startSpan(
    {
      op: "gen_ai.handoff",
      name: `handoff from ${myAgent.name} to ${otherAgent.name}`,
      attributes: {
        "gen_ai.request.model": myAgent.model,
      },
    },
    () => {
      // the handoff span just marks the handoff
      // no actual work is done here
    },
  );

  // Execute the other agent
  await Sentry.startSpan(
    { op: "gen_ai.invoke_agent", name: `invoke_agent ${otherAgent.name}` },
    () => otherAgent.run(),
  );
}