Tracing OpenAI calls without changing them
Importing OpenAI from langfuse.openai instead of openai records every chat call as a generation, with the messages, model, parameters and token usage.
View the code here
"""A stand-in chat model for the shop's support desk: no key, no network."""
import re
def reply(messages):
"""Answer the last message, and count words as tokens the way a chat model reports usage."""
last = messages[-1]["content"]
lookup = re.search(r"Lookup: (\S+) (.+)", last)
if "refund" in last.lower():
text = "Refunds need approval, so a person will review your request."
elif lookup and lookup.group(2) == "not found":
text = f"I could not find order {lookup.group(1)}. Please check the number."
elif lookup:
text = f"Order {lookup.group(1)} {lookup.group(2)}."
else:
text = "Could you send your order number?"
used = sum(len(message["content"].split()) for message in messages)
return text, {"input": used, "output": len(text.split())}
from langfuse import Langfuse, observe
from langfuse.openai import OpenAI
import local_langfuse
from shop_model import reply
url = local_langfuse.start(model=reply)
langfuse = Langfuse(public_key="pk-lf-local", secret_key="sk-lf-local", base_url=url)
client = OpenAI(base_url=f"{url}/v1", api_key="not-a-real-key")
import re
ORDERS = {"A17": "shipped on 3 March"}
SYSTEM = "You answer customers of a small online shop in one short sentence."
@observe(name="lookup-order", as_type="tool")
def lookup_order(order_id):
return ORDERS.get(order_id, "not found")
def ask_model(messages):
response = client.chat.completions.create(model="shop-model", messages=messages, name="write-reply")
return response.choices[0].message.content
@observe(name="answer-ticket", as_type="agent")
def answer(ticket):
order = re.search(r"[A-Z]\d{2}", ticket)
facts = f"\nLookup: {order.group()} {lookup_order(order.group())}" if order else ""
return ask_model([{"role": "system", "content": SYSTEM}, {"role": "user", "content": ticket + facts}])
print(answer("Where is my order A17?"))
langfuse.flush()
local_langfuse.tree()
print(local_langfuse.REQUESTS)
generation = next(span for span in local_langfuse.SPANS if span["name"] == "write-reply")
print(generation["attributes"]["langfuse.observation.usage_details"])
"""A stand-in for the Langfuse server, on your own machine.
The Langfuse SDK sends observations to /api/public/otel/v1/traces and uses a
REST API for scores, prompts and datasets. This server answers those requests
on 127.0.0.1 and keeps what it receives in lists you can print. Given a model
function, it also answers like OpenAI's chat API. start() blocks every
connection to another computer for the rest of the program.
"""
import json
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, unquote, urlparse
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
SPANS, SCORES, REQUESTS = [], [], [] # observations, scores, "METHOD /path" of each request
PROMPTS, DATASETS = {}, {} # prompt name -> its versions, dataset name -> dataset
MODEL = None # answers /v1/chat/completions when set
NOW = "2026-03-03T09:00:00Z"
def _value(v):
kind = v.WhichOneof("value")
if kind == "array_value":
return [_value(item) for item in v.array_value.values]
return getattr(v, kind) if kind else None
def _spans(body):
request = ExportTraceServiceRequest()
request.ParseFromString(body)
for resource in request.resource_spans:
for scope in resource.scope_spans:
for span in scope.spans:
SPANS.append({
"name": span.name, "id": span.span_id.hex(), "trace": span.trace_id.hex(),
"parent": span.parent_span_id.hex(), "scope": scope.scope.name,
"attributes": {a.key: _value(a.value) for a in span.attributes},
})
return {}
def _scores(batch):
for event in batch:
if event["type"] == "score-create":
SCORES.append(event["body"])
return {"successes": [{"id": e["id"], "status": 201} for e in batch], "errors": []}
def _prompt(name, query, body):
versions = PROMPTS.setdefault(name, [])
if body is not None: # a new version: it takes "latest" and any labels it asks for
labels = body.get("labels", []) + ["latest"]
for old in versions:
old["labels"] = [label for label in old["labels"] if label not in labels]
versions.append({"name": name, "type": body.get("type", "text"), "prompt": body["prompt"],
"config": body.get("config") or {}, "labels": labels, "tags": [],
"version": len(versions) + 1})
return versions[-1]
if "version" in query:
return versions[int(query["version"]) - 1]
label = query.get("label", "production")
return next(v for v in versions if label in v["labels"])
def _relabel(path, body):
name, _, version = path.split("/")[-3:]
versions = PROMPTS[unquote(name)]
for old in versions:
old["labels"] = [label for label in old["labels"] if label not in body["newLabels"]]
versions[int(version) - 1]["labels"] += body["newLabels"]
return versions[int(version) - 1]
def _dataset(path, query, body):
if path == "/api/public/v2/datasets":
DATASETS[body["name"]] = {"id": body["name"], "name": body["name"], "items": [],
"metadata": body.get("metadata"), "projectId": "local",
"createdAt": NOW, "updatedAt": NOW}
return DATASETS[body["name"]]
if path.startswith("/api/public/v2/datasets/"):
return DATASETS[unquote(path.rsplit("/", 1)[1])]
if path == "/api/public/dataset-items" and body is not None:
dataset = DATASETS[body["datasetName"]]
item = {"id": body.get("id") or f"item-{len(dataset['items']) + 1}", "status": "ACTIVE",
"input": body.get("input"), "expectedOutput": body.get("expectedOutput"),
"metadata": body.get("metadata"), "sourceTraceId": body.get("sourceTraceId"),
"datasetId": dataset["id"], "datasetName": dataset["name"],
"createdAt": NOW, "updatedAt": NOW, "mediaReferences": []}
dataset["items"].append(item)
return item
if path == "/api/public/dataset-items":
items = DATASETS[query["datasetName"]]["items"]
return {"data": items, "meta": {"page": 1, "limit": 50, "totalItems": len(items), "totalPages": 1}}
return {"id": "run-item", "datasetRunId": body["runName"], "datasetRunName": body["runName"],
"datasetItemId": body["datasetItemId"], "traceId": body["traceId"],
"observationId": body.get("observationId"), "createdAt": NOW, "updatedAt": NOW}
def _chat(body):
text, usage = MODEL(body["messages"])
return {"id": "chatcmpl-1", "object": "chat.completion", "created": 0, "model": body["model"],
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}],
"usage": {"prompt_tokens": usage["input"], "completion_tokens": usage["output"],
"total_tokens": usage["input"] + usage["output"]}}
class Handler(BaseHTTPRequestHandler):
def _handle(self):
url = urlparse(self.path)
query = {key: values[0] for key, values in parse_qs(url.query).items()}
raw = self.rfile.read(int(self.headers.get("Content-Length") or 0))
body = json.loads(raw) if raw and "json" in self.headers.get("Content-Type", "") else None
REQUESTS.append(f"{self.command} {url.path}")
try:
if url.path.endswith("/otel/v1/traces"):
return self._reply(200, _spans(raw))
if url.path == "/api/public/ingestion":
return self._reply(207, _scores(body["batch"]))
if url.path == "/api/public/projects":
return self._reply(200, {"data": [{"id": "local", "name": "shop-desk", "metadata": {},
"organization": {"id": "local", "name": "local"}}]})
if url.path.startswith("/api/public/v2/prompts/") and self.command == "PATCH":
return self._reply(200, _relabel(url.path, body))
if url.path.startswith("/api/public/v2/prompts"):
name = body["name"] if body else unquote(url.path.rsplit("/", 1)[1])
return self._reply(200, _prompt(name, query, body))
if "dataset" in url.path:
return self._reply(200, _dataset(url.path, query, body))
if url.path.endswith("/chat/completions") and MODEL:
return self._reply(200, _chat(body))
except (KeyError, StopIteration):
pass
self._reply(404, {"message": f"not found: {url.path}"})
do_GET = do_POST = do_PATCH = _handle
def _reply(self, code, data):
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(data).encode())
def log_message(self, *args):
pass
def _local_only():
"""Refuse to look up or connect to any other computer, so nothing leaves by mistake."""
lookup, connect = socket.getaddrinfo, socket.socket.connect
def guarded_lookup(host, *args, **kwargs):
if host not in ("127.0.0.1", "localhost", "::1"):
raise ConnectionRefusedError(f"blocked a connection to {host}: this lab stays on your machine")
return lookup(host, *args, **kwargs)
def guarded_connect(sock, address):
if isinstance(address, tuple) and address[0] not in ("127.0.0.1", "::1"):
raise ConnectionRefusedError(f"blocked a connection to {address[0]}: this lab stays on your machine")
return connect(sock, address)
socket.getaddrinfo, socket.socket.connect = guarded_lookup, guarded_connect
def start(model=None):
"""Start the server in the background and return its address."""
global MODEL
_local_only()
MODEL = model
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
return f"http://127.0.0.1:{server.server_address[1]}"
def tree(*fields):
"""Print the observations received so far, each under its parent, with any fields asked for."""
ids = {span["id"] for span in SPANS}
def walk(span, depth):
attributes = span["attributes"]
line = " " * depth + f"{span['name']} ({attributes.get('langfuse.observation.type', 'span')})"
for field in fields:
line += f" {field}={attributes.get('langfuse.observation.' + field)!r}"
print(line)
for child in SPANS:
if child["parent"] == span["id"]:
walk(child, depth + 1)
for root in [span for span in SPANS if span["parent"] not in ids]:
walk(root, 0)
Lesson 6 wrote the generation by hand. Most applications call a model through a client library, and Langfuse ships drop-in versions of some of them. The OpenAI one is a changed import; the calls stay the same. The model here is still your reply, served by the local server behind an OpenAI-style API.
from langfuse import Langfuse, observe
from langfuse.openai import OpenAI
import local_langfuse
from shop_model import reply
url = local_langfuse.start(model=reply)
langfuse = Langfuse(public_key="pk-lf-local", secret_key="sk-lf-local", base_url=url)
client = OpenAI(base_url=f"{url}/v1", api_key="not-a-real-key")start(model=reply) makes the local server answer /v1/chat/completions with your function. The client's base_url points there, so the key can be any string; with a real provider it would be that provider's address and key.
import re
ORDERS = {"A17": "shipped on 3 March"}
SYSTEM = "You answer customers of a small online shop in one short sentence."
@observe(name="lookup-order", as_type="tool")
def lookup_order(order_id):
return ORDERS.get(order_id, "not found")def ask_model(messages):
response = client.chat.completions.create(model="shop-model", messages=messages, name="write-reply")
return response.choices[0].message.contentThis is an ordinary OpenAI call. name is the one addition, a keyword the Langfuse wrapper accepts to name the generation; the answer function is lesson 6's, unchanged.
print(answer("Where is my order A17?"))
langfuse.flush()
local_langfuse.tree()
print(local_langfuse.REQUESTS)
generation = next(span for span in local_langfuse.SPANS if span["name"] == "write-reply")
print(generation["attributes"]["langfuse.observation.usage_details"])python openai_desk.pyOrder A17 shipped on 3 March.
answer-ticket (agent)
lookup-order (tool)
write-reply (generation)
['POST /v1/chat/completions', 'POST /api/public/otel/v1/traces']
{"completion_tokens": 6, "prompt_tokens": 23, "total_tokens": 29, "completion_tokens_details": null, "prompt_tokens_details": null}The chat call became a generation under the agent without any Langfuse code around it. REQUESTS shows the OpenAI client's request going to the local server, then the traces export.
Usage arrived in OpenAI's own field names, prompt_tokens and completion_tokens. Langfuse maps these to its input and output when it stores them. The wrapper also recorded the full message list as input and the default model parameters, so everything in the prompt, including a customer's text, is part of the trace.
- Print the generation's
langfuse.observation.model.parametersattribute, then passtemperature=0to the call and print it again. - Call
ask_modeldirectly, outsideanswer, and see where its generation lands. - Pass
metadata={"langfuse_tags": ["orders"]}to the call and look for the tag.
You understood something today that you didn't yesterday.