Linking a prompt to its generations
Passing the prompt object to a generation records its name and version on the trace. Langfuse then compares latency, cost and quality per prompt version.
A version is only useful if you can tell how it did. That needs every generation to say which prompt version produced it, which the SDK does when you hand it the prompt.
langfuse.create_prompt(
name="desk-system",
type="text",
prompt="You answer customers of a small online shop in one short sentence.",
labels=["production"],
)The desk's prompt, with the shop filled in so it has no variables.
prompt = langfuse.get_prompt("desk-system")
@observe(name="write-reply", as_type="generation")
def ask_model(messages):
text, usage = reply(messages)
langfuse.update_current_generation(model="shop-model", usage_details=usage, prompt=prompt)
return text
ask_model([{"role": "system", "content": prompt.compile()}, {"role": "user", "content": "Hello?"}])
langfuse.flush()
attributes = local_langfuse.SPANS[0]["attributes"]
print({key: value for key, value in attributes.items() if ".prompt." in key})View the code here
from langfuse import Langfuse, observe
import local_langfuse
url = local_langfuse.start()
langfuse = Langfuse(public_key="pk-lf-local", secret_key="sk-lf-local", base_url=url)
from shop_model import reply
langfuse.create_prompt(
name="desk-system",
type="text",
prompt="You answer customers of a small online shop in one short sentence.",
labels=["production"],
)
prompt = langfuse.get_prompt("desk-system")
@observe(name="write-reply", as_type="generation")
def ask_model(messages):
text, usage = reply(messages)
langfuse.update_current_generation(model="shop-model", usage_details=usage, prompt=prompt)
return text
ask_model([{"role": "system", "content": prompt.compile()}, {"role": "user", "content": "Hello?"}])
langfuse.flush()
attributes = local_langfuse.SPANS[0]["attributes"]
print({key: value for key, value in attributes.items() if ".prompt." in key})
"""A stand-in chat model for the shop's support desk: no key, no network."""
import re
def reply(messages):
"""Answer the last message, and count words as tokens the way a chat model reports usage."""
last = messages[-1]["content"]
lookup = re.search(r"Lookup: (\S+) (.+)", last)
if "refund" in last.lower():
text = "Refunds need approval, so a person will review your request."
elif lookup and lookup.group(2) == "not found":
text = f"I could not find order {lookup.group(1)}. Please check the number."
elif lookup:
text = f"Order {lookup.group(1)} {lookup.group(2)}."
else:
text = "Could you send your order number?"
used = sum(len(message["content"].split()) for message in messages)
return text, {"input": used, "output": len(text.split())}
"""A stand-in for the Langfuse server, on your own machine.
The Langfuse SDK sends observations to /api/public/otel/v1/traces and uses a
REST API for scores, prompts and datasets. This server answers those requests
on 127.0.0.1 and keeps what it receives in lists you can print. Given a model
function, it also answers like OpenAI's chat API. start() blocks every
connection to another computer for the rest of the program.
"""
import json
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, unquote, urlparse
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
SPANS, SCORES, REQUESTS = [], [], [] # observations, scores, "METHOD /path" of each request
PROMPTS, DATASETS = {}, {} # prompt name -> its versions, dataset name -> dataset
MODEL = None # answers /v1/chat/completions when set
NOW = "2026-03-03T09:00:00Z"
def _value(v):
kind = v.WhichOneof("value")
if kind == "array_value":
return [_value(item) for item in v.array_value.values]
return getattr(v, kind) if kind else None
def _spans(body):
request = ExportTraceServiceRequest()
request.ParseFromString(body)
for resource in request.resource_spans:
for scope in resource.scope_spans:
for span in scope.spans:
SPANS.append({
"name": span.name, "id": span.span_id.hex(), "trace": span.trace_id.hex(),
"parent": span.parent_span_id.hex(), "scope": scope.scope.name,
"attributes": {a.key: _value(a.value) for a in span.attributes},
})
return {}
def _scores(batch):
for event in batch:
if event["type"] == "score-create":
SCORES.append(event["body"])
return {"successes": [{"id": e["id"], "status": 201} for e in batch], "errors": []}
def _prompt(name, query, body):
versions = PROMPTS.setdefault(name, [])
if body is not None: # a new version: it takes "latest" and any labels it asks for
labels = body.get("labels", []) + ["latest"]
for old in versions:
old["labels"] = [label for label in old["labels"] if label not in labels]
versions.append({"name": name, "type": body.get("type", "text"), "prompt": body["prompt"],
"config": body.get("config") or {}, "labels": labels, "tags": [],
"version": len(versions) + 1})
return versions[-1]
if "version" in query:
return versions[int(query["version"]) - 1]
label = query.get("label", "production")
return next(v for v in versions if label in v["labels"])
def _relabel(path, body):
name, _, version = path.split("/")[-3:]
versions = PROMPTS[unquote(name)]
for old in versions:
old["labels"] = [label for label in old["labels"] if label not in body["newLabels"]]
versions[int(version) - 1]["labels"] += body["newLabels"]
return versions[int(version) - 1]
def _dataset(path, query, body):
if path == "/api/public/v2/datasets":
DATASETS[body["name"]] = {"id": body["name"], "name": body["name"], "items": [],
"metadata": body.get("metadata"), "projectId": "local",
"createdAt": NOW, "updatedAt": NOW}
return DATASETS[body["name"]]
if path.startswith("/api/public/v2/datasets/"):
return DATASETS[unquote(path.rsplit("/", 1)[1])]
if path == "/api/public/dataset-items" and body is not None:
dataset = DATASETS[body["datasetName"]]
item = {"id": body.get("id") or f"item-{len(dataset['items']) + 1}", "status": "ACTIVE",
"input": body.get("input"), "expectedOutput": body.get("expectedOutput"),
"metadata": body.get("metadata"), "sourceTraceId": body.get("sourceTraceId"),
"datasetId": dataset["id"], "datasetName": dataset["name"],
"createdAt": NOW, "updatedAt": NOW, "mediaReferences": []}
dataset["items"].append(item)
return item
if path == "/api/public/dataset-items":
items = DATASETS[query["datasetName"]]["items"]
return {"data": items, "meta": {"page": 1, "limit": 50, "totalItems": len(items), "totalPages": 1}}
return {"id": "run-item", "datasetRunId": body["runName"], "datasetRunName": body["runName"],
"datasetItemId": body["datasetItemId"], "traceId": body["traceId"],
"observationId": body.get("observationId"), "createdAt": NOW, "updatedAt": NOW}
def _chat(body):
text, usage = MODEL(body["messages"])
return {"id": "chatcmpl-1", "object": "chat.completion", "created": 0, "model": body["model"],
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}],
"usage": {"prompt_tokens": usage["input"], "completion_tokens": usage["output"],
"total_tokens": usage["input"] + usage["output"]}}
class Handler(BaseHTTPRequestHandler):
def _handle(self):
url = urlparse(self.path)
query = {key: values[0] for key, values in parse_qs(url.query).items()}
raw = self.rfile.read(int(self.headers.get("Content-Length") or 0))
body = json.loads(raw) if raw and "json" in self.headers.get("Content-Type", "") else None
REQUESTS.append(f"{self.command} {url.path}")
try:
if url.path.endswith("/otel/v1/traces"):
return self._reply(200, _spans(raw))
if url.path == "/api/public/ingestion":
return self._reply(207, _scores(body["batch"]))
if url.path == "/api/public/projects":
return self._reply(200, {"data": [{"id": "local", "name": "shop-desk", "metadata": {},
"organization": {"id": "local", "name": "local"}}]})
if url.path.startswith("/api/public/v2/prompts/") and self.command == "PATCH":
return self._reply(200, _relabel(url.path, body))
if url.path.startswith("/api/public/v2/prompts"):
name = body["name"] if body else unquote(url.path.rsplit("/", 1)[1])
return self._reply(200, _prompt(name, query, body))
if "dataset" in url.path:
return self._reply(200, _dataset(url.path, query, body))
if url.path.endswith("/chat/completions") and MODEL:
return self._reply(200, _chat(body))
except (KeyError, StopIteration):
pass
self._reply(404, {"message": f"not found: {url.path}"})
do_GET = do_POST = do_PATCH = _handle
def _reply(self, code, data):
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(data).encode())
def log_message(self, *args):
pass
def _local_only():
"""Refuse to look up or connect to any other computer, so nothing leaves by mistake."""
lookup, connect = socket.getaddrinfo, socket.socket.connect
def guarded_lookup(host, *args, **kwargs):
if host not in ("127.0.0.1", "localhost", "::1"):
raise ConnectionRefusedError(f"blocked a connection to {host}: this lab stays on your machine")
return lookup(host, *args, **kwargs)
def guarded_connect(sock, address):
if isinstance(address, tuple) and address[0] not in ("127.0.0.1", "::1"):
raise ConnectionRefusedError(f"blocked a connection to {address[0]}: this lab stays on your machine")
return connect(sock, address)
socket.getaddrinfo, socket.socket.connect = guarded_lookup, guarded_connect
def start(model=None):
"""Start the server in the background and return its address."""
global MODEL
_local_only()
MODEL = model
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
return f"http://127.0.0.1:{server.server_address[1]}"
def tree(*fields):
"""Print the observations received so far, each under its parent, with any fields asked for."""
ids = {span["id"] for span in SPANS}
def walk(span, depth):
attributes = span["attributes"]
line = " " * depth + f"{span['name']} ({attributes.get('langfuse.observation.type', 'span')})"
for field in fields:
line += f" {field}={attributes.get('langfuse.observation.' + field)!r}"
print(line)
for child in SPANS:
if child["parent"] == span["id"]:
walk(child, depth + 1)
for root in [span for span in SPANS if span["parent"] not in ids]:
walk(root, 0)
from langfuse import Langfuse
import local_langfuse
url = local_langfuse.start()
langfuse = Langfuse(public_key="pk-lf-local", secret_key="sk-lf-local", base_url=url)
import random
langfuse.create_prompt(name="desk-ab", prompt="Answer in one short sentence.", labels=["prod-a"])
langfuse.create_prompt(name="desk-ab", prompt="Answer briefly. Thank the customer.", labels=["prod-b"])
variants = [langfuse.get_prompt("desk-ab", label="prod-a"), langfuse.get_prompt("desk-ab", label="prod-b")]
random.seed(3)
for ticket in ["Where is A17?", "Is B22 late?", "Hello?", "Where is A17 now?"]:
prompt = random.choice(variants)
print(ticket, "-> version", prompt.version)
python linked.py{'langfuse.observation.prompt.name': 'desk-system', 'langfuse.observation.prompt.version': 1}The generation carries the prompt's name and version as two attributes. That link is what Langfuse's metrics for each prompt version are built from: median latency, tokens and cost, and the scores that part 6 adds. With the OpenAI wrapper from lesson 7 the same link is one keyword, langfuse_prompt=prompt, on the chat call; the project in lesson 30 uses it.
Only generations are linked, and a fallback prompt is never linked, because it is not a version stored in Langfuse. propagate_attributes(prompt=prompt) links every generation started inside it, for code where the generation is created by another library.
Two versions at once
Linking also makes an A/B test possible: two versions carry two labels, each request picks one, and Langfuse compares their metrics.
import random
langfuse.create_prompt(name="desk-ab", prompt="Answer in one short sentence.", labels=["prod-a"])
langfuse.create_prompt(name="desk-ab", prompt="Answer briefly. Thank the customer.", labels=["prod-b"])
variants = [langfuse.get_prompt("desk-ab", label="prod-a"), langfuse.get_prompt("desk-ab", label="prod-b")]
random.seed(3)
for ticket in ["Where is A17?", "Is B22 late?", "Hello?", "Where is A17 now?"]:
prompt = random.choice(variants)
print(ticket, "-> version", prompt.version)python ab_test.pyWhere is A17? -> version 1 Is B22 late? -> version 1 Hello? -> version 2 Where is A17 now? -> version 2
Each ticket used one of the two versions, chosen at random; random.seed only makes the choice repeat here. Linked to the generations, the two groups of traces can then be compared. The documentation suggests A/B tests after a version has passed tests on a dataset, which part 6 builds.
- Link the fallback prompt from lesson 21 and print the prompt attributes.
- Wrap the call in
propagate_attributes(prompt=prompt)instead of passingprompt=. - Choose the variant from the ticket's number, so the same customer always gets the same version.
You understood something today that you didn't yesterday.