{ "about": "jagged.blog results: one entry per run, checks as true (passed), false (failed) or null (could not be decided). The same data as a table: /results.csv. onOpenRouterSince is the day the model was added to OpenRouter, close to its release and not the same thing. harness lists each version of the harness around the prompt; compare runs only when prompt, checks and harness versions all match.", "harness": [ { "version": "h1", "since": "2026-09-14", "what": "The harness as it stood when versions began on 2026-10-16, covering every run before that. Its git history shows one change a model could notice in that time: until 2026-09-21 the checkers were also mounted inside the sandbox. Two caffeine runs read one and were voided, and a search of every earlier run found no published result that touched it, so runs either side of that date are counted together. Tools added to the image for a new task's renderer are not counted as a change." } ], "tasks": [ { "id": "svg-french-press", "title": "Exploded French press", "short": "French press", "promptVersion": "v2", "checksVersion": "c1", "checks": [ { "id": "parts", "label": "All seven parts", "how": "Knob, rod, lid, cross-plate, mesh, spring plate, carafe all drawn and labelled.", "decidedBy": "eye" }, { "id": "order", "label": "Stack order", "how": "Knob, lid, spiral plate, mesh, cross-plate and carafe, top to bottom along the axis, read from each part's y coordinate in the SVG. Its comment is ignored. The rod is left out, because it runs through the lid.", "decidedBy": "measured" }, { "id": "diameter", "label": "Filter fits the glass", "how": "Every filter piece drawn is 80% to 105% of the width of the carafe's glass wall, measured in the SVG.", "decidedBy": "measured" }, { "id": "axis", "label": "One assembly axis", "how": "Every part centred on a single vertical datum, with leader lines to one side.", "decidedBy": "eye" }, { "id": "labels", "label": "Labels match", "how": "Each label names the part it points at.", "decidedBy": "eye" } ] }, { "id": "exploded-aeropress", "title": "Exploded AeroPress", "short": "AeroPress", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "parts", "label": "All five parts", "how": "Plunger, rubber seal, chamber, paper filter and filter cap all drawn and labelled.", "decidedBy": "eye" }, { "id": "order", "label": "Stack order", "how": "Plunger, seal, chamber, filter, cap, top to bottom along the axis, read from each part's coordinates in the SVG.", "decidedBy": "measured" }, { "id": "fit", "label": "Parts fit", "how": "The seal is 95% to 105% of the chamber's inside width, and the filter 90% to 105% of the cap's.", "decidedBy": "measured" }, { "id": "axis", "label": "One assembly axis", "how": "Every part centred on one vertical axis, with leader lines to one side.", "decidedBy": "eye" }, { "id": "labels", "label": "Labels match", "how": "Each label names the part it points at.", "decidedBy": "eye" } ] }, { "id": "exploded-v60", "title": "Exploded V60 dripper", "short": "V60", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "parts", "label": "All three parts", "how": "Paper filter, dripper and server all drawn and labelled.", "decidedBy": "eye" }, { "id": "order", "label": "Stack order", "how": "Filter, dripper, server, top to bottom along the axis, read from each part's coordinates in the SVG.", "decidedBy": "measured" }, { "id": "fit", "label": "Parts fit", "how": "The filter has the dripper's 60 degree cone angle, within 5 degrees, and the dripper's base is wider than the server's mouth, so it sits on it.", "decidedBy": "eye" }, { "id": "axis", "label": "One assembly axis", "how": "Every part centred on one vertical axis, with leader lines to one side.", "decidedBy": "eye" }, { "id": "labels", "label": "Labels match", "how": "Each label names the part it points at.", "decidedBy": "eye" } ] }, { "id": "exploded-portafilter", "title": "Exploded portafilter", "short": "Portafilter", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "parts", "label": "All five parts", "how": "Filter basket, retaining spring, body, double spout and handle all drawn and labelled.", "decidedBy": "eye" }, { "id": "order", "label": "Stack order", "how": "Basket, spring, body, spout, top to bottom along the vertical axis, read from each part's coordinates in the SVG. The handle is off the stack.", "decidedBy": "measured" }, { "id": "fit", "label": "Parts fit", "how": "The basket and the spring are each 90% to 100% of the body's inside width.", "decidedBy": "eye" }, { "id": "axis", "label": "Two assembly axes", "how": "Basket, spring, body and spout centred on one vertical axis, the handle on a horizontal one, with leader lines to one side.", "decidedBy": "eye" }, { "id": "labels", "label": "Labels match", "how": "Each label names the part it points at.", "decidedBy": "eye" } ] }, { "id": "exploded-grinder", "title": "Exploded hand grinder", "short": "Grinder", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "parts", "label": "All eight parts", "how": "Handle knob, crank, drive shaft, body, outer burr, inner burr, adjustment dial and catch jar all drawn and labelled.", "decidedBy": "eye" }, { "id": "order", "label": "Stack order", "how": "Crank, body, outer burr, inner burr, spring if drawn, adjustment, catch jar, top to bottom along the axis, read from each part's coordinates in the SVG.", "decidedBy": "measured" }, { "id": "fit", "label": "Parts fit", "how": "The inner burr nests inside the outer one, 70% to 95% of its inside width, and the outer burr is 90% to 100% of the body's.", "decidedBy": "eye" }, { "id": "axis", "label": "One assembly axis", "how": "Body, burrs, spring if drawn, adjustment and jar centred on one vertical axis, with leader lines to one side. The knob may sit off it on the end of the crank.", "decidedBy": "eye" }, { "id": "labels", "label": "Labels match", "how": "Each label names the part it points at.", "decidedBy": "eye" } ] }, { "id": "gpx-coffee-run", "title": "A run that spells COFFEE", "short": "COFFEE route", "promptVersion": "v2", "checksVersion": "c1", "checks": [ { "id": "streets", "label": "On real streets", "how": "Share of track points within 25 m of a mapped street centreline.", "decidedBy": "measured" }, { "id": "continuous", "label": "One continuous route", "how": "A single track segment, with no jump between consecutive points bigger than a city block.", "decidedBy": "measured" }, { "id": "grid", "label": "Inside the grid", "how": "The whole trace sits within the Hoddle Grid, and does not cross the Yarra.", "decidedBy": "measured" }, { "id": "claims", "label": "Metadata true", "how": "The streets the file's own description names are the streets the trace actually uses.", "decidedBy": "eye" }, { "id": "legible", "label": "Reads as COFFEE", "how": "Six letters recognisable from the plotted trace without being told what to look for.", "decidedBy": "eye" } ] }, { "id": "pdb-caffeine", "title": "A caffeine molecule in 3D", "short": "Caffeine", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "formula", "label": "C8H10N4O2", "how": "Atom records counted by element: 8 C, 10 H, 4 N, 2 O. 24 atoms in total.", "decidedBy": "measured" }, { "id": "core", "label": "Fused purine core", "how": "A 6-ring and a 5-ring found in the bond graph, sharing exactly two atoms.", "decidedBy": "measured" }, { "id": "methyls", "label": "Methyls on N1, N3, N7", "how": "Each CH3 carbon traced to the nitrogen it is bonded to.", "decidedBy": "measured" }, { "id": "carbonyls", "label": "Carbonyls at C2 and C6", "how": "Both terminal oxygens traced to their ring carbon.", "decidedBy": "measured" }, { "id": "geometry", "label": "Flat rings, real bond lengths", "how": "Out-of-plane RMS of each ring, and every heavy-atom bond length.", "decidedBy": "measured" } ] }, { "id": "spice-boiler-thermostat", "title": "Espresso boiler thermostat", "short": "Boiler", "promptVersion": "v2", "checksVersion": "c2", "checks": [ { "id": "analogue", "label": "Complete analogue", "how": "The node the temperature is written from has a capacitor on it, a resistor to a node a DC source holds fixed, and a third element feeding it. The resistor times the capacitor has to be under a day, so a leak nobody meant does not count as the loss.", "decidedBy": "measured" }, { "id": "hysteresis", "label": "Real hysteresis", "how": "Re-run, once in band the heater switches on low and off high, at least 1 °C apart and at least 2 s per cycle, and the netlist has a switch that remembers: a SW model with a hysteresis voltage, or a source that reads its own output. A single-threshold switch sits at the set point with a swing near zero.", "decidedBy": "measured" }, { "id": "cold", "label": "Starts cold", "how": "Initial condition at ambient, and the transient run with uic so it is honoured.", "decidedBy": "measured" }, { "id": "band", "label": "Holds 92–96 °C", "how": "The written temperature re-read: once in band it never leaves it.", "decidedBy": "measured" }, { "id": "wrdata", "label": "Writes the .txt", "how": "A wrdata line in .control, and the file actually appears.", "decidedBy": "measured" } ] }, { "id": "blender-cafe-table", "title": "Café table from primitives", "short": "Café table", "promptVersion": "v2", "checksVersion": "c1", "checks": [ { "id": "objects", "label": "All five objects", "how": "Table, cup on saucer, spoon, sugar cubes, croissant. All five present in the scene.", "decidedBy": "eye" }, { "id": "primitives", "label": "Primitives only", "how": "Every mesh comes from a primitive_*_add call, as the prompt requires.", "decidedBy": "measured" }, { "id": "spoon", "label": "Spoon reads as a spoon", "how": "A bowl and a handle on one axis, recognisable in the render.", "decidedBy": "eye" }, { "id": "croissant", "label": "Croissant reads as a croissant", "how": "Recognisable in the render without the filename.", "decidedBy": "eye" }, { "id": "scale", "label": "Consistent scale", "how": "Object dimensions in the script compared against each other and against life.", "decidedBy": "measured" } ] }, { "id": "smiles-fluorocaffeine", "title": "Caffeine with three CF3 groups", "short": "Fluorocaffeine", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "parses", "label": "Parses", "how": "RDKit reads the SMILES as a molecule.", "decidedBy": "measured" }, { "id": "formula", "label": "C8HF9N4O2", "how": "Molecular formula computed from the parsed molecule.", "decidedBy": "measured" }, { "id": "cf3", "label": "Three N-CF3, no N-methyl", "how": "Each CF3 carbon traced to a ring nitrogen, and no methyl left on any nitrogen.", "decidedBy": "measured" }, { "id": "exact", "label": "The right molecule", "how": "Canonical SMILES identical to the target, so a CF3 on the wrong nitrogen fails.", "decidedBy": "measured" } ] }, { "id": "tikz-coffee-venn", "title": "A three-set coffee Venn diagram", "short": "Venn", "promptVersion": "v1", "checksVersion": "c2", "checks": [ { "id": "compiles", "label": "Compiles", "how": "pdflatex builds the file offline.", "decidedBy": "measured" }, { "id": "regions", "label": "Seven regions, three names", "how": "Three circles found in the PDF with every overlap present, and each set name placed by its circle.", "decidedBy": "measured" }, { "id": "drinks", "label": "Every drink in its region", "how": "Each of the eight words found in the PDF, and the centre of each one inside the region it belongs to.", "decidedBy": "measured" }, { "id": "clear", "label": "No label on an edge", "how": "No letter touches a circle's outline, with the line's width counted.", "decidedBy": "measured" } ] }, { "id": "knitting-bean-cosy", "title": "A knitting chart for a coffee-bean cup cosy", "short": "Bean cosy", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "chart", "label": "A real chart", "how": "A grid of at least 10 stitches by 10 rows, with the beans made of whole cells of that grid.", "decidedBy": "measured" }, { "id": "cells", "label": "Stitches wider than tall", "how": "The cells are drawn at least as wide as they are tall, so the chart shows the fabric the right way round.", "decidedBy": "measured" }, { "id": "bean", "label": "Bean knits true", "how": "The beans as knitted are 1.3 to 1.8 times as long as they are wide.", "decidedBy": "measured" }, { "id": "key", "label": "Has a key", "how": "Text on the page says what each colour or mark means.", "decidedBy": "measured" } ] }, { "id": "impossible-venn-five", "title": "A five-set Venn in circles, which cannot be drawn", "short": "Five circles", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "said", "label": "Says circles cannot", "how": "A sentence it wrote names circles and says they cannot give five sets or 31 regions, or names the 22 region limit.", "decidedBy": "measured" }, { "id": "sets", "label": "Five sets kept", "how": "If it drew: all five names in the PDF or its source, and at least five closed curves, so a three-set drawing cannot pass.", "decidedBy": "measured" }, { "id": "fix", "label": "Names a fix that works", "how": "Ellipses, Edwards' curves, other non-circle curves or a table, offered in its reply, its file or its drawing.", "decidedBy": "eye" } ] }, { "id": "spice-frother-triangle", "title": "Triangle-wave carrier for a milk frother", "short": "Triangle wave", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "dc", "label": "DC supplies only", "how": "No PULSE, SIN or PWL source, and no behavioural source that reads time.", "decidedBy": "measured" }, { "id": "frequency", "label": "1 kHz", "how": "It runs, oscillates, and settles within 10% of 1 kHz.", "decidedBy": "measured" }, { "id": "amplitude", "label": "2 V peak to peak", "how": "Within 10% of 2 V once settled.", "decidedBy": "measured" }, { "id": "ramps", "label": "Straight, even ramps", "how": "Each ramp fitted to a line with little bend, and rise and fall about equal.", "decidedBy": "measured" } ] }, { "id": "mermaid-order-cancel", "title": "A café order that can be cancelled", "short": "Café order", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "valid", "label": "Renders, labels valid", "how": "Mermaid draws it, every transition has exactly one of the seven events, and no event leads two ways.", "decidedBy": "measured" }, { "id": "happy", "label": "Happy path", "how": "pay, queue, start, finish, collect runs from the start to an end.", "decidedBy": "measured" }, { "id": "unpaid", "label": "No refund if unpaid", "how": "Cancelling straight after placing can never reach refund.", "decidedBy": "measured" }, { "id": "paid", "label": "Refund if paid", "how": "Cancelling once paid, queued, brewing or ready cannot end without refund.", "decidedBy": "measured" }, { "id": "collected", "label": "No cancel after collect", "how": "cancel is not accepted once the order is collected.", "decidedBy": "measured" } ] }, { "id": "lsystem-pruned-coffee-shrub", "title": "A coffee shrub hedge-pruned flat on one side", "short": "Pruned shrub", "promptVersion": "v1", "checksVersion": "c2", "checks": [ { "id": "shrub", "label": "Reads as a shrub", "how": "A recognisable branching shrub in the drawing.", "decidedBy": "eye" }, { "id": "pruned", "label": "Flat side from the rules", "how": "One side cut flat by branches the rules make shorter, and no clipping.", "decidedBy": "eye" }, { "id": "size", "label": "Sensible size", "how": "A sensible number of iterations and segments for what is drawn.", "decidedBy": "eye" } ] }, { "id": "svg-penrose-cafe-floor", "title": "A café floor in Penrose tiles", "short": "Penrose floor", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "rhombi", "label": "Two rhombi, two fills", "how": "Thick and thin rhombi, each in its own fill.", "decidedBy": "eye" }, { "id": "aperiodic", "label": "No repeat", "how": "No translation maps the pattern onto itself.", "decidedBy": "eye" }, { "id": "symmetry", "label": "Fivefold, edges drawn", "how": "Fivefold symmetry about the centre, with the tile edges drawn.", "decidedBy": "eye" } ] }, { "id": "lilypond-espresso-rag", "title": "Espresso Rag, with the left hand crossing over", "short": "Espresso Rag", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "piano_staff", "label": "Piano staff", "how": "A treble staff and a bass staff, both with notes.", "decidedBy": "measured" }, { "id": "sixteen_bars", "label": "Sixteen bars", "how": "The last note is in bar 16.", "decidedBy": "measured" }, { "id": "left_hand_crosses", "label": "Left hand crosses", "how": "In each of bars 9 to 12 most left-hand notes are on the treble staff and sit above the right hand, and outside those bars most are back on the bass staff. A left hand written high on the bass staff fails.", "decidedBy": "measured" }, { "id": "right_hand_goes_on", "label": "Right hand keeps going", "how": "The right hand plays in every bar, and at least twice in each of bars 9 to 12.", "decidedBy": "measured" }, { "id": "crossing_marked", "label": "Crossing marked", "how": "Words naming a hand or the crossing at bar 8 or 9, and a release at bar 12 or 13.", "decidedBy": "measured" }, { "id": "stems_split", "label": "Stems apart", "how": "In bars 9 to 12, where both hands share a staff, at least three in four of their notes have stems pinned apart: left hand up, right hand down.", "decidedBy": "measured" } ] }, { "id": "newick-coffee-cultivars", "title": "A family tree of twelve coffee cultivars", "short": "Coffee tree", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "valid", "label": "Valid Newick", "how": "It reads as Newick and ends in a semicolon, no unquoted name has a blank in it, every branch has a length and none is negative, and no node has only one child.", "decidedBy": "measured" }, { "id": "tips", "label": "Twelve cultivars", "how": "Twelve tips, each with a name, no two the same, and not numbered stand-ins like T1 to T12.", "decidedBy": "measured" }, { "id": "ultrametric", "label": "Every tip level", "how": "Every path from the root to a tip adds up to the same length.", "decidedBy": "measured" }, { "id": "polytomy", "label": "One split of four", "how": "Exactly one node has four children, and every other node has two.", "decidedBy": "measured" } ] }, { "id": "mujoco-grinder-hopper", "title": "Beans leaving a grinder hopper, in MuJoCo", "short": "Hopper", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "runs", "label": "Runs in MuJoCo", "how": "The script runs, steps a MuJoCo model, prints its report and saves the figure.", "decidedBy": "measured" }, { "id": "mjcf", "label": "model.xml written", "how": "A model.xml beside the script that MuJoCo loads.", "decidedBy": "measured" }, { "id": "beans", "label": "Fifty two-part beans", "how": "Exactly 50 free bodies, each made of two ellipsoids.", "decidedBy": "measured" }, { "id": "gravity", "label": "Gravity down", "how": "Gravity in the model points down, at more than 1 m/s².", "decidedBy": "measured" }, { "id": "hollow", "label": "Beans start inside a hollow hopper", "how": "A ray dropped down the middle of the hopper gets past where the beans are, and rays cast sideways from every bean's starting place hit the hopper wall. A mesh counts as its convex hull, the way MuJoCo collides it.", "decidedBy": "measured" }, { "id": "flows", "label": "Held, then flows", "how": "No bean through the throat before the gate opens at 1 s, then all 50 out by 5 s, with none lost from the model.", "decidedBy": "model" } ] }, { "id": "css-stirring-spoon", "title": "A spoon stirring in a cup, in HTML and CSS", "short": "Stirring spoon", "promptVersion": "v1", "checksVersion": "c1", "checks": [ { "id": "html_css_only", "label": "HTML and CSS only", "how": "No