Spatial feedback to the AI + manual add-board control

Spatial feedback (direction #1): scene.spatial_summary() reports each board's
world bounding box (Part.bbox) and flags interpenetrating pairs; the GUI feeds
it into the interpreter prompt. The SYSTEM prompt now tells the AI to use the
layout to position boards flush against each other (computed wood-move) and to
fix overlaps — so relative commands like "position the left side flush against
p2" work. Verified live: "stack p2 on top of p1" moved p2 onto p1's top face.

Manual add: the Parts tab gained a stock dropdown + length + "Add board" button
to place a board instantly without the AI (controller.place).

88 tests pass (bbox axis-aligned, spatial_summary flags/clears overlap).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
rob 2026-05-30 12:17:41 -03:00
parent 35adf5ee0d
commit 52be9ee5b1
6 changed files with 97 additions and 5 deletions

View File

@ -60,6 +60,12 @@ operations + interpreter:
- `numpad.py` — a numberpad control panel (2/4/6/8 move, 1/3/7/9 rotate, +/ - `numpad.py` — a numberpad control panel (2/4/6/8 move, 1/3/7/9 rotate, +/
raise/lower, 0/. front/iso, 5 fit) that also responds to the **physical raise/lower, 0/. front/iso, 5 fit) that also responds to the **physical
numpad keys** (MainWindow.keyPressEvent forwards them when not typing). numpad keys** (MainWindow.keyPressEvent forwards them when not typing).
- **Spatial feedback**: `scene.spatial_summary()` (each board's world bounding
box via `Part.bbox()` + flagged interpenetrations) is fed into the interpreter
prompt, so the AI can reason about where boards are — e.g. "position the left
side flush against p2" becomes a computed `wood-move`, and it can fix overlaps.
- **Manual add**: the Parts tab has a stock dropdown + length + "Add board" to
place a board instantly without the AI (`controller.place`).
- **Multi-selection**: `controller.selected` is a list driven by 3D Ctrl+click - **Multi-selection**: `controller.selected` is a list driven by 3D Ctrl+click
(`viewport.picked` carries an additive flag) and list multi-select. Group ops (`viewport.picked` carries an additive flag) and list multi-select. Group ops
(`move_selected`/`rotate_selected`/stand/lay/sand/delete) apply to all selected (`move_selected`/`rotate_selected`/stand/lay/sand/delete) apply to all selected

View File

@ -59,6 +59,12 @@ Rules:
- Refer to boards that ALREADY exist by their real id (p1, p2, ...) or their name. - Refer to boards that ALREADY exist by their real id (p1, p2, ...) or their name.
- For a board you place earlier in THIS response, refer to it later as $1, $2, ... - For a board you place earlier in THIS response, refer to it later as $1, $2, ...
numbered by the order you place boards in this response (the first wood-place is $1). numbered by the order you place boards in this response (the first wood-place is $1).
- A "Layout" section gives each board's bounding box (in inches). Use it to
reason about where boards are. Boards should TOUCH at faces, never overlap or
leave gaps. To position one board flush against / next to another, compute the
offset from the two bounding boxes and emit wood-move with the relative
dx/dy/dz that makes their faces meet (e.g. move so the moving board's x-max
equals the target board's x-min). Fix any "Interpenetrating" pairs the same way.
- "these" / "them" / "the selected ones" refer to the currently-selected boards - "these" / "them" / "the selected ones" refer to the currently-selected boards
listed under the scene; emit one call per selected board (e.g. wood-move for each). listed under the scene; emit one call per selected board (e.g. wood-move for each).
- Legs and uprights must be stood up: place the board, then wood-stand it. - Legs and uprights must be stood up: place the board, then wood-stand it.

View File

@ -418,10 +418,12 @@ class Controller(QObject):
def run_command(self, text: str) -> str: def run_command(self, text: str) -> str:
"""Interpret a spoken/typed command and apply it. Returns a spoken summary. """Interpret a spoken/typed command and apply it. Returns a spoken summary.
(Slow call from a worker thread.)""" (Slow call from a worker thread.)"""
from ..scene import spatial_summary
self.save() # ensure disk reflects current state self.save() # ensure disk reflects current state
sel = ", ".join(self.selected) if self.selected else "none" sel = ", ".join(self.selected) if self.selected else "none"
scene_text = (cli.cmd_status(self.scene, None) scene_text = (cli.cmd_status(self.scene, None)
+ f"\nCurrently selected ('these' / 'them' / 'the selected'): {sel}") + f"\nCurrently selected ('these' / 'them' / 'the selected'): {sel}"
+ "\n" + spatial_summary(self.scene))
calls = driver.interpret(text, self.schemas(), scene_text=scene_text) calls = driver.interpret(text, self.schemas(), scene_text=scene_text)
messages = driver.dispatch(calls, verbose=False, executor=self.execute_call) messages = driver.dispatch(calls, verbose=False, executor=self.execute_call)
self._commit() self._commit()

View File

@ -4,11 +4,12 @@ that solves "delete that" ambiguity)."""
from __future__ import annotations from __future__ import annotations
from PySide6.QtCore import Qt from PySide6.QtCore import Qt
from PySide6.QtWidgets import (QAbstractItemView, QDoubleSpinBox, QFormLayout, from PySide6.QtWidgets import (QAbstractItemView, QComboBox, QDoubleSpinBox,
QGridLayout, QGroupBox, QInputDialog, QLabel, QFormLayout, QGridLayout, QGroupBox, QHBoxLayout,
QMenu, QPushButton, QTreeWidget, QTreeWidgetItem, QInputDialog, QLabel, QMenu, QPushButton, QTreeWidget,
QVBoxLayout, QWidget) QTreeWidgetItem, QVBoxLayout, QWidget)
from ..lumber import NOMINAL_TO_ACTUAL
from .controller import Controller from .controller import Controller
@ -21,6 +22,18 @@ class PartsPanel(QWidget):
root = QVBoxLayout(self) root = QVBoxLayout(self)
root.addWidget(QLabel("<b>Parts</b> <span style='color:#888'>(connected boards group into assemblies)</span>")) root.addWidget(QLabel("<b>Parts</b> <span style='color:#888'>(connected boards group into assemblies)</span>"))
# Quick manual add — no AI needed.
add = QHBoxLayout()
self.stock = QComboBox()
self.stock.addItems(sorted(NOMINAL_TO_ACTUAL, key=lambda s: (s[0], len(s), s)))
self.stock.setCurrentText("2x4")
self.add_len = QDoubleSpinBox(); self.add_len.setRange(0.5, 480)
self.add_len.setValue(24); self.add_len.setSuffix(" in")
add_btn = QPushButton("+ Add board")
add_btn.clicked.connect(self._add_board)
add.addWidget(self.stock); add.addWidget(self.add_len); add.addWidget(add_btn)
root.addLayout(add)
self.tree = QTreeWidget() self.tree = QTreeWidget()
self.tree.setHeaderHidden(True) self.tree.setHeaderHidden(True)
self.tree.setSelectionMode(QAbstractItemView.ExtendedSelection) # Ctrl/Shift multi-select self.tree.setSelectionMode(QAbstractItemView.ExtendedSelection) # Ctrl/Shift multi-select
@ -127,6 +140,9 @@ class PartsPanel(QWidget):
ids += [it.child(i).data(0, Qt.UserRole) for i in range(it.childCount())] ids += [it.child(i).data(0, Qt.UserRole) for i in range(it.childCount())]
return list(dict.fromkeys(ids)) return list(dict.fromkeys(ids))
def _add_board(self) -> None:
self.c.place(self.stock.currentText(), self.add_len.value())
def _on_row_selected(self) -> None: def _on_row_selected(self) -> None:
if self._loading: if self._loading:
return return

View File

@ -167,6 +167,32 @@ class Feature:
return self.kind in CUT_KINDS return self.kind in CUT_KINDS
def _aabb_overlap(b1, b2, eps=0.05) -> bool:
(lo1, hi1), (lo2, hi2) = b1, b2
return all(min(hi1[i], hi2[i]) - max(lo1[i], lo2[i]) > eps for i in range(3))
def spatial_summary(scene) -> str:
"""A compact text description of where each board sits (bounding boxes) and
which boards interpenetrate fed to the AI so it can reason spatially."""
if not scene.parts:
return "empty"
boxes = {p.id: p.bbox() for p in scene.parts}
lines = ["Layout (inch bounding boxes, x=length-ish, z=up):"]
for p in scene.parts:
lo, hi = boxes[p.id]
label = f"{p.id}" + (f" ({p.name})" if p.name else "")
lines.append(f" {label}: x[{lo[0]:.1f},{hi[0]:.1f}] "
f"y[{lo[1]:.1f},{hi[1]:.1f}] z[{lo[2]:.1f},{hi[2]:.1f}]")
ids = list(boxes)
overlaps = [f"{ids[i]}&{ids[j]}" for i in range(len(ids)) for j in range(i + 1, len(ids))
if _aabb_overlap(boxes[ids[i]], boxes[ids[j]])]
if overlaps:
lines.append("Interpenetrating (usually unintended — boards should touch, not overlap): "
+ ", ".join(overlaps))
return "\n".join(lines)
@dataclass @dataclass
class Part: class Part:
id: str id: str
@ -204,6 +230,22 @@ class Part:
cl, cw, ct = self.local_frame() cl, cw, ct = self.local_frame()
return [[cl[i], cw[i], ct[i]] for i in range(3)] return [[cl[i], cw[i], ct[i]] for i in range(3)]
def bbox(self) -> tuple:
"""World axis-aligned bounding box (min_xyz, max_xyz) of the board."""
t, w = self.section_in
L = self.length_in
R = self.rotation_matrix()
corners = []
for x in (0.0, L):
for y in (-w / 2, w / 2):
for z in (-t / 2, t / 2):
loc = (x, y, z)
corners.append(tuple(self.position_in[i] + sum(R[i][j] * loc[j] for j in range(3))
for i in range(3)))
lo = tuple(min(c[i] for c in corners) for i in range(3))
hi = tuple(max(c[i] for c in corners) for i in range(3))
return lo, hi
def feature_world_frame(self, feat) -> tuple: def feature_world_frame(self, feat) -> tuple:
"""(contact point, outward normal, u, v) of a feature, in world space, """(contact point, outward normal, u, v) of a feature, in world space,
with the feature's own rotation about its normal applied to u/v.""" with the feature's own rotation about its normal applied to u/v."""

View File

@ -321,6 +321,26 @@ def test_connect_needs_two_boards():
s.connect("f1", "f2") s.connect("f1", "f2")
def test_bbox_axis_aligned():
s = Scene()
p = s.place("2x4", 24) # 1.5 x 3.5 section
lo, hi = p.bbox()
assert lo == pytest.approx((0, -1.75, -0.75))
assert hi == pytest.approx((24, 1.75, 0.75))
def test_spatial_summary_flags_overlap():
from woodshop.scene import spatial_summary
s = Scene()
s.place("2x4", 24) # p1 at origin
s.place("2x4", 24) # p2 at origin -> overlaps p1
summ = spatial_summary(s)
assert "p1" in summ and "p2" in summ
assert "p1&p2" in summ # interpenetration flagged
s.move("p2", dy=10) # slide clear
assert "p1&p2" not in spatial_summary(s)
def test_clear(): def test_clear():
s = Scene() s = Scene()
s.place("2x4", 24) s.place("2x4", 24)