Add vendor assessment agent with structured output

GNUmakefile

@@ -38,11 +38,24 @@ GO_TOOL=	$(GO_BASE) tool
 
 TEST_FLAGS?=	-race -cover -coverprofile=coverage.out
 
+E2E_CONFIG ?= $(CURDIR)/e2e/console/testdata/config.yaml
 E2E_COVER_DIR ?= $(CURDIR)/coverage/e2e
 
 DOCKER_IMAGE_NAME=	ghcr.io/getprobo/probo
 DOCKER_TAG_NAME?=	latest
 
+PROBOD_BIN_DEPS= pkg/server/api/connect/v1/schema/schema.go \
+	pkg/server/api/connect/v1/types/types.go \
+	pkg/server/api/console/v1/schema/schema.go \
+	pkg/server/api/console/v1/types/types.go \
+	pkg/server/api/trust/v1/schema/schema.go \
+	pkg/server/api/trust/v1/types/types.go \
+	pkg/server/api/mcp/v1/server/server.go \
+	pkg/server/api/mcp/v1/types/types.go \
+	apps/console/dist/index.html \
+	apps/trust/dist/index.html \
+	@probo/emails
+
 PROBOD_BIN_EXTRA_DEPS=
 PROBOD_BIN=	bin/probod
 PROBOD_SRC=	cmd/probod/main.go
@@ -126,8 +139,9 @@ test-bench: test ## Run benchmark tests
 
 .PHONY: test-e2e
 test-e2e: CGO_ENABLED=1
-test-e2e: bin/probod ## Run console e2e tests
-	PROBO_E2E_BINARY=$(CURDIR)/bin/probod \
+test-e2e: $(PROBOD_BIN) ## Run console e2e tests
+	PROBO_E2E_BINARY=$(CURDIR)/$(PROBOD_BIN) \
+	PROBO_E2E_CONFIG=$(E2E_CONFIG) \
 	GOTESTSUM_FORMAT=testname $(GO_TEST) -count=1 ./e2e/console/...
 
 bin/probod-coverage:
@@ -138,6 +152,7 @@ test-e2e-coverage: bin/probod-coverage ## Run e2e tests with coverage
 	@$(RM) -rf $(E2E_COVER_DIR) && $(MKDIR) -p $(E2E_COVER_DIR)
 	PROBO_E2E_BINARY=$(CURDIR)/bin/probod-coverage \
 	PROBO_E2E_COVERDIR=$(E2E_COVER_DIR) \
+	PROBO_E2E_CONFIG=$(E2E_CONFIG) \
 	CGO_ENABLED=1 $(GO) test -count=1 -v ./e2e/console/...
 	$(GO) tool covdata textfmt -i=$(E2E_COVER_DIR) -o=coverage-e2e.out
 	$(GO) tool cover -html=coverage-e2e.out -o=coverage-e2e.html
@@ -149,7 +164,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera
 	$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
 
 .PHONY: build
-build: bin/probod bin/prb bin/probod-bootstrap
+build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap
 
 CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem
 DEV_ENV            = .env
@@ -217,19 +232,8 @@ scan-license: ## Check dependencies licenses compliance
 docker-build:
 	$(DOCKER_BUILD) --tag $(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME) --file Dockerfile .
 
-.PHONY: bin/probod
-bin/probod: pkg/server/api/connect/v1/schema/schema.go \
-	pkg/server/api/connect/v1/types/types.go \
-	pkg/server/api/console/v1/schema/schema.go \
-	pkg/server/api/console/v1/types/types.go \
-	pkg/server/api/trust/v1/schema/schema.go \
-	pkg/server/api/trust/v1/types/types.go \
-	pkg/server/api/mcp/v1/server/server.go \
-	pkg/server/api/mcp/v1/types/types.go \
-	apps/console/dist/index.html \
-	apps/trust/dist/index.html \
-	$(PROBOD_BIN_EXTRA_DEPS) \
-	@probo/emails
+.PHONY: $(PROBOD_BIN)
+$(PROBOD_BIN): $(PROBOD_BIN_DEPS) $(PROBOD_BIN_EXTRA_DEPS)
 	$(GO_BUILD) -o $(PROBOD_BIN) $(PROBOD_SRC)
 
 .PHONY: bin/prb

e2e/console/vendor_test.go

@@ -986,6 +986,118 @@ func TestVendor_OmittableWebsiteUrl(t *testing.T) {
 	})
 }
 
+// TestVendor_Assess exercises the assessVendor mutation through authorization
+// and tenant-isolation paths without running the real LLM/browser pipeline.
+// The e2e config deliberately omits `llm.vendor-assessor.provider`, so an
+// authorized call reaches DisabledVendorAssessor and surfaces a stable
+// UNAVAILABLE error. Happy-path payload shape is covered by unit tests in
+// pkg/probo.
+func TestVendor_Assess(t *testing.T) {
+	t.Parallel()
+
+	const query = `
+		mutation AssessVendor($input: AssessVendorInput!) {
+			assessVendor(input: $input) {
+				vendor {
+					id
+				}
+			}
+		}
+	`
+
+	type resultShape struct {
+		AssessVendor struct {
+			Vendor struct {
+				ID string `json:"id"`
+			} `json:"vendor"`
+		} `json:"assessVendor"`
+	}
+
+	t.Run("owner call surfaces the disabled error", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(owner).WithName("Unconfigured assess").Create()
+
+		var result resultShape
+		err := owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://vendor.example.com",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+
+	t.Run("admin call surfaces the disabled error", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		admin := testutil.NewClientInOrg(t, testutil.RoleAdmin, owner)
+		vendorID := factory.NewVendor(owner).WithName("Admin-assessed vendor").Create()
+
+		var result resultShape
+		err := admin.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://admin.example.com",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+
+	t.Run("viewer cannot assess a vendor", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		viewer := testutil.NewClientInOrg(t, testutil.RoleViewer, owner)
+		vendorID := factory.NewVendor(owner).WithName("Viewer attempt").Create()
+
+		var result resultShape
+		err := viewer.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://viewer.example.com",
+			},
+		}, &result)
+		testutil.RequireForbiddenError(t, err)
+	})
+
+	t.Run("cannot assess vendor from another organization", func(t *testing.T) {
+		t.Parallel()
+
+		org1Owner := testutil.NewClient(t, testutil.RoleOwner)
+		org2Owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(org1Owner).WithName("Org1 vendor").Create()
+
+		var result resultShape
+		err := org2Owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://cross-tenant.example.com",
+			},
+		}, &result)
+		require.Error(t, err, "vendor assess must not cross tenant boundaries")
+	})
+
+	t.Run("procedure is accepted on the input", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(owner).WithName("Procedure test").Create()
+
+		var result resultShape
+		err := owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://procedure.example.com",
+				"procedure":  "Focus on SOC 2 controls and data residency",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+}
+
 func TestVendor_TenantIsolation(t *testing.T) {
 	t.Parallel()
 

pkg/agent/agent.go

@@ -23,48 +23,53 @@ import (
 	"go.probo.inc/probo/pkg/llm"
 )
 
-const DefaultMaxTurns = 10
+const (
+	DefaultMaxTurns              = 10
+	DefaultMaxEmptyOutputRetries = 2
+)
 
 type (
 	Option func(*Agent)
 
 	Agent struct {
-		name               string
-		handoffDescription string
-		instructions       string
-		instructionsFunc   func(ctx context.Context, a *Agent) string
-		model              string
-		modelSettings      ModelSettings
-		tools              []Tool
-		handoffs           []*Handoff
-		mcpServers         []*MCPServer
-		maxTurns           int
-		maxToolDepth       int
-		client             *llm.Client
-		logger             *log.Logger
-		hooks              []RunHooks
-		agentHooks         AgentHooks
-		inputGuardrails    []InputGuardrail
-		outputGuardrails   []OutputGuardrail
-		session            Session
-		sessionID          string
-		outputType         *OutputType
-		toolUseBehavior    ToolUseBehavior
-		resetToolChoice    bool
-		responseFormat     *llm.ResponseFormat
-		approval           *ApprovalConfig
+		name                  string
+		handoffDescription    string
+		instructions          string
+		instructionsFunc      func(ctx context.Context, a *Agent) string
+		model                 string
+		modelSettings         ModelSettings
+		tools                 []Tool
+		handoffs              []*Handoff
+		mcpServers            []*MCPServer
+		maxTurns              int
+		maxEmptyOutputRetries int
+		maxToolDepth          int
+		client                *llm.Client
+		logger                *log.Logger
+		hooks                 []RunHooks
+		agentHooks            AgentHooks
+		inputGuardrails       []InputGuardrail
+		outputGuardrails      []OutputGuardrail
+		session               Session
+		sessionID             string
+		outputType            *OutputType
+		toolUseBehavior       ToolUseBehavior
+		resetToolChoice       bool
+		responseFormat        *llm.ResponseFormat
+		approval              *ApprovalConfig
 	}
 )
 
 func New(name string, client *llm.Client, opts ...Option) *Agent {
 	a := &Agent{
-		name:            name,
-		client:          client,
-		maxTurns:        DefaultMaxTurns,
-		maxToolDepth:    DefaultMaxToolDepth,
-		toolUseBehavior: RunLLMAgain(),
-		resetToolChoice: true,
-		logger:          log.NewLogger(log.WithOutput(io.Discard)),
+		name:                  name,
+		client:                client,
+		maxTurns:              DefaultMaxTurns,
+		maxEmptyOutputRetries: DefaultMaxEmptyOutputRetries,
+		maxToolDepth:          DefaultMaxToolDepth,
+		toolUseBehavior:       RunLLMAgain(),
+		resetToolChoice:       true,
+		logger:                log.NewLogger(log.WithOutput(io.Discard)),
 	}
 
 	for _, opt := range opts {
@@ -204,6 +209,18 @@ func WithMaxTurns(n int) Option {
 	}
 }
 
+// WithMaxEmptyOutputRetries bounds the number of times the core loop
+// will re-ask the model to produce a structured output after it
+// returned a thinking-only empty response on a synthesis turn.
+func WithMaxEmptyOutputRetries(n int) Option {
+	return func(a *Agent) {
+		if n < 0 {
+			n = 0
+		}
+		a.maxEmptyOutputRetries = n
+	}
+}
+
 func WithMaxToolDepth(n int) Option {
 	return func(a *Agent) {
 		if n < 1 {
@@ -255,6 +272,15 @@ func WithParallelToolCalls(enabled bool) Option {
 	}
 }
 
+func WithThinking(budgetTokens int) Option {
+	return func(a *Agent) {
+		a.modelSettings.Thinking = &llm.ThinkingConfig{
+			Enabled:      true,
+			BudgetTokens: budgetTokens,
+		}
+	}
+}
+
 func WithLogger(l *log.Logger) Option {
 	return func(a *Agent) {
 		a.logger = l