{{ message }}
Repository navigation
improvement(chat): show browser agent site favicons #7747
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next
Next commit
improvement(chat): show browser agent site favicons
- Loading branch information
commit c730028874e47ea06b92d72f20f4db22fbca6731
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
214 changes: 214 additions & 0 deletions
214
...aceId]/home/components/message-content/components/agent-group/browser-agent-icon.test.tsx
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| /** | ||
| * @vitest-environment jsdom | ||
| */ | ||
| import { act } from 'react' | ||
| import { createRoot, type Root } from 'react-dom/client' | ||
| import { afterEach, beforeEach, describe, expect, it } from 'vitest' | ||
| import type { AgentGroupItem } from '@/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group-view' | ||
| import { | ||
| BrowserAgentIcon, | ||
| getBrowserAgentHostname, | ||
| } from '@/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/browser-agent-icon' | ||
| import type { ToolCallData } from '@/app/workspace/[workspaceId]/home/types' | ||
|
|
||
| function tool(overrides: Partial<ToolCallData> = {}): AgentGroupItem { | ||
| return { | ||
| type: 'tool', | ||
| data: { | ||
| id: 'browser-tool', | ||
| toolName: 'browser_navigate', | ||
| displayTitle: 'Opening page', | ||
| status: 'success', | ||
| result: { success: true, output: { url: 'https://example.com/document' } }, | ||
| ...overrides, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| describe('getBrowserAgentHostname', () => { | ||
| it('uses the pending destination, then the observed redirect, and retains it during editing', () => { | ||
| const navigating = tool({ | ||
| status: 'executing', | ||
| params: { url: 'https://example.org/start' }, | ||
| result: undefined, | ||
| }) | ||
| expect(getBrowserAgentHostname([tool(), navigating])).toBe('example.org') | ||
| const redirected = tool({ params: { url: 'https://example.org/start' } }) | ||
| expect(getBrowserAgentHostname([redirected])).toBe('example.com') | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| redirected, | ||
| tool({ toolName: 'browser_type', status: 'executing', result: undefined }), | ||
| ]) | ||
| ).toBe('example.com') | ||
| }) | ||
|
|
||
| it.each(['error', 'cancelled', 'rejected', 'awaiting_approval'] as const)( | ||
| 'does not adopt a destination from a %s tool', | ||
| (status) => { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ status, params: { url: 'https://example.org' }, result: undefined }), | ||
| ]) | ||
| ).toBe('example.com') | ||
| } | ||
| ) | ||
|
|
||
| it.each(['', 'about:blank', 'file:///document', 'data:text/html,hello', 'not a URL'])( | ||
| 'clears the previous site for a page without an HTTP hostname: %s', | ||
| (url) => { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ toolName: 'browser_switch_tab', result: { success: true, output: { url } } }), | ||
| ]) | ||
| ).toBeNull() | ||
| } | ||
| ) | ||
|
|
||
| it.each(['browser_open_tab', 'browser_switch_tab', 'browser_close_tab', 'browser_go_back'])( | ||
| 'clears an obsolete site while %s has no known destination', | ||
| (toolName) => { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ toolName, status: 'executing', result: undefined }), | ||
| ]) | ||
| ).toBeNull() | ||
| } | ||
| ) | ||
|
|
||
| it('uses the agent tab from a tab list, including legacy results', () => { | ||
| const tabs = [ | ||
| { tabId: 'visible', url: 'https://example.org' }, | ||
| { tabId: 'agent', url: 'https://example.com' }, | ||
| ] | ||
| for (const output of [ | ||
| { tabs, activeTabId: 'visible', automationTabId: 'agent' }, | ||
| { tabs, activeTabId: 'agent' }, | ||
| ]) { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool({ toolName: 'browser_list_tabs', result: { success: true, output } }), | ||
| ]) | ||
| ).toBe('example.com') | ||
| } | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ | ||
| toolName: 'browser_list_tabs', | ||
| result: { | ||
| success: true, | ||
| output: { tabs, activeTabId: 'visible', automationTabId: null }, | ||
| }, | ||
| }), | ||
| ]) | ||
| ).toBeNull() | ||
| }) | ||
|
|
||
| it('follows a click into a new tab and screenshot page metadata', () => { | ||
| for (const [toolName, output] of [ | ||
| [ | ||
| 'browser_click', | ||
| { activeTab: { url: 'https://example.org' }, effect: { tabChanged: true } }, | ||
| ], | ||
| ['browser_screenshot', { viewport: { url: 'https://example.org' } }], | ||
| ['browser_extract', { page: { url: 'https://example.org' } }], | ||
| ] as const) { | ||
| expect( | ||
| getBrowserAgentHostname([tool(), tool({ toolName, result: { success: true, output } })]) | ||
| ).toBe('example.org') | ||
| } | ||
| }) | ||
|
|
||
| it('clears a stale site when an action navigated without reporting its destination', () => { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ | ||
| toolName: 'browser_click', | ||
| result: { success: true, output: { effect: { urlChanged: true } } }, | ||
| }), | ||
| ]) | ||
| ).toBeNull() | ||
| }) | ||
|
|
||
| it('ignores URLs from other tools and nested agent runs', () => { | ||
| expect( | ||
| getBrowserAgentHostname([ | ||
| tool(), | ||
| tool({ | ||
| toolName: 'browser_read_text', | ||
| params: { elementId: 4 }, | ||
| result: { success: true, output: { url: 'https://example.org/embedded' } }, | ||
| }), | ||
| tool({ | ||
| toolName: 'web_search', | ||
| result: { success: true, output: { url: 'https://example.org' } }, | ||
| }), | ||
| { | ||
| type: 'agent_group', | ||
| group: { | ||
| id: 'other-run', | ||
| agentName: 'browser', | ||
| agentLabel: 'Browser', | ||
| isOpen: true, | ||
| isDelegating: true, | ||
| items: [tool({ result: { success: true, output: { url: 'https://example.org' } } })], | ||
| }, | ||
| }, | ||
| ]) | ||
| ).toBe('example.com') | ||
| }) | ||
| }) | ||
|
|
||
| describe('BrowserAgentIcon', () => { | ||
| let container: HTMLDivElement | ||
| let root: Root | ||
|
|
||
| beforeEach(() => { | ||
| ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true | ||
| container = document.createElement('div') | ||
| root = createRoot(container) | ||
| }) | ||
|
|
||
| afterEach(() => act(() => root.unmount())) | ||
|
|
||
| const render = (url: string) => { | ||
| act(() => | ||
| root.render( | ||
| <BrowserAgentIcon items={[tool({ result: { success: true, output: { url } } })]} /> | ||
| ) | ||
| ) | ||
| } | ||
|
|
||
| it('keeps the globe until load and resets image state when the hostname changes', () => { | ||
| render('https://example.com/document?token=private#section') | ||
| const firstImage = container.querySelector('img')! | ||
| expect(firstImage.src).toBe('https://www.google.com/s2/favicons?domain=example.com&sz=32') | ||
| expect(container.querySelector('svg')).not.toBeNull() | ||
| act(() => firstImage.dispatchEvent(new Event('load'))) | ||
| expect(container.querySelector('svg')).toBeNull() | ||
|
|
||
| render('https://example.com/another-document') | ||
| expect(container.querySelector('img')).toBe(firstImage) | ||
| expect(container.querySelector('svg')).toBeNull() | ||
|
|
||
| render('https://example.org/document') | ||
| expect(container.querySelector('img')).not.toBe(firstImage) | ||
| expect(container.querySelector('svg')).not.toBeNull() | ||
| act(() => firstImage.dispatchEvent(new Event('error'))) | ||
| expect(container.querySelector('img')).not.toBeNull() | ||
| act(() => container.querySelector('img')!.dispatchEvent(new Event('error'))) | ||
| expect(container.querySelector('img')).toBeNull() | ||
| expect(container.querySelector('svg')).not.toBeNull() | ||
|
|
||
| render('https://example.com') | ||
| expect(container.querySelector('img')).not.toBeNull() | ||
| render('about:blank') | ||
| expect(container.querySelector('img')).toBeNull() | ||
| expect(container.querySelector('svg')).not.toBeNull() | ||
| }) | ||
| }) |
114 changes: 114 additions & 0 deletions
114
...orkspaceId]/home/components/message-content/components/agent-group/browser-agent-icon.tsx
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| 'use client' | ||
|
|
||
| import { useState } from 'react' | ||
| import { isBrowserToolName } from '@sim/browser-protocol' | ||
| import { cn } from '@sim/emcn' | ||
| import { Globe } from '@sim/emcn/icons' | ||
| import { isRecordLike } from '@sim/utils/object' | ||
| import { browserTabHostname } from '@/lib/browser-agent/tab-label' | ||
| import { faviconUrl } from '@/lib/core/utils/favicon' | ||
| import type { AgentGroupItem } from '@/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group-view' | ||
| import { ToolCallStatus } from '@/app/workspace/[workspaceId]/home/types' | ||
|
|
||
| /** Uses this run's page observations so historical icons never follow another run's live tab. */ | ||
| export function getBrowserAgentHostname(items: AgentGroupItem[]): string | null { | ||
| for (let index = items.length - 1; index >= 0; index--) { | ||
| const item = items[index] | ||
| if (item.type !== 'tool' || !isBrowserToolName(item.data.toolName)) continue | ||
| const { toolName, status, params, result } = item.data | ||
| if (status !== ToolCallStatus.executing && status !== ToolCallStatus.success) continue | ||
| /** Element-scoped reads can report an iframe URL rather than the browser page. */ | ||
| if (toolName === 'browser_read_text' && params?.elementId !== undefined) continue | ||
|
|
||
| const navigatesToUrl = | ||
| toolName === 'browser_navigate' || | ||
| toolName === 'browser_open_url' || | ||
| toolName === 'browser_open_tab' | ||
| if (status === ToolCallStatus.executing && navigatesToUrl) { | ||
| return typeof params?.url === 'string' ? browserTabHostname(params.url) : null | ||
| } | ||
|
|
||
| const output = result?.success && isRecordLike(result.output) ? result.output : null | ||
| if (output) { | ||
| if (isRecordLike(output.activeTab) && typeof output.activeTab.url === 'string') { | ||
| return browserTabHostname(output.activeTab.url) | ||
| } | ||
| if (typeof output.url === 'string') return browserTabHostname(output.url) | ||
| if ( | ||
| toolName === 'browser_extract' && | ||
| isRecordLike(output.page) && | ||
| typeof output.page.url === 'string' | ||
| ) { | ||
| return browserTabHostname(output.page.url) | ||
| } | ||
| if ( | ||
| toolName === 'browser_screenshot' && | ||
| isRecordLike(output.viewport) && | ||
| typeof output.viewport.url === 'string' | ||
| ) { | ||
| return browserTabHostname(output.viewport.url) | ||
| } | ||
| if (toolName === 'browser_list_tabs' && Array.isArray(output.tabs)) { | ||
| const tabId = 'automationTabId' in output ? output.automationTabId : output.activeTabId | ||
| const tab = output.tabs.find((tab) => isRecordLike(tab) && tab.tabId === tabId) | ||
| return isRecordLike(tab) && typeof tab.url === 'string' ? browserTabHostname(tab.url) : null | ||
| } | ||
| /** A navigation without a destination invalidates the previous page observation. */ | ||
| if ( | ||
| isRecordLike(output.effect) && | ||
| (output.effect.urlChanged === true || | ||
| output.effect.topUrlChanged === true || | ||
| output.effect.tabChanged === true) | ||
| ) { | ||
| return null | ||
| } | ||
| } | ||
| if ( | ||
| navigatesToUrl || | ||
| toolName === 'browser_switch_tab' || | ||
| toolName === 'browser_close_tab' || | ||
| toolName === 'browser_go_back' || | ||
| toolName === 'browser_go_forward' | ||
| ) { | ||
| return null | ||
| } | ||
| } | ||
| return null | ||
| } | ||
|
|
||
| interface BrowserAgentIconProps { | ||
| items: AgentGroupItem[] | ||
| } | ||
|
|
||
| export function BrowserAgentIcon({ items }: BrowserAgentIconProps) { | ||
| const hostname = getBrowserAgentHostname(items) | ||
| return <BrowserAgentFavicon key={hostname} hostname={hostname} /> | ||
| } | ||
|
|
||
| interface BrowserAgentFaviconProps { | ||
| hostname: string | null | ||
| } | ||
|
|
||
| function BrowserAgentFavicon({ hostname }: BrowserAgentFaviconProps) { | ||
| const [status, setStatus] = useState<'loading' | 'loaded' | 'failed'>('loading') | ||
|
|
||
| return ( | ||
| <span className='relative flex size-[16px] items-center justify-center' aria-hidden='true'> | ||
| {hostname && status !== 'failed' && ( | ||
| <img | ||
| src={faviconUrl(hostname, 32)} | ||
| alt='' | ||
|
waleedlatif1 marked this conversation as resolved.
|
||
| className={cn( | ||
| 'size-[16px] rounded-[3px]', | ||
| status !== 'loaded' && 'pointer-events-none absolute opacity-0' | ||
| )} | ||
| onLoad={() => setStatus('loaded')} | ||
| onError={() => setStatus('failed')} | ||
| /> | ||
| )} | ||
| {(!hostname || status !== 'loaded') && ( | ||
| <Globe className='size-[16px] text-[var(--text-icon)]' /> | ||
| )} | ||
| </span> | ||
| ) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
You can’t perform that action at this time.

Uh oh!
There was an error while loading. Please reload this page.