improvement(search): fall back to Jira summaries holding half the ter… · simstudioai/sim@f67d024 · GitHub
Skip to content

Commit f67d024

Browse files
authored
improvement(search): fall back to Jira summaries holding half the terms of a longer query (#9115)
* improvement(search): fall back to summaries holding half the terms of a longer query * fix(search): count Jira half-term summaries by stemmed term, from a wider pool, most terms first * improvement(search): share the half-terms threshold and convert Jira descriptions only when filtered
1 parent 5eb0b33 commit f67d024

3 files changed

Lines changed: 193 additions & 68 deletions

File tree

‎apps/sim/lib/sim-search/live/atlassian.test.ts‎

Lines changed: 88 additions & 10 deletions

‎apps/sim/lib/sim-search/live/atlassian.ts‎

Lines changed: 67 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ import { providerText } from '@/lib/sim-search/live/text'
1212
import {
1313
confluenceTextClause,
1414
holdsHalfTheTerms,
15-
jiraSummaryClause,
1615
jiraTextClause,
16+
stemmedTerms,
1717
} from '@/lib/sim-search/live/text-match'
1818
import type {
1919
NativeClient,
@@ -117,6 +117,13 @@ async function sites(client: NativeClient) {
117117
return array(await client.json('/oauth/token/accessible-resources', { memo: true }))
118118
}
119119

120+
/**
121+
* Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first,
122+
* not by relevance, and the page is then filtered to summaries holding enough terms, so it starts
123+
* from a wider pool than the caller's limit or the any-term fallback's.
124+
*/
125+
const HALF_TERM_CANDIDATES = 50
126+
120127
/**
121128
* One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the
122129
* all-terms search.
@@ -176,55 +183,82 @@ export async function searchAtlassian(
176183
const cloudId = string(site.id)
177184
const origin = string(site.url).replace(/\/$/, '')
178185
if (provider === 'jira') {
179-
const searchIssues = async (jql: string, cursor?: string): Promise<SitePage> => {
186+
/** One JQL page, each issue with its summary and raw fields. */
187+
const searchIssues = async (
188+
jql: string,
189+
{ cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {}
190+
) => {
180191
const data = object(
181192
await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
182193
body: {
183194
jql: scopeAtlassianQuery(jql, scope, order),
184-
maxResults: input.limit,
195+
maxResults,
185196
fields: ['summary', 'description', 'updated', 'creator', 'status'],
186197
...(cursor ? { nextPageToken: cursor } : {}),
187198
},
188199
signal: input.signal,
189200
})
190201
)
191202
return {
192-
documents: array(data.issues)
193-
.filter((row) => {
194-
const fields = object(row.fields)
195-
return (
196-
!relevant ||
197-
relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
198-
)
199-
})
200-
.map((row) => issue(row, cloudId, origin)),
201-
excluded: false,
203+
issues: array(data.issues).map((row) => {
204+
const fields = object(row.fields)
205+
return {
206+
document: issue(row, cloudId, origin),
207+
summary: string(fields.summary),
208+
fields,
209+
}
210+
}),
202211
next: string(data.nextPageToken) || undefined,
203212
}
204213
}
205214
const exact = await searchIssues(
206215
input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
207-
input.native?.cursor && single ? input.native.cursor : undefined
216+
{ cursor: input.native?.cursor && single ? input.native.cursor : undefined }
208217
)
218+
const exactPage: SitePage = {
219+
documents: exact.issues
220+
.filter(
221+
({ summary, fields }) =>
222+
!relevant || relevant(`${summary}\n${textContent(fields.description)}`)
223+
)
224+
.map(({ document }) => document),
225+
excluded: false,
226+
next: exact.next,
227+
}
209228
// JQL has no relevance order, so an all-terms match larger than a page comes back newest
210-
// first; issues whose summary holds every term lead it instead.
211-
if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact
212-
try {
213-
const titled = await searchIssues(jiraSummaryClause(text))
214-
if (!titled.documents.length) return exact
215-
const leading = new Set(titled.documents.map((document) => document.id))
216-
return {
217-
documents: [
218-
...titled.documents,
219-
...exact.documents.filter((document) => !leading.has(document.id)),
220-
].slice(0, input.limit),
221-
excluded: false,
222-
ranking: 'relevance',
223-
}
224-
} catch {
225-
// A cancelled search stops here; any other failure keeps the all-terms page.
226-
input.signal?.throwIfAborted()
227-
return exact
229+
// first; issues whose summary holds every term, or else at least half of three or more
230+
// terms (most terms first), lead it instead.
231+
if (!text || input.native || order || input.textMatch === 'any' || !exact.next)
232+
return exactPage
233+
const terms = stemmedTerms(text)
234+
const [every, half] = await Promise.allSettled([
235+
searchIssues(jiraTextClause(text, 'all', 'summary')),
236+
terms.count >= 3
237+
? searchIssues(jiraTextClause(text, 'any', 'summary'), {
238+
maxResults: HALF_TERM_CANDIDATES,
239+
})
240+
: undefined,
241+
])
242+
// A cancelled search stops here; any other failure keeps the all-terms page.
243+
input.signal?.throwIfAborted()
244+
let titled =
245+
every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : []
246+
if (!titled.length && half.status === 'fulfilled' && half.value) {
247+
titled = half.value.issues
248+
.map(({ document, summary }) => ({ document, held: terms.held(summary) }))
249+
.filter(({ held }) => held >= terms.required)
250+
.sort((left, right) => right.held - left.held)
251+
.map(({ document }) => document)
252+
}
253+
if (!titled.length) return exactPage
254+
const leading = new Set(titled.map((document) => document.id))
255+
return {
256+
documents: [
257+
...titled,
258+
...exactPage.documents.filter((document) => !leading.has(document.id)),
259+
].slice(0, input.limit),
260+
excluded: false,
261+
ranking: 'relevance',
228262
}
229263
}
230264
const searchPages = async (clause: string, cursor?: string) => {
@@ -289,7 +323,7 @@ export async function searchAtlassian(
289323
: '') +
290324
(pages.some((result) => result.ranking === 'relevance')
291325
? provider === 'jira'
292-
? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.'
326+
? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.'
293327
: ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
294328
: '') +
295329
(pages.some((result) => result.ranking === 'unavailable')

‎apps/sim/lib/sim-search/live/text-match.ts‎

Lines changed: 38 additions & 25 deletions

0 commit comments

Comments
 (0)