FilesExpand file tree

 main

/

readability-basic-compare.ts

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

66 lines (59 loc) · 1.68 KB

 main

/

readability-basic-compare.ts

Copy path

Top

File metadata and controls

• Code
• Blame

66 lines (59 loc) · 1.68 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

import { createWebFetchTool } from "../src/agents/tools/web-tools.js";

const DEFAULT_URLS = [

"https://example.com/",

"https://news.ycombinator.com/",

"https://www.reddit.com/r/javascript/",

"https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent",

"https://httpbin.org/html",

];

const urls = process.argv.slice(2);

const targets = urls.length > 0 ? urls : DEFAULT_URLS;

async function runFetch(url: string, readability: boolean) {

if (!readability) {

throw new Error("Basic extraction removed. Set readability=true or enable Firecrawl.");

}

const tool = createWebFetchTool({

config: {

tools: {

web: { fetch: { readability, cacheTtlMinutes: 0, firecrawl: { enabled: false } } },

},

},

sandboxed: false,

});

if (!tool) {

throw new Error("web_fetch tool is disabled");

}

const result = await tool.execute("test", { url, extractMode: "markdown" });

return result.details as {

text?: string;

title?: string;

extractor?: string;

length?: number;

truncated?: boolean;

};

}

function truncate(value: string, max = 160): string {

if (!value) {

return "";

}

return value.length > max ? `${value.slice(0, max)}…` : value;

}

async function run() {

for (const url of targets) {

console.log(`\n=== ${url}`);

const readable = await runFetch(url, true);

console.log(

`readability: ${readable.extractor ?? "unknown"} len=${readable.length ?? 0} title=${truncate(

readable.title ?? "",

80,

)}`,

);

if (readable.text) {

console.log(`readability sample: ${truncate(readable.text)}`);

}

}

}

run().catch((error) => {

console.error(error);

process.exit(1);

});