FilesExpand file tree

main

/

wet-processor.sh

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

executable file

·

97 lines (85 loc) · 2.39 KB

main

/

wet-processor.sh

Copy path

Top

File metadata and controls

• Code
• Blame

executable file

·

97 lines (85 loc) · 2.39 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

#!/bin/bash

# Common Crawl WET Processor -- Medical + CS Corpus Import

# Processes pre-extracted text (no HTML parsing needed)

# Usage: ./wet-processor.sh [CRAWL_INDEX] [SEGMENT_NUM]

set -euo pipefail

CRAWL_INDEX="${1:-CC-MAIN-2026-08}"

SEGMENT_NUM="${2:-0}"

BRAIN_URL="${BRAIN_URL:-https://pi.ruv.io}"

AUTH="Authorization: Bearer ruvector-crawl-2026"

WORK_DIR="/tmp/wet-processing"

BATCH_SIZE=10 # items per batch inject call

# Medical + CS domains to filter for

DOMAINS=(

"pubmed.ncbi.nlm.nih.gov"

"ncbi.nlm.nih.gov"

"who.int"

"cancer.org"

"aad.org"

"skincancer.org"

"dermnetnz.org"

"melanoma.org"

"mayoclinic.org"

"clevelandclinic.org"

"medlineplus.gov"

"cdc.gov"

"nih.gov"

"nejm.org"

"thelancet.com"

"bmj.com"

"nature.com/articles"

"sciencedirect.com"

"arxiv.org"

"acm.org"

"ieee.org"

"dl.acm.org"

"proceedings.mlr.press"

"openreview.net"

"paperswithcode.com"

"github.com"

"stackoverflow.com"

"medium.com"

"towardsdatascience.com"

"distill.pub"

)

mkdir -p "$WORK_DIR"

echo "=== WET Processor ==="

echo "Crawl: $CRAWL_INDEX"

echo "Domains: ${#DOMAINS[@]}"

echo ""

# Step 1: Get WET file list for this crawl

echo "--- Fetching WET file paths ---"

PATHS_URL="https://data.commoncrawl.org/crawl-data/${CRAWL_INDEX}/wet.paths.gz"

curl -sL "$PATHS_URL" | gunzip > "$WORK_DIR/wet-paths.txt" 2>/dev/null || {

echo "ERROR: Could not fetch WET paths for $CRAWL_INDEX"

exit 1

}

TOTAL_SEGMENTS=$(wc -l < "$WORK_DIR/wet-paths.txt")

echo "Total WET segments: $TOTAL_SEGMENTS"

# Select segment to process

WET_PATH=$(sed -n "$((SEGMENT_NUM + 1))p" "$WORK_DIR/wet-paths.txt")

if [ -z "$WET_PATH" ]; then

echo "ERROR: Segment $SEGMENT_NUM not found"

exit 1

fi

echo "Processing segment $SEGMENT_NUM: $WET_PATH"

# Step 2: Download and decompress WET file

echo "--- Downloading WET segment ---"

WET_FILE="$WORK_DIR/segment.wet.gz"

curl -sL "https://data.commoncrawl.org/$WET_PATH" -o "$WET_FILE" --max-time 300 || {

echo "ERROR: Download failed"

exit 1

}

echo "Downloaded: $(du -h "$WET_FILE" | cut -f1)"

# Step 3: Extract, filter by domain, and inject

echo "--- Processing and filtering ---"

gunzip -c "$WET_FILE" | node "$(dirname "$0")/wet-filter-inject.js" \

--brain-url "$BRAIN_URL" \

--auth "$AUTH" \

--batch-size "$BATCH_SIZE" \

--domains "$(IFS=,; echo "${DOMAINS[*]}")" \

--crawl-index "$CRAWL_INDEX"

# Cleanup

rm -f "$WET_FILE"

echo ""

echo "=== Done ==="