bundle and improve runtime perf · vectorize-io/pg0@a565d5b · GitHub
Skip to content

Commit a565d5b

Browse files
committed
bundle and improve runtime perf
1 parent c5565c8 commit a565d5b

8 files changed

Lines changed: 537 additions & 40 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 127 additions & 0 deletions

‎.github/workflows/release-cli.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ jobs:
7676
echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
7777
7878
- name: Build
79+
env:
80+
BUNDLE_POSTGRESQL: "true"
7981
run: cargo build --release --target ${{ matrix.target }}
8082

8183
- name: Prepare artifact (Unix)
@@ -144,9 +146,9 @@ jobs:
144146
echo "- \`pg0-windows-x86_64.exe\` - Windows x64" >> release_notes.md
145147
echo "" >> release_notes.md
146148
echo "### Bundled Components" >> release_notes.md
147-
echo "On first run, the CLI automatically downloads:" >> release_notes.md
148-
echo "- PostgreSQL ${PG_VERSION} (from theseus-rs)" >> release_notes.md
149-
echo "- pgvector ${PGVECTOR_VERSION}" >> release_notes.md
149+
echo "PostgreSQL is bundled directly in the binary - no downloads required on first run!" >> release_notes.md
150+
echo "- PostgreSQL ${PG_VERSION}" >> release_notes.md
151+
echo "- pgvector ${PGVECTOR_VERSION} (downloaded on first run)" >> release_notes.md
150152
echo "" >> release_notes.md
151153
echo "### Installation (macOS/Linux)" >> release_notes.md
152154
echo "\`\`\`bash" >> release_notes.md

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ dirs = "5"
2020
thiserror = "1"
2121
tracing = "0.1"
2222
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
23+
flate2 = "1"
24+
tar = "0.4"
2325

2426
[profile.release]
2527
strip = true

‎README.md‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,10 +408,67 @@ pg0 start -c shared_buffers=1GB -c maintenance_work_mem=2GB
408408

409409
## How It Works
410410

411-
On first run, pg0 downloads PostgreSQL from [theseus-rs](https://github.com/theseus-rs/postgresql-binaries) and pgvector from pre-compiled binaries. These are cached in `~/.pg0/installation/` for subsequent runs.
411+
PostgreSQL is **bundled directly** into the pg0 binary - no downloads required on first run! On first start, pg0 extracts PostgreSQL to `~/.pg0/installation/` and initializes the database.
412+
413+
pgvector is downloaded on first run from pre-compiled binaries and cached in the installation directory.
412414

413415
Data is stored in `~/.pg0/instances/<name>/data/` (or your custom `--data-dir`) and persists between restarts.
414416

417+
## Troubleshooting
418+
419+
### GitHub API Rate Limit Exceeded
420+
421+
pgvector is downloaded from GitHub on first run. If you see this error:
422+
423+
```
424+
Error: HTTP status client error (403 rate limit exceeded) for url (https://api.github.com/...)
425+
```
426+
427+
This means you've hit GitHub's API rate limit for unauthenticated requests (60 requests/hour).
428+
429+
**Solution:** Set a GitHub token to increase the rate limit to 5,000 requests/hour:
430+
431+
```bash
432+
# Create a token at https://github.com/settings/tokens (no scopes needed)
433+
export GITHUB_TOKEN=ghp_your_token_here
434+
pg0 start
435+
```
436+
437+
Or inline:
438+
439+
```bash
440+
GITHUB_TOKEN=ghp_your_token_here pg0 start
441+
```
442+
443+
The token doesn't need any special permissions - a basic token with no scopes works fine.
444+
445+
### PostgreSQL Cannot Run as Root
446+
447+
PostgreSQL refuses to run as root for security reasons. If you see permission errors in Docker or Linux:
448+
449+
```bash
450+
# Create a non-root user
451+
useradd -m pguser
452+
su - pguser -c "pg0 start"
453+
```
454+
455+
See the [Docker](#docker) section for complete examples.
456+
457+
### Port Already in Use
458+
459+
If port 5432 is already in use, pg0 will automatically find the next available port when starting without an explicit `--port`:
460+
461+
```bash
462+
pg0 start --name second-instance
463+
# Output: Port 5432 is in use, using port 5433 instead.
464+
```
465+
466+
To use a specific port, specify it explicitly:
467+
468+
```bash
469+
pg0 start --port 5433
470+
```
471+
415472
## Build from Source
416473

417474
```bash

‎build.rs‎

Lines changed: 123 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,136 @@
1+
use std::env;
12
use std::fs;
3+
use std::io;
4+
use std::path::PathBuf;
25

36
fn main() {
4-
// Read versions from versions.env
5-
let versions = fs::read_to_string("versions.env").expect("Failed to read versions.env");
7+
println!("cargo:rerun-if-changed=versions.env");
8+
println!("cargo:rerun-if-env-changed=BUNDLE_POSTGRESQL");
9+
10+
// Load versions from versions.env
11+
let versions_env = fs::read_to_string("versions.env").expect("Failed to read versions.env");
12+
let mut pg_version = String::new();
13+
let mut pgvector_version = String::new();
14+
let mut pgvector_tag = String::new();
15+
let mut pgvector_repo = String::new();
616

7-
for line in versions.lines() {
17+
for line in versions_env.lines() {
818
let line = line.trim();
919
if line.is_empty() || line.starts_with('#') {
1020
continue;
1121
}
1222
if let Some((key, value)) = line.split_once('=') {
13-
println!("cargo:rustc-env={}={}", key.trim(), value.trim());
23+
match key.trim() {
24+
"PG_VERSION" => pg_version = value.trim().to_string(),
25+
"PGVECTOR_VERSION" => pgvector_version = value.trim().to_string(),
26+
"PGVECTOR_COMPILED_TAG" => pgvector_tag = value.trim().to_string(),
27+
"PGVECTOR_COMPILED_REPO" => pgvector_repo = value.trim().to_string(),
28+
_ => {}
29+
}
1430
}
1531
}
1632

17-
// Re-run if versions.env changes
18-
println!("cargo:rerun-if-changed=versions.env");
33+
println!("cargo:rustc-env=PG_VERSION={}", pg_version);
34+
println!("cargo:rustc-env=PGVECTOR_VERSION={}", pgvector_version);
35+
println!("cargo:rustc-env=PGVECTOR_COMPILED_TAG={}", pgvector_tag);
36+
println!("cargo:rustc-env=PGVECTOR_COMPILED_REPO={}", pgvector_repo);
37+
38+
// Check if we should bundle PostgreSQL
39+
let bundle = env::var("BUNDLE_POSTGRESQL")
40+
.map(|v| v == "1" || v == "true")
41+
.unwrap_or(false);
42+
43+
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
44+
45+
if bundle {
46+
bundle_postgresql(&pg_version, &out_dir);
47+
} else {
48+
// Create an empty marker file so include_bytes! doesn't fail
49+
let marker = out_dir.join("postgresql_bundle.tar.gz");
50+
if !marker.exists() {
51+
fs::write(&marker, b"").expect("Failed to create empty bundle marker");
52+
}
53+
println!(
54+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
55+
marker.display()
56+
);
57+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=false");
58+
}
59+
}
60+
61+
fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) {
62+
let target = env::var("TARGET").unwrap();
63+
64+
// Map Rust target to theseus-rs binary name
65+
let pg_target = match target.as_str() {
66+
"aarch64-apple-darwin" => "aarch64-apple-darwin",
67+
"x86_64-apple-darwin" => "x86_64-apple-darwin",
68+
"x86_64-unknown-linux-gnu" => "x86_64-unknown-linux-gnu",
69+
"x86_64-unknown-linux-musl" => "x86_64-unknown-linux-musl",
70+
"aarch64-unknown-linux-gnu" => "aarch64-unknown-linux-gnu",
71+
"aarch64-unknown-linux-musl" => "aarch64-unknown-linux-musl",
72+
"x86_64-pc-windows-msvc" => "x86_64-pc-windows-msvc",
73+
_ => {
74+
eprintln!(
75+
"Warning: Unknown target {}, PostgreSQL will not be bundled",
76+
target
77+
);
78+
let marker = out_dir.join("postgresql_bundle.tar.gz");
79+
fs::write(&marker, b"").expect("Failed to create empty bundle marker");
80+
println!(
81+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
82+
marker.display()
83+
);
84+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=false");
85+
return;
86+
}
87+
};
88+
89+
let ext = if target.contains("windows") {
90+
"zip"
91+
} else {
92+
"tar.gz"
93+
};
94+
let filename = format!("postgresql-{}-{}.{}", pg_version, pg_target, ext);
95+
let url = format!(
96+
"https://github.com/theseus-rs/postgresql-binaries/releases/download/{}/{}",
97+
pg_version, filename
98+
);
99+
100+
let bundle_path = out_dir.join(&filename);
101+
102+
// Download if not already cached
103+
if !bundle_path.exists() {
104+
eprintln!(
105+
"Downloading PostgreSQL {} for {}...",
106+
pg_version, pg_target
107+
);
108+
download_file(&url, &bundle_path).expect("Failed to download PostgreSQL bundle");
109+
eprintln!("Downloaded to {}", bundle_path.display());
110+
} else {
111+
eprintln!("Using cached PostgreSQL bundle: {}", bundle_path.display());
112+
}
113+
114+
println!(
115+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
116+
bundle_path.display()
117+
);
118+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=true");
119+
}
120+
121+
fn download_file(url: &str, dest: &PathBuf) -> io::Result<()> {
122+
// Use curl for downloading (available on all CI platforms)
123+
let status = std::process::Command::new("curl")
124+
.args(["-fsSL", url, "-o"])
125+
.arg(dest)
126+
.status()?;
127+
128+
if !status.success() {
129+
return Err(io::Error::new(
130+
io::ErrorKind::Other,
131+
format!("Failed to download {}", url),
132+
));
133+
}
134+
135+
Ok(())
19136
}

‎docker-tests/test_debian_amd64.sh‎

Lines changed: 12 additions & 0 deletions

0 commit comments

Comments
 (0)