ci: hydrate vendored deps before build jobs

nikivdev · nikivdev · commit a868228dd75c · 2026-02-21T15:43:12.000+02:00
diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
@@ -41,6 +41,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Hydrate vendored deps
+        run: scripts/vendor/vendor-repo.sh hydrate
+
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -102,6 +105,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Hydrate vendored deps
+        run: scripts/vendor/vendor-repo.sh hydrate
+
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -131,7 +137,7 @@ jobs:
           path: dist/flow-x86_64-unknown-linux-gnu-linux-host-simd.tar.gz
 
   release:
-    needs: build
+    needs: [build, build-linux-host-simd]
     timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -36,6 +36,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Hydrate vendored deps
+        run: scripts/vendor/vendor-repo.sh hydrate
+
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -97,6 +100,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Hydrate vendored deps
+        run: scripts/vendor/vendor-repo.sh hydrate
+
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -126,7 +132,7 @@ jobs:
           path: dist/flow-x86_64-unknown-linux-gnu-linux-host-simd.tar.gz
 
   release:
-    needs: build
+    needs: [build, build-linux-host-simd]
     timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
diff --git a/docs/ci-cd-runbook.md b/docs/ci-cd-runbook.md
@@ -7,12 +7,16 @@ This runbook documents how Flow CI/CD is wired today and how to debug it quickly
 - Workflows:
   - `.github/workflows/canary.yml`: runs on every push to `main`, publishes/updates the `canary` release/tag.
   - `.github/workflows/release.yml`: runs on tag pushes matching `v*`, publishes stable releases.
+- Vendored deps bootstrap:
+  - Both workflows run `scripts/vendor/vendor-repo.sh hydrate` immediately after checkout in each build job.
+  - This materializes `lib/vendor/*` from the pinned commit in `vendor.lock.toml` before Cargo builds.
 - Build jobs in both workflows:
   - Matrix build: macOS + Linux targets.
   - SIMD build: `build-linux-host-simd` (Linux x64 with `--features linux-host-simd-json`).
 - Release jobs:
   - Gather all build artifacts.
   - Publish release assets (and in Canary, force-move `canary` tag to current `main` commit).
+  - `release` waits for both `build` and `build-linux-host-simd`.
 
 ## Runner Modes
 
@@ -92,6 +96,14 @@ gh run view <run-id> --log-failed
 gh run watch <run-id>
 ```
 
+If failure shows:
+
+- `failed to load source for dependency 'axum'`
+- `Unable to update .../lib/vendor/axum`
+- `No such file or directory (os error 2)`
+
+then vendored deps were not hydrated before build.
+
 ### 2) SIMD lane queued forever
 
 Usually means self-hosted runner routing issue.
@@ -131,6 +143,25 @@ f ci-blacksmith-enable-apply
 f ci-blacksmith-disable-apply
 ```
 
+### 3b) Vendored repo hydrate issues
+
+Hydration depends on `vendor.lock.toml` pin and vendor repo availability.
+
+Quick checks:
+
+```bash
+scripts/vendor/vendor-repo.sh status
+scripts/vendor/vendor-repo.sh hydrate
+```
+
+Expected:
+
+- pinned commit in `vendor.lock.toml` is non-empty,
+- hydrate logs `hydrated <crate> -> lib/vendor/<crate>`,
+- `lib/vendor/axum/Cargo.toml` and `lib/vendor/reqwest/Cargo.toml` exist after hydrate.
+
+If CI cannot clone SSH URL from lock, `vendor-repo.sh` now falls back to HTTPS clone for GitHub URLs.
+
 ### 4) `curl ... install.sh` does not fetch expected fresh build
 
 Flow installer defaults to `canary` unless `FLOW_VERSION` is set differently. Check if `canary` moved:
diff --git a/scripts/vendor/vendor-repo.sh b/scripts/vendor/vendor-repo.sh
@@ -134,11 +134,21 @@ ensure_checkout() {
   if git clone "$repo_url" "$checkout" >/dev/null 2>&1; then
     echo "cloned $repo_url -> $checkout" >&2
   else
-    echo "warning: failed to clone $repo_url" >&2
-    echo "initializing local checkout at $checkout (set remote for later push)" >&2
-    git init "$checkout" >/dev/null
-    git -C "$checkout" checkout -q -B "$branch"
-    git -C "$checkout" remote add origin "$repo_url"
+    https_url=""
+    if [[ "$repo_url" =~ ^git@github\.com:(.+)\.git$ ]]; then
+      https_url="https://github.com/${BASH_REMATCH[1]}.git"
+    fi
+
+    if [[ -n "$https_url" ]] && git clone "$https_url" "$checkout" >/dev/null 2>&1; then
+      echo "cloned $https_url -> $checkout (fallback from SSH URL)" >&2
+      git -C "$checkout" remote set-url origin "$repo_url" >/dev/null 2>&1 || true
+    else
+      echo "warning: failed to clone $repo_url" >&2
+      echo "initializing local checkout at $checkout (set remote for later push)" >&2
+      git init "$checkout" >/dev/null
+      git -C "$checkout" checkout -q -B "$branch"
+      git -C "$checkout" remote add origin "$repo_url"
+    fi
   fi
 
   if ! git -C "$checkout" rev-parse --verify "$branch" >/dev/null 2>&1; then
@@ -357,8 +367,11 @@ cmd_status() {
 
   if [[ -d "$checkout/.git" ]]; then
     local head_sha
-    head_sha="$(git -C "$checkout" rev-parse HEAD)"
-    echo "head:      $head_sha"
+    if head_sha="$(git -C "$checkout" rev-parse --verify HEAD 2>/dev/null)"; then
+      echo "head:      $head_sha"
+    else
+      echo "head:      <no commits yet>"
+    fi
 
     if git -C "$checkout" remote get-url origin >/dev/null 2>&1; then
       if git -C "$checkout" fetch -q origin "$branch" >/dev/null 2>&1; then
