From f07df2838e41b86390b9ccf95203bf0b33beec05 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 13 Jul 2026 12:29:39 +1000 Subject: [PATCH] =?UTF-8?q?kb:=20fix=20scripts=20after=20Grok=20review=20?= =?UTF-8?q?=E2=80=94=20add=20status.sh,=20fix=20inbox/search/lint=20bugs,?= =?UTF-8?q?=20update=20README=20with=20script=20reference=20+=20manual=20w?= =?UTF-8?q?orkflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 69 ++++++++++++++++++++++++++++ scripts/inbox.sh | 89 ++++++++++++++++++++++++------------ scripts/lint.sh | 93 +++++++++++++++++++++++++++++--------- scripts/search.sh | 113 ++++++++++++++++++++++++++-------------------- scripts/stats.sh | 6 +-- scripts/status.sh | 56 +++++++++++++++++++++++ 6 files changed, 323 insertions(+), 103 deletions(-) create mode 100755 scripts/status.sh diff --git a/README.md b/README.md index c1f2893..584f8be 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,75 @@ rg "tags:.*inv/thesis" wiki/ | 每天 22:00 | Hermes 对话扫描 → `raw/YYYY-MM/` 提取知识 | | 每周日 | Review inbox → 决定 absorb 哪些 | +## 脚本参考 + +```bash +# 查看 inbox +./scripts/inbox.sh # 列出所有待处理 +./scripts/inbox.sh --count # 只看数量 +./scripts/inbox.sh --recent 10 # 最近 10 条 +./scripts/inbox.sh --month 2026-07 # 只看某月 + +# 搜索 +./scripts/search.sh "NVIDIA" # 全文搜索（固定字符串） +./scripts/search.sh "k3s" --wiki-only # 只搜 wiki +./scripts/search.sh "earnings" --raw-only # 只搜 raw +./scripts/search.sh "C\+\+" --regex # 使用正则 +./scripts/search.sh "NVIDIA" --files-only # 只列文件名 + +# 标记状态 +./scripts/status.sh absorb raw/2026-07/article.md # 标记为已吸收 +./scripts/status.sh skip raw/2026-07/old-news.md # 跳过 +./scripts/status.sh inbox raw/2026-07/file.md # 重置为待处理 + +# 统计 +./scripts/stats.sh + +# 健康检查（退出码非零 = 有问题） +./scripts/lint.sh +``` + +## 前置依赖 + +- `bash` 4+ +- `ripgrep` (`rg`) — `apt install ripgrep` 或 `brew install ripgrep` +- `git` + +## 不依赖 Hermes 的手动操作 + +### 手动 Ingest（你自己加一篇文章） + +1. 创建文件 `raw/YYYY-MM/YYYY-MM-DD-slug.md` +2. 写 frontmatter： +```yaml +--- +source_url: https://example.com/article +ingested: 2026-07-13 +status: inbox +--- +``` +3. 正文放下面 +4. `git add` + `git commit` + +### 手动 Absorb（从 raw 编译到 wiki） + +1. `./scripts/inbox.sh` 看看有哪些待处理 +2. 读 raw 文件，理解内容 +3. 搜 wiki 找相关页面：`./scripts/search.sh "关键词" --wiki-only` +4. 编辑或创建 wiki 页面 +5. `./scripts/status.sh absorb raw/路径` 标记已完成 +6. 更新页面 `[[wikilinks]]` 交叉引用 + +### Status 生命周期 + +``` +inbox → absorbed (内容已编译到 wiki/) +inbox → skipped (看过但不值得编译) +inbox → failed_fetch (链接打不开/抓取失败) +``` + +raw 文件标记 absorbed/skipped 后不会再出现在 inbox，但文件保留在 raw/ 中可搜索。 + ## 隐私 - GitHub repo = **private** diff --git a/scripts/inbox.sh b/scripts/inbox.sh index d983fd3..fa74435 100755 --- a/scripts/inbox.sh +++ b/scripts/inbox.sh @@ -1,23 +1,55 @@ #!/bin/bash -# KB Inbox — list all raw files with status: inbox -# Usage: ./scripts/inbox.sh [--count] [--recent N] +# KB Inbox — list raw files with status: inbox +# Usage: ./scripts/inbox.sh [--count] [--recent N] [--month YYYY-MM] + +set -euo pipefail KB_DIR="$(cd "$(dirname "$0")/.." && pwd)" COUNT_ONLY=false RECENT=0 +MONTH="" -for arg in "$@"; do - case $arg in - --count) COUNT_ONLY=true ;; - --recent) RECENT=1 ;; +# Check dependencies +if ! command -v rg &>/dev/null; then + echo "❌ ripgrep (rg) is required. Install: apt install ripgrep / brew install ripgrep" >&2 + exit 1 +fi + +while [ $# -gt 0 ]; do + case $1 in + --count) + COUNT_ONLY=true ;; + --recent) + if [[ ! "${2-}" =~ ^[0-9]+$ ]]; then + echo "Error: --recent requires a number" >&2 + exit 1 + fi + RECENT="$2"; shift ;; + --month) + MONTH="$2"; shift ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: ./scripts/inbox.sh [--count] [--recent N] [--month YYYY-MM]" >&2 + exit 1 ;; esac + shift done cd "$KB_DIR" +# Use frontmatter-only match: status: inbox in the YAML header +match_inbox() { + rg -l --glob '*.md' '^status:\s*inbox\s*$' "$@" +} + if $COUNT_ONLY; then - count=$(rg -l "status: inbox" raw/ 2>/dev/null | wc -l) - echo "📥 Inbox: $count items pending" + if [ -n "$MONTH" ]; then + count=$(match_inbox "raw/${MONTH}/" 2>/dev/null | wc -l) + echo "📥 Inbox ($MONTH): $count items pending" + else + count=$(match_inbox raw/ 2>/dev/null | wc -l) + echo "📥 Inbox total: $count items pending" + fi exit 0 fi @@ -25,36 +57,37 @@ echo "📥 KB Inbox — pending review" echo "============================" echo "" -if [ "$RECENT" -gt 0 ]; then - # Show most recent first - items=$(rg -l "status: inbox" raw/ 2>/dev/null | sort -r | head -"$RECENT") -else - items=$(rg -l "status: inbox" raw/ 2>/dev/null | sort -r) +# Collect items safely +search_path="raw/" +[ -n "$MONTH" ] && search_path="raw/${MONTH}/" + +mapfile -t items < <(match_inbox "$search_path" 2>/dev/null | sort -r) +if [ "$RECENT" -gt 0 ] && [ "${#items[@]}" -gt "$RECENT" ]; then + items=("${items[@]:0:$RECENT}") fi -count=0 -for f in $items; do - title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//') - url=$(grep "^source_url:" "$f" 2>/dev/null | head -1 | sed 's/source_url: *"//;s/"$//') - ingested=$(grep "^ingested:" "$f" 2>/dev/null | head -1 | sed 's/ingested: *//') +if [ "${#items[@]}" -eq 0 ]; then + echo "✅ No inbox items — all caught up!" + exit 0 +fi + +for f in "${items[@]}"; do + # Extract frontmatter fields + title=$(awk '/^---/{f++;next} f==1 && /^title:/{gsub(/^title: *"?/,""); gsub(/"$/,""); print; exit}' "$f" 2>/dev/null) + url=$(awk '/^---/{f++;next} f==1 && /^source_url:/{gsub(/^source_url: *"?/,""); gsub(/"$/,""); print; exit}' "$f" 2>/dev/null) + ingested=$(awk '/^---/{f++;next} f==1 && /^ingested:/{gsub(/^ingested: */,""); print; exit}' "$f" 2>/dev/null) if [ -z "$title" ]; then - title=$(basename "$f" .md | sed 's/^[0-9-]*//' | tr '-' ' ') + title=$(basename "$f" .md | sed 's/^[0-9-]*//' | tr '-' ' ' | sed 's/^ *//') fi - echo "[$ingested] 📄 ${title:0:80}" + echo "[${ingested:-?}] 📄 ${title:0:80}" if [ -n "$url" ]; then echo " 🔗 ${url:0:100}" fi echo " 📁 $f" echo "" - - count=$((count + 1)) done -if [ "$count" -eq 0 ]; then - echo "✅ No inbox items — all caught up!" -else - echo "---" - echo "Total: $count inbox items" -fi +echo "---" +echo "Total: ${#items[@]} inbox items" diff --git a/scripts/lint.sh b/scripts/lint.sh index 3436a43..0f43a05 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,46 +1,95 @@ #!/bin/bash -# KB Lint — basic health checks -# Usage: ./scripts/lint.sh [--fix-broken-links] +# KB Lint — health checks with exit codes +# Usage: ./scripts/lint.sh + +set -euo pipefail KB_DIR="$(cd "$(dirname "$0")/.." && pwd)" cd "$KB_DIR" +if ! command -v rg &>/dev/null; then + echo "❌ ripgrep (rg) is required." >&2 + exit 1 +fi + +broken=0 +stale=0 +orphan=0 + echo "🔍 KB Lint" echo "===========" echo "" -issues=0 - -# P0: Broken wikilinks +# ── P0: Broken wikilinks ────────────────────────────────── echo "--- P0: Broken Wikilinks ---" -# Find all [[...]] links in wiki/ -grep -roh '\[\[[^]]*\]\]' wiki/ 2>/dev/null | sed 's/\[\[//;s/\]\]//' | sort -u | while read link; do - # Check if the target file exists (case-insensitive) - found=$(find wiki/ -iname "${link}.md" 2>/dev/null | head -1) + +while IFS= read -r match; do + file="${match%%:*}" + raw_link="${match#*:}" + + # Strip [[ and ]] + link="${raw_link#\[\[}" + link="${link%\]\]}" + + # Strip alias: [[page|display]] → page + link="${link%%|*}" + # Strip anchor: [[page#heading]] → page + link="${link%%#*}" + link="$(echo "$link" | xargs)" # trim whitespace + + [ -z "$link" ] && continue + + # Look for matching .md file (case-insensitive) + found=$(find wiki/ -iname "${link}.md" -print -quit 2>/dev/null) if [ -z "$found" ]; then - echo "❌ Broken: [[$link]] → no matching file found" - issues=$((issues + 1)) + echo "❌ $file: [[$link]] → not found" + broken=1 fi -done +done < <(rg -noH '\[\[([^]]+)\]\]' wiki/ 2>/dev/null) + +if [ "$broken" -eq 0 ]; then + echo "✅ No broken links" +fi echo "" -# P1: Stale investing pages (>60 days) +# ── P1: Stale investing pages (>60 days) ───────────────── echo "--- P1: Stale Investing Pages (>60 days) ---" -find wiki/investing/ -name "*.md" -mtime +60 2>/dev/null | while read f; do - echo "⚠️ ${f} — last modified $(stat -c %y "$f" 2>/dev/null | cut -d' ' -f1)" -done +if [ -d wiki/investing ]; then + while IFS= read -r f; do + [ -z "$f" ] && continue + modified=$(date -r "$f" +%Y-%m-%d 2>/dev/null || stat -f %Sm -t %Y-%m-%d "$f" 2>/dev/null || echo "?") + echo "⚠️ $f — last modified $modified" + stale=1 + done < <(find wiki/investing -name "*.md" -mtime +60 2>/dev/null) +fi +if [ "$stale" -eq 0 ]; then + echo "✅ All investing pages up to date" +fi echo "" -# P2: Orphan pages (no incoming links from other wiki pages) +# ── P2: Orphan pages ────────────────────────────────────── echo "--- P2: Possible Orphans ---" -for page in wiki/**/*.md; do - [ ! -f "$page" ] && continue +while IFS= read -r page; do + [ -z "$page" ] && continue slug=$(basename "$page" .md) - refs=$(rg -l "\[\[${slug}(\]\]|\|)" wiki/ 2>/dev/null | grep -v "$page" | wc -l) + refs=$(rg -l "\[\[${slug}(\]\]|\||#)" wiki/ 2>/dev/null | grep -cv "$page" || true) if [ "$refs" -eq 0 ]; then echo "🔸 $page — no incoming links" + orphan=1 fi -done +done < <(find wiki -name "*.md" 2>/dev/null) + +if [ "$orphan" -eq 0 ]; then + echo "✅ No orphan pages" +fi echo "" -echo "✅ Lint complete" +# ── Result ───────────────────────────────────────────────── +issues=$((broken + stale + orphan)) +if [ "$issues" -gt 0 ]; then + echo "❌ Lint found issues: $broken broken links, $stale stale pages, $orphan orphans" + exit 1 +else + echo "✅ Lint clean — no issues found" + exit 0 +fi diff --git a/scripts/search.sh b/scripts/search.sh index 0430e38..7c4f5d8 100755 --- a/scripts/search.sh +++ b/scripts/search.sh @@ -1,76 +1,89 @@ #!/bin/bash -# KB Search — search across raw/ and wiki/ -# Usage: ./scripts/search.sh "query" [--wiki-only] [--raw-only] [--recent N] +# KB Search — search across raw/ and wiki/ with context +# Usage: ./scripts/search.sh "query" [--wiki-only|--raw-only] [--regex] [--recent N] + +set -euo pipefail KB_DIR="$(cd "$(dirname "$0")/.." && pwd)" +if ! command -v rg &>/dev/null; then + echo "❌ ripgrep (rg) is required. Install: apt install ripgrep / brew install ripgrep" >&2 + exit 1 +fi + if [ $# -lt 1 ]; then - echo "Usage: ./scripts/search.sh [options]" - echo "" - echo "Options:" - echo " --wiki-only Search only wiki/ pages" - echo " --raw-only Search only raw/ files" - echo " --recent N Show only N most recent results" - echo "" - echo "Examples:" - echo " ./scripts/search.sh NVIDIA" - echo " ./scripts/search.sh 'position sizing' --wiki-only" - echo " ./scripts/search.sh k3s --recent 10" + echo "Usage: ./scripts/search.sh [options]" >&2 + echo "" >&2 + echo "Options:" >&2 + echo " --wiki-only Search only wiki/ pages" >&2 + echo " --raw-only Search only raw/ files" >&2 + echo " --regex Treat query as regex (default: fixed string)" >&2 + echo " --files-only Show only matching file paths (no line context)" >&2 + echo "" >&2 + echo "Examples:" >&2 + echo " ./scripts/search.sh NVIDIA" >&2 + echo " ./scripts/search.sh 'k3s cluster' --wiki-only" >&2 exit 1 fi QUERY="$1" shift + WIKI_ONLY=false RAW_ONLY=false -RECENT=0 -ARGS="" +USE_REGEX=false +FILES_ONLY=false while [ $# -gt 0 ]; do case $1 in --wiki-only) WIKI_ONLY=true ;; --raw-only) RAW_ONLY=true ;; - --recent) RECENT="$2"; shift ;; - *) ARGS="$ARGS $1" ;; + --regex) USE_REGEX=true ;; + --files-only) FILES_ONLY=true ;; + *) + echo "Unknown option: $1" >&2 + exit 1 ;; esac shift done +if $WIKI_ONLY && $RAW_ONLY; then + echo "Error: cannot use both --wiki-only and --raw-only" >&2 + exit 1 +fi + cd "$KB_DIR" -if $WIKI_ONLY; then - echo "🔍 Searching wiki/ for: $QUERY" - echo "================================" - rg --color=always -l "$QUERY" wiki/ | while read f; do - title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//') - [ -z "$title" ] && title=$(basename "$f" .md) - echo "📘 [$title] — $f" - done -elif $RAW_ONLY; then - echo "🔍 Searching raw/ for: $QUERY" - echo "================================" - rg --color=always -l "$QUERY" raw/ | while read f; do - status=$(grep "^status:" "$f" 2>/dev/null | head -1 | sed 's/status: *//') - title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//') - [ -z "$title" ] && title=$(basename "$f" .md) - echo "📥 [$status] $title — $f" - done +# Build rg args +RG_ARGS=(-n --color=always --glob '*.md') +if $USE_REGEX; then + RG_ARGS+=(-e "$QUERY") else - echo "🔍 Searching all KB for: $QUERY" + RG_ARGS+=(-F -e "$QUERY") +fi +if $FILES_ONLY; then + RG_ARGS+=(-l) +fi + +search() { + local label="$1" + local dir="$2" + local prefix="$3" + + echo "" + echo "--- $label ---" + rg "${RG_ARGS[@]}" "$dir" 2>/dev/null | while IFS= read -r line; do + echo "${prefix} ${line}" + done || true +} + +if $WIKI_ONLY; then + search "Wiki Pages" "wiki/" "📘" +elif $RAW_ONLY; then + search "Raw Files" "raw/" "📥" +else + echo "🔍 Searching KB for: $QUERY" echo "================================" - echo "" - echo "--- Wiki Pages ---" - rg --color=always -l "$QUERY" wiki/ 2>/dev/null | while read f; do - title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//') - [ -z "$title" ] && title=$(basename "$f" .md) - echo "📘 $title — $f" - done - echo "" - echo "--- Raw Files ---" - rg --color=always -l "$QUERY" raw/ 2>/dev/null | while read f; do - status=$(grep "^status:" "$f" 2>/dev/null | head -1 | sed 's/status: *//') - title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//') - [ -z "$title" ] && title=$(basename "$f" .md) - echo "📥 [$status] $title — $f" - done + search "Wiki Pages" "wiki/" "📘" + search "Raw Files" "raw/" "📥" fi diff --git a/scripts/stats.sh b/scripts/stats.sh index e788bf5..8417ccd 100755 --- a/scripts/stats.sh +++ b/scripts/stats.sh @@ -11,9 +11,9 @@ echo "" # Raw counts raw_total=$(find raw/ -name "*.md" 2>/dev/null | wc -l) -raw_inbox=$(rg -l "status: inbox" raw/ 2>/dev/null | wc -l) -raw_absorbed=$(rg -l "status: absorbed" raw/ 2>/dev/null | wc -l) -raw_skipped=$(rg -l "status: skipped" raw/ 2>/dev/null | wc -l) +raw_inbox=$(rg -l --glob '*.md' '^status:\s*inbox\s*$' raw/ 2>/dev/null | wc -l) +raw_absorbed=$(rg -l --glob '*.md' '^status:\s*absorbed\s*$' raw/ 2>/dev/null | wc -l) +raw_skipped=$(rg -l --glob '*.md' '^status:\s*skipped\s*$' raw/ 2>/dev/null | wc -l) echo "📥 Raw Files" echo " Total: $raw_total" diff --git a/scripts/status.sh b/scripts/status.sh new file mode 100755 index 0000000..461853b --- /dev/null +++ b/scripts/status.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# KB Status — mark raw files as absorbed or skipped +# Usage: ./scripts/status.sh absorb or ./scripts/status.sh skip + +set -euo pipefail + +KB_DIR="$(cd "$(dirname "$0")/.." && pwd)" +cd "$KB_DIR" + +if [ $# -lt 2 ]; then + echo "Usage: ./scripts/status.sh " >&2 + echo "" >&2 + echo "Examples:" >&2 + echo " ./scripts/status.sh absorb raw/2026-07/2026-07-13-article.md" >&2 + echo " ./scripts/status.sh skip raw/2026-07/2026-07-13-old-news.md" >&2 + echo " ./scripts/status.sh inbox raw/2026-07/file.md # reset to inbox" >&2 + exit 1 +fi + +ACTION="$1" +FILE="$2" + +case "$ACTION" in + absorb|skip|inbox) ;; + *) + echo "Error: action must be 'absorb', 'skip', or 'inbox'" >&2 + exit 1 ;; +esac + +if [ ! -f "$FILE" ]; then + echo "Error: file not found: $FILE" >&2 + exit 1 +fi + +# Check it's a raw file +if [[ "$FILE" != raw/* ]]; then + echo "Error: only raw/ files can have status changed (not wiki/)" >&2 + exit 1 +fi + +# Update the status line in frontmatter +if grep -q '^status:' "$FILE"; then + sed -i "s/^status:.*/status: ${ACTION}/" "$FILE" +else + # No status field — add after the first --- line + sed -i "0,/^---$/s//---\nstatus: ${ACTION}/" "$FILE" +fi + +echo "✅ $FILE → status: ${ACTION}" + +# Append to log.md +TIMESTAMP=$(date +%Y-%m-%d) +TITLE=$(awk '/^---/{f++;next} f==1 && /^title:/{gsub(/^title: *"?/,""); gsub(/"$/,""); print; exit}' "$FILE" 2>/dev/null || basename "$FILE" .md) +echo "## [${TIMESTAMP}] ${ACTION} | ${TITLE}" >> log.md +echo " - ${FILE}" >> log.md +echo "📝 Logged to log.md"