kb: fix scripts after Grok review — add status.sh, fix inbox/search/lint bugs, update README with script reference + manual workflow

This commit is contained in:
hermes
2026-07-13 12:29:39 +10:00
parent 9bb0f5da8b
commit f07df2838e
6 changed files with 323 additions and 103 deletions
+69
View File
@@ -191,6 +191,75 @@ rg "tags:.*inv/thesis" wiki/
| 每天 22:00 | Hermes 对话扫描 → `raw/YYYY-MM/` 提取知识 |
| 每周日 | Review inbox → 决定 absorb 哪些 |
## 脚本参考
```bash
# 查看 inbox
./scripts/inbox.sh # 列出所有待处理
./scripts/inbox.sh --count # 只看数量
./scripts/inbox.sh --recent 10 # 最近 10 条
./scripts/inbox.sh --month 2026-07 # 只看某月
# 搜索
./scripts/search.sh "NVIDIA" # 全文搜索(固定字符串)
./scripts/search.sh "k3s" --wiki-only # 只搜 wiki
./scripts/search.sh "earnings" --raw-only # 只搜 raw
./scripts/search.sh "C\+\+" --regex # 使用正则
./scripts/search.sh "NVIDIA" --files-only # 只列文件名
# 标记状态
./scripts/status.sh absorb raw/2026-07/article.md # 标记为已吸收
./scripts/status.sh skip raw/2026-07/old-news.md # 跳过
./scripts/status.sh inbox raw/2026-07/file.md # 重置为待处理
# 统计
./scripts/stats.sh
# 健康检查(退出码非零 = 有问题)
./scripts/lint.sh
```
## 前置依赖
- `bash` 4+
- `ripgrep` (`rg`) — `apt install ripgrep``brew install ripgrep`
- `git`
## 不依赖 Hermes 的手动操作
### 手动 Ingest(你自己加一篇文章)
1. 创建文件 `raw/YYYY-MM/YYYY-MM-DD-slug.md`
2. 写 frontmatter
```yaml
---
source_url: https://example.com/article
ingested: 2026-07-13
status: inbox
---
```
3. 正文放下面
4. `git add` + `git commit`
### 手动 Absorb(从 raw 编译到 wiki
1. `./scripts/inbox.sh` 看看有哪些待处理
2. 读 raw 文件,理解内容
3. 搜 wiki 找相关页面:`./scripts/search.sh "关键词" --wiki-only`
4. 编辑或创建 wiki 页面
5. `./scripts/status.sh absorb raw/路径` 标记已完成
6. 更新页面 `[[wikilinks]]` 交叉引用
### Status 生命周期
```
inbox → absorbed (内容已编译到 wiki/)
inbox → skipped (看过但不值得编译)
inbox → failed_fetch (链接打不开/抓取失败)
```
raw 文件标记 absorbed/skipped 后不会再出现在 inbox,但文件保留在 raw/ 中可搜索。
## 隐私
- GitHub repo = **private**
+60 -27
View File
@@ -1,23 +1,55 @@
#!/bin/bash
# KB Inbox — list all raw files with status: inbox
# Usage: ./scripts/inbox.sh [--count] [--recent N]
# KB Inbox — list raw files with status: inbox
# Usage: ./scripts/inbox.sh [--count] [--recent N] [--month YYYY-MM]
set -euo pipefail
KB_DIR="$(cd "$(dirname "$0")/.." && pwd)"
COUNT_ONLY=false
RECENT=0
MONTH=""
for arg in "$@"; do
case $arg in
--count) COUNT_ONLY=true ;;
--recent) RECENT=1 ;;
# Check dependencies
if ! command -v rg &>/dev/null; then
echo "❌ ripgrep (rg) is required. Install: apt install ripgrep / brew install ripgrep" >&2
exit 1
fi
while [ $# -gt 0 ]; do
case $1 in
--count)
COUNT_ONLY=true ;;
--recent)
if [[ ! "${2-}" =~ ^[0-9]+$ ]]; then
echo "Error: --recent requires a number" >&2
exit 1
fi
RECENT="$2"; shift ;;
--month)
MONTH="$2"; shift ;;
*)
echo "Unknown option: $1" >&2
echo "Usage: ./scripts/inbox.sh [--count] [--recent N] [--month YYYY-MM]" >&2
exit 1 ;;
esac
shift
done
cd "$KB_DIR"
# Use frontmatter-only match: status: inbox in the YAML header
match_inbox() {
rg -l --glob '*.md' '^status:\s*inbox\s*$' "$@"
}
if $COUNT_ONLY; then
count=$(rg -l "status: inbox" raw/ 2>/dev/null | wc -l)
echo "📥 Inbox: $count items pending"
if [ -n "$MONTH" ]; then
count=$(match_inbox "raw/${MONTH}/" 2>/dev/null | wc -l)
echo "📥 Inbox ($MONTH): $count items pending"
else
count=$(match_inbox raw/ 2>/dev/null | wc -l)
echo "📥 Inbox total: $count items pending"
fi
exit 0
fi
@@ -25,36 +57,37 @@ echo "📥 KB Inbox — pending review"
echo "============================"
echo ""
if [ "$RECENT" -gt 0 ]; then
# Show most recent first
items=$(rg -l "status: inbox" raw/ 2>/dev/null | sort -r | head -"$RECENT")
else
items=$(rg -l "status: inbox" raw/ 2>/dev/null | sort -r)
# Collect items safely
search_path="raw/"
[ -n "$MONTH" ] && search_path="raw/${MONTH}/"
mapfile -t items < <(match_inbox "$search_path" 2>/dev/null | sort -r)
if [ "$RECENT" -gt 0 ] && [ "${#items[@]}" -gt "$RECENT" ]; then
items=("${items[@]:0:$RECENT}")
fi
count=0
for f in $items; do
title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//')
url=$(grep "^source_url:" "$f" 2>/dev/null | head -1 | sed 's/source_url: *"//;s/"$//')
ingested=$(grep "^ingested:" "$f" 2>/dev/null | head -1 | sed 's/ingested: *//')
if [ "${#items[@]}" -eq 0 ]; then
echo "✅ No inbox items — all caught up!"
exit 0
fi
for f in "${items[@]}"; do
# Extract frontmatter fields
title=$(awk '/^---/{f++;next} f==1 && /^title:/{gsub(/^title: *"?/,""); gsub(/"$/,""); print; exit}' "$f" 2>/dev/null)
url=$(awk '/^---/{f++;next} f==1 && /^source_url:/{gsub(/^source_url: *"?/,""); gsub(/"$/,""); print; exit}' "$f" 2>/dev/null)
ingested=$(awk '/^---/{f++;next} f==1 && /^ingested:/{gsub(/^ingested: */,""); print; exit}' "$f" 2>/dev/null)
if [ -z "$title" ]; then
title=$(basename "$f" .md | sed 's/^[0-9-]*//' | tr '-' ' ')
title=$(basename "$f" .md | sed 's/^[0-9-]*//' | tr '-' ' ' | sed 's/^ *//')
fi
echo "[$ingested] 📄 ${title:0:80}"
echo "[${ingested:-?}] 📄 ${title:0:80}"
if [ -n "$url" ]; then
echo " 🔗 ${url:0:100}"
fi
echo " 📁 $f"
echo ""
count=$((count + 1))
done
if [ "$count" -eq 0 ]; then
echo "✅ No inbox items — all caught up!"
else
echo "---"
echo "Total: $count inbox items"
fi
echo "Total: ${#items[@]} inbox items"
+71 -22
View File
@@ -1,46 +1,95 @@
#!/bin/bash
# KB Lint — basic health checks
# Usage: ./scripts/lint.sh [--fix-broken-links]
# KB Lint — health checks with exit codes
# Usage: ./scripts/lint.sh
set -euo pipefail
KB_DIR="$(cd "$(dirname "$0")/.." && pwd)"
cd "$KB_DIR"
if ! command -v rg &>/dev/null; then
echo "❌ ripgrep (rg) is required." >&2
exit 1
fi
broken=0
stale=0
orphan=0
echo "🔍 KB Lint"
echo "==========="
echo ""
issues=0
# P0: Broken wikilinks
# ── P0: Broken wikilinks ──────────────────────────────────
echo "--- P0: Broken Wikilinks ---"
# Find all [[...]] links in wiki/
grep -roh '\[\[[^]]*\]\]' wiki/ 2>/dev/null | sed 's/\[\[//;s/\]\]//' | sort -u | while read link; do
# Check if the target file exists (case-insensitive)
found=$(find wiki/ -iname "${link}.md" 2>/dev/null | head -1)
while IFS= read -r match; do
file="${match%%:*}"
raw_link="${match#*:}"
# Strip [[ and ]]
link="${raw_link#\[\[}"
link="${link%\]\]}"
# Strip alias: [[page|display]] → page
link="${link%%|*}"
# Strip anchor: [[page#heading]] → page
link="${link%%#*}"
link="$(echo "$link" | xargs)" # trim whitespace
[ -z "$link" ] && continue
# Look for matching .md file (case-insensitive)
found=$(find wiki/ -iname "${link}.md" -print -quit 2>/dev/null)
if [ -z "$found" ]; then
echo "Broken: [[$link]] → no matching file found"
issues=$((issues + 1))
echo "$file: [[$link]] → not found"
broken=1
fi
done < <(rg -noH '\[\[([^]]+)\]\]' wiki/ 2>/dev/null)
if [ "$broken" -eq 0 ]; then
echo "✅ No broken links"
fi
done
echo ""
# P1: Stale investing pages (>60 days)
# ── P1: Stale investing pages (>60 days) ─────────────────
echo "--- P1: Stale Investing Pages (>60 days) ---"
find wiki/investing/ -name "*.md" -mtime +60 2>/dev/null | while read f; do
echo "⚠️ ${f} — last modified $(stat -c %y "$f" 2>/dev/null | cut -d' ' -f1)"
done
if [ -d wiki/investing ]; then
while IFS= read -r f; do
[ -z "$f" ] && continue
modified=$(date -r "$f" +%Y-%m-%d 2>/dev/null || stat -f %Sm -t %Y-%m-%d "$f" 2>/dev/null || echo "?")
echo "⚠️ $f — last modified $modified"
stale=1
done < <(find wiki/investing -name "*.md" -mtime +60 2>/dev/null)
fi
if [ "$stale" -eq 0 ]; then
echo "✅ All investing pages up to date"
fi
echo ""
# P2: Orphan pages (no incoming links from other wiki pages)
# ── P2: Orphan pages ──────────────────────────────────────
echo "--- P2: Possible Orphans ---"
for page in wiki/**/*.md; do
[ ! -f "$page" ] && continue
while IFS= read -r page; do
[ -z "$page" ] && continue
slug=$(basename "$page" .md)
refs=$(rg -l "\[\[${slug}(\]\]|\|)" wiki/ 2>/dev/null | grep -v "$page" | wc -l)
refs=$(rg -l "\[\[${slug}(\]\]|\||#)" wiki/ 2>/dev/null | grep -cv "$page" || true)
if [ "$refs" -eq 0 ]; then
echo "🔸 $page — no incoming links"
orphan=1
fi
done < <(find wiki -name "*.md" 2>/dev/null)
if [ "$orphan" -eq 0 ]; then
echo "✅ No orphan pages"
fi
done
echo ""
echo "✅ Lint complete"
# ── Result ─────────────────────────────────────────────────
issues=$((broken + stale + orphan))
if [ "$issues" -gt 0 ]; then
echo "❌ Lint found issues: $broken broken links, $stale stale pages, $orphan orphans"
exit 1
else
echo "✅ Lint clean — no issues found"
exit 0
fi
+63 -50
View File
@@ -1,76 +1,89 @@
#!/bin/bash
# KB Search — search across raw/ and wiki/
# Usage: ./scripts/search.sh "query" [--wiki-only] [--raw-only] [--recent N]
# KB Search — search across raw/ and wiki/ with context
# Usage: ./scripts/search.sh "query" [--wiki-only|--raw-only] [--regex] [--recent N]
set -euo pipefail
KB_DIR="$(cd "$(dirname "$0")/.." && pwd)"
if ! command -v rg &>/dev/null; then
echo "❌ ripgrep (rg) is required. Install: apt install ripgrep / brew install ripgrep" >&2
exit 1
fi
if [ $# -lt 1 ]; then
echo "Usage: ./scripts/search.sh <query> [options]"
echo ""
echo "Options:"
echo " --wiki-only Search only wiki/ pages"
echo " --raw-only Search only raw/ files"
echo " --recent N Show only N most recent results"
echo ""
echo "Examples:"
echo " ./scripts/search.sh NVIDIA"
echo " ./scripts/search.sh 'position sizing' --wiki-only"
echo " ./scripts/search.sh k3s --recent 10"
echo "Usage: ./scripts/search.sh <query> [options]" >&2
echo "" >&2
echo "Options:" >&2
echo " --wiki-only Search only wiki/ pages" >&2
echo " --raw-only Search only raw/ files" >&2
echo " --regex Treat query as regex (default: fixed string)" >&2
echo " --files-only Show only matching file paths (no line context)" >&2
echo "" >&2
echo "Examples:" >&2
echo " ./scripts/search.sh NVIDIA" >&2
echo " ./scripts/search.sh 'k3s cluster' --wiki-only" >&2
exit 1
fi
QUERY="$1"
shift
WIKI_ONLY=false
RAW_ONLY=false
RECENT=0
ARGS=""
USE_REGEX=false
FILES_ONLY=false
while [ $# -gt 0 ]; do
case $1 in
--wiki-only) WIKI_ONLY=true ;;
--raw-only) RAW_ONLY=true ;;
--recent) RECENT="$2"; shift ;;
*) ARGS="$ARGS $1" ;;
--regex) USE_REGEX=true ;;
--files-only) FILES_ONLY=true ;;
*)
echo "Unknown option: $1" >&2
exit 1 ;;
esac
shift
done
if $WIKI_ONLY && $RAW_ONLY; then
echo "Error: cannot use both --wiki-only and --raw-only" >&2
exit 1
fi
cd "$KB_DIR"
if $WIKI_ONLY; then
echo "🔍 Searching wiki/ for: $QUERY"
echo "================================"
rg --color=always -l "$QUERY" wiki/ | while read f; do
title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//')
[ -z "$title" ] && title=$(basename "$f" .md)
echo "📘 [$title] — $f"
done
elif $RAW_ONLY; then
echo "🔍 Searching raw/ for: $QUERY"
echo "================================"
rg --color=always -l "$QUERY" raw/ | while read f; do
status=$(grep "^status:" "$f" 2>/dev/null | head -1 | sed 's/status: *//')
title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//')
[ -z "$title" ] && title=$(basename "$f" .md)
echo "📥 [$status] $title$f"
done
# Build rg args
RG_ARGS=(-n --color=always --glob '*.md')
if $USE_REGEX; then
RG_ARGS+=(-e "$QUERY")
else
echo "🔍 Searching all KB for: $QUERY"
RG_ARGS+=(-F -e "$QUERY")
fi
if $FILES_ONLY; then
RG_ARGS+=(-l)
fi
search() {
local label="$1"
local dir="$2"
local prefix="$3"
echo ""
echo "--- $label ---"
rg "${RG_ARGS[@]}" "$dir" 2>/dev/null | while IFS= read -r line; do
echo "${prefix} ${line}"
done || true
}
if $WIKI_ONLY; then
search "Wiki Pages" "wiki/" "📘"
elif $RAW_ONLY; then
search "Raw Files" "raw/" "📥"
else
echo "🔍 Searching KB for: $QUERY"
echo "================================"
echo ""
echo "--- Wiki Pages ---"
rg --color=always -l "$QUERY" wiki/ 2>/dev/null | while read f; do
title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//')
[ -z "$title" ] && title=$(basename "$f" .md)
echo "📘 $title$f"
done
echo ""
echo "--- Raw Files ---"
rg --color=always -l "$QUERY" raw/ 2>/dev/null | while read f; do
status=$(grep "^status:" "$f" 2>/dev/null | head -1 | sed 's/status: *//')
title=$(grep "^title:" "$f" 2>/dev/null | head -1 | sed 's/title: *"//;s/"$//')
[ -z "$title" ] && title=$(basename "$f" .md)
echo "📥 [$status] $title$f"
done
search "Wiki Pages" "wiki/" "📘"
search "Raw Files" "raw/" "📥"
fi
+3 -3
View File
@@ -11,9 +11,9 @@ echo ""
# Raw counts
raw_total=$(find raw/ -name "*.md" 2>/dev/null | wc -l)
raw_inbox=$(rg -l "status: inbox" raw/ 2>/dev/null | wc -l)
raw_absorbed=$(rg -l "status: absorbed" raw/ 2>/dev/null | wc -l)
raw_skipped=$(rg -l "status: skipped" raw/ 2>/dev/null | wc -l)
raw_inbox=$(rg -l --glob '*.md' '^status:\s*inbox\s*$' raw/ 2>/dev/null | wc -l)
raw_absorbed=$(rg -l --glob '*.md' '^status:\s*absorbed\s*$' raw/ 2>/dev/null | wc -l)
raw_skipped=$(rg -l --glob '*.md' '^status:\s*skipped\s*$' raw/ 2>/dev/null | wc -l)
echo "📥 Raw Files"
echo " Total: $raw_total"
+56
View File
@@ -0,0 +1,56 @@
#!/bin/bash
# KB Status — mark raw files as absorbed or skipped
# Usage: ./scripts/status.sh absorb <path> or ./scripts/status.sh skip <path>
set -euo pipefail
KB_DIR="$(cd "$(dirname "$0")/.." && pwd)"
cd "$KB_DIR"
if [ $# -lt 2 ]; then
echo "Usage: ./scripts/status.sh <absorb|skip|inbox> <path-to-raw-file>" >&2
echo "" >&2
echo "Examples:" >&2
echo " ./scripts/status.sh absorb raw/2026-07/2026-07-13-article.md" >&2
echo " ./scripts/status.sh skip raw/2026-07/2026-07-13-old-news.md" >&2
echo " ./scripts/status.sh inbox raw/2026-07/file.md # reset to inbox" >&2
exit 1
fi
ACTION="$1"
FILE="$2"
case "$ACTION" in
absorb|skip|inbox) ;;
*)
echo "Error: action must be 'absorb', 'skip', or 'inbox'" >&2
exit 1 ;;
esac
if [ ! -f "$FILE" ]; then
echo "Error: file not found: $FILE" >&2
exit 1
fi
# Check it's a raw file
if [[ "$FILE" != raw/* ]]; then
echo "Error: only raw/ files can have status changed (not wiki/)" >&2
exit 1
fi
# Update the status line in frontmatter
if grep -q '^status:' "$FILE"; then
sed -i "s/^status:.*/status: ${ACTION}/" "$FILE"
else
# No status field — add after the first --- line
sed -i "0,/^---$/s//---\nstatus: ${ACTION}/" "$FILE"
fi
echo "$FILE → status: ${ACTION}"
# Append to log.md
TIMESTAMP=$(date +%Y-%m-%d)
TITLE=$(awk '/^---/{f++;next} f==1 && /^title:/{gsub(/^title: *"?/,""); gsub(/"$/,""); print; exit}' "$FILE" 2>/dev/null || basename "$FILE" .md)
echo "## [${TIMESTAMP}] ${ACTION} | ${TITLE}" >> log.md
echo " - ${FILE}" >> log.md
echo "📝 Logged to log.md"