diff --git a/.scripts/sync-links-to-wiki.py b/.scripts/sync-links-to-wiki.py new file mode 100755 index 0000000..016fd27 --- /dev/null +++ b/.scripts/sync-links-to-wiki.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +Sync new posts from Links App to the wiki vault. +Run daily via cron to keep wiki in sync with Links App. + +Usage: + python3 sync-links-to-wiki.py [--dry-run] +""" + +import urllib.request +import urllib.parse +import json +import os +import sys +import re +import subprocess +from datetime import datetime + +# Config +WIKI_DIR = os.path.expanduser("~/junv/one-knowledge") +LINKS_API = "http://go/api/posts" +SYNC_STATE_FILE = os.path.join(WIKI_DIR, ".sync-state") +POSTS_DIR = os.path.join(WIKI_DIR, "links-posts") + +def get_last_sync_time(): + """Get the last sync timestamp from state file.""" + if os.path.exists(SYNC_STATE_FILE): + with open(SYNC_STATE_FILE, 'r') as f: + return f.read().strip() + return None + +def save_last_sync_time(timestamp): + """Save the sync timestamp to state file.""" + with open(SYNC_STATE_FILE, 'w') as f: + f.write(timestamp) + +def slugify(title, max_len=50): + """Convert title to a valid filename slug.""" + # Remove special characters, keep Chinese and alphanumeric + slug = re.sub(r'[^\w\s\u4e00-\u9fff-]', '', title) + slug = re.sub(r'[-\s]+', '-', slug) + slug = slug.lower().strip('-')[:max_len] + return slug or 'untitled' + +def fetch_posts_since(since_timestamp=None, limit=100): + """Fetch posts from Links App API, stop when we hit posts older than since_timestamp.""" + posts = [] + page = 1 + stop = False + + while True and not stop: + url = f"{LINKS_API}?limit={limit}&page={page}" + print(f"Fetching page {page}...", end=" ", flush=True) + + try: + with urllib.request.urlopen(url, timeout=30) as resp: + data = json.loads(resp.read().decode('utf-8')) + except Exception as e: + print(f"Error fetching {url}: {e}") + break + + results = data.get('results', []) + if not results: + print("no results, stopping") + break + + # Filter by timestamp if provided + if since_timestamp: + for post in results: + if post['created_at'] > since_timestamp: + posts.append(post) + else: + stop = True + print(f"reached cutoff at post {post['id']}") + break + else: + posts.extend(results) + + # Check if there are more pages + if not data.get('next') or stop: + print("no more pages" if not stop else "") + break + + page += 1 + + # Safety limit + if page > 500: + print("safety limit reached") + break + + return posts + +def post_to_markdown(post): + """Convert a post object to Obsidian-compatible markdown.""" + post_id = post['id'] + title = post.get('title', 'Untitled') + content = post.get('content', post.get('summary', '')) + created = post.get('created_at', '')[:10] # Just the date + updated = post.get('updated_at', '')[:10] + tags = [t['name'] for t in post.get('tag_details', [])] + external_url = f"http://go/ui/posts/{post_id}/" + + # Build frontmatter + safe_title = title.replace('"', '\\"') + frontmatter = f"""--- +title: "{safe_title}" +created: {created} +updated: {updated} +type: summary +tags: [{', '.join(tags)}] +external: {external_url} +--- + +{content}""" + + return frontmatter + +def main(): + dry_run = '--dry-run' in sys.argv + + print(f"[{datetime.now().isoformat()}] Starting Links App sync...") + + # Get last sync time + last_sync = get_last_sync_time() + if last_sync: + print(f"Last sync: {last_sync}") + else: + print("First sync - will fetch all posts") + + # Fetch posts + posts = fetch_posts_since(last_sync) + print(f"Found {len(posts)} new posts") + + if not posts: + print("No new posts to sync") + return + + if dry_run: + print(f"[DRY RUN] Would sync {len(posts)} posts:") + for p in posts[:5]: + print(f" - {p['title']}") + if len(posts) > 5: + print(f" ... and {len(posts) - 5} more") + return + + # Create posts directory if needed + os.makedirs(POSTS_DIR, exist_ok=True) + + # Write posts + newest_timestamp = None + for post in posts: + post_id = post['id'] + title = post.get('title', 'Untitled') + slug = slugify(title) + filename = f"{post_id}-{slug}.md" + filepath = os.path.join(POSTS_DIR, filename) + + # Write markdown + with open(filepath, 'w', encoding='utf-8') as f: + f.write(post_to_markdown(post)) + + # Track newest timestamp + created = post.get('created_at', '') + if not newest_timestamp or created > newest_timestamp: + newest_timestamp = created + + print(f" Wrote: {filename}") + + # Update sync state + if newest_timestamp: + save_last_sync_time(newest_timestamp) + print(f"Updated sync state to {newest_timestamp}") + + # Git operations + print("Committing and pushing to GitHub...") + os.chdir(WIKI_DIR) + + # Check git config + subprocess.run(["git", "config", "user.name", "hermes"], capture_output=True) + subprocess.run(["git", "config", "user.email", "hermes@junv"], capture_output=True) + + # Add, commit, push + subprocess.run(["git", "add", "links-posts/"], capture_output=True) + subprocess.run(["git", "add", ".sync-state"], capture_output=True) + + result = subprocess.run(["git", "status", "--porcelain"], capture_output=True, text=True) + if result.stdout.strip(): + commit_msg = f"Sync: {len(posts)} new posts from Links App ({datetime.now().strftime('%Y-%m-%d %H:%M')})" + subprocess.run(["git", "commit", "-m", commit_msg], capture_output=True) + push_result = subprocess.run(["git", "push"], capture_output=True, text=True) + if push_result.returncode == 0: + print(f"✅ Pushed: {commit_msg}") + else: + print(f"⚠️ Push failed: {push_result.stderr}") + else: + print("No changes to commit") + + print(f"[{datetime.now().isoformat()}] Sync complete!") + +if __name__ == "__main__": + main() diff --git a/.sync-state b/.sync-state new file mode 100644 index 0000000..dd4854b --- /dev/null +++ b/.sync-state @@ -0,0 +1 @@ +2026-04-08T10:05:43.651150+10:00 \ No newline at end of file