Add sync script and state tracker

.scripts/sync-links-to-wiki.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+Sync new posts from Links App to the wiki vault.
+Run daily via cron to keep wiki in sync with Links App.
+
+Usage:
+    python3 sync-links-to-wiki.py [--dry-run]
+"""
+
+import urllib.request
+import urllib.parse
+import json
+import os
+import sys
+import re
+import subprocess
+from datetime import datetime
+
+# Config
+WIKI_DIR = os.path.expanduser("~/junv/one-knowledge")
+LINKS_API = "http://go/api/posts"
+SYNC_STATE_FILE = os.path.join(WIKI_DIR, ".sync-state")
+POSTS_DIR = os.path.join(WIKI_DIR, "links-posts")
+
+def get_last_sync_time():
+    """Get the last sync timestamp from state file."""
+    if os.path.exists(SYNC_STATE_FILE):
+        with open(SYNC_STATE_FILE, 'r') as f:
+            return f.read().strip()
+    return None
+
+def save_last_sync_time(timestamp):
+    """Save the sync timestamp to state file."""
+    with open(SYNC_STATE_FILE, 'w') as f:
+        f.write(timestamp)
+
+def slugify(title, max_len=50):
+    """Convert title to a valid filename slug."""
+    # Remove special characters, keep Chinese and alphanumeric
+    slug = re.sub(r'[^\w\s\u4e00-\u9fff-]', '', title)
+    slug = re.sub(r'[-\s]+', '-', slug)
+    slug = slug.lower().strip('-')[:max_len]
+    return slug or 'untitled'
+
+def fetch_posts_since(since_timestamp=None, limit=100):
+    """Fetch posts from Links App API, stop when we hit posts older than since_timestamp."""
+    posts = []
+    page = 1
+    stop = False
+    
+    while True and not stop:
+        url = f"{LINKS_API}?limit={limit}&page={page}"
+        print(f"Fetching page {page}...", end=" ", flush=True)
+        
+        try:
+            with urllib.request.urlopen(url, timeout=30) as resp:
+                data = json.loads(resp.read().decode('utf-8'))
+        except Exception as e:
+            print(f"Error fetching {url}: {e}")
+            break
+        
+        results = data.get('results', [])
+        if not results:
+            print("no results, stopping")
+            break
+        
+        # Filter by timestamp if provided
+        if since_timestamp:
+            for post in results:
+                if post['created_at'] > since_timestamp:
+                    posts.append(post)
+                else:
+                    stop = True
+                    print(f"reached cutoff at post {post['id']}")
+                    break
+        else:
+            posts.extend(results)
+        
+        # Check if there are more pages
+        if not data.get('next') or stop:
+            print("no more pages" if not stop else "")
+            break
+        
+        page += 1
+        
+        # Safety limit
+        if page > 500:
+            print("safety limit reached")
+            break
+    
+    return posts
+
+def post_to_markdown(post):
+    """Convert a post object to Obsidian-compatible markdown."""
+    post_id = post['id']
+    title = post.get('title', 'Untitled')
+    content = post.get('content', post.get('summary', ''))
+    created = post.get('created_at', '')[:10]  # Just the date
+    updated = post.get('updated_at', '')[:10]
+    tags = [t['name'] for t in post.get('tag_details', [])]
+    external_url = f"http://go/ui/posts/{post_id}/"
+    
+    # Build frontmatter
+    safe_title = title.replace('"', '\\"')
+    frontmatter = f"""---
+title: "{safe_title}"
+created: {created}
+updated: {updated}
+type: summary
+tags: [{', '.join(tags)}]
+external: {external_url}
+---
+
+{content}"""
+
+    return frontmatter
+
+def main():
+    dry_run = '--dry-run' in sys.argv
+    
+    print(f"[{datetime.now().isoformat()}] Starting Links App sync...")
+    
+    # Get last sync time
+    last_sync = get_last_sync_time()
+    if last_sync:
+        print(f"Last sync: {last_sync}")
+    else:
+        print("First sync - will fetch all posts")
+    
+    # Fetch posts
+    posts = fetch_posts_since(last_sync)
+    print(f"Found {len(posts)} new posts")
+    
+    if not posts:
+        print("No new posts to sync")
+        return
+    
+    if dry_run:
+        print(f"[DRY RUN] Would sync {len(posts)} posts:")
+        for p in posts[:5]:
+            print(f"  - {p['title']}")
+        if len(posts) > 5:
+            print(f"  ... and {len(posts) - 5} more")
+        return
+    
+    # Create posts directory if needed
+    os.makedirs(POSTS_DIR, exist_ok=True)
+    
+    # Write posts
+    newest_timestamp = None
+    for post in posts:
+        post_id = post['id']
+        title = post.get('title', 'Untitled')
+        slug = slugify(title)
+        filename = f"{post_id}-{slug}.md"
+        filepath = os.path.join(POSTS_DIR, filename)
+        
+        # Write markdown
+        with open(filepath, 'w', encoding='utf-8') as f:
+            f.write(post_to_markdown(post))
+        
+        # Track newest timestamp
+        created = post.get('created_at', '')
+        if not newest_timestamp or created > newest_timestamp:
+            newest_timestamp = created
+        
+        print(f"  Wrote: {filename}")
+    
+    # Update sync state
+    if newest_timestamp:
+        save_last_sync_time(newest_timestamp)
+        print(f"Updated sync state to {newest_timestamp}")
+    
+    # Git operations
+    print("Committing and pushing to GitHub...")
+    os.chdir(WIKI_DIR)
+    
+    # Check git config
+    subprocess.run(["git", "config", "user.name", "hermes"], capture_output=True)
+    subprocess.run(["git", "config", "user.email", "hermes@junv"], capture_output=True)
+    
+    # Add, commit, push
+    subprocess.run(["git", "add", "links-posts/"], capture_output=True)
+    subprocess.run(["git", "add", ".sync-state"], capture_output=True)
+    
+    result = subprocess.run(["git", "status", "--porcelain"], capture_output=True, text=True)
+    if result.stdout.strip():
+        commit_msg = f"Sync: {len(posts)} new posts from Links App ({datetime.now().strftime('%Y-%m-%d %H:%M')})"
+        subprocess.run(["git", "commit", "-m", commit_msg], capture_output=True)
+        push_result = subprocess.run(["git", "push"], capture_output=True, text=True)
+        if push_result.returncode == 0:
+            print(f"✅ Pushed: {commit_msg}")
+        else:
+            print(f"⚠️ Push failed: {push_result.stderr}")
+    else:
+        print("No changes to commit")
+    
+    print(f"[{datetime.now().isoformat()}] Sync complete!")
+
+if __name__ == "__main__":
+    main()