From 41a8726fd34ec40581c2bffdeac461733a7027a1 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Tue, 4 Nov 2025 20:15:43 +1100 Subject: [PATCH] Update worker to be apscheduler --- CELERY_TO_APSCHEDULER_MIGRATION.md | 179 ++++++++++++++++++++++++ agents.md | 213 +++++++++++++++++++---------- core/__init__.py | 4 + core/apps.py | 33 +++++ core/celery.py | 14 -- core/scheduler.py | 40 ++++++ core/settings.py | 26 +--- data/db.sqlite3 | Bin 389120 -> 389120 bytes docker-compose.yml | 71 ---------- docker.sh | 12 +- k8s/manifest.yaml | 96 ------------- links/page_views.py | 22 ++- links/tasks.py | 72 ++++++---- pyproject.toml | 4 +- uv.lock | 210 +++------------------------- 15 files changed, 490 insertions(+), 506 deletions(-) create mode 100644 CELERY_TO_APSCHEDULER_MIGRATION.md create mode 100644 core/__init__.py create mode 100644 core/apps.py delete mode 100644 core/celery.py create mode 100644 core/scheduler.py diff --git a/CELERY_TO_APSCHEDULER_MIGRATION.md b/CELERY_TO_APSCHEDULER_MIGRATION.md new file mode 100644 index 0000000..2d8791d --- /dev/null +++ b/CELERY_TO_APSCHEDULER_MIGRATION.md @@ -0,0 +1,179 @@ +# Migration from Celery to APScheduler + +## Summary + +This project has been migrated from using Celery (with Redis broker) to APScheduler for background task processing. This simplifies the deployment architecture by running all background tasks in-process with the Django application, eliminating the need for separate worker containers and Redis infrastructure. + +## Key Changes + +### 1. Dependencies (`pyproject.toml`) +- **Removed**: `celery>=5.3.0`, `redis>=4.5.0`, `flower>=2.0.0` +- **Added**: `apscheduler>=3.10.0` + +### 2. New Files Created +- `core/scheduler.py`: APScheduler configuration and initialization +- `core/apps.py`: Django app configuration to start scheduler on app ready +- `core/__init__.py`: App configuration reference + +### 3. Modified Files + +#### `links/tasks.py` +- Removed `@shared_task` decorators +- Changed function signatures to accept `retry_count` parameter +- Replaced Celery's `.delay()` and `.retry()` with: + - Threading for immediate background execution + - APScheduler for delayed/scheduled retries +- Updated retry logic to use APScheduler's job scheduling + +#### `links/page_views.py` +- Added `from threading import Thread` import +- Replaced all `task.delay()` calls with thread-based execution +- Updated screenshot creation to use threading + +#### `core/settings.py` +- Removed Celery configuration variables: + - `CELERY_BROKER_URL` + - `CELERY_RESULT_BACKEND` + - `CELERY_ACCEPT_CONTENT` + - `CELERY_TASK_SERIALIZER` + - `CELERY_RESULT_SERIALIZER` + - `CELERY_TIMEZONE` + - `CELERYBEAT_SCHEDULE_FILENAME` + - `CELERY_BEAT_SCHEDULE` +- Added `core.apps.CoreConfig` to `INSTALLED_APPS` + +#### `docker-compose.yml` +- Removed services: + - `celery_worker` + - `celery_beat` + - `celery_flower` + - `redis` +- Removed Celery environment variables from `web` service +- Removed volume mount for `/app/data/celery` + +#### `k8s/manifest.yaml` +- Removed environment variables: + - `CELERY_BROKER_URL` + - `CELERY_RESULT_BACKEND` +- Commented out worker deployment (lines 271-362) + +#### `core/celery.py` +- **Deleted**: No longer needed + +### 4. Documentation Updates (`agents.md`) +- Updated architecture diagram +- Replaced Celery sections with APScheduler documentation +- Updated troubleshooting guide +- Updated command reference +- Updated environment variables documentation + +## How It Works Now + +### Task Execution + +#### Immediate Background Execution +Tasks that need to run immediately use Python threading: +```python +from threading import Thread +thread = Thread(target=process_page, args=(page_id,)) +thread.daemon = True +thread.start() +``` + +#### Delayed/Scheduled Execution +Tasks that need to be retried or scheduled use APScheduler: +```python +from core.scheduler import scheduler +from datetime import datetime, timedelta +run_date = datetime.now() + timedelta(seconds=delay) +scheduler.add_job( + capture_screenshot, + 'date', + run_date=run_date, + args=[page_id, screenshot_id, retry_count + 1] +) +``` + +#### Periodic Tasks +Registered in `core/apps.py` when Django starts: +```python +scheduler.add_job( + schedule_pending_pages, + 'interval', + seconds=120, + id='schedule_pending_pages', + replace_existing=True +) +``` + +### Scheduler Configuration +- **Executor**: ThreadPoolExecutor with 20 threads +- **Max Instances**: 3 per job +- **Misfire Grace Time**: 300 seconds +- **Timezone**: Inherited from Django's `TIME_ZONE` setting + +## Benefits + +1. **Simplified Deployment**: No need for separate worker containers +2. **Reduced Infrastructure**: No Redis dependency for task queue +3. **Easier Development**: Everything runs in one process +4. **Lower Resource Usage**: Fewer containers to manage +5. **Simpler Debugging**: All logs in one place + +## Migration Checklist + +- [x] Update dependencies in `pyproject.toml` +- [x] Create APScheduler configuration +- [x] Update task functions +- [x] Update task callers +- [x] Remove Celery configuration from settings +- [x] Update Docker Compose +- [x] Update Kubernetes manifests +- [x] Update documentation +- [x] Test local development +- [ ] Test Docker build +- [ ] Test Kubernetes deployment + +## Testing + +### Local Development +```bash +source .venv/bin/activate +uv sync +python manage.py migrate +python manage.py runserver +``` + +The scheduler will start automatically and you should see log messages indicating: +- "APScheduler started" +- "Scheduled periodic task: schedule_pending_pages" + +### Docker Development +```bash +docker-compose up web +``` + +### Production (Kubernetes) +```bash +kubectl apply -f k8s/manifest.yaml +``` + +Monitor logs to ensure scheduler starts correctly. + +## Rollback Plan + +If needed to rollback: +1. Revert changes to `pyproject.toml` (restore Celery dependencies) +2. Restore `core/celery.py` +3. Revert changes to `links/tasks.py` (restore `@shared_task` decorators) +4. Revert changes to `links/page_views.py` (restore `.delay()` calls) +5. Restore Celery configuration in `core/settings.py` +6. Restore worker services in `docker-compose.yml` and `k8s/manifest.yaml` +7. Run `uv sync` to install Celery dependencies + +## Notes + +- APScheduler stores job state in memory, so scheduled jobs are lost on restart +- For production with multiple replicas, consider using a persistent job store if needed +- Thread pool size (20) can be adjusted in `core/scheduler.py` based on workload +- The periodic task runs every 120 seconds to check for pending pages diff --git a/agents.md b/agents.md index adccb5a..61c0fbc 100644 --- a/agents.md +++ b/agents.md @@ -8,10 +8,10 @@ This guide is designed to help AI coding agents understand and work with the URL ### Tech Stack -- **Backend**: Django 5.1, Django REST Framework, Celery +- **Backend**: Django 5.1, Django REST Framework, APScheduler - **Frontend**: Tailwind CSS, AlpineJS (minimal JS) - **Database**: SQLite (default, configurable) -- **Task Queue**: Celery with Redis broker +- **Task Queue**: APScheduler (in-process background scheduler) - **Storage**: Local filesystem or AWS S3/Cloudflare R2 - **Web Automation**: Selenium with Chromium (for screenshots and scraping) - **Mobile**: SwiftUI iOS app with Core Data @@ -39,10 +39,10 @@ This guide is designed to help AI coding agents understand and work with the URL └──────────────────┬──────────────────────────────────────────┘ │ ┌──────────────────┴──────────────────────────────────────────┐ -│ Background Tasks (Celery) │ -│ - Screenshot capture │ -│ - Page metadata extraction │ -│ - Periodic cleanup tasks │ +│ Background Tasks (APScheduler + Threading) │ +│ - Screenshot capture (background threads) │ +│ - Page metadata extraction (background threads) │ +│ - Periodic task scheduling (APScheduler) │ └─────────────────────────────────────────────────────────────┘ │ ┌──────────────────┴──────────────────────────────────────────┐ @@ -132,53 +132,92 @@ Located in various `*_views.py` files with corresponding `*_urls.py`: - OpenAPI 3.1 spec available at `/static/openapi.yaml` - Documents all endpoints, request/response schemas, and authentication -## Background Task System (Celery) +## Background Task System (APScheduler) + +The application uses APScheduler for background task scheduling, running in-process with the Django application. This eliminates the need for separate worker processes and Redis broker. ### Tasks (links/tasks.py) -#### @shared_task: fetch_page_async +#### process_page(page_id, retry_count=0) Asynchronously fetches page metadata: 1. Downloads page HTML 2. Extracts title, description, meta tags 3. Stores content in Page model 4. Updates processing status +5. Triggers screenshot capture **Features**: - Retry logic with exponential backoff (Fibonacci sequence) - Timeout protection (20s) - BeautifulSoup for HTML parsing - Multiple metadata extraction strategies (title, h1, og:tags) +- Runs in background thread for non-blocking execution -#### @shared_task: capture_screenshot_async +#### capture_screenshot(page_id, screenshot_id, retry_count=0) Captures website screenshots: 1. Launches headless Chromium via Selenium 2. Configures viewport and options 3. Takes screenshot 4. Uploads to storage (R2/S3 or local) -5. Creates Screenshot model +5. Updates Screenshot model **Configuration**: - Headless mode - Custom user agent - 60s timeout - Supports custom viewport sizes -- Error handling and retry logic +- Error handling and retry logic via APScheduler +- Runs in background thread -#### @shared_task: cleanup_old_screenshots +#### schedule_pending_pages() Periodic maintenance task: -- Removes screenshots older than configured retention period -- Cleans up orphaned files -- Maintains storage efficiency +- Runs every 120 seconds +- Checks for pages with PENDING status +- Schedules processing for pending pages +- Respects retry limits and backoff -### Celery Configuration +### APScheduler Configuration ```python -# core/celery.py -CELERY_BROKER_URL = redis://redis:6379/0 -CELERY_RESULT_BACKEND = redis://redis:6379/0 +# core/scheduler.py +scheduler = BackgroundScheduler( + executors={'default': ThreadPoolExecutor(20)}, + job_defaults={ + 'coalesce': False, + 'max_instances': 3, + 'misfire_grace_time': 300 + }, + timezone=settings.TIME_ZONE +) ``` -Beat schedule in settings.py for periodic tasks. +**Initialization** in `core/apps.py`: +- Scheduler starts automatically when Django application starts +- Periodic jobs registered in `CoreConfig.ready()` method +- No separate worker process needed + +### Task Execution + +Tasks are executed using Python threading for immediate background execution: +```python +from threading import Thread +thread = Thread(target=process_page, args=(page_id,)) +thread.daemon = True +thread.start() +``` + +For scheduled/delayed tasks, APScheduler is used: +```python +from core.scheduler import scheduler +from datetime import datetime, timedelta +run_date = datetime.now() + timedelta(seconds=delay) +scheduler.add_job( + capture_screenshot, + 'date', + run_date=run_date, + args=[page_id, screenshot_id, retry_count + 1] +) +``` ## Key Features & Implementation @@ -192,14 +231,14 @@ Extracts and processes URL parameters: ``` ### 2. Asynchronous Page Processing -**Files**: `links/tasks.py`, `links/views.py` +**Files**: `links/tasks.py`, `links/page_views.py` When a page is bookmarked: 1. Page record created with `processing_status=PENDING` -2. Celery task dispatched for metadata extraction +2. Background thread started for metadata extraction 3. Background worker fetches and parses page 4. Title, description, content extracted -5. Optional screenshot capture triggered +5. Screenshot capture triggered in separate thread 6. Status updated to COMPLETED ### 3. Search System @@ -252,11 +291,31 @@ Translation files in `locale/` directory. Uses Django's i18n framework. 5. Add tests #### Adding a Background Task -1. Define task in `links/tasks.py` with `@shared_task` decorator -2. Call task with `.delay()` or `.apply_async()` -3. Configure retry logic and timeouts -4. Add logging for debugging -5. For periodic tasks, add to Celery beat schedule +1. Define task function in `links/tasks.py` (no decorator needed) +2. For immediate execution, use threading: + ```python + from threading import Thread + thread = Thread(target=task_function, args=(arg1,)) + thread.daemon = True + thread.start() + ``` +3. For delayed/scheduled execution, use APScheduler: + ```python + from core.scheduler import scheduler + scheduler.add_job(task_function, 'date', run_date=run_time, args=[arg1]) + ``` +4. For periodic tasks, add to `core/apps.py` in `CoreConfig.ready()`: + ```python + scheduler.add_job( + task_function, + 'interval', + seconds=120, + id='task_id', + replace_existing=True + ) + ``` +5. Add logging for debugging +6. Implement retry logic if needed #### Adding a New Template View 1. Create view function in `links/views.py` or create new view file @@ -320,11 +379,10 @@ python manage.py collectstatic # Collect to staticfiles/ ### docker-compose.yml Services -- **web**: Django development server (port 8000) +- **web**: Django development server (port 8000) with APScheduler running in-process - **node**: Tailwind CSS compiler -- **celery_worker**: Background task processor -- **celery_beat**: Periodic task scheduler -- **redis**: Message broker and cache + +Note: Redis, Celery worker, and Celery beat services have been removed as they are no longer needed. ### Helper Script: docker.sh @@ -346,10 +404,6 @@ SECRET_KEY=your-secret-key DEBUG=True ALLOWED_HOSTS=* -# Celery -CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 - # Optional: R2/S3 Storage R2_ENDPOINT_URL=https://... R2_ACCESS_KEY_ID=... @@ -389,15 +443,16 @@ Multi-stage build: **Runtime requirements**: - Python 3.12 - Chromium + ChromeDriver (for screenshots) -- Redis (external service) ### Kubernetes Manifests in `k8s/manifest.yaml`: -- Deployment for web service +- Deployment for web service (includes APScheduler) - Service for load balancing - ConfigMap for configuration -- Optional: Redis StatefulSet, PersistentVolumeClaim for data +- PersistentVolumeClaim for data + +Note: Worker deployment is no longer needed as tasks run in-process. ### Environment Setup @@ -406,7 +461,6 @@ Production checklist: - [ ] Configure `SECRET_KEY` (strong random value) - [ ] Set `ALLOWED_HOSTS` to actual domains - [ ] Configure database (PostgreSQL recommended for production) -- [ ] Set up Redis instance - [ ] Configure R2/S3 for media storage - [ ] Set up SSL/TLS termination - [ ] Configure backup strategy @@ -416,16 +470,16 @@ Production checklist: ### Screenshots Not Generating 1. Check Chromium installation: `which chromium` -2. Verify Celery worker is running: `docker ps` or check logs -3. Check Redis connection -4. Review Celery worker logs for errors +2. Check Django application logs for task errors +3. Verify APScheduler is running (check startup logs) +4. Review application logs for screenshot task errors 5. Verify storage configuration (R2/S3 credentials) ### Metadata Extraction Failing 1. Check website accessibility (some sites block bots) 2. Verify timeout settings (increase if needed) 3. Check for JavaScript-heavy sites (may need Selenium instead of requests) -4. Review error logs in Celery worker +4. Review error logs in application logs ### Migration Issues 1. Check for unapplied migrations: `python manage.py showmigrations` @@ -435,8 +489,8 @@ Production checklist: ### Performance Issues 1. Add database indexes for frequently queried fields -2. Implement caching (Django cache framework + Redis) -3. Optimize Celery task execution (tune concurrency) +2. Implement caching (Django cache framework) +3. Tune APScheduler thread pool size if needed (default: 20 threads) 4. Use database connection pooling 5. Enable query optimization (select_related, prefetch_related) @@ -448,7 +502,8 @@ Production checklist: core/ ├── settings.py # Django settings, installed apps, middleware ├── urls.py # Main URL routing -├── celery.py # Celery configuration +├── scheduler.py # APScheduler configuration +├── apps.py # App configuration and scheduler initialization └── middleware.py # Custom middleware (locale, etc.) links/ @@ -463,7 +518,7 @@ links/ ├── tag_views.py # Tag management ├── forms.py # Django forms ├── serializers.py # DRF serializers -├── tasks.py # Celery background tasks +├── tasks.py # Background task functions (APScheduler) ├── storage.py # Storage abstraction (R2/S3) ├── urls.py # Links app URL routing ├── *_urls.py # Feature-specific URL routing @@ -538,13 +593,27 @@ if result is None: cache.set('key', result, timeout=3600) ``` -### Celery Task Optimization +### APScheduler Task Optimization ```python -@shared_task(bind=True, max_retries=3) -def my_task(self): - # Use task routing for different queues - # Set time limits - # Implement idempotency +# Configure thread pool size in core/scheduler.py +executors = { + 'default': ThreadPoolExecutor(20), # Adjust based on workload +} + +# Add job with proper configuration +scheduler.add_job( + my_task, + 'interval', + seconds=60, + max_instances=3, # Limit concurrent instances + id='unique_job_id', + replace_existing=True +) + +# For one-time delayed tasks +from datetime import datetime, timedelta +run_date = datetime.now() + timedelta(seconds=300) +scheduler.add_job(my_task, 'date', run_date=run_date, args=[arg1]) ``` ## Security Considerations @@ -579,20 +648,25 @@ LOGGING = { } ``` -### Celery Monitoring -- Use Flower: `celery -A core flower` (port 5555) -- Monitor task success/failure rates -- Track task execution times +### APScheduler Monitoring +- Check scheduler status: `scheduler.running` returns True/False +- List all jobs: `scheduler.get_jobs()` +- Monitor task execution via Django logs +- Track task execution times in application logs ### Health Checks Implement health check endpoint: ```python # views.py def health_check(request): + from core.scheduler import scheduler # Check database - # Check Redis - # Check Celery workers - return JsonResponse({'status': 'healthy'}) + # Check APScheduler status + scheduler_running = scheduler.running + return JsonResponse({ + 'status': 'healthy', + 'scheduler_running': scheduler_running + }) ``` ## Contributing Guidelines @@ -612,7 +686,7 @@ When contributing code: ### Django Management Make sure enable enable `.venv` before running any commands through `source .venv/bin/activate` ```bash -python manage.py runserver # Development server +python manage.py runserver # Development server (APScheduler starts automatically) python manage.py shell # Django shell python manage.py dbshell # Database shell python manage.py createsuperuser # Create admin user @@ -622,13 +696,14 @@ python manage.py compilemessages # Compile translations python manage.py makemessages -l zh_Hans # Extract translation strings ``` -### Celery -```bash -celery -A core worker -l info # Start worker -celery -A core beat -l info # Start beat scheduler -celery -A core worker --purge # Clear all tasks -celery -A core inspect active # Show active tasks -celery -A core inspect stats # Worker statistics +### APScheduler (In-Process) +APScheduler starts automatically with Django. To interact with it: +```python +# In Django shell +from core.scheduler import scheduler +scheduler.get_jobs() # List all scheduled jobs +scheduler.print_jobs() # Print job details +scheduler.running # Check if scheduler is running ``` ### UV Package Manager diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..63a63d4 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1,4 @@ +""" +Core package initialization +""" +default_app_config = 'core.apps.CoreConfig' diff --git a/core/apps.py b/core/apps.py new file mode 100644 index 0000000..41857ea --- /dev/null +++ b/core/apps.py @@ -0,0 +1,33 @@ +""" +Core app configuration +""" +from django.apps import AppConfig +import logging + +logger = logging.getLogger(__name__) + + +class CoreConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'core' + + def ready(self): + """ + Initialize APScheduler when Django starts + """ + from core.scheduler import scheduler, start_scheduler + from links.tasks import schedule_pending_pages + + # Start the scheduler + start_scheduler() + + # Add periodic job for checking pending pages + if not scheduler.get_job('schedule_pending_pages'): + scheduler.add_job( + schedule_pending_pages, + 'interval', + seconds=120, + id='schedule_pending_pages', + replace_existing=True + ) + logger.info("Scheduled periodic task: schedule_pending_pages") diff --git a/core/celery.py b/core/celery.py deleted file mode 100644 index e05efd8..0000000 --- a/core/celery.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -from celery import Celery -from django.conf import settings - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') - -app = Celery('core') -app.config_from_object('django.conf:settings', namespace='CELERY') -app.autodiscover_tasks() - -# Optional: For debugging -@app.task(bind=True) -def debug_task(self): - print(f'Request: {self.request!r}') diff --git a/core/scheduler.py b/core/scheduler.py new file mode 100644 index 0000000..65c34e1 --- /dev/null +++ b/core/scheduler.py @@ -0,0 +1,40 @@ +""" +APScheduler configuration for background tasks +""" +import logging +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.executors.pool import ThreadPoolExecutor +from django.conf import settings + +logger = logging.getLogger(__name__) + +# Configure executors +executors = { + 'default': ThreadPoolExecutor(20), +} + +# Configure job defaults +job_defaults = { + 'coalesce': False, + 'max_instances': 3, + 'misfire_grace_time': 300, # 5 minutes +} + +# Create scheduler instance +scheduler = BackgroundScheduler( + executors=executors, + job_defaults=job_defaults, + timezone=settings.TIME_ZONE +) + +def start_scheduler(): + """Start the scheduler if not already running""" + if not scheduler.running: + scheduler.start() + logger.info("APScheduler started") + +def shutdown_scheduler(): + """Shutdown the scheduler gracefully""" + if scheduler.running: + scheduler.shutdown() + logger.info("APScheduler shutdown") diff --git a/core/settings.py b/core/settings.py index 31c87cf..ea6609d 100644 --- a/core/settings.py +++ b/core/settings.py @@ -14,6 +14,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', # Make sure this is here only once 'widget_tweaks', + 'core.apps.CoreConfig', 'links', 'tailwind', 'new_theme', @@ -198,37 +199,12 @@ REST_FRAMEWORK = { TIME_ZONE = 'Australia/Melbourne' USE_TZ = True -# Make sure Celery uses the same timezone -CELERY_TIMEZONE = TIME_ZONE - -# Celery Configuration -CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') -CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0') -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_SERIALIZER = 'json' -CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined - -# Celery Beat settings -CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celery', 'celerybeat-schedule') - -# Celery Beat Schedule -CELERY_BEAT_SCHEDULE = { - 'check-pending-pages': { - 'task': 'links.tasks.schedule_pending_pages', - 'schedule': 120.0, # Run every 60 seconds - }, -} - # Media files configuration MEDIA_URL = '/media/' MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media') # Ensure media files are served in all environments SERVE_MEDIA = True -CELERY_RESULT_BACKEND -# Celery Beat settings -CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celerybeat-schedule') # Cloudflare R2 Settings R2_ENDPOINT_URL = os.environ.get('R2_ENDPOINT_URL') diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 954793e366a92c2ce61505fe0c0561a11f980c46..ba4edb53e18674f14b395c0c79da846d79b4d476 100644 GIT binary patch delta 575 zcmZp8Al~pme1bG%@nke9hlviZ!u&{bd+E%P~%`ooNnmBELLx37;kJ5Z)9L(YG`O+5^rE>XlQJ%SCE&U zoS$2elUkCR0+Q1;G}JXPQ82KyGBmU@Fx4|PH8eIe$B;2G&@(VIurM^Uby4MDV31~T zbYv_@%}dG5OUF=aV4`PXYAI)AY>Gq5$lT0!yMiP06Fz1Ye$DCZ0nDx}{QCTwn*|eY z^6{`TGcww$fl1Zvrt!?@9Yi=7Rx@xN;knM1&M}W~DGLYVN`}>w3k4?Cb2V1m3NYkm zHtO;^I`T5e1D(d3QBqQ1rLSLJUanV}nVp}Pu2+SdJV=2T{z7#f)8CgznU=EOgnyYWTeyr(;wp3Rv2tZ(_V6$_uv>wY$;^X1G5Pg~bK z?caf-pdPD={s~X^%|K|EEuCW? d-%=J1#+9231tv4LuUyCs#4OuaE@Tb&0{|dUCcFRu diff --git a/docker-compose.yml b/docker-compose.yml index b343578..a81a6b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,12 @@ services: - .:/app - venv:/app/.venv - ./data/media:/app/data/media - - ./data/celery:/app/data/celery ports: - "8000:8000" env_file: - .env environment: - DJANGO_SETTINGS_MODULE=core.settings - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - redis restart: unless-stopped networks: - app-network @@ -52,72 +47,6 @@ services: tty: true stdin_open: true - celery_worker: - image: web-local-image - command: uv run celery -A core worker --loglevel=info - volumes: - - .:/app - - venv:/app/.venv - - ./data/media:/app/data/media - - ./data/celery:/app/data/celery - environment: - - DJANGO_SETTINGS_MODULE=core.settings - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - redis - - web - restart: unless-stopped - networks: - - app-network - - celery_beat: - image: web-local-image - command: uv run celery -A core beat -s /app/data/celery/celerybeat-schedule --loglevel=info - volumes: - - .:/app - - venv:/app/.venv - - ./data/media:/app/data/media - - ./data/celery:/app/data/celery - environment: - - DJANGO_SETTINGS_MODULE=core.settings - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - redis - - web - restart: unless-stopped - networks: - - app-network - - celery_flower: - image: web-local-image - command: uv run celery -A core flower --port=5555 --loglevel=info - volumes: - - .:/app - - venv:/app/.venv - - ./data/media:/app/data/media - - ./data/celery:/app/data/celery - ports: - - "5555:5555" - environment: - - DJANGO_SETTINGS_MODULE=core.settings - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - redis - - web - restart: unless-stopped - networks: - - app-network - - redis: - image: redis:alpine - ports: - - "6379:6379" - networks: - - app-network - volumes: venv: node_modules: diff --git a/docker.sh b/docker.sh index 31bbde6..8bb65b9 100755 --- a/docker.sh +++ b/docker.sh @@ -7,7 +7,6 @@ case "$1" in docker-compose up -d echo "Services started. Access:" echo "- Web: http://localhost:8000" - echo "- Flower (Celery Monitor): http://localhost:5555" ;; "stop") docker-compose down @@ -15,9 +14,6 @@ case "$1" in "restart") docker-compose restart ;; - "restart-worker") - docker-compose restart celery_worker celery_beat - ;; "logs") if [ "$2" ]; then docker-compose logs -f "$2" @@ -44,16 +40,12 @@ case "$1" in "tailwind-build") docker-compose exec tailwind npm run build ;; - "flower") - echo "Opening Flower dashboard in default browser..." - # docker-compose exec flower open http://localhost:5555 - ;; *) - echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|makemigrations|tailwind-logs|tailwind-build|flower}" + echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|static|tailwind-logs|tailwind-build}" echo "" echo "Additional commands:" echo " logs [service] View logs of all services or specific service" - echo " flower Open Flower dashboard in browser" + echo " static Collect static files" exit 1 ;; esac diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index 1c7a613..d50efb9 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -104,10 +104,6 @@ spec: # { secretKeyRef: { name: database-credentials, key: password } } - name: CSRF_TRUSTED_ORIGINS value: "https://to.junv.cc,https://go.junv.cc,http://go" - - name: CELERY_BROKER_URL - value: "redis://192.168.1.2:6379" - - name: CELERY_RESULT_BACKEND - value: "redis://192.168.1.2:6379" - name: UV_CACHE_DIR value: "/app/.cache/uv" - name: R2_BUCKET_NAME @@ -267,95 +263,3 @@ spec: name: links port: number: 80 ---- -# apiVersion: apps/v1 -# kind: Deployment -# metadata: -# name: links-worker -# labels: -# tags.datadoghq.com/env: "prod" -# tags.datadoghq.com/service: "links-worker" -# spec: -# selector: -# matchLabels: -# app: links-worker -# replicas: 1 -# template: -# metadata: -# labels: -# app: links-worker -# tags.datadoghq.com/env: "prod" -# tags.datadoghq.com/service: "links-worker" -# spec: -# volumes: -# - name: data -# persistentVolumeClaim: -# claimName: links-pvc -# - name: cache -# emptyDir: {} -# containers: -# - name: celery-worker -# image: "ghcr.io/wahyd4/links:main" -# command: ["sh", "-c", "uv run celery -A core worker --loglevel=info"] -# securityContext: -# runAsUser: 1000 -# imagePullPolicy: Always -# volumeMounts: -# - name: data -# mountPath: /app/data -# - name: cache -# mountPath: /app/.cache -# env: -# - name: DB_HOST -# value: new-postgres-postgresql.db.svc.cluster.local -# - name: DB_NAME -# value: badges -# - name: DB_USERNAME -# value: postgres -# - name: CELERY_BROKER_URL -# value: "redis://redis-master.db.svc.cluster.local:6379" -# - name: CELERY_RESULT_BACKEND -# value: "redis://redis-master.db.svc.cluster.local:6379" -# - name: UV_CACHE_DIR -# value: "/app/.cache/uv" -# resources: -# requests: -# cpu: 200m -# memory: 256Mi -# limits: -# cpu: 2000m -# memory: 2000Mi - -# - name: celery-beat -# image: "ghcr.io/wahyd4/links:main" -# command: ["sh", "-c", "uv run celery -A core beat -s /app/data/celerybeat-schedule --loglevel=info"] -# securityContext: -# runAsUser: 1000 -# imagePullPolicy: Always -# volumeMounts: -# - name: data -# mountPath: /app/data -# - name: cache -# mountPath: /app/.cache -# env: -# - name: DB_HOST -# value: new-postgres-postgresql.db.svc.cluster.local -# - name: DB_NAME -# value: badges -# - name: DB_USERNAME -# value: postgres -# - name: CELERY_BROKER_URL -# value: "redis://redis-master.db.svc.cluster.local:6379" -# - name: CELERY_RESULT_BACKEND -# value: "redis://redis-master.db.svc.cluster.local:6379" -# - name: UV_CACHE_DIR -# value: "/app/.cache/uv" -# resources: -# requests: -# cpu: 200m -# memory: 256Mi -# limits: -# cpu: 1000m -# memory: 1000Mi -# imagePullSecrets: -# - name: github-image-pull-secret diff --git a/links/page_views.py b/links/page_views.py index 5532257..289ed6f 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -25,6 +25,7 @@ from .tasks import capture_screenshot, process_page from pathlib import Path from datetime import datetime from rest_framework.decorators import action +from threading import Thread class PageListView(ListView): model = Page @@ -59,7 +60,9 @@ class PageCreateView(CreateView): # Start processing the page try: - process_page.delay(page.id) + thread = Thread(target=process_page, args=(page.id,)) + thread.daemon = True + thread.start() messages.success(self.request, _('Page created successfully and processing started.')) except Exception as e: messages.error(self.request, _('Page created but processing failed to start.')) @@ -87,7 +90,9 @@ class PageUpdateView(UpdateView): # Trigger processing task from .tasks import process_page - process_page.delay(self.object.id) + thread = Thread(target=process_page, args=(self.object.id,)) + thread.daemon = True + thread.start() return response @@ -204,7 +209,14 @@ class PageViewSet(viewsets.ModelViewSet): # Trigger screenshot capture if needed from .tasks import capture_screenshot - capture_screenshot.delay(serializer.instance.id, serializer.instance.id) # Use page_id as screenshot_id + # Create screenshot record first + screenshot = Screenshot.objects.create( + page=serializer.instance, + status=Screenshot.Status.PENDING + ) + thread = Thread(target=capture_screenshot, args=(serializer.instance.id, screenshot.id)) + thread.daemon = True + thread.start() return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) @@ -229,7 +241,9 @@ class PageViewSet(viewsets.ModelViewSet): # Trigger screenshot capture task try: - capture_screenshot.delay(page.id, screenshot.id) + thread = Thread(target=capture_screenshot, args=(page.id, screenshot.id)) + thread.daemon = True + thread.start() return Response({ 'status': 'success', 'message': 'Screenshot capture initiated', diff --git a/links/tasks.py b/links/tasks.py index 26163ed..a0f26f6 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -1,4 +1,3 @@ -from celery import shared_task from django.utils import timezone from datetime import timedelta import requests @@ -10,12 +9,13 @@ import os from django.conf import settings from urllib.parse import quote import time -from celery.exceptions import MaxRetriesExceededError import random +from threading import Thread logger = logging.getLogger(__name__) SCREENSHOT_TIMEOUT = 60 FETCH_PAGE_TIMEOUT = 20 +MAX_RETRIES = 3 def fibonacci(n): if n <= 0: @@ -56,8 +56,7 @@ def extract_description_from_soup(soup): return text return None -@shared_task(bind=True, max_retries=3) -def capture_screenshot(self, page_id, screenshot_id): +def capture_screenshot(page_id, screenshot_id, retry_count=0): """Task to capture full-page screenshot with improved Chinese website support""" from .models import Page, Screenshot @@ -199,19 +198,30 @@ def capture_screenshot(self, page_id, screenshot_id): except Exception as exc: logger.error(f"Failed to capture screenshot for page {page_id}: {exc}") - try: - retry_delay = fibonacci(self.request.retries or 0) + if retry_count < MAX_RETRIES: + retry_delay = fibonacci(retry_count) screenshot.error = f"Error: {str(exc)}. Retrying in {retry_delay} seconds..." screenshot.save() - self.retry(exc=exc, countdown=retry_delay) - except MaxRetriesExceededError: + # Schedule retry using APScheduler + from core.scheduler import scheduler + from datetime import datetime, timedelta + run_date = datetime.now() + timedelta(seconds=retry_delay) + scheduler.add_job( + capture_screenshot, + 'date', + run_date=run_date, + args=[page_id, screenshot_id, retry_count + 1], + id=f'capture_screenshot_{screenshot_id}_retry_{retry_count + 1}', + replace_existing=True + ) + logger.info(f"Scheduled retry {retry_count + 1} for screenshot {screenshot_id} in {retry_delay} seconds") + else: logger.error(f"Max retries exceeded for page {page_id}") screenshot.status = Screenshot.Status.FAILED - screenshot.error = f"Failed after {self.request.retries} attempts. Last error: {str(exc)}" + screenshot.error = f"Failed after {retry_count} attempts. Last error: {str(exc)}" screenshot.save() -@shared_task(bind=True, max_retries=3) -def process_page(self, page_id): +def process_page(page_id, retry_count=0): """Task to process a page and extract its metadata""" from .models import Page, Screenshot @@ -256,8 +266,10 @@ def process_page(self, page_id): status=Screenshot.Status.PENDING ) - # Trigger screenshot capture with the new screenshot ID - capture_screenshot.delay(page.id, screenshot.id) + # Trigger screenshot capture asynchronously + thread = Thread(target=capture_screenshot, args=(page.id, screenshot.id)) + thread.daemon = True + thread.start() page.process_status = Page.ProcessStatus.COMPLETED page.save() @@ -269,18 +281,29 @@ def process_page(self, page_id): page.retry_count += 1 page.last_retry_at = timezone.now() - if page.retry_count >= 3: + if page.retry_count >= MAX_RETRIES: page.process_status = Page.ProcessStatus.FAILED page.error_message = str(exc) + page.save() else: page.process_status = Page.ProcessStatus.PENDING - retry_delay = fibonacci(page.retry_count) - self.retry(exc=exc, countdown=retry_delay) + page.save() + + # Schedule retry using APScheduler + retry_delay = fibonacci(retry_count) + from core.scheduler import scheduler + from datetime import datetime, timedelta + run_date = datetime.now() + timedelta(seconds=retry_delay) + scheduler.add_job( + process_page, + 'date', + run_date=run_date, + args=[page_id, retry_count + 1], + id=f'process_page_{page_id}_retry_{retry_count + 1}', + replace_existing=True + ) + logger.info(f"Scheduled retry {retry_count + 1} for page {page_id} in {retry_delay} seconds") - page.save() - raise exc - -@shared_task def schedule_pending_pages(): """Periodic task to schedule processing of pending pages""" from .models import Page @@ -288,11 +311,14 @@ def schedule_pending_pages(): logger.info("Checking for pending pages...") pending_pages = Page.objects.filter( process_status=Page.ProcessStatus.PENDING, - retry_count__lt=3 + retry_count__lt=MAX_RETRIES ).exclude( - last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(3)) + last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(MAX_RETRIES)) ) for page in pending_pages: logger.info(f"Scheduling processing for page {page.id}") - process_page.delay(page.id) + # Run in background thread + thread = Thread(target=process_page, args=(page.id,)) + thread.daemon = True + thread.start() diff --git a/pyproject.toml b/pyproject.toml index 73ec3f7..eed6435 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,7 @@ dependencies = [ "requests==2.32.4", "beautifulsoup4==4.12.3", "djangorestframework==3.15.2", - "celery>=5.3.0", - "redis>=4.5.0", - "flower>=2.0.0", + "apscheduler>=3.10.0", "selenium>=4.0.0", "boto3>=1.35.0", "python-magic>=0.4.27", diff --git a/uv.lock b/uv.lock index 39ea227..a3a4a7a 100644 --- a/uv.lock +++ b/uv.lock @@ -3,15 +3,15 @@ revision = 3 requires-python = ">=3.12" [[package]] -name = "amqp" -version = "5.3.1" +name = "apscheduler" +version = "3.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "vine" }, + { name = "tzlocal" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/81/192db4f8471de5bc1f0d098783decffb1e6e69c4f8b4bc6711094691950b/apscheduler-3.11.1.tar.gz", hash = "sha256:0db77af6400c84d1747fe98a04b8b58f0080c77d11d338c4f507a9752880f221", size = 108044, upload-time = "2025-10-31T18:55:42.819Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, + { url = "https://files.pythonhosted.org/packages/58/9f/d3c76f76c73fcc959d28e9def45b8b1cc3d7722660c5003b19c1022fd7f4/apscheduler-3.11.1-py3-none-any.whl", hash = "sha256:6162cb5683cb09923654fa9bdd3130c4be4bfda6ad8990971c9597ecd52965d2", size = 64278, upload-time = "2025-10-31T18:55:41.186Z" }, ] [[package]] @@ -44,15 +44,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/fe/e8c672695b37eecc5cbf43e1d0638d88d66ba3a44c4d321c796f4e59167f/beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed", size = 147925, upload-time = "2024-01-17T16:53:12.779Z" }, ] -[[package]] -name = "billiard" -version = "4.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031, upload-time = "2024-09-21T13:40:22.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766, upload-time = "2024-09-21T13:40:20.188Z" }, -] - [[package]] name = "black" version = "25.1.0" @@ -105,26 +96,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/0c/a3eeca35b22ac8f441d412881582a5f3b8665de0269baf9fdeb8e86d7f1c/botocore-1.36.26-py3-none-any.whl", hash = "sha256:4e3f19913887a58502e71ef8d696fe7eaa54de7813ff73390cd5883f837dfa6e", size = 13360675, upload-time = "2025-02-21T20:28:02.987Z" }, ] -[[package]] -name = "celery" -version = "5.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "billiard" }, - { name = "click" }, - { name = "click-didyoumean" }, - { name = "click-plugins" }, - { name = "click-repl" }, - { name = "kombu" }, - { name = "python-dateutil" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/9c/cf0bce2cc1c8971bf56629d8f180e4ca35612c7e79e6e432e785261a8be4/celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706", size = 1575692, upload-time = "2024-04-17T20:29:43.675Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/c4/6a4d3772e5407622feb93dd25c86ce3c0fee746fa822a777a627d56b4f2a/celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64", size = 425983, upload-time = "2024-04-17T20:29:39.406Z" }, -] - [[package]] name = "certifi" version = "2025.1.31" @@ -196,43 +167,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, ] -[[package]] -name = "click-didyoumean" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, -] - -[[package]] -name = "click-plugins" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5f/1d/45434f64ed749540af821fd7e42b8e4d23ac04b1eda7c26613288d6cd8a8/click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", size = 8164, upload-time = "2019-04-04T04:27:04.82Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/da/824b92d9942f4e472702488857914bdd50f73021efea15b4cad9aca8ecef/click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8", size = 7497, upload-time = "2019-04-04T04:27:03.36Z" }, -] - -[[package]] -name = "click-repl" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "prompt-toolkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -312,22 +246,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f8/08d37b2cd89da306e3520bd27f8a85692122b42b56c0c2c3784ff09c022f/flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a", size = 57745, upload-time = "2025-02-16T18:45:42.351Z" }, ] -[[package]] -name = "flower" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "celery" }, - { name = "humanize" }, - { name = "prometheus-client" }, - { name = "pytz" }, - { name = "tornado" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/a1/357f1b5d8946deafdcfdd604f51baae9de10aafa2908d0b7322597155f92/flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0", size = 3220408, upload-time = "2023-08-13T14:37:46.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/ff/ee2f67c0ff146ec98b5df1df637b2bc2d17beeb05df9f427a67bd7a7d79c/flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2", size = 383553, upload-time = "2023-08-13T14:37:41.552Z" }, -] - [[package]] name = "fontawesome-free" version = "5.15.3" @@ -357,15 +275,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259, upload-time = "2022-09-25T15:39:59.68Z" }, ] -[[package]] -name = "humanize" -version = "4.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/8c/4f2f0784d08a383b5de3d3b1d65a6f204cc5dc487621c91c550388d756af/humanize-4.12.1.tar.gz", hash = "sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232", size = 80827, upload-time = "2025-02-18T17:07:48.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/30/5ef5994b090398f9284d2662f56853e5183ae2cb5d8e3db67e4f4cfea407/humanize-4.12.1-py3-none-any.whl", hash = "sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea", size = 127409, upload-time = "2025-02-18T17:07:47.027Z" }, -] - [[package]] name = "idna" version = "3.10" @@ -402,20 +311,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, ] -[[package]] -name = "kombu" -version = "5.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/4d/b93fcb353d279839cc35d0012bee805ed0cf61c07587916bfc35dbfddaf1/kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf", size = 442858, upload-time = "2024-09-19T12:25:37.261Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/ec/7811a3cf9fdfee3ee88e54d08fcbc3fabe7c1b6e4059826c59d7b795651c/kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763", size = 201349, upload-time = "2024-09-19T12:25:34.926Z" }, -] - [[package]] name = "markdown" version = "3.7" @@ -532,27 +427,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload-time = "2024-04-20T21:34:40.434Z" }, ] -[[package]] -name = "prometheus-client" -version = "0.21.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/62/14/7d0f567991f3a9af8d1cd4f619040c93b68f09a02b6d0b6ab1b2d1ded5fe/prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb", size = 78551, upload-time = "2024-12-03T14:59:12.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/c2/ab7d37426c179ceb9aeb109a85cda8948bb269b7561a0be870cc656eefe4/prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301", size = 54682, upload-time = "2024-12-03T14:59:10.935Z" }, -] - -[[package]] -name = "prompt-toolkit" -version = "3.0.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/e1/bd15cb8ffdcfeeb2bdc215de3c3cffca11408d829e4b8416dcfe71ba8854/prompt_toolkit-3.0.50.tar.gz", hash = "sha256:544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab", size = 429087, upload-time = "2025-01-20T15:55:35.072Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/ea/d836f008d33151c7a1f62caf3d8dd782e4d15f6a43897f64480c2b8de2ad/prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198", size = 387816, upload-time = "2025-01-20T15:55:29.98Z" }, -] - [[package]] name = "pycodestyle" version = "2.12.1" @@ -637,24 +511,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" }, ] -[[package]] -name = "pytz" -version = "2025.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/57/df1c9157c8d5a05117e455d66fd7cf6dbc46974f832b1058ed4856785d8a/pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e", size = 319617, upload-time = "2025-01-31T01:54:48.615Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930, upload-time = "2025-01-31T01:54:45.634Z" }, -] - -[[package]] -name = "redis" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355, upload-time = "2024-12-06T09:50:41.956Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502, upload-time = "2024-12-06T09:50:39.656Z" }, -] - [[package]] name = "requests" version = "2.32.4" @@ -753,24 +609,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/a5/b2860373aa8de1e626b2bdfdd6df4355f0565b47e51f7d0c54fe70faf8fe/sqlparse-0.5.1-py3-none-any.whl", hash = "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4", size = 44156, upload-time = "2024-07-15T19:30:25.033Z" }, ] -[[package]] -name = "tornado" -version = "6.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/59/45/a0daf161f7d6f36c3ea5fc0c2de619746cc3dd4c76402e9db545bd920f63/tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b", size = 501135, upload-time = "2024-11-22T03:06:38.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/7e/71f604d8cea1b58f82ba3590290b66da1e72d840aeb37e0d5f7291bd30db/tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1", size = 436299, upload-time = "2024-11-22T03:06:20.162Z" }, - { url = "https://files.pythonhosted.org/packages/96/44/87543a3b99016d0bf54fdaab30d24bf0af2e848f1d13d34a3a5380aabe16/tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803", size = 434253, upload-time = "2024-11-22T03:06:22.39Z" }, - { url = "https://files.pythonhosted.org/packages/cb/fb/fdf679b4ce51bcb7210801ef4f11fdac96e9885daa402861751353beea6e/tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec", size = 437602, upload-time = "2024-11-22T03:06:24.214Z" }, - { url = "https://files.pythonhosted.org/packages/4f/3b/e31aeffffc22b475a64dbeb273026a21b5b566f74dee48742817626c47dc/tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946", size = 436972, upload-time = "2024-11-22T03:06:25.559Z" }, - { url = "https://files.pythonhosted.org/packages/22/55/b78a464de78051a30599ceb6983b01d8f732e6f69bf37b4ed07f642ac0fc/tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf", size = 437173, upload-time = "2024-11-22T03:06:27.584Z" }, - { url = "https://files.pythonhosted.org/packages/79/5e/be4fb0d1684eb822c9a62fb18a3e44a06188f78aa466b2ad991d2ee31104/tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634", size = 437892, upload-time = "2024-11-22T03:06:28.933Z" }, - { url = "https://files.pythonhosted.org/packages/f5/33/4f91fdd94ea36e1d796147003b490fe60a0215ac5737b6f9c65e160d4fe0/tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73", size = 437334, upload-time = "2024-11-22T03:06:30.428Z" }, - { url = "https://files.pythonhosted.org/packages/2b/ae/c1b22d4524b0e10da2f29a176fb2890386f7bd1f63aacf186444873a88a0/tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c", size = 437261, upload-time = "2024-11-22T03:06:32.458Z" }, - { url = "https://files.pythonhosted.org/packages/b5/25/36dbd49ab6d179bcfc4c6c093a51795a4f3bed380543a8242ac3517a1751/tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482", size = 438463, upload-time = "2024-11-22T03:06:34.71Z" }, - { url = "https://files.pythonhosted.org/packages/61/cc/58b1adeb1bb46228442081e746fcdbc4540905c87e8add7c277540934edb/tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38", size = 438907, upload-time = "2024-11-22T03:06:36.71Z" }, -] - [[package]] name = "trio" version = "0.29.0" @@ -820,27 +658,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/dd/84f10e23edd882c6f968c21c2434fe67bd4a528967067515feca9e611e5e/tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639", size = 346762, upload-time = "2025-01-21T19:49:37.187Z" }, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "url-manager" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "apscheduler" }, { name = "asgiref" }, { name = "beautifulsoup4" }, { name = "boto3" }, - { name = "celery" }, { name = "django" }, { name = "django-simplemde" }, { name = "django-tailwind" }, { name = "django-widget-tweaks" }, { name = "djangorestframework" }, - { name = "flower" }, { name = "fontawesome-free" }, { name = "gunicorn" }, { name = "markdown" }, { name = "pillow" }, { name = "python-magic" }, - { name = "redis" }, { name = "requests" }, { name = "selenium" }, { name = "sqlparse" }, @@ -858,18 +706,17 @@ dev = [ [package.metadata] requires-dist = [ + { name = "apscheduler", specifier = ">=3.10.0" }, { name = "asgiref", specifier = "==3.8.1" }, { name = "beautifulsoup4", specifier = "==4.12.3" }, { name = "black", marker = "extra == 'dev'", specifier = ">=23.0" }, { name = "boto3", specifier = ">=1.35.0" }, - { name = "celery", specifier = ">=5.3.0" }, { name = "django", specifier = "==5.1.13" }, { name = "django-simplemde", specifier = "==0.1.4" }, { name = "django-tailwind", specifier = "==3.8.0" }, { name = "django-widget-tweaks", specifier = "==1.5.0" }, { name = "djangorestframework", specifier = "==3.15.2" }, { name = "flake8", marker = "extra == 'dev'", specifier = ">=6.0" }, - { name = "flower", specifier = ">=2.0.0" }, { name = "fontawesome-free", specifier = "==5.15.3" }, { name = "gunicorn", specifier = "==23.0.0" }, { name = "isort", marker = "extra == 'dev'", specifier = ">=5.0" }, @@ -878,7 +725,6 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" }, { name = "pytest-django", marker = "extra == 'dev'", specifier = ">=4.5" }, { name = "python-magic", specifier = ">=0.4.27" }, - { name = "redis", specifier = ">=4.5.0" }, { name = "requests", specifier = "==2.32.4" }, { name = "selenium", specifier = ">=4.0.0" }, { name = "sqlparse", specifier = "==0.5.1" }, @@ -900,24 +746,6 @@ socks = [ { name = "pysocks" }, ] -[[package]] -name = "vine" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.2.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, -] - [[package]] name = "websocket-client" version = "1.8.0"