home/links
2
0
Fork 0
mirror of https://github.com/wahyd4/links.git synced 2026-08-08 21:04:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

Update work to process page

Browse Source
This commit is contained in:
junv
2024-11-04 13:58:57 +11:00
parent 827b54d719
commit c76e47590a
7 changed files with 49 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
celerybeat-schedule
BIN
View File
Binary file not shown.
data/db.sqlite3
BIN
View File
Binary file not shown.
docker-compose.yml
+3 -5
View File
@@ -1,5 +1,3 @@
version: '3.8'
services:
web:
build: .
@@ -22,7 +20,7 @@ services:
celery_worker:
build: .
command: celery -A url_manager worker -l info
command: celery -A url_manager worker --loglevel=info
volumes:
- .:/app
environment:
@@ -37,7 +35,7 @@ services:
celery_beat:
build: .
command: celery -A url_manager beat -l info
command: celery -A url_manager beat --loglevel=info
volumes:
- .:/app
environment:
@@ -46,7 +44,7 @@ services:
- CELERY_RESULT_BACKEND=redis://redis:6379/0
depends_on:
- redis
- web
- celery_worker
networks:
- app-network
links/tasks.py
+5
View File
@@ -82,6 +82,7 @@ def process_page(self, page_id):
page.process_status = Page.ProcessStatus.COMPLETED
page.save()
logger.info(f"Successfully processed page {page_id}")
except Exception as exc:
page.retry_count += 1
@@ -90,10 +91,12 @@ def process_page(self, page_id):
if page.retry_count >= 3:
page.process_status = Page.ProcessStatus.FAILED
page.error_message = str(exc)
logger.error(f"Failed to process page {page_id} after 3 retries: {exc}")
else:
page.process_status = Page.ProcessStatus.PENDING
# Schedule retry with fibonacci backoff
retry_delay = fibonacci(page.retry_count)
logger.info(f"Scheduling retry for page {page_id} in {retry_delay} seconds")
self.retry(exc=exc, countdown=retry_delay)
page.save()
@@ -102,6 +105,7 @@ def process_page(self, page_id):
@shared_task
def schedule_pending_pages():
"""Periodic task to schedule processing of pending pages"""
logger.info("Checking for pending pages...")
pending_pages = Page.objects.filter(
process_status=Page.ProcessStatus.PENDING,
retry_count__lt=3
@@ -110,4 +114,5 @@ def schedule_pending_pages():
)
for page in pending_pages:
logger.info(f"Scheduling processing for page {page.id}")
process_page.delay(page.id)
links/templates/links/page_list.html
+39 -14
View File
@@ -18,26 +18,51 @@
<div class="flex items-center justify-between">
<div class="flex-1">
<a href="{{ page.get_absolute_url }}" class="text-lg font-medium text-blue-600 hover:text-blue-800">
{{ page.title }}
{{ page.title|default:"Untitled" }}
</a>
<a href="{{ page.url }}" target="_blank" rel="noopener noreferrer"
class="block mt-1 text-sm text-gray-600 hover:text-gray-900 break-all">
{{ page.url }}
</a>
</div>
<div class="flex space-x-2 ml-4">
<a href="{% url 'page-update' page.pk %}" class="text-gray-600 hover:text-gray-900">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"/>
</svg>
</a>
<a href="{% url 'page-delete' page.pk %}" class="text-red-600 hover:text-red-800">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
</svg>
</a>
<div class="flex items-center space-x-4">
<!-- Status Badge -->
{% if page.process_status == 'completed' %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-green-100 text-green-800">
{% trans "Completed" %}
</span>
{% elif page.process_status == 'processing' %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-yellow-100 text-yellow-800">
{% trans "Processing" %}
</span>
{% elif page.process_status == 'failed' %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-red-100 text-red-800">
{% trans "Failed" %}
{% if page.retry_count > 0 %}
({{ page.retry_count }}/3)
{% endif %}
</span>
{% else %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-gray-100 text-gray-800">
{% trans "Pending" %}
</span>
{% endif %}
<!-- Action Buttons -->
<div class="flex space-x-2">
<a href="{% url 'page-update' page.pk %}" class="text-gray-600 hover:text-gray-900">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"/>
</svg>
</a>
<a href="{% url 'page-delete' page.pk %}" class="text-red-600 hover:text-red-800">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
</svg>
</a>
</div>
</div>
</div>
<div class="mt-2">
manage-docker.sh
+1 -1
View File
@@ -40,7 +40,7 @@ case "$1" in
;;
"flower")
echo "Opening Flower dashboard in default browser..."
docker-compose exec flower open http://localhost:5555
# docker-compose exec flower open http://localhost:5555
;;
*)
echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|makemigrations|tailwind-logs|tailwind-build|flower}"
url_manager/settings.py
+1 -1
View File
@@ -209,6 +209,6 @@ CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined
CELERY_BEAT_SCHEDULE = {
'check-pending-pages': {
'task': 'links.tasks.schedule_pending_pages',
'schedule': 300.0, # Run every 5 minutes
'schedule': 60.0, # 每60秒运行一次
},
}