From c76e47590aab33d9931a833c6aa0ca09ff1746b5 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Mon, 4 Nov 2024 13:58:57 +1100 Subject: [PATCH] Update work to process page --- celerybeat-schedule | Bin 16384 -> 16384 bytes data/db.sqlite3 | Bin 204800 -> 204800 bytes docker-compose.yml | 8 ++-- links/tasks.py | 5 +++ links/templates/links/page_list.html | 53 ++++++++++++++++++++------- manage-docker.sh | 2 +- url_manager/settings.py | 2 +- 7 files changed, 49 insertions(+), 21 deletions(-) diff --git a/celerybeat-schedule b/celerybeat-schedule index 30eeaa0005c0a5776035f7c130958ad271d09219..c3f159d9add3334ddad4b7344449342e3e418350 100644 GIT binary patch delta 78 zcmZo@U~Fh$++d+FSwq2ObB@9WUPgz_vU>TZ@CDnZd zv{fvQG!}xvRxE;z92SCqfxU3xpW)u+Lvqb9@AL5do_S|D)rV94ZCe?{TPgI=`Ap7KOF5ex3-$#&W%jUQbd+jRh20-y5Em^lE;PFVLhv$m9PZmITe~jifl1s(~6tqh_5(?I%7o;8&5|*r@JneVkqFca}vmT5&>lewQu425KtXbtcwlOgf zGHe2E9Jy|Ud^4Df-np}R)$%;s^XpB^E|oMH-CNOt|8jfjXC$XZBC|^{N-PR_5ch!SU91$-r{7+vs|s1{N8HHqO_w-+h?YoaPKb%?T#`=q)agG{o3wiX delta 1113 zcmZ{i%Wl&^6o#E}>m})~AXFBos;Go$9eeCJc0^UBQB@Eiq^4DcPzzIgk_?@CT+bu{ zHtA*2Ev%}(01I}AMYrAZ1gzK~@g9urUT7(jWslD}^Pm5GGuy|7?c>77vE0_Dd-dGb z*Ec`(+`yh&+ILI2Be#_Q4x^f;-5Na9M!t+#g`dNFL%)Y+2M@oE!`|S?&cr}L^Ik6$ zYqg>_cCfXYznv=`?62k*XUr-xtCpdoa@naE$UyeAX-*q980Q(sG0ljC0S#L_6Sw}O zq-2t(tRv4g9rTZq%2*Z8@X)E!*v*6DTK?_9;;X#b-(9&;)ShhZ4QCby+EPZMS}OH= zJ>AC9hZ&gb4NaHRY!hfKy{JEhT73y{7(hUU&$}dsWPdXz1O#UtN&9L5$&>#61}rzujj3=p z_8CtLkX5x%)i8AjS(f2kow1#YdqR4*aeYMFneOf0xdJXDUBX0PZ@#CcS;aQOF_~H5 zyv4}WbLz)j@TQ!qMG+&DslAzo3iMRfePqmsEgBLMKTAat4F$e|8ICCtFbPYXGm#9I z7Qk`@FO$LRYBz}+DKW4NW8+~T#ObaCZ$O((5+`6&N1;-ig13AfB$X^#`>6tC=BHA$ zjQdzp9ts`i-$jhaB!w(bEwl-Xl5>L=kq}`Tz^1Ah&*=5lmCwK$RnyhW<+5v6E`dGC zRFGqxfbCo>Xtlc!E>T^_K}+3J)%Zgj^W?tHC6Rby0_HlD1<)X}6I~R4WEoY%)orJ2 zqksFR8OQd#vjfc+wcg4u&pIC+t>s5^DD{uLs*UuD=NO)Mu|<|~Z4()%I-lRKa& diff --git a/docker-compose.yml b/docker-compose.yml index cbbbd53..411cba3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: web: build: . @@ -22,7 +20,7 @@ services: celery_worker: build: . - command: celery -A url_manager worker -l info + command: celery -A url_manager worker --loglevel=info volumes: - .:/app environment: @@ -37,7 +35,7 @@ services: celery_beat: build: . - command: celery -A url_manager beat -l info + command: celery -A url_manager beat --loglevel=info volumes: - .:/app environment: @@ -46,7 +44,7 @@ services: - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: - redis - - web + - celery_worker networks: - app-network diff --git a/links/tasks.py b/links/tasks.py index 953cf4c..da475c9 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -82,6 +82,7 @@ def process_page(self, page_id): page.process_status = Page.ProcessStatus.COMPLETED page.save() + logger.info(f"Successfully processed page {page_id}") except Exception as exc: page.retry_count += 1 @@ -90,10 +91,12 @@ def process_page(self, page_id): if page.retry_count >= 3: page.process_status = Page.ProcessStatus.FAILED page.error_message = str(exc) + logger.error(f"Failed to process page {page_id} after 3 retries: {exc}") else: page.process_status = Page.ProcessStatus.PENDING # Schedule retry with fibonacci backoff retry_delay = fibonacci(page.retry_count) + logger.info(f"Scheduling retry for page {page_id} in {retry_delay} seconds") self.retry(exc=exc, countdown=retry_delay) page.save() @@ -102,6 +105,7 @@ def process_page(self, page_id): @shared_task def schedule_pending_pages(): """Periodic task to schedule processing of pending pages""" + logger.info("Checking for pending pages...") pending_pages = Page.objects.filter( process_status=Page.ProcessStatus.PENDING, retry_count__lt=3 @@ -110,4 +114,5 @@ def schedule_pending_pages(): ) for page in pending_pages: + logger.info(f"Scheduling processing for page {page.id}") process_page.delay(page.id) diff --git a/links/templates/links/page_list.html b/links/templates/links/page_list.html index 6add56c..4e69af0 100644 --- a/links/templates/links/page_list.html +++ b/links/templates/links/page_list.html @@ -18,26 +18,51 @@
- {{ page.title }} + {{ page.title|default:"Untitled" }} {{ page.url }}
-
- - - - - - - - - - +
+ + {% if page.process_status == 'completed' %} + + {% trans "Completed" %} + + {% elif page.process_status == 'processing' %} + + {% trans "Processing" %} + + {% elif page.process_status == 'failed' %} + + {% trans "Failed" %} + {% if page.retry_count > 0 %} + ({{ page.retry_count }}/3) + {% endif %} + + {% else %} + + {% trans "Pending" %} + + {% endif %} + + +
+ + + + + + + + + + +
diff --git a/manage-docker.sh b/manage-docker.sh index 071c27d..b108633 100755 --- a/manage-docker.sh +++ b/manage-docker.sh @@ -40,7 +40,7 @@ case "$1" in ;; "flower") echo "Opening Flower dashboard in default browser..." - docker-compose exec flower open http://localhost:5555 + # docker-compose exec flower open http://localhost:5555 ;; *) echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|makemigrations|tailwind-logs|tailwind-build|flower}" diff --git a/url_manager/settings.py b/url_manager/settings.py index de2d442..715b446 100644 --- a/url_manager/settings.py +++ b/url_manager/settings.py @@ -209,6 +209,6 @@ CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined CELERY_BEAT_SCHEDULE = { 'check-pending-pages': { 'task': 'links.tasks.schedule_pending_pages', - 'schedule': 300.0, # Run every 5 minutes + 'schedule': 60.0, # 每60秒运行一次 }, }