From bd37986b2182c6a21256000763052e61dcf9c67a Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Mon, 30 Mar 2026 19:51:12 +1100 Subject: [PATCH] Update jobs --- links/apps.py | 172 ++++++++++++++++ links/job_registry.py | 50 +++++ links/templates/links/jobs.html | 299 ++++++++++++++-------------- links/templates/links/settings.html | 4 +- links/views.py | 153 +++++++------- netscan/apps.py | 69 +++++++ qdrant_sync.py | 12 +- 7 files changed, 524 insertions(+), 235 deletions(-) create mode 100644 links/job_registry.py diff --git a/links/apps.py b/links/apps.py index e133061..702d4fc 100644 --- a/links/apps.py +++ b/links/apps.py @@ -8,3 +8,175 @@ class LinksConfig(AppConfig): def ready(self): # Import signals to register them import links.signals + self._register_job_types() + + def _register_job_types(self): + from . import job_registry + if job_registry.all_types(): + return # already registered (e.g. double-init in test runner) + + from .models import Screenshot, Page, FileUpload + from django.urls import reverse + + # ── Screenshots ────────────────────────────────────────────────── + def ss_stats(): + return { + 'total': Screenshot.objects.count(), + 'pending': Screenshot.objects.filter(status=Screenshot.Status.PENDING).count(), + 'processing': Screenshot.objects.filter(status=Screenshot.Status.PROCESSING).count(), + 'completed': Screenshot.objects.filter(status=Screenshot.Status.COMPLETED).count(), + 'failed': Screenshot.objects.filter(status=Screenshot.Status.FAILED).count(), + } + + def ss_queryset(sf): + qs = Screenshot.objects.select_related('page').order_by('-updated_at') + return qs if sf == 'all' else qs.filter(status=sf) + + def ss_serialize(obj): + return { + 'id': str(obj.id), + 'title': (obj.page.title or obj.page.url) if obj.page else '?', + 'detail_url': reverse('page-detail', args=[obj.page.pk]) if obj.page else '#', + 'status': obj.status, + 'retry': obj.retry_count, + 'retry_max': 3, + 'error': obj.error or '', + 'updated_at': obj.updated_at, + 'extra': {}, + } + + job_registry.register({ + 'id': 'screenshots', + 'label': 'Screenshots', + 'icon_color': 'text-indigo-500', + 'icon_path': ( + 'M3 9a2 2 0 012-2h.93a2 2 0 001.664-.89l.812-1.22A2 2 0 0110.07 4h3.86' + 'a2 2 0 011.664.89l.812 1.22A2 2 0 0018.07 7H19a2 2 0 012 2v9a2 2 0 01-2' + ' 2H5a2 2 0 01-2-2V9z M15 13a3 3 0 11-6 0 3 3 0 016 0z' + ), + 'title_label': 'Page', + 'status_choices': [ + ('all', 'All'), ('pending', 'Pending'), ('processing', 'Processing'), + ('completed', 'Completed'), ('failed', 'Failed'), + ], + 'columns': ['id', 'title', 'status', 'retry', 'error', 'updated'], + 'get_stats': ss_stats, + 'get_queryset': ss_queryset, + 'serialize': ss_serialize, + 'bulk_actions': { + 'retry': 'bulk_retry_screenshots', + 'fail': 'bulk_fail_screenshots', + 'delete': 'bulk_delete_screenshots', + }, + }) + + # ── Page Processing ───────────────────────────────────────────── + def pg_stats(): + return { + 'total': Page.objects.count(), + 'pending': Page.objects.filter(process_status=Page.ProcessStatus.PENDING).count(), + 'processing': Page.objects.filter(process_status=Page.ProcessStatus.PROCESSING).count(), + 'completed': Page.objects.filter(process_status=Page.ProcessStatus.COMPLETED).count(), + 'failed': Page.objects.filter(process_status=Page.ProcessStatus.FAILED).count(), + } + + def pg_queryset(sf): + qs = Page.objects.order_by('-updated_at') + return qs if sf == 'all' else qs.filter(process_status=sf) + + def pg_serialize(obj): + return { + 'id': str(obj.id), + 'title': obj.title or obj.url, + 'detail_url': reverse('page-detail', args=[obj.pk]), + 'status': obj.process_status, + 'retry': obj.retry_count, + 'retry_max': 3, + 'error': obj.error_message or '', + 'updated_at': obj.updated_at, + 'extra': {}, + } + + job_registry.register({ + 'id': 'pages', + 'label': 'Page Processing', + 'icon_color': 'text-purple-500', + 'icon_path': ( + 'M19 20H5a2 2 0 01-2-2V6a2 2 0 012-2h10a2 2 0 012 2v1m2 13a2 2 0 01-2-2' + 'V7m2 13a2 2 0 002-2V9.5a2 2 0 00-2-2h-2' + ), + 'title_label': 'Title / URL', + 'status_choices': [ + ('all', 'All'), ('pending', 'Pending'), ('processing', 'Processing'), + ('completed', 'Completed'), ('failed', 'Failed'), + ], + 'columns': ['id', 'title', 'status', 'retry', 'error', 'updated'], + 'get_stats': pg_stats, + 'get_queryset': pg_queryset, + 'serialize': pg_serialize, + 'bulk_actions': { + 'retry': 'bulk_retry_pages', + 'fail': 'bulk_fail_pages', + 'delete': 'bulk_delete_pages', + }, + }) + + # ── Image Imports ────────────────────────────────────────────── + def ii_stats(): + base = FileUpload.objects.filter(source_url__isnull=False).exclude(source_url='') + return { + 'total': base.count(), + 'pending': base.filter(size=0).count(), + 'completed': base.filter(size__gt=0).count(), + } + + def ii_queryset(sf): + base = FileUpload.objects.filter( + source_url__isnull=False, + ).exclude(source_url='').order_by('-updated_at') + if sf == 'pending': + return base.filter(size=0) + if sf == 'completed': + return base.filter(size__gt=0) + return base + + def ii_serialize(obj): + return { + 'id': str(obj.id), + 'title': obj.name, + 'detail_url': getattr(obj, 'download_url', '#') or '#', + 'status': 'completed' if obj.size > 0 else 'pending', + 'retry': None, + 'retry_max': None, + 'error': '', + 'updated_at': obj.updated_at, + 'extra': { + 'source_url': obj.source_url or '', + 'formatted_size': ( + getattr(obj, 'formatted_size', '') if obj.size > 0 else '' + ), + }, + } + + job_registry.register({ + 'id': 'image_imports', + 'label': 'Image Imports', + 'icon_color': 'text-green-500', + 'icon_path': ( + 'M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828' + ' 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12' + 'a2 2 0 002 2z' + ), + 'title_label': 'Filename', + 'status_choices': [ + ('all', 'All'), ('pending', 'Pending'), ('completed', 'Done'), + ], + 'columns': ['id', 'title', 'source_url', 'status', 'size', 'updated'], + 'get_stats': ii_stats, + 'get_queryset': ii_queryset, + 'serialize': ii_serialize, + 'bulk_actions': { + 'retry': 'bulk_retry_image_imports', + 'delete': 'bulk_delete_image_imports', + }, + }) diff --git a/links/job_registry.py b/links/job_registry.py new file mode 100644 index 0000000..a4a0349 --- /dev/null +++ b/links/job_registry.py @@ -0,0 +1,50 @@ +""" +Job type registry for the /ui/jobs/ page. + +To add a new job tab: + 1. Call ``register(job_type_dict)`` from ``LinksConfig.ready()`` in links/apps.py. + 2. Add the corresponding bulk action handler in ``JobsView.post()``. + +Each job type dict must have: + id (str) unique tab slug, used as ?tab= value + label (str) display name shown in the tab bar + icon_color (str) Tailwind text-colour class, e.g. 'text-indigo-500' + icon_path (str) SVG value for the stats card icon + title_label (str) column header for the title/name column + status_choices (list) [(value, label), ...]; first entry should be ('all', 'All') + columns (list[str]) ordered column ids from: + ['id','title','source_url','status','retry','error','size','updated'] + get_stats (callable) () -> dict with 'total' plus any of: + pending / processing / completed / failed + (omit keys that do not apply to this type) + get_queryset (callable) (status_filter: str) -> QuerySet + serialize (callable) (obj) -> dict with keys: + id, title, detail_url, status, + retry (int|None), retry_max (int|None), + error (str), updated_at (datetime), + extra (dict – for source_url, formatted_size, etc.) + bulk_actions (dict) subset of {'retry': action_name, + 'fail': action_name, + 'delete': action_name} +""" +import logging + +logger = logging.getLogger(__name__) + +_registry: list[dict] = [] + + +def register(job_type: dict) -> None: + """Register a job type. Call from LinksConfig.ready().""" + _registry.append(job_type) + logger.debug("Registered job type: %s", job_type.get("id")) + + +def all_types() -> list[dict]: + """Return all registered job types in registration order.""" + return list(_registry) + + +def get_type(type_id: str) -> dict | None: + """Return the job type dict with the given id, or None.""" + return next((j for j in _registry if j["id"] == type_id), None) diff --git a/links/templates/links/jobs.html b/links/templates/links/jobs.html index 58ff95d..dea09cb 100644 --- a/links/templates/links/jobs.html +++ b/links/templates/links/jobs.html @@ -41,102 +41,67 @@ {% endif %} -
- +
+ {% for stat in job_type_stats %}

- - - + + - {% trans "Screenshots" %} + {{ stat.label }}

-
- -
{{ stats.screenshots.total }}
+
+ +
{{ stat.total }}
{% trans "Total" %}
 - -
{{ stats.screenshots.pending }}
+ {% if stat.pending is not None %} + +
{{ stat.pending }}
{% trans "Pending" %}
 - -
{{ stats.screenshots.processing }}
+ {% endif %} + {% if stat.processing is not None %} + +
{{ stat.processing }}
{% trans "Running" %}
 - -
{{ stats.screenshots.completed }}
+ {% endif %} + {% if stat.completed is not None %} + +
{{ stat.completed }}
{% trans "Done" %}
 - -
{{ stats.screenshots.failed }}
-
{% trans "Failed" %}
- -
-
- - -
-
-

- - - - {% trans "Page Processing" %} -

-
-
- -
{{ stats.pages.total }}
-
{% trans "Total" %}
- - -
{{ stats.pages.pending }}
-
{% trans "Pending" %}
- - -
{{ stats.pages.processing }}
-
{% trans "Running" %}
- - -
{{ stats.pages.completed }}
-
{% trans "Done" %}
- - -
{{ stats.pages.failed }}
+ {% endif %} + {% if stat.failed is not None %} + +
{{ stat.failed }}
{% trans "Failed" %}
 + {% endif %}
+ {% endfor %}
- +
- - {% trans "Screenshots" %} + {% if tab_info.is_active %}border-blue-500 text-blue-600 bg-white{% else %}border-transparent text-gray-500 hover:text-gray-700 hover:bg-gray-50{% endif %}"> + {{ tab_info.label }} - {{ stats.screenshots.total }} - - - - {% trans "Pages" %} - - {{ stats.pages.total }} + {% if tab_info.is_active %}bg-blue-100 text-blue-600{% else %}bg-gray-100 text-gray-500{% endif %}"> + {{ tab_info.count }} + {% endfor %} @@ -146,32 +111,29 @@ {{ scheduled_jobs|length }} - - {% trans "Image Imports" %} - - {{ stats.image_imports.total }} - -
- - {% if tab == 'screenshots' or tab == 'pages' %} + + {% if status_choices %}
- {% for s, label, pill_active_style, pill_inactive_style in filter_options %} - - {{ label }} - - {% endfor %} -
- {% elif tab == 'image_imports' %} -
- {% for s, label, pill_active_style, pill_inactive_style in import_filter_options %} - + {% for value, label in status_choices %} + {{ label }} {% endfor %} @@ -193,7 +155,7 @@ style="background:#1d4ed8;color:#fff;padding:.3rem .85rem;border-radius:.25rem;font-size:.8rem;cursor:pointer;font-weight:500;"> ↺ {% trans "Retry" %} - @@ -220,6 +182,19 @@ {% if tab == 'scheduler' %} +
+ + + + ( {% trans "job(s)" %}) + +
{% if scheduled_jobs %}
@@ -232,26 +207,28 @@ - {% for job in scheduled_jobs %} - - - - - - - {% endfor %} +
{{ job.id }}{{ job.name }}{{ job.trigger }} - {% if job.next_run %}{{ job.next_run }}{% else %}—{% endif %} -
+
+ {% trans "No jobs match this filter." %} +
{% else %}
{% trans "No scheduled jobs." %}
{% endif %} - {% elif tab == 'screenshots' %} - - {% if page_obj.object_list %} + {% else %} + + {% if page_rows %}
@@ -262,64 +239,87 @@ :checked="allIds.length > 0 && selectedIds.length === allIds.length"> - - - - + + {% if 'source_url' in current_tab_config.columns %}{% endif %} + {% if 'status' in current_tab_config.columns %}{% endif %} + {% if 'retry' in current_tab_config.columns %}{% endif %} + {% if 'error' in current_tab_config.columns %}{% endif %} + {% if 'size' in current_tab_config.columns %}{% endif %} - {% for ss in page_obj.object_list %} - + {% for row in page_rows %} + - + + {% if 'source_url' in current_tab_config.columns %} + + {% endif %} + {% if 'status' in current_tab_config.columns %} - + {% endif %} + {% if 'retry' in current_tab_config.columns %} + {% endif %} + {% if 'error' in current_tab_config.columns %} + - + {% endif %} + {% if 'size' in current_tab_config.columns %} + + {% endif %} + {% endfor %}
ID{% trans "Page" %}{% trans "Status" %}{% trans "Retries" %}{% trans "Error" %}{{ current_tab_config.title_label }}{% trans "Source URL" %}{% trans "Status" %}{% trans "Retries" %}{% trans "Error" %}{% trans "Size" %}{% trans "Updated" %}
+ value="{{ row.id }}" + @change="toggle('{{ row.id }}')" + :checked="selectedIds.includes('{{ row.id }}')"> {{ ss.id }}{{ row.id|truncatechars:12 }} - - {{ ss.page.title|default:ss.page.url|truncatechars:55 }} + {% if row.detail_url and row.detail_url != '#' %} + + {{ row.title|truncatechars:55 }} + + {% else %} + {{ row.title|truncatechars:55 }} + {% endif %} + + + {{ row.extra.source_url|truncatechars:50 }} - {{ ss.status }} + {{ row.status }} {{ ss.retry_count }}/3 - {% if ss.error %} + {% if row.retry is not None %}{{ row.retry }}/{{ row.retry_max }}{% else %}—{% endif %} + + {% if row.error %} {% else %}—{% endif %} {{ ss.updated_at|timesince }} {% trans "ago" %} + {% if row.extra.formatted_size %}{{ row.extra.formatted_size }}{% else %}—{% endif %} + {{ row.updated_at|timesince }} {% trans "ago" %}
{% else %} -
{% trans "No screenshots match this filter." %}
+
{% trans "No items match this filter." %}
{% endif %} - {% elif tab == 'pages' %} - - {% if page_obj.object_list %} + {% comment %}pages table removed — unified table above handles all types{% endcomment %} + {% if False %}
@@ -385,8 +385,8 @@
{% trans "No pages match this filter." %}
{% endif %} - {% elif tab == 'image_imports' %} - + {% comment %}image_imports table removed — unified table above handles all types{% endcomment %} + {% if False %} {% if page_obj.object_list %}
@@ -450,6 +450,7 @@
{% trans "No image imports match this filter." %}
{% endif %} {% endif %} + {% endif %} {% if page_obj.has_other_pages %} @@ -510,6 +511,8 @@ {{ all_ids|json_script:"jobs-all-ids" }} +{{ scheduled_jobs|json_script:"scheduled-jobs-data" }} +{{ bulk_actions|json_script:"bulk-actions-data" }}