From bbda32887ed589d11f8b0470abffa13dc17ebe58 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sun, 17 Nov 2024 12:26:23 +1100 Subject: [PATCH] be able to upload and view images --- core/settings.py | 21 ++++- data/db.sqlite3 | Bin 241664 -> 241664 bytes docker-compose.yml | 2 + links/api_views.py | 30 ++++++- links/models.py | 9 +++ links/storage.py | 81 +++++++++++++------ links/templates/links/collection_detail.html | 24 ++++-- 7 files changed, 132 insertions(+), 35 deletions(-) diff --git a/core/settings.py b/core/settings.py index 4f409bf..e9c405b 100644 --- a/core/settings.py +++ b/core/settings.py @@ -223,7 +223,7 @@ MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media') # Ensure media files are served in all environments SERVE_MEDIA = True - +CELERY_RESULT_BACKEND # Celery Beat settings CELERYBEAT_SCHEDULE_FILENAME = os.path.join(BASE_DIR, 'data', 'celerybeat-schedule') @@ -232,3 +232,22 @@ R2_ENDPOINT_URL = os.environ.get('R2_ENDPOINT_URL') R2_ACCESS_KEY_ID = os.environ.get('R2_ACCESS_KEY_ID') R2_SECRET_ACCESS_KEY = os.environ.get('R2_SECRET_ACCESS_KEY') R2_BUCKET_NAME = os.environ.get('R2_BUCKET_NAME') + +# For debugging +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'level': 'DEBUG', + }, + }, + 'loggers': { + 'links': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} diff --git a/data/db.sqlite3 b/data/db.sqlite3 index ac239175d77d16859902ae8c5cc7c06d5e9fef3e..3ba7c3e40780a033a788c669da87b0457f120707 100644 GIT binary patch delta 2248 zcmb7`KW}426vb^PUONe~sXzgwO_VegW8XV-=g$odt8GGIQKiq!yqRbh1qD$cs;rAN zRAebuqNCvp5TK>tD^RS2#EJrKPeF$mKgVlB5+Zja-{(8?dvoqN@5zhalNY@&-)_FV zc;{jB<@o6QsQLX->zfa^`oA{1y~{@b$Ntm)N&ld~+xxrsym#5V{0_Ij?mavCt+m%) z+glkmpN$8t?RIy*?97+ zdvxNDlN>xgJ+4=u4lBzVO+XPVVnGRlV67?kY964o&ZsuBqoq7|6+ zArg{NGOOW#dJi9;rsG!gz641EQJ~!h+?RS^+JR;wvbR_2S5nZr3uqZ76u4a2n4Fiv zSXLA&Z2^={RBw;VqJh^b7+`@3y`Y$+VA5JR?+#IV6QY-XiZ<_EqcxuH-voeG6Txj2 zOEa|20-BN1s^Fjsg~J9H5RHkX5DIU!rbbiCq6H{uP&pxeOy$rs0F ziF3lJrLnbvLkO@nXiSuHB#gE zFK#+GmBScwOGN?B1h~2&!0g;)3QUA#s0>p805myjn{$*3>if~l3b1IILX2F@g0i6C zqj1RuksKUi*#U~(T-nCF_f&wt9NYw;ii^n`(IyowXJ{*nXkd)eIpRdTrbwewwvxn9(I6Q#d56Kq6Y0&epv56zz{UuA|YwYVF_#+n83| z?AB6iWAx_cnD4iW*;lBtXOqY=P-V1d24Lwys$wCIad7(aQa%sGhumAbh`c<-) zJiRQrTJp8`yd>;yf45H7vq?7I6>W9!-W3vcB;RGXbyzPN*V#?ZRprrvYFBz_S71{~xA618{wx B5e)zU diff --git a/docker-compose.yml b/docker-compose.yml index ab98bfa..b343578 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: - ./data/celery:/app/data/celery ports: - "8000:8000" + env_file: + - .env environment: - DJANGO_SETTINGS_MODULE=core.settings - CELERY_BROKER_URL=redis://redis:6379/0 diff --git a/links/api_views.py b/links/api_views.py index 7f68d02..5e0527e 100644 --- a/links/api_views.py +++ b/links/api_views.py @@ -5,6 +5,9 @@ from .models import ImageCollection, Image from .serializers import ImageCollectionSerializer, ImageSerializer from .storage import R2Storage import uuid +import logging +import os +logger = logging.getLogger(__name__) class ImageCollectionViewSet(viewsets.ModelViewSet): queryset = ImageCollection.objects.all() @@ -12,18 +15,38 @@ class ImageCollectionViewSet(viewsets.ModelViewSet): @action(detail=True, methods=['post']) def upload_images(self, request, pk=None): + """API endpoint to upload images to a collection""" collection = self.get_object() - files = request.FILES.getlist('files') + files = request.FILES.getlist('file') storage = R2Storage() + logger.debug(f"Processing upload request for collection {collection.id}") + logger.debug(f"Number of files: {len(files)}") + uploaded_images = [] for file in files: try: + logger.info(os.environ.get('R2_ENDPOINT_URL')) + logger.debug(f"Processing file: {file.name}") + logger.debug(f"File size: {file.size}") + logger.debug(f"Content type: {file.content_type}") + # Generate unique file key file_key = f"images/{collection.id}/{uuid.uuid4()}/{file.name}" + # Validate file + if not hasattr(file, 'read'): + raise ValueError("Invalid file object - missing read method") + + if not file.content_type.startswith('image/'): + raise ValueError("Invalid file type. Only images are allowed.") + # Upload to R2 - storage.upload_file(file, file_key, file.content_type) + try: + storage.upload_file(file, file_key, file.content_type) + except Exception as e: + logger.error(f"Storage upload failed", exc_info=True) + raise ValueError(f"Storage upload failed: {str(e)}") # Create image record image = Image.objects.create( @@ -34,8 +57,11 @@ class ImageCollectionViewSet(viewsets.ModelViewSet): size=file.size ) + logger.debug(f"Created image record: {image.id}") uploaded_images.append(ImageSerializer(image).data) + except Exception as e: + logger.error(f"Upload failed for {file.name}", exc_info=True) return Response({ 'error': str(e) }, status=status.HTTP_400_BAD_REQUEST) diff --git a/links/models.py b/links/models.py index 2997440..cac8549 100644 --- a/links/models.py +++ b/links/models.py @@ -7,6 +7,8 @@ from django.utils import timezone from django.conf import settings import os import uuid +from links.storage import R2Storage +from datetime import timedelta logger = logging.getLogger(__name__) @@ -272,3 +274,10 @@ class Image(models.Model): def __str__(self): return self.title + + def get_url(self, expires_in=3600): + """Get a signed URL for the image that expires after the specified time""" + if not self.file_key: + return None + storage = R2Storage() + return storage.get_url(self.file_key, expires_in=expires_in) diff --git a/links/storage.py b/links/storage.py index 0adf3c5..51801d2 100644 --- a/links/storage.py +++ b/links/storage.py @@ -1,9 +1,14 @@ import boto3 from django.conf import settings from botocore.config import Config +import logging +from django.core.files.base import File + +logger = logging.getLogger(__name__) class R2Storage: def __init__(self): + logger.debug(f"Initializing R2Storage with endpoint: {settings.R2_ENDPOINT_URL}") self.client = boto3.client('s3', endpoint_url=settings.R2_ENDPOINT_URL, aws_access_key_id=settings.R2_ACCESS_KEY_ID, @@ -12,35 +17,61 @@ class R2Storage: region_name='auto' ) self.bucket = settings.R2_BUCKET_NAME + logger.debug(f"Using bucket: {self.bucket}") def upload_file(self, file_obj, key, content_type=None): """Upload a file to R2""" - extra_args = {} - if content_type: - extra_args['ContentType'] = content_type + try: + logger.debug(f"Starting file upload: {key}") + logger.debug(f"File object type: {type(file_obj)}") - self.client.upload_fileobj( - file_obj, - self.bucket, - key, - ExtraArgs=extra_args - ) - return key + # Reset file pointer and read content + file_obj.seek(0) + file_content = file_obj.read() - def delete_file(self, key): - """Delete a file from R2""" - self.client.delete_object( - Bucket=self.bucket, - Key=key - ) + logger.debug(f"File content size: {len(file_content) if file_content else 'None'}") + + if not file_content: + raise ValueError("File content is empty") + + from io import BytesIO + file_data = BytesIO(file_content) + + extra_args = { + 'ContentType': content_type or 'application/octet-stream', + } + + logger.debug(f"Uploading to R2 with extra args: {extra_args}") + + self.client.upload_fileobj( + file_data, + self.bucket, + key, + ExtraArgs=extra_args + ) + + logger.debug(f"Successfully uploaded file: {key}") + return key + + except Exception as e: + logger.error(f"Upload failed: {str(e)}", exc_info=True) + raise def get_url(self, key, expires_in=3600): - """Generate a presigned URL for the file""" - return self.client.generate_presigned_url( - 'get_object', - Params={ - 'Bucket': self.bucket, - 'Key': key - }, - ExpiresIn=expires_in - ) + """Generate a signed URL that expires after the specified time""" + if not key: + return None + try: + url = self.client.generate_presigned_url( + 'get_object', + Params={ + 'Bucket': self.bucket, + 'Key': key + }, + ExpiresIn=expires_in # URL expires in 1 hour by default + ) + logger.debug(f"Generated signed URL for {key}") + return url + except Exception as e: + logger.error(f"Failed to generate signed URL: {str(e)}") + return None diff --git a/links/templates/links/collection_detail.html b/links/templates/links/collection_detail.html index d6188e9..94b5932 100644 --- a/links/templates/links/collection_detail.html +++ b/links/templates/links/collection_detail.html @@ -150,14 +150,15 @@