bleese14/004.erp.generic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: reduce memory usage for monthly ZIP archive flow

Browse Source 
Replace generated archive download response with FileResponse streaming so ZIP files are not fully loaded into process memory.

Move monthly archive task output to a temporary file and upload that file to storage, and write invoice attachments into ZIP archives in chunks.

Add Celery memory guard settings, a memory-aware base task hook, and periodic /tmp cleanup for prefixed archive files older than one day.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
mariomitte
2026-09-05 08:15:30 +02:00
parent b985c285f9
commit 61bc772054
6 changed files with 260 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
backend/modules/fleet/views.py
View File

@@ -2450,6 +2450,26 @@ def _file_attachment(file_field, fallback_name):
return (filename, content, _guess_content_type(filename))
def _write_file_field_to_zip(archive, *, file_field, entry_name, chunk_size=64 * 1024):
if not file_field:
return False
file_field.open('rb')
try:
first_chunk = file_field.read(chunk_size)
if not first_chunk:
return False
with archive.open(entry_name, mode='w') as destination:
destination.write(first_chunk)
while True:
chunk = file_field.read(chunk_size)
if not chunk:
break
destination.write(chunk)
finally:
file_field.close()
return True
def _build_image_attachments_for_work_order(work_order):
attachments = []
photos = WorkOrderPhoto.objects.filter(is_active=True, work_order=work_order).order_by('created_at')
@@ -3008,7 +3028,7 @@ def _parse_year_month_params(request):
return year, month
def _build_monthly_service_tasks_archive_content(*, user, year, month):
def _write_monthly_service_tasks_archive_entries(archive, *, user, year, month):
from modules.task_management.models import Task
tasks = list(
@@ -3028,24 +3048,37 @@ def _build_monthly_service_tasks_archive_content(*, user, year, month):
raise DRFValidationError({'detail': 'Nema servisnih taskova za odabrani mjesec.'})
used_names = set()
entries_written = 0
for task in tasks:
work_order = task.work_order
if work_order is None:
continue
docx_bytes = _build_work_order_service_records_docx_bytes(work_order, related_tasks=[task])
base_name = _service_records_docx_filename(work_order, task)
entry_name = _unique_zip_entry_name(base_name, used_names)
archive.writestr(entry_name, docx_bytes)
entries_written += 1
if entries_written == 0:
raise DRFValidationError({'detail': 'Nije moguće kreirati ZIP za odabrani mjesec.'})
return entries_written
def _build_monthly_service_tasks_archive_content(*, user, year, month):
archive_buffer = BytesIO()
with zipfile.ZipFile(archive_buffer, mode='w', compression=zipfile.ZIP_DEFLATED) as archive:
for task in tasks:
work_order = task.work_order
if work_order is None:
continue
docx_bytes = _build_work_order_service_records_docx_bytes(work_order, related_tasks=[task])
base_name = _service_records_docx_filename(work_order, task)
entry_name = _unique_zip_entry_name(base_name, used_names)
archive.writestr(entry_name, docx_bytes)
_write_monthly_service_tasks_archive_entries(
archive,
user=user,
year=year,
month=month,
)
archive_content = archive_buffer.getvalue()
if not archive_content:
raise DRFValidationError({'detail': 'Nije moguće kreirati ZIP za odabrani mjesec.'})
return archive_content
def _build_monthly_work_orders_archive_content(*, user, year, month):
def _write_monthly_work_orders_archive_entries(archive, *, user, year, month):
from modules.task_management.models import Task
monthly_tasks = (
@@ -3084,23 +3117,34 @@ def _build_monthly_work_orders_archive_content(*, user, year, month):
raise DRFValidationError({'detail': 'Nema putnih naloga za odabrani mjesec.'})
used_names = set()
entries_written = 0
for work_order in work_orders:
pdf_bytes = _build_work_order_pdf(work_order)
work_order_pdf_name = _unique_zip_entry_name(_pdf_filename(work_order, 'work_order'), used_names)
archive.writestr(work_order_pdf_name, pdf_bytes)
entries_written += 1
display_code = _work_order_display_code(work_order)
invoices = work_order.invoices.filter(is_active=True).order_by('datum', 'created_at')
for index, invoice in enumerate(invoices, start=1):
file_name = Path(str(getattr(getattr(invoice, 'image', None), 'name', '') or f"invoice-{index}.bin")).name
archive_path = _unique_zip_entry_name(f"Racuni/{display_code}/{file_name}", used_names)
if _write_file_field_to_zip(archive, file_field=invoice.image, entry_name=archive_path):
entries_written += 1
if entries_written == 0:
raise DRFValidationError({'detail': 'Nije moguće kreirati ZIP za odabrani mjesec.'})
return entries_written
def _build_monthly_work_orders_archive_content(*, user, year, month):
archive_buffer = BytesIO()
with zipfile.ZipFile(archive_buffer, mode='w', compression=zipfile.ZIP_DEFLATED) as archive:
for work_order in work_orders:
pdf_bytes = _build_work_order_pdf(work_order)
work_order_pdf_name = _unique_zip_entry_name(_pdf_filename(work_order, 'work_order'), used_names)
archive.writestr(work_order_pdf_name, pdf_bytes)
display_code = _work_order_display_code(work_order)
invoices = work_order.invoices.filter(is_active=True).order_by('datum', 'created_at')
for index, invoice in enumerate(invoices, start=1):
attachment = _file_attachment(invoice.image, fallback_name=f"invoice-{index}.bin")
if not attachment:
continue
invoice_filename, content, _content_type = attachment
archive_path = f"Racuni/{display_code}/{invoice_filename}"
archive.writestr(_unique_zip_entry_name(archive_path, used_names), content)
_write_monthly_work_orders_archive_entries(
archive,
user=user,
year=year,
month=month,
)
archive_content = archive_buffer.getvalue()
if not archive_content:
raise DRFValidationError({'detail': 'Nije moguće kreirati ZIP za odabrani mjesec.'})
@@ -3411,12 +3455,18 @@ def generated_archive_download(request, archive_id):
if generated_archive is None:
raise DRFValidationError({'detail': 'ZIP arhiva nije dostupna ili je istekla.'})
generated_archive.file.open('rb')
file_name = getattr(generated_archive.file, 'name', '')
storage = getattr(generated_archive.file, 'storage', None)
if not file_name or storage is None:
raise DRFValidationError({'detail': 'ZIP arhiva je prazna ili nedostupna.'})
try:
archive_bytes = generated_archive.file.read()
finally:
generated_archive.file.close()
if not archive_bytes:
if not storage.exists(file_name):
raise DRFValidationError({'detail': 'ZIP arhiva je prazna ili nedostupna.'})
except (FileNotFoundError, OSError, ValueError):
raise DRFValidationError({'detail': 'ZIP arhiva je prazna ili nedostupna.'})
try:
generated_archive.file.open('rb')
except (FileNotFoundError, OSError, ValueError):
raise DRFValidationError({'detail': 'ZIP arhiva je prazna ili nedostupna.'})
filename = generated_archive.filename or _generated_archive_filename_for_user(
request.user,
@@ -3424,10 +3474,13 @@ def generated_archive_download(request, archive_id):
month=generated_archive.month,
archive_type=generated_archive.archive_type,
)
response = HttpResponse(archive_bytes, content_type='application/zip')
response = FileResponse(generated_archive.file, content_type='application/zip')
response['Content-Disposition'] = f'attachment; filename="{filename}"'
response['Cache-Control'] = 'private, max-age=3600'
response['Content-Length'] = str(len(archive_bytes))
try:
response['Content-Length'] = str(generated_archive.file.size)
except (OSError, ValueError, TypeError):
pass
return response