from django.db import transaction from apps.authentication.models import City from apps.herd.models import Rancher, Herd class HerdRancherSyncService: @classmethod def bulk_sync(cls, queryset, batch_size=1000): """ optimized bulk sync for large datasets """ city_map = { name.strip(): id for id, name in City.objects.all().values_list('id', 'name') } rancher_map = { r.national_code: r for r in Rancher.objects.filter( national_code__in=queryset.values_list( 'rancher_national_code', flat=True ) ).only('id', 'national_code') } new_ranchers = [] new_herds = [] existing_herds = set( Herd.objects.filter( rancher__national_code__in=queryset.values_list( 'rancher_national_code', flat=True ) ).values_list( 'rancher__national_code', 'code' ) ) seen_in_batch = set() for temp in queryset.iterator(chunk_size=batch_size): rancher = rancher_map.get(temp.rancher_national_code) if not rancher: rancher = Rancher( first_name=temp.rancher_name, mobile=temp.mobile, national_code=temp.rancher_national_code, rancher_type='N', city_id=city_map.get(temp.city.strip()), province_id=30 ) new_ranchers.append(rancher) rancher_map[temp.rancher_national_code] = rancher herd_key = (temp.rancher_national_code, temp.herd_code) if herd_key in existing_herds: continue if herd_key in seen_in_batch: continue seen_in_batch.add(herd_key) new_herds.append({ "rancher_code": temp.rancher_national_code, "herd": Herd( name=temp.herd_name, code=temp.herd_code, epidemiologic=temp.epidemiologic, latitude=temp.latitude, longitude=temp.longitude, postal=temp.postal_code, unit_unique_id=temp.unit_unique_id, city_id=city_map.get(temp.city.strip()), province_id=30 ) } ) with transaction.atomic(): Rancher.objects.bulk_create( new_ranchers, ignore_conflicts=True, batch_size=batch_size ) # refresh ranchers with ids rancher_map = { r.national_code: r for r in Rancher.objects.filter( national_code__in=rancher_map.keys() ) } final_herds = [] for item in new_herds: rancher = rancher_map.get(item["rancher_code"]) if not rancher: continue herd = item["herd"] herd.rancher = rancher final_herds.append(herd) Herd.objects.bulk_create( final_herds, ignore_conflicts=True, batch_size=batch_size )