118 lines
3.4 KiB
Python
118 lines
3.4 KiB
Python
from django.db import transaction
|
|
|
|
from apps.authentication.models import City
|
|
from apps.herd.models import Rancher, Herd
|
|
|
|
|
|
class HerdRancherSyncService:
|
|
|
|
@classmethod
|
|
def bulk_sync(cls, queryset, batch_size=1000):
|
|
"""
|
|
optimized bulk sync for large datasets
|
|
"""
|
|
|
|
city_map = {
|
|
name.strip(): id
|
|
for id, name in City.objects.all().values_list('id', 'name')
|
|
}
|
|
|
|
rancher_map = {
|
|
r.national_code: r
|
|
for r in Rancher.objects.filter(
|
|
national_code__in=queryset.values_list(
|
|
'rancher_national_code', flat=True
|
|
)
|
|
).only('id', 'national_code')
|
|
}
|
|
|
|
new_ranchers = []
|
|
new_herds = []
|
|
|
|
existing_herds = set(
|
|
Herd.objects.filter(
|
|
rancher__national_code__in=queryset.values_list(
|
|
'rancher_national_code', flat=True
|
|
)
|
|
).values_list(
|
|
'rancher__national_code', 'code'
|
|
)
|
|
)
|
|
|
|
seen_in_batch = set()
|
|
|
|
for temp in queryset.iterator(chunk_size=batch_size):
|
|
|
|
rancher = rancher_map.get(temp.rancher_national_code)
|
|
|
|
if not rancher:
|
|
rancher = Rancher(
|
|
first_name=temp.rancher_name,
|
|
mobile=temp.mobile,
|
|
national_code=temp.rancher_national_code,
|
|
rancher_type='N',
|
|
city_id=city_map.get(temp.city.strip()),
|
|
province_id=30
|
|
)
|
|
new_ranchers.append(rancher)
|
|
rancher_map[temp.rancher_national_code] = rancher
|
|
|
|
herd_key = (temp.rancher_national_code, temp.herd_code)
|
|
|
|
if herd_key in existing_herds:
|
|
continue
|
|
|
|
if herd_key in seen_in_batch:
|
|
continue
|
|
|
|
seen_in_batch.add(herd_key)
|
|
|
|
new_herds.append({
|
|
"rancher_code": temp.rancher_national_code,
|
|
"herd": Herd(
|
|
name=temp.herd_name,
|
|
code=temp.herd_code,
|
|
epidemiologic=temp.epidemiologic,
|
|
latitude=temp.latitude,
|
|
longitude=temp.longitude,
|
|
postal=temp.postal_code,
|
|
unit_unique_id=temp.unit_unique_id,
|
|
city_id=city_map.get(temp.city.strip()),
|
|
province_id=30
|
|
)
|
|
}
|
|
)
|
|
|
|
with transaction.atomic():
|
|
Rancher.objects.bulk_create(
|
|
new_ranchers,
|
|
ignore_conflicts=True,
|
|
batch_size=batch_size
|
|
)
|
|
|
|
# refresh ranchers with ids
|
|
rancher_map = {
|
|
r.national_code: r
|
|
for r in Rancher.objects.filter(
|
|
national_code__in=rancher_map.keys()
|
|
)
|
|
}
|
|
|
|
final_herds = []
|
|
|
|
for item in new_herds:
|
|
rancher = rancher_map.get(item["rancher_code"])
|
|
|
|
if not rancher:
|
|
continue
|
|
|
|
herd = item["herd"]
|
|
herd.rancher = rancher
|
|
final_herds.append(herd)
|
|
|
|
Herd.objects.bulk_create(
|
|
final_herds,
|
|
ignore_conflicts=True,
|
|
batch_size=batch_size
|
|
)
|