import - whole system of syncing ranchers/herds/livestocks

apps/herd/services/herd_rancher_sync.py

@@ -0,0 +1,132 @@
+from django.db import transaction
+
+from apps.authentication.models import City
+from apps.herd.models import Rancher, Herd
+
+
+class HerdRancherSyncService:
+
+    @classmethod
+    def bulk_sync(cls, queryset, batch_size=1000):
+        """
+        optimized bulk sync for large datasets
+        """
+
+        # -------------------------
+        # Cache Cities
+        # -------------------------
+        city_map = {
+            name.strip(): id
+            for id, name in City.objects.all().values_list('id', 'name')
+        }
+
+        # -------------------------
+        # Cache existing ranchers
+        # -------------------------
+        rancher_map = {
+            r.national_code: r
+            for r in Rancher.objects.filter(
+                national_code__in=queryset.values_list(
+                    'rancher_national_code', flat=True
+                )
+            ).only('id', 'national_code')
+        }
+
+        new_ranchers = []
+        new_herds = []
+
+        existing_herds = set(
+            Herd.objects.filter(
+                rancher__national_code__in=queryset.values_list(
+                    'rancher_national_code', flat=True
+                )
+            ).values_list(
+                'rancher__national_code', 'code'
+            )
+        )
+
+        seen_in_batch = set()
+
+        for temp in queryset.iterator(chunk_size=batch_size):
+
+            # -------------------------
+            # Rancher
+            # -------------------------
+            rancher = rancher_map.get(temp.rancher_national_code)
+
+            if not rancher:
+                rancher = Rancher(
+                    first_name=temp.rancher_name,
+                    mobile=temp.mobile,
+                    national_code=temp.rancher_national_code,
+                    rancher_type='N',
+                    city_id=city_map.get(temp.city.strip()),
+                    province_id=30
+                )
+                new_ranchers.append(rancher)
+                rancher_map[temp.rancher_national_code] = rancher
+
+            # -------------------------
+            # Herd
+            # -------------------------
+            herd_key = (temp.rancher_national_code, temp.herd_code)
+
+            if herd_key in existing_herds:
+                continue
+
+            if herd_key in seen_in_batch:
+                continue
+
+            seen_in_batch.add(herd_key)
+
+            new_herds.append({
+                "rancher_code": temp.rancher_national_code,
+                "herd": Herd(
+                    name=temp.herd_name,
+                    code=temp.herd_code,
+                    epidemiologic=temp.epidemiologic,
+                    latitude=temp.latitude,
+                    longitude=temp.longitude,
+                    postal=temp.postal_code,
+                    unit_unique_id=temp.unit_unique_id,
+                    city_id=city_map.get(temp.city.strip()),
+                    province_id=30
+                )
+            }
+            )
+
+        # -------------------------
+        # Bulk DB Operations
+        # -------------------------
+        with transaction.atomic():
+            Rancher.objects.bulk_create(
+                new_ranchers,
+                ignore_conflicts=True,
+                batch_size=batch_size
+            )
+
+            # refresh ranchers with ids
+            rancher_map = {
+                r.national_code: r
+                for r in Rancher.objects.filter(
+                    national_code__in=rancher_map.keys()
+                )
+            }
+
+            final_herds = []
+
+            for item in new_herds:
+                rancher = rancher_map.get(item["rancher_code"])
+
+                if not rancher:
+                    continue  # یا raise error
+
+                herd = item["herd"]
+                herd.rancher = rancher
+                final_herds.append(herd)
+
+            Herd.objects.bulk_create(
+                final_herds,
+                ignore_conflicts=True,
+                batch_size=batch_size
+            )