import - whole system of syncing ranchers/herds/livestocks

apps/herd/management/commands/sync_herd_rancher.py

@@ -0,0 +1,70 @@
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from apps.herd.models import (
+    HerdRancherTemporary,
+)
+from apps.herd.services.herd_rancher_sync import HerdRancherSyncService
+
+
+class Command(BaseCommand):
+    help = "Bulk Sync Rancher & Herd from HerdRancherTemporary"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--limit',
+            type=int,
+            help='limit number of records'
+        )
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='run without saving data'
+        )
+        parser.add_argument(
+            '--batch-size',
+            type=int,
+            default=1000,
+            help='batch size'
+        )
+
+    def handle(self, *args, **options):
+        limit = options['limit']
+        dry_run = options['dry_run']
+        batch_size = options['batch_size']
+
+        qs = HerdRancherTemporary.objects.all()
+
+        if limit:
+            qs = qs[:limit]
+
+        total = qs.count()
+
+        self.stdout.write(
+            self.style.NOTICE(f"Start bulk syncing {total} records")
+        )
+
+        if dry_run:
+            self.stdout.write(
+                self.style.WARNING("Running in DRY-RUN mode (no DB writes)")
+            )
+            return
+
+        try:
+            with transaction.atomic():
+                HerdRancherSyncService.bulk_sync(
+                    queryset=qs,
+                    batch_size=batch_size
+                )
+
+                qs.update(sync_status='S')
+
+        except Exception as e:
+            qs.update(sync_status='F')
+            raise e
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Done. synced={total}"
+            )
+        )

apps/herd/migrations/0025_herdranchertemporary_sync_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2025-12-28 05:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('herd', '0024_alter_herdranchertemporary_agent_code_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='herdranchertemporary',
+            name='sync_status',
+            field=models.CharField(max_length=50, null=True),
+        ),
+    ]

apps/herd/migrations/0026_herdranchertemporary_city.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2025-12-28 06:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('herd', '0025_herdranchertemporary_sync_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='herdranchertemporary',
+            name='city',
+            field=models.CharField(max_length=150, null=True),
+        ),
+    ]

apps/herd/migrations/0027_alter_herd_name_alter_herd_photo.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0 on 2025-12-28 06:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('herd', '0026_herdranchertemporary_city'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='herd',
+            name='name',
+            field=models.CharField(max_length=200),
+        ),
+        migrations.AlterField(
+            model_name='herd',
+            name='photo',
+            field=models.CharField(max_length=200, null=True),
+        ),
+    ]

apps/herd/models.py

@@ -24,8 +24,8 @@ class Herd(BaseModel):
         related_name='herd',
         null=True
     )
-    name = models.CharField(max_length=50)
-    photo = models.CharField(max_length=50, null=True)
+    name = models.CharField(max_length=200)
+    photo = models.CharField(max_length=200, null=True)
     code = models.CharField(max_length=20)
     heavy_livestock_number = models.BigIntegerField(default=0)
     light_livestock_number = models.BigIntegerField(default=0)
@@ -182,6 +182,8 @@ class HerdRancherTemporary(BaseModel):
     mobile = models.CharField(max_length=150, null=True)
     agent_code = models.CharField(max_length=150, null=True)
     registerer_user = models.CharField(max_length=150, null=True)
+    sync_status = models.CharField(max_length=50, null=True)
+    city = models.CharField(max_length=150, null=True)
 
     def save(self, *args, **kwargs):
         return super(HerdRancherTemporary, self).save(*args, **kwargs)

apps/herd/services/herd_rancher_sync.py

@@ -0,0 +1,132 @@
+from django.db import transaction
+
+from apps.authentication.models import City
+from apps.herd.models import Rancher, Herd
+
+
+class HerdRancherSyncService:
+
+    @classmethod
+    def bulk_sync(cls, queryset, batch_size=1000):
+        """
+        optimized bulk sync for large datasets
+        """
+
+        # -------------------------
+        # Cache Cities
+        # -------------------------
+        city_map = {
+            name.strip(): id
+            for id, name in City.objects.all().values_list('id', 'name')
+        }
+
+        # -------------------------
+        # Cache existing ranchers
+        # -------------------------
+        rancher_map = {
+            r.national_code: r
+            for r in Rancher.objects.filter(
+                national_code__in=queryset.values_list(
+                    'rancher_national_code', flat=True
+                )
+            ).only('id', 'national_code')
+        }
+
+        new_ranchers = []
+        new_herds = []
+
+        existing_herds = set(
+            Herd.objects.filter(
+                rancher__national_code__in=queryset.values_list(
+                    'rancher_national_code', flat=True
+                )
+            ).values_list(
+                'rancher__national_code', 'code'
+            )
+        )
+
+        seen_in_batch = set()
+
+        for temp in queryset.iterator(chunk_size=batch_size):
+
+            # -------------------------
+            # Rancher
+            # -------------------------
+            rancher = rancher_map.get(temp.rancher_national_code)
+
+            if not rancher:
+                rancher = Rancher(
+                    first_name=temp.rancher_name,
+                    mobile=temp.mobile,
+                    national_code=temp.rancher_national_code,
+                    rancher_type='N',
+                    city_id=city_map.get(temp.city.strip()),
+                    province_id=30
+                )
+                new_ranchers.append(rancher)
+                rancher_map[temp.rancher_national_code] = rancher
+
+            # -------------------------
+            # Herd
+            # -------------------------
+            herd_key = (temp.rancher_national_code, temp.herd_code)
+
+            if herd_key in existing_herds:
+                continue
+
+            if herd_key in seen_in_batch:
+                continue
+
+            seen_in_batch.add(herd_key)
+
+            new_herds.append({
+                "rancher_code": temp.rancher_national_code,
+                "herd": Herd(
+                    name=temp.herd_name,
+                    code=temp.herd_code,
+                    epidemiologic=temp.epidemiologic,
+                    latitude=temp.latitude,
+                    longitude=temp.longitude,
+                    postal=temp.postal_code,
+                    unit_unique_id=temp.unit_unique_id,
+                    city_id=city_map.get(temp.city.strip()),
+                    province_id=30
+                )
+            }
+            )
+
+        # -------------------------
+        # Bulk DB Operations
+        # -------------------------
+        with transaction.atomic():
+            Rancher.objects.bulk_create(
+                new_ranchers,
+                ignore_conflicts=True,
+                batch_size=batch_size
+            )
+
+            # refresh ranchers with ids
+            rancher_map = {
+                r.national_code: r
+                for r in Rancher.objects.filter(
+                    national_code__in=rancher_map.keys()
+                )
+            }
+
+            final_herds = []
+
+            for item in new_herds:
+                rancher = rancher_map.get(item["rancher_code"])
+
+                if not rancher:
+                    continue  # یا raise error
+
+                herd = item["herd"]
+                herd.rancher = rancher
+                final_herds.append(herd)
+
+            Herd.objects.bulk_create(
+                final_herds,
+                ignore_conflicts=True,
+                batch_size=batch_size
+            )