import - whole system of syncing ranchers/herds/livestocks

apps/tag/management/commands/sync_livestock.py

@@ -0,0 +1,173 @@
+import time
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from apps.herd.models import Herd
+from apps.livestock.models import LiveStock, LiveStockType
+from apps.tag.models import Tag, TemporaryTags
+from common.generics import parse_birthdate
+
+BATCH_SIZE = 5000
+CHUNK_SIZE = 10000
+
+
+class Command(BaseCommand):
+    help = "Create Livestock if missing and assign Tag if missing"
+
+    def handle(self, *args, **options):
+
+        self.stdout.write(
+            self.style.NOTICE(f"Start bulk syncing .... records")
+        )
+
+        qs = (
+            TemporaryTags.objects
+            .filter(sync_status__isnull=True)
+            .only('herd_code', 'birthdate', 'gender', 'tag')
+        )
+
+        total = qs.count()
+        processed = 0
+        start_time = time.time()
+
+        LOG_EVERY = 10_000  # هر چند رکورد لاگ بده
+
+        buffer = []
+        for temp in qs.iterator(chunk_size=CHUNK_SIZE):
+            buffer.append(temp)
+
+            if len(buffer) >= BATCH_SIZE:
+                self.process_batch(buffer)
+                processed += len(buffer)
+                buffer.clear()
+
+                if processed % LOG_EVERY == 0:
+                    elapsed = time.time() - start_time
+                    speed = processed / elapsed if elapsed else 0
+                    percent = (processed / total) * 100
+                    remaining = total - processed
+                    eta = remaining / speed if speed else 0
+
+                    self.stdout.write(
+                        self.style.NOTICE(
+                            f"[SYNC] {processed:,}/{total:,} "
+                            f"({percent:.2f}%) | "
+                            f"{speed:.0f} rec/s | "
+                            f"ETA: {eta / 60:.1f} min"
+                        )
+                    )
+
+        if buffer:
+            self.process_batch(buffer)
+
+        self.stdout.write(self.style.SUCCESS("DONE ✅"))
+
+    # ----------------------------------------------------
+
+    def process_batch(self, temps):
+        herd_codes = {t.herd_code for t in temps if t.herd_code}
+
+        herds = {
+            h.code: h
+            for h in Herd.objects.filter(code__in=herd_codes)
+        }
+
+        livestocks = LiveStock.objects.filter(
+            herd__code__in=herd_codes
+        ).select_related('herd').only(
+            'id', 'herd_id', 'birthdate', 'gender', 'tag'
+        )
+
+        livestock_map = {
+            (ls.herd.code, ls.birthdate, ls.gender): ls
+            for ls in livestocks
+        }
+
+        livestock_types = {
+            stock_type.name: stock_type
+            for stock_type in LiveStockType.objects.all()
+        }
+
+        existing_tags = {
+            t.tag_code: t
+            for t in Tag.objects.filter(
+                tag_code__in=[t.tag for t in temps if t.tag]
+            )
+        }
+
+        new_livestock = []
+        updated_livestock = []
+        new_tags = []
+
+        for temp in temps:
+            herd = herds.get(temp.herd_code)
+            if not herd:
+                continue  # گله باید وجود داشته باشد
+
+            birthdate = parse_birthdate(temp.birthdate)
+            gender = 1 if temp.gender == 'M' else 2
+            livestock_type = livestock_types.get(temp.type)
+            weight_type = livestock_type.weight_type
+
+            key = (temp.herd_code, birthdate, gender)
+            livestock = livestock_map.get(key)
+
+            # ---------- دام وجود ندارد ----------
+            if not livestock:
+                if not temp.tag:
+                    continue
+
+                tag = existing_tags.get(temp.tag)
+
+                if not tag:
+                    tag = Tag(tag_code=temp.tag, status='A')
+                    new_tags.append(tag)
+                    existing_tags[temp.tag] = tag
+
+                livestock = LiveStock(
+                    herd=herd,
+                    birthdate=birthdate,
+                    gender=gender,
+                    tag=tag,
+                    weight_type=weight_type,
+                    type=livestock_type
+                )
+                new_livestock.append(livestock)
+                livestock_map[key] = livestock
+
+                temp.sync_status = 'S'
+                continue
+
+            # ---------- دام وجود دارد ولی پلاک ندارد ----------
+            if livestock.tag is None and temp.tag:
+                tag = existing_tags.get(temp.tag)
+
+                if not tag:
+                    tag = Tag(tag_code=temp.tag, status='A')
+                    new_tags.append(tag)
+                    existing_tags[temp.tag] = tag
+
+                livestock.tag = tag
+                updated_livestock.append(livestock)
+
+                temp.sync_status = 'S'
+
+        # ---------- BULK ----------
+        with transaction.atomic():
+            Tag.objects.bulk_create(new_tags, batch_size=BATCH_SIZE)
+            LiveStock.objects.bulk_create(
+                new_livestock,
+                batch_size=BATCH_SIZE,
+                ignore_conflicts=True
+            )
+            LiveStock.objects.bulk_update(
+                updated_livestock,
+                ['tag'],
+                batch_size=BATCH_SIZE
+            )
+            TemporaryTags.objects.bulk_update(
+                temps,
+                ['sync_status'],
+                batch_size=BATCH_SIZE
+            )

apps/tag/migrations/0026_temporarytags_sync_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2025-12-28 10:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tag', '0025_temporarytags_agriculture_unique_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='temporarytags',
+            name='sync_status',
+            field=models.CharField(max_length=150, null=True),
+        ),
+    ]

apps/tag/models.py

@@ -135,6 +135,7 @@ class TemporaryTags(BaseModel):
     agriculture_unique_id = models.CharField(max_length=150, null=True)
     keeper_agent = models.CharField(max_length=150, null=True)
     registerer_user = models.CharField(max_length=150, null=True)
+    sync_status = models.CharField(max_length=150, null=True)
 
     def save(self, *args, **kwargs):
         super(TemporaryTags, self).save(*args, **kwargs)

apps/tag/services/temporary_tags_sync.py

@@ -0,0 +1,105 @@
+from apps.tags.models import Tag
+from django.db import transaction
+
+from apps.herd.models import Herd
+from apps.livestock.models import LiveStock, LiveStockType
+from apps.tag.models import TemporaryTags
+
+BATCH_SIZE = 1000
+
+
+class TemporaryTagsSyncService:
+
+    @classmethod
+    @transaction.atomic
+    def sync_tags_to_livestock(cls):
+        # Preload herds
+        herds = {h.code: h for h in Herd.objects.all()}
+
+        # Preload existing LiveStocks for speed
+        live_stocks_map = {}
+        for ls in LiveStock.objects.select_related('herd', 'type').all():
+            key = (ls.herd.code, ls.birthdate.date() if ls.birthdate else None, ls.gender)
+            live_stocks_map.setdefault(key, []).append(ls)
+
+        # Preload LiveStockType
+        gender_type_map = {t.name.lower(): t for t in LiveStockType.objects.all()}
+
+        # Preload existing tags
+        existing_tags = {t.tag_code: t for t in Tag.objects.all()}
+
+        new_herds = []
+        new_livestock = []
+        new_tags = []
+
+        for temp in TemporaryTags.objects.iterator():
+            herd = herds.get(temp.herd_code)
+            if not herd:
+                herd = Herd(code=temp.herd_code, name=temp.herd_code, province_id=30)
+                new_herds.append(herd)
+                herds[temp.herd_code] = herd
+
+            # Gender mapping
+            gender_lower = temp.gender.lower() if temp.gender else 'male'
+            ls_type = gender_type_map.get(gender_lower)
+
+            birthdate = temp.birthdate  # convert to datetime if needed
+
+            key = (herd.code, birthdate, ls_type.id if ls_type else None)
+            live_stock_list = live_stocks_map.get(key, [])
+
+            if live_stock_list:
+                # دام موجود است، فقط Tag ایجاد می‌کنیم
+                ls = live_stock_list[0]  # می‌توانیم چندتا هم داشته باشیم
+            else:
+                # دام جدید ایجاد می‌کنیم
+                ls = LiveStock(
+                    herd=herd,
+                    type=ls_type,
+                    gender=1 if gender_lower == 'male' else 2,
+                    birthdate=birthdate
+                )
+                new_livestock.append(ls)
+                live_stocks_map[key] = [ls]
+
+            # ----------------------
+            # Tag برای هر دام
+            # ----------------------
+            if temp.serial:  # یا temp.tag_code
+                tag_code = f"{temp.country_code}{temp.static_code}{temp.ownership_code}{temp.species_code}{temp.serial}"
+                if tag_code not in existing_tags:
+                    tag = Tag(
+                        serial=temp.serial,
+                        country_code=getattr(temp, 'country_code', 364),
+                        static_code=getattr(temp, 'static_code', 0),
+                        ownership_code=getattr(temp, 'ownership_code', 0),
+                        species_code=getattr(temp, 'species_code', 0),
+                        tag_code=tag_code,
+                        status='F'
+                    )
+                    new_tags.append(tag)
+                    existing_tags[tag_code] = tag
+                else:
+                    tag = existing_tags[tag_code]
+
+                # Assign Tag to LiveStock
+                ls.tag = tag
+
+            # Batch insert
+            if len(new_livestock) >= BATCH_SIZE:
+                cls._bulk_insert(new_herds, new_livestock, new_tags)
+                new_herds.clear()
+                new_livestock.clear()
+                new_tags.clear()
+
+        # insert remaining
+        cls._bulk_insert(new_herds, new_livestock, new_tags)
+
+    @staticmethod
+    def _bulk_insert(herds, livestock, tags):
+        if herds:
+            Herd.objects.bulk_create(herds, ignore_conflicts=True, batch_size=BATCH_SIZE)
+        if tags:
+            Tag.objects.bulk_create(tags, ignore_conflicts=True, batch_size=BATCH_SIZE)
+        if livestock:
+            LiveStock.objects.bulk_create(livestock, ignore_conflicts=True, batch_size=BATCH_SIZE)