Merge pull request #194 from ArtemFilin1990/fix/type-first-42585-prefix-20261021

Fix numeric prefixes and verify all 42,585 EVEREST staging rows

.github/workflows/everest-type-first.yml

@@ -40,7 +40,7 @@ jobs:
             --input data/nomenclature.csv \
             --output output/everest_nomenclature_type_first.sqlite \
             --report output/type_first_report.json \
-            --min-rows 40000 --min-brands 74
+            --min-rows 42585 --min-brands 74
       - name: Check complete and non-promoted data
         run: |
           python - <<'PY'
@@ -48,7 +48,8 @@ jobs:
           data = json.load(open('output/type_first_report.json'))
           c = sqlite3.connect('output/everest_nomenclature_type_first.sqlite')
           assert data['brands'] >= 74
-          assert data['rows'] >= 40000
+          assert data['rows'] >= 42585
+          assert data['duplicate_source_rows'] == 0
           assert c.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
           assert c.execute('SELECT COUNT(*) FROM stage_designations').fetchone()[0] == data['rows']
           assert c.execute("SELECT COUNT(*) FROM stage_designations WHERE semantic_status != 'UNVERIFIED'").fetchone()[0] == 0

tools/scripts/everest_catalog/normalize_snapshot.py

@@ -10,13 +10,24 @@ import unicodedata
 from collections import Counter
 from pathlib import Path
 
-RULESET = "EVEREST-TYPE-FIRST-3.0"
+RULESET = "EVEREST-TYPE-FIRST-3.1"
 EXPECTED_HEADER = ["Brand","Product Name","Product Link","Analog Name","Analog Link","Factory"]
 JOINT = re.compile(r"^(?:[0-9]|G|Е)?(ШСЛ|ШСП|ШС|Ш)\s*(\d{1,5})(.*)$")
 SERIES = ("UCFL","UCFC","UCPA","UCPH","UCFB","UCP","UCT","UCF","UC","UK",
           "RNAO","RNA","NKI","NKIS","NKS","NK","NAO","NA","AXK","HK","BK",
           "NUP","NU","NJ","NF","QJ","N","LME","LMB","LMK","LM")
 
+def render_prefix_normalized(original, prefix):
+    """Render a safe prefix rewrite without guessing suffix separators."""
+    value = unicodedata.normalize("NFKC", original or "").upper().strip()
+    value = re.sub(r"\s+", " ", value.translate(str.maketrans("–—−‑", "----")))
+    value = value.strip('"')
+    if prefix:
+        parenthesized = re.fullmatch(r"\(" + re.escape(prefix) + r"\)\s*-?\s*(.+)", value)
+        if parenthesized:
+            return f"{prefix}-{parenthesized.group(1)}"
+    return value
+
 def split_designation(original):
     value = unicodedata.normalize("NFKC",original or "").upper().strip()
     value = re.sub(r"\s+"," ",value.translate(str.maketrans("–—−‑","----")))
@@ -28,12 +39,19 @@ def split_designation(original):
         value = value.strip('"')
     if re.search("[A-Z]",value) and re.search("[А-ЯЁ]",value):
         issues.append("MIXED_ALPHABETS")
+    prefix = ""
+    # A leading numeric value in parentheses is a prefix, not part of the
+    # bearing body: (70)32134Л -> 70-32134Л.  Other parentheses still need
+    # manual review because they may describe assemblies or options.
+    match = re.fullmatch(r"\((\d{1,4})\)\s*-?\s*(.+)", value)
+    if match:
+        prefix, value = match.groups()
+    else:
+        match = re.fullmatch(r"(\d[\dА-ЯЁA-Z]{0,3})-(.+)", value)
+        if match:
+            prefix, value = match.groups()
     if re.search(r"[+()\[\]]",value):
         issues.append("ASSEMBLY_OR_PARENTHESIS")
-    prefix = ""
-    match = re.fullmatch(r"(\d[\dА-ЯЁA-Z]{0,3})-(.+)",value)
-    if match:
-        prefix,value=match.groups()
     match=JOINT.fullmatch(value)
     if match:
         body=value[:match.end(2)].replace(" ","")
@@ -74,6 +92,7 @@ def build(source,output,report,min_rows=40000,min_brands=74):
         db.executescript("""
         CREATE TABLE stage_designations (
           source_line INTEGER PRIMARY KEY, brand TEXT NOT NULL, original_designation TEXT NOT NULL,
+          normalized_designation TEXT NOT NULL,
           prefix TEXT NOT NULL, body TEXT NOT NULL, suffix TEXT NOT NULL, bearing_type_hint TEXT NOT NULL,
           family TEXT NOT NULL, parse_status TEXT NOT NULL, issues_json TEXT NOT NULL,
           semantic_status TEXT NOT NULL CHECK(semantic_status='UNVERIFIED'),
@@ -84,6 +103,7 @@ def build(source,output,report,min_rows=40000,min_brands=74):
         """)
         brands=Counter()
         statuses=Counter()
+        type_hints=Counter()
         unique=set()
         with source.open(encoding="utf-8-sig",newline="") as stream:
             reader=csv.DictReader(stream)
@@ -95,13 +115,15 @@ def build(source,output,report,min_rows=40000,min_brands=74):
                 if not brand or not raw:
                     raise ValueError(f"Blank brand/designation at source line {reader.line_num}")
                 prefix,body,suffix,kind,family,status,issues=split_designation(raw)
+                normalized=render_prefix_normalized(raw,prefix)
                 db.execute("""INSERT INTO stage_designations VALUES
-                  (?,?,?,?,?,?,?,?,?,?,'UNVERIFIED',?,?,?,?,?,?,?)""",(
-                  reader.line_num,brand,raw,prefix,body,suffix,kind,family,status,
+                  (?,?,?,?,?,?,?,?,?,?,?,'UNVERIFIED',?,?,?,?,?,?,?)""",(
+                  reader.line_num,brand,raw,normalized,prefix,body,suffix,kind,family,status,
                   json.dumps(issues,ensure_ascii=False),row["Analog Name"],row["Product Link"],
                   row["Analog Link"],row["Factory"],"data/nomenclature.csv",blob,RULESET))
                 brands[brand]+=1
                 statuses[status]+=1
+                type_hints[kind]+=1
                 unique.add((brand,raw))
         rows=sum(brands.values())
         if rows<min_rows or len(brands)<min_brands:
@@ -112,6 +134,8 @@ def build(source,output,report,min_rows=40000,min_brands=74):
         assert db.execute("SELECT COUNT(*) FROM stage_designations WHERE semantic_status!='UNVERIFIED'").fetchone()[0]==0
         result={"rows":rows,"brands":len(brands),"unique_brand_designations":len(unique),
                 "duplicate_source_rows":rows-len(unique),"parse_status_counts":dict(statuses),
+                "bearing_type_hint_counts":dict(sorted(type_hints.items())),
+                "unknown_type_hint_rows":type_hints["UNKNOWN"],
                 "brand_row_counts":dict(sorted(brands.items())),"source_blob_sha":blob,
                 "ruleset":RULESET,"semantic_status":"ALL_UNVERIFIED",
                 "production_database_changed":False}

tools/scripts/everest_catalog/test_normalize_snapshot.py

@@ -4,7 +4,7 @@ import sqlite3
 import tempfile
 import unittest
 from pathlib import Path
-from normalize_snapshot import split_designation, build
+from normalize_snapshot import split_designation, render_prefix_normalized, build
 
 class TypeFirstTests(unittest.TestCase):
     def test_type_first(self):
@@ -13,6 +13,9 @@ class TypeFirstTests(unittest.TestCase):
             "2ШС 20": ("","2ШС20","","SPHERICAL_PLAIN"),
             "ШСЛ90К1": ("","ШСЛ90","К1","SPHERICAL_PLAIN"),
             "6-180205": ("6","180205","","UNKNOWN"),
+            "(70)32134Л": ("70","32134","Л","UNKNOWN"),
+            "(70) 32134Л": ("70","32134","Л","UNKNOWN"),
+            "(70)-32134Л": ("70","32134","Л","UNKNOWN"),
             "NU 205 ECP/C3": ("","NU205","ECP/C3","CYLINDRICAL_ROLLER"),
             "HK1616": ("","HK1616","","NEEDLE"),
             "UCP205": ("","UCP205","","MOUNTED_UNIT_OR_INSERT"),
@@ -22,6 +25,30 @@ class TypeFirstTests(unittest.TestCase):
             with self.subTest(designation=designation):
                 self.assertEqual(split_designation(designation)[:4],expected)
 
+    def test_parenthesized_numeric_prefix_is_safe_but_assembly_is_not(self):
+        prefix, body, suffix, _, _, status, issues = split_designation("(70)32134Л")
+        self.assertEqual(f"{prefix}-{body}{suffix}", "70-32134Л")
+        self.assertEqual(render_prefix_normalized("(70)32134Л",prefix),"70-32134Л")
+        self.assertEqual(render_prefix_normalized("(70) 32134Л",prefix),"70-32134Л")
+        self.assertEqual((status, issues), ("PARSED", []))
+
+        *_, status, issues = split_designation("(70)32134Л (комплект)")
+        self.assertEqual(status, "REVIEW")
+        self.assertIn("ASSEMBLY_OR_PARENTHESIS", issues)
+
+    def test_prefix_normalization_is_materialized_in_staging(self):
+        with tempfile.TemporaryDirectory() as folder:
+            root=Path(folder)
+            source=root/"source.csv"
+            with source.open("w",encoding="utf-8",newline="") as handle:
+                writer=csv.writer(handle)
+                writer.writerow(["Brand","Product Name","Product Link","Analog Name","Analog Link","Factory"])
+                writer.writerow(["EVEREST","(70)32134Л","","","",""])
+            build(source,root/"snapshot.sqlite",root/"report.json",min_rows=1,min_brands=1)
+            with sqlite3.connect(root/"snapshot.sqlite") as db:
+                row=db.execute("SELECT original_designation,normalized_designation,prefix,body,suffix,semantic_status FROM stage_designations").fetchone()
+            self.assertEqual(row,("(70)32134Л","70-32134Л","70","32134","Л","UNVERIFIED"))
+
     def test_all_74_brands_staged_only(self):
         with tempfile.TemporaryDirectory() as folder:
             root = Path(folder)
@@ -34,6 +61,10 @@ class TypeFirstTests(unittest.TestCase):
             output, report = root/"snapshot.sqlite",root/"report.json"
             result=build(source,output,report,min_rows=74,min_brands=74)
             self.assertEqual((result["rows"],result["brands"]),(74,74))
+            self.assertEqual(result["ruleset"],"EVEREST-TYPE-FIRST-3.1")
+            self.assertEqual(result["bearing_type_hint_counts"],
+                             {"CYLINDRICAL_ROLLER":37,"SPHERICAL_PLAIN":37})
+            self.assertEqual(result["unknown_type_hint_rows"],0)
             db=sqlite3.connect(output)
             self.assertEqual(db.execute("SELECT COUNT(*) FROM stage_designations WHERE semantic_status='UNVERIFIED'").fetchone()[0],74)
             self.assertEqual(db.execute("PRAGMA integrity_check").fetchone()[0],"ok")