diff --git a/.github/workflows/everest-type-first.yml b/.github/workflows/everest-type-first.yml index c99e0d0..24a7905 100644 --- a/.github/workflows/everest-type-first.yml +++ b/.github/workflows/everest-type-first.yml @@ -40,7 +40,7 @@ jobs: --input data/nomenclature.csv \ --output output/everest_nomenclature_type_first.sqlite \ --report output/type_first_report.json \ - --min-rows 40000 --min-brands 74 + --min-rows 42585 --min-brands 74 - name: Check complete and non-promoted data run: | python - <<'PY' @@ -48,7 +48,8 @@ jobs: data = json.load(open('output/type_first_report.json')) c = sqlite3.connect('output/everest_nomenclature_type_first.sqlite') assert data['brands'] >= 74 - assert data['rows'] >= 40000 + assert data['rows'] >= 42585 + assert data['duplicate_source_rows'] == 0 assert c.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' assert c.execute('SELECT COUNT(*) FROM stage_designations').fetchone()[0] == data['rows'] assert c.execute("SELECT COUNT(*) FROM stage_designations WHERE semantic_status != 'UNVERIFIED'").fetchone()[0] == 0 diff --git a/tools/scripts/everest_catalog/normalize_snapshot.py b/tools/scripts/everest_catalog/normalize_snapshot.py index abdee58..2055590 100644 --- a/tools/scripts/everest_catalog/normalize_snapshot.py +++ b/tools/scripts/everest_catalog/normalize_snapshot.py @@ -10,13 +10,24 @@ import unicodedata from collections import Counter from pathlib import Path -RULESET = "EVEREST-TYPE-FIRST-3.0" +RULESET = "EVEREST-TYPE-FIRST-3.1" EXPECTED_HEADER = ["Brand","Product Name","Product Link","Analog Name","Analog Link","Factory"] JOINT = re.compile(r"^(?:[0-9]|G|Е)?(ШСЛ|ШСП|ШС|Ш)\s*(\d{1,5})(.*)$") SERIES = ("UCFL","UCFC","UCPA","UCPH","UCFB","UCP","UCT","UCF","UC","UK", "RNAO","RNA","NKI","NKIS","NKS","NK","NAO","NA","AXK","HK","BK", "NUP","NU","NJ","NF","QJ","N","LME","LMB","LMK","LM") +def render_prefix_normalized(original, prefix): + """Render a safe prefix rewrite without guessing suffix separators.""" + value = unicodedata.normalize("NFKC", original or "").upper().strip() + value = re.sub(r"\s+", " ", value.translate(str.maketrans("–—−‑", "----"))) + value = value.strip('"') + if prefix: + parenthesized = re.fullmatch(r"\(" + re.escape(prefix) + r"\)\s*-?\s*(.+)", value) + if parenthesized: + return f"{prefix}-{parenthesized.group(1)}" + return value + def split_designation(original): value = unicodedata.normalize("NFKC",original or "").upper().strip() value = re.sub(r"\s+"," ",value.translate(str.maketrans("–—−‑","----"))) @@ -28,12 +39,19 @@ def split_designation(original): value = value.strip('"') if re.search("[A-Z]",value) and re.search("[А-ЯЁ]",value): issues.append("MIXED_ALPHABETS") + prefix = "" + # A leading numeric value in parentheses is a prefix, not part of the + # bearing body: (70)32134Л -> 70-32134Л. Other parentheses still need + # manual review because they may describe assemblies or options. + match = re.fullmatch(r"\((\d{1,4})\)\s*-?\s*(.+)", value) + if match: + prefix, value = match.groups() + else: + match = re.fullmatch(r"(\d[\dА-ЯЁA-Z]{0,3})-(.+)", value) + if match: + prefix, value = match.groups() if re.search(r"[+()\[\]]",value): issues.append("ASSEMBLY_OR_PARENTHESIS") - prefix = "" - match = re.fullmatch(r"(\d[\dА-ЯЁA-Z]{0,3})-(.+)",value) - if match: - prefix,value=match.groups() match=JOINT.fullmatch(value) if match: body=value[:match.end(2)].replace(" ","") @@ -74,6 +92,7 @@ def build(source,output,report,min_rows=40000,min_brands=74): db.executescript(""" CREATE TABLE stage_designations ( source_line INTEGER PRIMARY KEY, brand TEXT NOT NULL, original_designation TEXT NOT NULL, + normalized_designation TEXT NOT NULL, prefix TEXT NOT NULL, body TEXT NOT NULL, suffix TEXT NOT NULL, bearing_type_hint TEXT NOT NULL, family TEXT NOT NULL, parse_status TEXT NOT NULL, issues_json TEXT NOT NULL, semantic_status TEXT NOT NULL CHECK(semantic_status='UNVERIFIED'), @@ -84,6 +103,7 @@ def build(source,output,report,min_rows=40000,min_brands=74): """) brands=Counter() statuses=Counter() + type_hints=Counter() unique=set() with source.open(encoding="utf-8-sig",newline="") as stream: reader=csv.DictReader(stream) @@ -95,13 +115,15 @@ def build(source,output,report,min_rows=40000,min_brands=74): if not brand or not raw: raise ValueError(f"Blank brand/designation at source line {reader.line_num}") prefix,body,suffix,kind,family,status,issues=split_designation(raw) + normalized=render_prefix_normalized(raw,prefix) db.execute("""INSERT INTO stage_designations VALUES - (?,?,?,?,?,?,?,?,?,?,'UNVERIFIED',?,?,?,?,?,?,?)""",( - reader.line_num,brand,raw,prefix,body,suffix,kind,family,status, + (?,?,?,?,?,?,?,?,?,?,?,'UNVERIFIED',?,?,?,?,?,?,?)""",( + reader.line_num,brand,raw,normalized,prefix,body,suffix,kind,family,status, json.dumps(issues,ensure_ascii=False),row["Analog Name"],row["Product Link"], row["Analog Link"],row["Factory"],"data/nomenclature.csv",blob,RULESET)) brands[brand]+=1 statuses[status]+=1 + type_hints[kind]+=1 unique.add((brand,raw)) rows=sum(brands.values()) if rows