Conversion thesaurus morphalous vs pleade-singulier-pluriel-masc-fem

Introduction

Un projet de lexique morpho-syntaxique, lemme, flexion et forme fléchie est mené par le laboratoire producteur ATILF - Analyse et traitement informatique de la langue française - UMR 7118 (Nancy, FR) https://www.ortolang.fr/market/lexicons/morphalou/v3.1. Morphalou3 est diffusé sous licence LGPL-LR

Voici un script qui convertie le fichier Morphalou3.1_LMF.xml en thesaurus singulier pluriel et masculin féminin pour ThesX.

On ne converse que les lemmes et les formes fléchies. Les conjugaisons ne sont pas conservées.

Le script morphalou31_lmf_to_pleade_sp_gender_option1.py

Version singulier-pluriel et masculin-féminin.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
morphalou31_lmf_to_pleade_sp_gender_option1.py

Morphalou 3.1 (LMF) -> thésaurus Pleade :
- Un concept par "groupe" (fusion masc/fém quand possible) + variantes sing/plur
- Option 1: masculin/féminin inclus dans le même concept

Heuristique de fusion masc/fém:
- On construit une clé à partir des <originatingEntry> du lemmatizedForm
  (target + texte). Si plusieurs lexicalEntry partagent une clé, on fusionne
  leurs formes (lemmes + pluriels, etc.).

Streaming XML (iterparse) : OK pour ~483 Mo.

Usage:
  python3 morphalou31_lmf_to_pleade_sp_gender_option1.py \
    --input Morphalou3.1_LMF.xml \
    --output pleade-sing-plur-genre-morphalou31.xml \
    --thesaurus-id pleade-sing-plur-genre-morphalou31
"""

from __future__ import annotations

import argparse
import datetime as dt
import re
import unicodedata
import xml.etree.ElementTree as ET
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Dict, Set, Optional, Iterable, Tuple, List


def norm_space(s: str) -> str:
    return re.sub(r"\s+", " ", s.strip())


def strip_ns(tag: str) -> str:
    return tag.split("}", 1)[1] if "}" in tag else tag


def slugify_fr(s: str) -> str:
    s = norm_space(s).lower()
    s = s.replace("’", "'")
    s = re.sub(r"[']/+", "-", s)
    s = re.sub(r"[\s_]+", "-", s)
    s = unicodedata.normalize("NFKD", s)
    s = "".join(ch for ch in s if not unicodedata.combining(ch))
    s = re.sub(r"[^a-z0-9-]+", "", s)
    s = re.sub(r"-{2,}", "-", s).strip("-")
    return s or "concept"


def text_of_first(parent: ET.Element, child_name: str) -> Optional[str]:
    for c in list(parent):
        if strip_ns(c.tag) == child_name and c.text and c.text.strip():
            return norm_space(c.text)
    return None


def normalize_number(v: Optional[str]) -> Optional[str]:
    if not v:
        return None
    v = v.strip().lower()
    if v in ("singular", "singulier", "sg", "s"):
        return "singular"
    if v in ("plural", "pluriel", "pl", "p"):
        return "plural"
    if v in ("invariable", "inv"):
        return "invariable"
    if "sing" in v:
        return "singular"
    if "plur" in v:
        return "plural"
    if "invar" in v:
        return "invariable"
    return None


def normalize_gender(v: Optional[str]) -> Optional[str]:
    if not v:
        return None
    v = v.strip().lower()
    if v in ("masculine", "m", "masc"):
        return "masculine"
    if v in ("feminine", "f", "fem"):
        return "feminine"
    return v


def iter_lexical_entries(path: str) -> Iterable[ET.Element]:
    for event, elem in ET.iterparse(path, events=("end",)):
        if strip_ns(elem.tag) == "lexicalEntry":
            yield elem
            elem.clear()


@dataclass
class EntryInfo:
    lemma: str
    category: Optional[str]
    gender: Optional[str]
    # All observed forms (orthography) from inflectedForm, with number
    forms: Set[Tuple[str, Optional[str]]] = field(default_factory=set)
    # Origin keys from lemmatizedForm/originatingEntry
    origin_keys: Set[str] = field(default_factory=set)


def extract_entry_info(entry: ET.Element) -> Optional[EntryInfo]:
    lemma = None
    category = None
    gender = None
    forms: Set[Tuple[str, Optional[str]]] = set()
    origin_keys: Set[str] = set()

    for formset in entry:
        if strip_ns(formset.tag) != "formSet":
            continue

        for form in list(formset):
            t = strip_ns(form.tag)

            if t == "lemmatizedForm":
                lemma = text_of_first(form, "orthography") or lemma
                category = text_of_first(form, "grammaticalCategory") or category
                gender = normalize_gender(text_of_first(form, "grammaticalGender")) or gender

                # originatingEntry keys (target + texte)
                for oe in list(form):
                    if strip_ns(oe.tag) == "originatingEntry":
                        target = oe.attrib.get("target", "").strip()
                        txt = norm_space(oe.text) if oe.text and oe.text.strip() else ""
                        # IMPORTANT : on ne garde la clé que si le texte n'est PAS vide
                        if target and txt:
                            origin_keys.add(f"{target}:{txt}")

            elif t == "inflectedForm":
                wf = text_of_first(form, "orthography")
                num = normalize_number(text_of_first(form, "grammaticalNumber"))
                if wf:
                    forms.add((wf, num))

    if not lemma:
        return None

    return EntryInfo(
        lemma=lemma,
        category=category,
        gender=gender,
        forms=forms,
        origin_keys=origin_keys,
    )


def make_unique_ids(keys: Iterable[str]) -> Dict[str, str]:
    used: Dict[str, int] = {}
    out: Dict[str, str] = {}
    for k in keys:
        base = slugify_fr(k)
        if base not in used:
            used[base] = 1
            out[k] = base
        else:
            used[base] += 1
            out[k] = f"{base}-{used[base]}"
    return out


def indent(elem: ET.Element, level: int = 0) -> None:
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        for e in elem:
            indent(e, level + 1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i


def build_groups(
    morphalou_xml: str,
    allowed_categories: Set[str],
    max_origin_key_freq: int,
) -> Dict[str, List[EntryInfo]]:
    """
    2-pass grouping:
      - pass1: count origin_key frequency (only keys with non-empty target+text)
      - pass2: group by an origin_key only if its global freq <= max_origin_key_freq
               else fallback to lemma (no fusion)
    """

    # PASS 1: frequency count
    key_freq: Dict[str, int] = defaultdict(int)
    for entry in iter_lexical_entries(morphalou_xml):
        info = extract_entry_info(entry)
        if not info or not info.category:
            continue
        if allowed_categories and info.category not in allowed_categories:
            continue
        for k in info.origin_keys:
            key_freq[k] += 1

    # PASS 2: grouping
    groups: Dict[str, List[EntryInfo]] = defaultdict(list)

    for entry in iter_lexical_entries(morphalou_xml):
        info = extract_entry_info(entry)
        if not info or not info.category:
            continue
        if allowed_categories and info.category not in allowed_categories:
            continue

        # keep only "safe" keys
        safe_keys = [k for k in info.origin_keys if key_freq.get(k, 0) <= max_origin_key_freq]

        if safe_keys:
            gk = sorted(safe_keys)[0]
        else:
            gk = f"lemma:{info.lemma}"

        groups[gk].append(info)

    return groups

def choose_primary_lemma(entries: List[EntryInfo]) -> str:
    # priorité: masculine, sinon premier par ordre alpha
    masc = [e.lemma for e in entries if e.gender == "masculine"]
    if masc:
        return sorted(masc, key=lambda s: s.lower())[0]
    return sorted([e.lemma for e in entries], key=lambda s: s.lower())[0]


def write_pleade(
    out_path: str,
    thesaurus_id: str,
    thesaurus_name: str,
    description: str,
    groups: Dict[str, List[EntryInfo]],
    keep_groups_without_plural: bool,
) -> None:
    root = ET.Element(
        "thesaurus",
        attrib={"id": thesaurus_id, "{http://www.w3.org/XML/1998/namespace}lang": "fr"},
    )

    infos = ET.SubElement(root, "informations")
    ET.SubElement(infos, "nom").text = thesaurus_name
    ET.SubElement(infos, "description").text = description
    mod = ET.SubElement(infos, "modification")
    ET.SubElement(mod, "date").text = dt.datetime.now().replace(microsecond=0).isoformat()

    # Prépare un id unique par "nom primaire" (pas par group_key) pour rester lisible
    primary_names = []
    merged_data = []  # list of (primary_lemma, variants_set)

    for gk, entries in groups.items():
        if not entries:
            continue

        primary = choose_primary_lemma(entries)

        variants: Set[str] = set()
        # 1) Ajouter les autres lemmes du groupe (masc/fém)
        for e in entries:
            if e.lemma != primary:
                variants.add(e.lemma)

        # 2) Ajouter les pluriels (de toutes les entrées fusionnées)
        for e in entries:
            for wf, num in e.forms:
                if num == "plural" and wf != primary:
                    variants.add(wf)

        # Option: si aucun variant (ni autre lemme, ni pluriel), on peut ignorer
        if not variants and not keep_groups_without_plural:
            continue

        primary_names.append(primary)
        merged_data.append((primary, variants))

    # IDs uniques par primary lemma
    primary_names_sorted = sorted(set(primary_names), key=lambda s: s.lower())
    primary_to_id = make_unique_ids(primary_names_sorted)

    # Écrit les concepts dans un ordre stable
    merged_data.sort(key=lambda t: t[0].lower())

    for primary, variants in merged_data:
        concept = ET.SubElement(root, "concept", attrib={"id": primary_to_id[primary]})
        relations = ET.SubElement(concept, "relations")
        ET.SubElement(relations, "nom").text = primary

        for v in sorted(variants, key=lambda s: s.lower()):
            va = ET.SubElement(relations, "va", attrib={"ref": v})
            va.text = v

    tree = ET.ElementTree(root)
    indent(root)
    tree.write(out_path, encoding="utf-8", xml_declaration=True)


DEFAULT_DESCRIPTION = (
    "Généré depuis Morphalou 3.1 (ATILF, ORTOLANG, LGPL-LR). Objectif : recherche sans distinguer singulier/pluriel.\n"
    "Analyse et traitement informatique de la langue française - UMR 7118 (ATILF) (2023). Morphalou [Lexique]. "
    "ORTOLANG (Open Resources and TOols for LANGuage) - www.ortolang.fr, v3.1, "
    "https://hdl.handle.net/11403/morphalou/v3.1."
)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--input", required=True, help="Morphalou3.1_LMF.xml")
    ap.add_argument("--output", required=True, help="XML Pleade de sortie")
    ap.add_argument("--thesaurus-id", required=True, help="ID du thésaurus Pleade (nouvel ID)")
    ap.add_argument("--thesaurus-name", default="Dictionnaire sing/plur + genre (Morphalou 3.1)", help="Nom")
    ap.add_argument("--description", default=DEFAULT_DESCRIPTION, help="Description (avec citation)")
    ap.add_argument(
        "--categories",
        default="commonNoun,adjective",
        help="grammaticalCategory à inclure (séparées par virgules), ex: commonNoun,adjective",
    )
    ap.add_argument(
        "--keep-groups-without-variant",
        action="store_true",
        help="Conserver aussi les groupes sans variantes (sinon ils sont ignorés)",
    )
    ap.add_argument(
		"--max-origin-key-freq",
		type=int,
		default=3,
		help="Fréquence max d'une clé originatingEntry pour autoriser une fusion (défaut: 3)",
	)

    args = ap.parse_args()

    allowed_categories = {norm_space(x) for x in args.categories.split(",") if norm_space(x)}

    groups = build_groups(args.input, allowed_categories, args.max_origin_key_freq)

    write_pleade(
        out_path=args.output,
        thesaurus_id=args.thesaurus_id,
        thesaurus_name=args.thesaurus_name,
        description=args.description,
        groups=groups,
        keep_groups_without_plural=args.keep_groups_without_variant,
    )

    # Stats sommaires
    total_groups = len(groups)
    print(f"OK: groupes traités={total_groups} -> {args.output}")


if __name__ == "__main__":
    main()

Les commandes

Changer les droits pour rendre exécutable :

chmod +x morphalou31_lmf_to_pleade_sp_gender_option1.py

Lancer le script python

python3 morphalou31_lmf_to_pleade_sp_gender_option1.py \
  --input chemin/vers/Morphalou3.1_LMF.xml \
  --output pleade-sing-plur-genre-morphalou31.xml \
  --thesaurus-id pleade-sing-plur-genre-morphalou31 \
  --max-origin-key-freq 2