Inflection-208 Add multisource and DMLex dictionary parsing support

tools/dictionary-parser/ParseWikidata

@@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then
         if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then
             OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"`
             echo Generating $LANGUAGE from $LEXEME_FILE
-            $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE
+            $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE
         else
             echo skipping $LANGUAGE
         fi

tools/dictionary-parser/README.md

@@ -20,5 +20,29 @@ These tools generate files that describes the grammatical properties of words fr
     ./ParseWikidata <THE_OPTIONS_FROM_STEP_3> \
     --inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \
     --dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \
-    <wikidata-NNNNNNNN-lexemes.json[.bz2]>
+    --wikidata <wikidata-NNNNNNNN-lexemes.json[.bz2]>
 ```
+
+## Usage for OASIS DMLex and Multi-Source Inputs
+
+`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`:
+
+- **Standalone DMLex (single or multiple files)**:
+```
+    ./ParseWikidata --language <lang> \
+    --inflections inflectional_<lang>.xml \
+    --dictionary dictionary_<lang>.lst \
+    --dmlex <custom1.json> [<custom2.json> ...]
+```
+- **Multi-Source (Wikidata + DMLex overlay)**:
+```
+    ./ParseWikidata <OPTIONS> \
+    --inflections inflectional_<lang>.xml \
+    --dictionary dictionary_<lang>.lst \
+    --wikidata <wikidata-lexemes.json[.bz2]> \
+    --dmlex <supplement1.json> [<supplement2.json> ...]
+```
+- **Conflict Resolution (`--merge-conflicts`)**:
+  - By default (`--merge-conflicts=false`), files are processed in CLI order and a conflicting surface form in a later source file overrides all previous grammemes and inflection patterns for that surface form.
+  - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge grammemes and inflection patterns across files, placing the later source's conflicting pattern first in priority.
+

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A lexical entry in OASIS DMLex v1.0 JSON format.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DMLexEntry {
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class Lexicon {
+        public String langCode;
+        public List<DMLexEntry> entries;
+    }
+
+    public String id;
+    public String headword;
+    public String langCode;
+
+    @JsonAlias({"pos"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public List<String> partOfSpeech;
+
+    public List<String> labels = new ArrayList<>();
+
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public List<String> pronunciations;
+
+    @JsonAlias({"forms"})
+    public List<DMLexInflectedForm> inflectedForms;
+
+    @JsonSetter("labels")
+    @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public void addLabels(List<String> incoming) {
+        if (incoming != null) {
+            this.labels.addAll(incoming);
+        }
+    }
+}

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An inflected form of a lexical entry in OASIS DMLex v1.0 JSON format.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DMLexInflectedForm {
+    public String id;
+
+    @JsonAlias({"text", "inflection"})
+    public String value;
+
+    public List<String> labels = new ArrayList<>();
+
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public List<String> pronunciations;
+
+    @JsonSetter("labels")
+    @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public void addLabels(List<String> incoming) {
+        if (incoming != null) {
+            this.labels.addAll(incoming);
+        }
+    }
+}

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.exc.MismatchedInputException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting a single string,
+ * an array of strings, or an object/array of objects containing {@code "transcription"},
+ * {@code "tag"}, {@code "value"}, {@code "label"}, or {@code "text"}.
+ */
+public class DMLexStringListDeserializer extends JsonDeserializer<List<String>> {
+    private static final String[] OBJECT_VALUE_KEYS = {
+        "transcription", "tag", "value", "label", "text"
+    };
+
+    @Override
+    public List<String> deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException {
+        JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+        List<String> result = new ArrayList<>();
+        extractValues(node, result, jsonParser);
+        return result;
+    }
+
+    private static void extractValues(JsonNode node, List<String> result, JsonParser jsonParser) throws IOException {
+        if (node == null || node.isNull()) {
+            return;
+        }
+        if (node.isTextual()) {
+            String text = node.textValue();
+            if (!text.isBlank()) {
+                result.add(text);
+            }
+        } else if (node.isArray()) {
+            for (JsonNode elem : node) {
+                extractValues(elem, result, jsonParser);
+            }
+        } else if (node.isObject()) {
+            JsonNode transcriptions = node.get("transcriptions");
+            if (transcriptions != null) {
+                extractValues(transcriptions, result, jsonParser);
+            } else {
+                for (String key : OBJECT_VALUE_KEYS) {
+                    JsonNode field = node.get(key);
+                    if (field != null && field.isTextual() && !field.textValue().isBlank()) {
+                        result.add(field.textValue());
+                        break;
+                    }
+                }
+            }
+        } else {
+            throw MismatchedInputException.from(
+                    jsonParser,
+                    List.class,
+                    "Expected string, object, or array for DMLex list field, got " + node.getNodeType());
+        }
+    }
+}

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java

@@ -4,12 +4,14 @@
  */
 package org.unicode.wikidata;
 
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import javax.annotation.Nonnull;
 
 /**
@@ -25,20 +27,27 @@ public class DictionaryEntry implements Comparable<DictionaryEntry> {
     private final TreeSet<Enum<?>> grammemes;
     private final List<InflectionPattern> inflectionalPatterns;
     private final List<InflectionPattern> rareInflectionalPatterns = new ArrayList<>();
+    private int sourceIndex;
+    private Map<InflectionPattern, Integer> patternSourceIndices = null;
 
     public static final Comparator<InflectionPattern> INFLECTION_PATTERN_COMPARATOR = Comparator
             .comparing(InflectionPattern::getCount)
             .reversed();
 
     public DictionaryEntry(String phrase, boolean isLemmaRare, TreeSet<Enum<?>> grammemes, InflectionPattern inflectionalPattern) {
-        this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern);
+        this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern, 0);
     }
 
     public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet<Enum<?>> grammemes, InflectionPattern inflectionalPattern) {
+        this(phrase, lemmas, isLemmaRare, grammemes, inflectionalPattern, 0);
+    }
+
+    public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet<Enum<?>> grammemes, InflectionPattern inflectionalPattern, int sourceIndex) {
         this.phrase = phrase;
         this.lemmas = new TreeSet<>(Collections.singleton(lemmas));
         this.grammemes = new TreeSet<>(grammemes.comparator());
         this.grammemes.addAll(grammemes);
+        this.sourceIndex = sourceIndex;
         if (inflectionalPattern != null) {
             this.inflectionalPatterns = new ArrayList<>(2);
             this.inflectionalPatterns.add(inflectionalPattern);
@@ -51,7 +60,53 @@ public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSe
         }
     }
 
+    private int getPatternSourceIndex(InflectionPattern pattern) {
+        if (patternSourceIndices != null) {
+            return patternSourceIndices.getOrDefault(pattern, sourceIndex);
+        }
+        return sourceIndex;
+    }
+
     public void merge(DictionaryEntry entry) {
+        merge(entry, false);
+    }
+
+    public void merge(DictionaryEntry entry, boolean mergeConflicts) {
+        if (!mergeConflicts) {
+            if (entry.sourceIndex > this.sourceIndex) {
+                this.sourceIndex = entry.sourceIndex;
+                this.grammemes.clear();
+                this.grammemes.addAll(entry.grammemes);
+                this.inflectionalPatterns.clear();
+                this.inflectionalPatterns.addAll(entry.inflectionalPatterns);
+                this.rareInflectionalPatterns.clear();
+                this.rareInflectionalPatterns.addAll(entry.rareInflectionalPatterns);
+                this.lemmas.clear();
+                this.lemmas.addAll(entry.lemmas);
+                this.patternSourceIndices = null;
+                return;
+            }
+            if (entry.sourceIndex < this.sourceIndex) {
+                return;
+            }
+        } else if (entry.sourceIndex != this.sourceIndex || this.patternSourceIndices != null || entry.patternSourceIndices != null) {
+            if (this.patternSourceIndices == null) {
+                this.patternSourceIndices = new IdentityHashMap<>();
+                for (InflectionPattern pattern : this.inflectionalPatterns) {
+                    this.patternSourceIndices.put(pattern, this.sourceIndex);
+                }
+            }
+            for (InflectionPattern pattern : entry.inflectionalPatterns) {
+                int existingSource = this.patternSourceIndices.getOrDefault(pattern, this.sourceIndex);
+                int incomingSource = entry.getPatternSourceIndex(pattern);
+                if (incomingSource > existingSource && !entry.rareInflectionalPatterns.contains(pattern)) {
+                    this.rareInflectionalPatterns.remove(pattern);
+                }
+                this.patternSourceIndices.put(pattern, Math.max(existingSource, incomingSource));
+            }
+            this.sourceIndex = Math.max(this.sourceIndex, entry.sourceIndex);
+        }
+
         this.grammemes.addAll(entry.grammemes);
         for (InflectionPattern inflectionalPattern : entry.inflectionalPatterns) {
             if (!this.inflectionalPatterns.contains(inflectionalPattern)) {
@@ -70,6 +125,10 @@ public Set<Enum<?>> getGrammemes() {
         return grammemes;
     }
 
+    List<InflectionPattern> getInflectionalPatterns() {
+        return inflectionalPatterns;
+    }
+
     @Override
     public String toString() {
         return this.toString(true);
@@ -82,10 +141,15 @@ public String toString(boolean isInflectional) {
             sb.append(' ').append(enumVal);
         }
         if (isInflectional) {
+            Comparator<InflectionPattern> patternComparator = (patternSourceIndices == null)
+                    ? INFLECTION_PATTERN_COMPARATOR
+                    : Comparator.<InflectionPattern, Integer>comparing(
+                            p -> patternSourceIndices.getOrDefault(p, sourceIndex), Comparator.reverseOrder())
+                            .thenComparing(INFLECTION_PATTERN_COMPARATOR);
             if (inflectionalPatterns.size() > 1) {
-                inflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR);
+                inflectionalPatterns.sort(patternComparator);
                 if (rareInflectionalPatterns.size() > 1) {
-                    rareInflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR);
+                    rareInflectionalPatterns.sort(patternComparator);
                 }
             }
             for (InflectionPattern inflectionPattern : inflectionalPatterns) {

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java

@@ -13,9 +13,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
@@ -48,17 +50,31 @@ private void sortInflectionPatterns(ArrayList<InflectionPattern> inflectionPatte
     }
 
     public void addDictionaryEntry(DictionaryEntry dictionaryEntry) {
+        addDictionaryEntry(dictionaryEntry, false);
+    }
+
+    public void addDictionaryEntry(DictionaryEntry dictionaryEntry, boolean mergeConflicts) {
         String phrase = dictionaryEntry.phrase;
         DictionaryEntry existingDictionaryEntry = dictionary.get(phrase);
         if (existingDictionaryEntry == null) {
             dictionary.put(phrase, dictionaryEntry);
         } else {
             mergedCount++;
-            existingDictionaryEntry.merge(dictionaryEntry);
+            existingDictionaryEntry.merge(dictionaryEntry, mergeConflicts);
         }
     }
 
     void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFoundException {
+        if (!inflectionPatterns.isEmpty()) {
+            Set<InflectionPattern> referencedPatterns = Collections.newSetFromMap(new IdentityHashMap<>());
+            for (DictionaryEntry dictionaryEntry : dictionary.values()) {
+                if (!dictionaryEntry.getGrammemes().isEmpty()) {
+                    referencedPatterns.addAll(dictionaryEntry.getInflectionalPatterns());
+                }
+            }
+            inflectionPatterns.removeIf(pattern -> !referencedPatterns.contains(pattern));
+        }
+
         TreeMap<Enum<?>, Integer> grammemeCounts = new TreeMap<>(EnumComparator.ENUM_COMPARATOR);
         int unclassifiedTerms = 0;
         if (isInflectional()) {

tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java

@@ -1175,9 +1175,18 @@ public String toString() {
         TYPEMAP.put("Q1909485", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // quantifier, but it might be used frequently as an indefinite noun
         TYPEMAP.put("Q694268", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // collective
         TYPEMAP.put("Q1450795", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // singulative
+
+        for (var entry : DEFAULTMAP.entrySet()) {
+            @SuppressWarnings({"unchecked", "rawtypes"})
+            Set<? extends Enum<?>> enumSet = EnumSet.of((Enum) entry.getValue());
+            TYPEMAP.putIfAbsent(entry.getKey(), enumSet);
+        }
     }
 
     static Set<? extends Enum<?>> getMappedGrammemes(String grammeme) {
+        if (grammeme == null) {
+            throw new IllegalArgumentException("Grammeme cannot be null");
+        }
         return TYPEMAP.get(grammeme);
     }
 }