Inflection-208 Add multisource and DMLex dictionary parsing support

tools/dictionary-parser/ParseWikidata

@@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then
         if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then
             OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"`
             echo Generating $LANGUAGE from $LEXEME_FILE
-            $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE
+            $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE
         else
             echo skipping $LANGUAGE
         fi

tools/dictionary-parser/README.md

@@ -20,5 +20,29 @@ These tools generate files that describes the grammatical properties of words fr
     ./ParseWikidata <THE_OPTIONS_FROM_STEP_3> \
     --inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \
     --dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \
-    <wikidata-NNNNNNNN-lexemes.json[.bz2]>
+    --wikidata <wikidata-NNNNNNNN-lexemes.json[.bz2]>
 ```
+
+## Usage for OASIS DMLex and Multi-Source Inputs
+
+`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`:
+
+- **Standalone DMLex (single or multiple files)**:
+```
+    ./ParseWikidata --language <lang> \
+    --inflections inflectional_<lang>.xml \
+    --dictionary dictionary_<lang>.lst \
+    --dmlex <custom1.json> [<custom2.json> ...]
+```
+- **Multi-Source (Wikidata + DMLex overlay)**:
+```
+    ./ParseWikidata <OPTIONS> \
+    --inflections inflectional_<lang>.xml \
+    --dictionary dictionary_<lang>.lst \
+    --wikidata <wikidata-lexemes.json[.bz2]> \
+    --dmlex <supplement1.json> [<supplement2.json> ...]
+```
+- **Conflict Resolution (`--merge-conflicts`)**:
+  - By default (`--merge-conflicts=false`), later source files take higher precedence per `(surfaceForm, PartOfSpeech)` and `(lemma, PartOfSpeech)` (overwriting conflicting entries of the same part of speech from earlier files while preserving distinct parts of speech).
+  - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge conflicting entries across files instead of overwriting.
+

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A lexical entry in OASIS DMLex v1.0 JSON format.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DMLexEntry {
+    public String id;
+    public String headword;
+    public String langCode;
+
+    public List<String> partOfSpeech = new ArrayList<>();
+
+    public List<String> labels = new ArrayList<>();
+
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public List<String> pronunciations;
+
+    @JsonAlias({"forms"})
+    public List<DMLexInflectedForm> inflectedForms;
+
+    @JsonSetter("partOfSpeech")
+    @JsonAlias({"pos"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public void addPartOfSpeech(List<String> incoming) {
+        if (incoming != null) {
+            this.partOfSpeech.addAll(incoming);
+        }
+    }
+
+    @JsonSetter("labels")
+    @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public void addLabels(List<String> incoming) {
+        if (incoming != null) {
+            this.labels.addAll(incoming);
+        }
+    }
+}

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An inflected form of a lexical entry in OASIS DMLex v1.0 JSON format.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DMLexInflectedForm {
+    public String id;
+
+    @JsonAlias({"text", "inflection"})
+    public String value;
+
+    public List<String> labels = new ArrayList<>();
+
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public List<String> pronunciations;
+
+    @JsonSetter("labels")
+    @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"})
+    @JsonDeserialize(using = DMLexStringListDeserializer.class)
+    public void addLabels(List<String> incoming) {
+        if (incoming != null) {
+            this.labels.addAll(incoming);
+        }
+    }
+}

tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
+ */
+package org.unicode.wikidata;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.exc.MismatchedInputException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting:
+ * <ul>
+ *   <li>a single string (e.g. {@code "noun"})</li>
+ *   <li>an array of strings (e.g. {@code ["singular", "nominative"]})</li>
+ *   <li>an object or array of objects containing {@code "tag"}, {@code "value"},
+ *       {@code "transcription"}, {@code "text"}, or {@code "label"}</li>
+ * </ul>
+ */
+public class DMLexStringListDeserializer extends JsonDeserializer<List<String>> {
+    @Override
+    public List<String> deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException {
+        List<String> result = new ArrayList<>();
+        JsonToken currentToken = jsonParser.currentToken();
+        if (currentToken == JsonToken.VALUE_STRING) {
+            String text = jsonParser.getText();
+            if (text != null && !text.isBlank()) {
+                result.add(text);
+            }
+            return result;
+        }
+        if (currentToken == JsonToken.START_OBJECT) {
+            readValuesFromObject(jsonParser, result);
+            return result;
+        }
+        if (currentToken != JsonToken.START_ARRAY) {
+            throw MismatchedInputException.from(
+                    jsonParser,
+                    List.class,
+                    "Expected string, object, or array for DMLex list field, got " + currentToken);
+        }
+
+        while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+            JsonToken elemToken = jsonParser.currentToken();
+            if (elemToken == JsonToken.VALUE_STRING) {
+                String text = jsonParser.getText();
+                if (text != null && !text.isBlank()) {
+                    result.add(text);
+                }
+            } else if (elemToken == JsonToken.START_OBJECT) {
+                readValuesFromObject(jsonParser, result);
+            } else {
+                throw MismatchedInputException.from(
+                        jsonParser,
+                        String.class,
+                        "Expected string or object element in DMLex array, got " + elemToken);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Extracts value(s) from a single JSON object. When an object carries multiple scalar string
+     * fields (for example {@code {"tag": "singular", "label": "Singular number"}} or
+     * {@code {"label": "US", "transcription": "ˈæp.əl"}}), only the highest-priority non-blank
+     * scalar field ({@code "transcription"} &rarr; {@code "tag"} &rarr; {@code "value"} &rarr;
+     * {@code "label"} &rarr; {@code "text"}) is selected, while also supporting nested
+     * {@code "transcriptions"} arrays/objects.
+     */
+    private static void readValuesFromObject(JsonParser jsonParser, List<String> result) throws IOException {
+        String transcription = null;
+        String tag = null;
+        String value = null;
+        String label = null;
+        String text = null;
+        List<String> nestedTranscriptions = null;
+
+        while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
+            String fieldName = jsonParser.currentName();
+            JsonToken valueToken = jsonParser.nextToken();
+            if (valueToken == JsonToken.VALUE_STRING) {
+                String str = jsonParser.getText();
+                if (str != null && !str.isBlank()) {
+                    switch (fieldName) {
+                        case "transcription" -> {
+                            if (transcription == null) {
+                                transcription = str;
+                            }
+                        }
+                        case "tag" -> {
+                            if (tag == null) {
+                                tag = str;
+                            }
+                        }
+                        case "value" -> {
+                            if (value == null) {
+                                value = str;
+                            }
+                        }
+                        case "label" -> {
+                            if (label == null) {
+                                label = str;
+                            }
+                        }
+                        case "text" -> {
+                            if (text == null) {
+                                text = str;
+                            }
+                        }
+                        default -> {}
+                    }
+                }
+            } else if ("transcriptions".equals(fieldName) || "transcription".equals(fieldName)) {
+                if (nestedTranscriptions == null) {
+                    nestedTranscriptions = new ArrayList<>();
+                }
+                if (valueToken == JsonToken.START_OBJECT) {
+                    readValuesFromObject(jsonParser, nestedTranscriptions);
+                } else if (valueToken == JsonToken.START_ARRAY) {
+                    while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+                        JsonToken elemToken = jsonParser.currentToken();
+                        if (elemToken == JsonToken.VALUE_STRING) {
+                            String str = jsonParser.getText();
+                            if (str != null && !str.isBlank()) {
+                                nestedTranscriptions.add(str);
+                            }
+                        } else if (elemToken == JsonToken.START_OBJECT) {
+                            readValuesFromObject(jsonParser, nestedTranscriptions);
+                        } else if (elemToken == JsonToken.START_ARRAY) {
+                            jsonParser.skipChildren();
+                        }
+                    }
+                }
+            } else if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) {
+                jsonParser.skipChildren();
+            }
+        }
+
+        if (transcription != null) {
+            result.add(transcription);
+            if (nestedTranscriptions != null) {
+                result.addAll(nestedTranscriptions);
+            }
+        } else if (nestedTranscriptions != null && !nestedTranscriptions.isEmpty()) {
+            result.addAll(nestedTranscriptions);
+        } else if (tag != null) {
+            result.add(tag);
+        } else if (value != null) {
+            result.add(value);
+        } else if (label != null) {
+            result.add(label);
+        } else if (text != null) {
+            result.add(text);
+        }
+    }
+}