Inflection-215 Minimize ICU4C data and code footprint via icu_filters.json and UCONFIG flags

.github/workflows/cmake-multi-platform.yml

@@ -11,7 +11,9 @@ on:
     - 'src/**'
     - 'resources/**'
     - 'tools/**'
+    - 'test/**'
     - 'CMakeLists.txt'
+    - 'icu_filters.json'
     - '.github/workflows/**'
     - '!tools/wikidata/**'
     - '!documents/**'
@@ -23,7 +25,9 @@ on:
       - 'src/**'
       - 'resources/**'
       - 'tools/**'
+      - 'test/**'
       - 'CMakeLists.txt'
+      - 'icu_filters.json'
       - '.github/workflows/**'
       - '!tools/wikidata/**'
       - '!documents/**'
@@ -32,38 +36,7 @@ env:
   ICU_MINOR: '1'
 
 jobs:
-  cache-icu:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/cache/restore@v4
-        id: cache
-        with:
-          path: ~/icu
-          key: ${{ runner.os }}-icu-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}
-      - name: Download and install icu
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          wget https://github.com/unicode-org/icu/releases/download/release-${ICU_MAJOR}.${ICU_MINOR}/icu4c-${ICU_MAJOR}.${ICU_MINOR}-Ubuntu24.04-x64.tgz
-          export ICU=~/icu/
-          mkdir -p $ICU
-          echo "ICU directory is $ICU"
-
-          # Get the release and unpack.
-          cp icu4c-${ICU_MAJOR}.${ICU_MINOR}-Ubuntu24.04-x64.tgz $ICU
-          pushd $ICU
-
-          pwd
-
-          tar xvfz *.tgz
-          rm *.tgz
-
-      - uses: actions/cache/save@v4
-        with:
-          path: ~/icu
-          key: ${{ runner.os }}-icu-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}
-
   build:
-    needs: cache-icu
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -130,30 +103,35 @@ jobs:
       id: cache
       with:
         path: ~/icu
-        key: ${{ runner.os }}-icu-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}
+        key: ${{ runner.os }}-icu-filtered-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}-${{ hashFiles('icu_filters.json') }}
 
-      # Install all the required dependencies for the macos
-    - name: Install ICU (Ubuntu/macos)
+    - name: Install ICU with data filter (Ubuntu/macOS)
       run: |
-        if [[ "${{ matrix.os }}" == "macos-latest" && "${{ steps.cache.outputs.cache-hit }}" != "true" ]]; then
+        if [[ "${{ steps.cache.outputs.cache-hit }}" != "true" ]]; then
+          if [[ "${{ matrix.os }}" == "macos-latest" ]]; then
+            ICU_PLATFORM="MacOSX"
+            NPROC=$(sysctl -n hw.ncpu)
+          else
+            ICU_PLATFORM="Linux"
+            NPROC=$(nproc)
+          fi
           pushd /tmp
-          curl -fLO https://github.com/unicode-org/icu/releases/download/release-${ICU_MAJOR}.${ICU_MINOR}/icu4c-${ICU_MAJOR}.${ICU_MINOR}-sources.tgz
-          tar -xzf icu4c-${ICU_MAJOR}.${ICU_MINOR}-sources.tgz
-          cd icu/source
-          ./runConfigureICU MacOSX --prefix="$HOME/icu/icu/usr/local" --enable-rpath --disable-samples --disable-tests --disable-extras --disable-icuio
-          make -j$(sysctl -n hw.ncpu)
+          git clone --depth 1 --branch release-${ICU_MAJOR}.${ICU_MINOR} https://github.com/unicode-org/icu.git
+          cd icu/icu4c/source
+          ICU_DATA_FILTER_FILE="${GITHUB_WORKSPACE}/icu_filters.json" ./runConfigureICU "${ICU_PLATFORM}" --prefix="$HOME/icu/icu/usr/local" --enable-rpath --disable-samples --disable-tests --disable-extras --disable-icuio
+          make -j"${NPROC}"
           make install
           popd
-          rm -rf /tmp/icu /tmp/icu4c-${ICU_MAJOR}.${ICU_MINOR}-sources.tgz
+          rm -rf /tmp/icu
         fi
         echo "ICU_ROOT=$HOME/icu/icu/usr/local" >> $GITHUB_ENV
 
-    - name: Save ICU cache (macOS)
-      if: matrix.os == 'macos-latest' && steps.cache.outputs.cache-hit != 'true'
+    - name: Save ICU cache
+      if: steps.cache.outputs.cache-hit != 'true'
       uses: actions/cache/save@v4
       with:
         path: ~/icu
-        key: ${{ runner.os }}-icu-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}
+        key: ${{ runner.os }}-icu-filtered-${{ env.ICU_MAJOR }}-${{ env.ICU_MINOR }}-${{ hashFiles('icu_filters.json') }}
 
     - name: Install lld (Ubuntu)
       # Linux Release builds link with -fuse-ld=lld to avoid gold-plugin LTO version mismatches.

README.md

@@ -118,6 +118,18 @@ Typical ICU installation requires:
 * ICU4C is installed to `/usr/local/lib/*icu*` by default.
 * `ICU_ROOT` should point to `/usr/local/lib/`. Omitting `/icu` applies to any other installation path.
 
+#### Minimizing ICU4C Data Size (Optional)
+For mobile, embedded, or size-constrained deployments, this repository provides an [ICU Data Build Tool filter configuration](https://unicode-org.github.io/icu/userguide/icu_data/buildtool.html) in [`icu_filters.json`](./icu_filters.json) that strips unused ICU data trees and locales not needed by Unicode Inflection (reducing `libicudata` from ~33 MB to ~1.7 MB uncompressed / ~310 KB compressed).
+
+To build ICU4C from a Git checkout of the ICU release tag with the data filter applied:
+```bash
+git clone --depth 1 --branch release-79.1 https://github.com/unicode-org/icu.git
+cd icu/icu4c/source
+ICU_DATA_FILTER_FILE=<PATH_TO_INFLECTION>/icu_filters.json ./runConfigureICU <Linux|MacOSX> --prefix=<PATH_TO_ICU>
+make -j 8 install
+```
+*(Note: Building with `ICU_DATA_FILTER_FILE` requires the full `icu4c/source/data/` source tree present in the ICU Git repository rather than the pre-built `.dat` archive bundled in `icu4c-*-sources.tgz`, as well as Python 3 for `icutools.databuilder`.)*
+
 ### Build Instructions
 
 ```bash

documents/how_to_add_new_language.md

@@ -19,6 +19,7 @@ Before you add new language support, go to the README.md in the inflection subfo
 * UPDATE: inflection/src/inflection/util/LocaleUtils.cpp
 * UPDATE: inflection/test/src/inflection/util/LocaleUtilsTest.cpp
 * UPDATE: inflection/resources/org/unicode/inflection/locale/supported-locales.properties
+* UPDATE: inflection/icu_filters.json (add the new language code to `localeFilter.includelist` so filtered ICU builds and `LocaleUtilsTest#testMinimalICUData` include its locale and RBNF data)
 * ADD: inflection/resources/org/unicode/inflection/tokenizer/config_*XX*.properties
 
 ## Define language grammar

documents/how_to_update_icu.md

@@ -8,6 +8,7 @@ Follow [ICU instructions](https://unicode-org.github.io/icu/download/) on how to
 
 ### Test the project
 Test your update locally and fix any problems - most likely to happen due to CLDR changes.
+Make sure to also test building ICU with `ICU_DATA_FILTER_FILE=<PATH_TO_INFLECTION>/icu_filters.json` (used by CI in `.github/workflows/cmake-multi-platform.yml`) and update `icu_filters.json` if the new ICU version introduces new required data resources.
 If there are errors, you'll need to update the CI actions before submitting.
 
 ### Update Minimum ICU Version and Actions on GitHub

icu_filters.json

@@ -0,0 +1,195 @@
+{
+  "localeFilter": {
+    "filterType": "locale",
+    "includeChildren": true,
+    "includeScripts": true,
+    "includelist": [
+      "ar",
+      "bg",
+      "bn",
+      "ca",
+      "cs",
+      "da",
+      "de",
+      "el",
+      "en",
+      "es",
+      "et",
+      "fi",
+      "fr",
+      "gu",
+      "he",
+      "hi",
+      "hr",
+      "hu",
+      "id",
+      "is",
+      "it",
+      "ja",
+      "kk",
+      "kn",
+      "ko",
+      "lt",
+      "ml",
+      "mr",
+      "ms",
+      "nb",
+      "nl",
+      "no",
+      "or",
+      "pa",
+      "pl",
+      "pt",
+      "ro",
+      "ru",
+      "sk",
+      "sr",
+      "sv",
+      "ta",
+      "te",
+      "th",
+      "tr",
+      "uk",
+      "ur",
+      "vi",
+      "wuu",
+      "yue",
+      "zh"
+    ]
+  },
+  "featureFilters": {
+    "brkitr_dictionaries": "exclude",
+    "brkitr_lstm": "exclude",
+    "brkitr_adaboost": "exclude",
+    "brkitr_rules": {
+      "includelist": [
+        "word"
+      ]
+    },
+    "brkitr_tree": {
+      "filterType": "locale",
+      "includeChildren": false,
+      "includelist": [
+        "root"
+      ]
+    },
+    "coll_ucadata": "exclude",
+    "coll_tree": "exclude",
+    "confusables": "exclude",
+    "conversion_mappings": "exclude",
+    "curr_supplemental": "exclude",
+    "curr_tree": {
+      "filterType": "locale",
+      "includeChildren": false,
+      "includelist": [
+        "root"
+      ]
+    },
+    "lang_tree": "exclude",
+    "normalization": {
+      "includelist": [
+        "nfkc"
+      ]
+    },
+    "region_tree": "exclude",
+    "stringprep": "exclude",
+    "translit": "exclude",
+    "unames": "exclude",
+    "unit_tree": "exclude",
+    "zone_supplemental": "exclude",
+    "zone_tree": "exclude",
+    "misc": {
+      "includelist": [
+        "keyTypeData",
+        "langInfo",
+        "likelySubtags",
+        "metadata",
+        "numberingSystems",
+        "pluralRanges",
+        "plurals",
+        "supplementalData"
+      ]
+    }
+  },
+  "resourceFilters": [
+    {
+      "categories": [
+        "brkitr_tree"
+      ],
+      "rules": [
+        "-/boundaries/*",
+        "+/boundaries/word",
+        "-/dictionaries",
+        "-/lstm"
+      ]
+    },
+    {
+      "categories": [
+        "curr_tree"
+      ],
+      "rules": [
+        "-/Currencies",
+        "-/Currencies%narrow",
+        "-/Currencies%variant",
+        "-/CurrencyUnitPatterns"
+      ]
+    },
+    {
+      "categories": [
+        "rbnf_tree"
+      ],
+      "rules": [
+        "-/RBNFRules/DurationRules"
+      ]
+    },
+    {
+      "categories": [
+        "locales_tree"
+      ],
+      "rules": [
+        "-/AuxExemplarCharacters",
+        "-/ExemplarCharactersIndex",
+        "-/ExemplarCharactersNumbers",
+        "-/ExemplarCharactersPunctuation",
+        "-/calendar",
+        "-/characterLabel",
+        "-/contextTransforms",
+        "-/fields",
+        "-/layout",
+        "-/measurementSystemNames",
+        "-/personNames",
+        "-/NumberElements/*/patternsLong",
+        "-/NumberElements/*/patternsShort"
+      ]
+    },
+    {
+      "categories": [
+        "misc"
+      ],
+      "files": {
+        "includelist": [
+          "supplementalData"
+        ]
+      },
+      "rules": [
+        "-/calendarData",
+        "-/calendarPreferenceData",
+        "-/characterFallbacks",
+        "-/codeMappings",
+        "-/currencyMap",
+        "-/idValidity",
+        "-/languageData",
+        "-/LanguageMatch",
+        "-/measurementData",
+        "-/metaZones",
+        "-/parentLocales",
+        "-/subdivisionContainment",
+        "-/territoryContainment",
+        "-/territoryInfo",
+        "-/timeData",
+        "-/unitPreferenceData",
+        "-/weekData"
+      ]
+    }
+  ]
+}