[bug-fix] Fix non-latin-feature-names: preserve Unicode feature names… · github/spec-kit@acf4347 · GitHub
Skip to content

Commit acf4347

Browse files
github-actions[bot]Copilotmnriem
authored
[bug-fix] Fix non-latin-feature-names: preserve Unicode feature names (#4780)
* Fix non-latin-feature-names: preserve Unicode feature names Apply the remediation from the bug assessment on issue #4574. Refs #4574 Assisted-by: GitHub Copilot (model: gpt-5.2-codex, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: complete Unicode feature naming across script variants Replace the initial proposed sanitizer with Unicode-aware name generation across Bash, PowerShell, and Python. Keep UTF-8 branch names within GitHub byte limits, preserve existing punctuation-only warnings, and update parity tests and documentation. Refs #4574. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: align Unicode categories and UTF-8 feature output Use Unicode letters and decimal digits consistently for feature names. Emit Python output as UTF-8 on Windows, decode parity subprocesses as UTF-8, and test the no-locale error and non-decimal number cases. Refs #4574. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: classify Unicode consistently on Bash across locales Use Python 3 for non-ASCII character classification where POSIX locale classes vary by platform. Preserve the shell-only ASCII path, report a clear error when a Unicode name lacks Python, and cover both paths. Refs #4574. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: select Python reliably for Unicode Bash feature names Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: respect LC_ALL and preserve Unicode stop words in Bash Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: sanitize ASCII controls and bound Unicode truncation Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * fix: read Unicode feature state as UTF-8 in Bash Use case-sensitive PowerShell stop-word matching after ASCII normalization. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * test: keep Bash feature-state regression portable Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Manfred Riem <15701806+mnriem@users.noreply.github.com>
1 parent d7dde18 commit acf4347

7 files changed

Lines changed: 711 additions & 109 deletions

File tree

‎docs/reference/core.md‎

Lines changed: 15 additions & 7 deletions

‎scripts/bash/common.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ read_feature_json_feature_directory() {
112112
fi
113113
if [[ -z "$_fd" ]] && command -v python3 >/dev/null 2>&1; then
114114
# Use Python so pretty-printed/multi-line JSON still parses correctly.
115-
if ! _fd=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); v=d.get('feature_directory'); print(v if v else '')" "$fj" 2>/dev/null); then
115+
if ! _fd=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1], encoding='utf-8')); v=d.get('feature_directory'); print(v if v else '')" "$fj" 2>/dev/null); then
116116
_fd=''
117117
fi
118118
fi

‎scripts/bash/create-new-feature.sh‎

Lines changed: 98 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -146,17 +146,82 @@ spec_prefix_exists() {
146146

147147
# Function to clean and format a branch name
148148
#
149-
# Three details keep this byte-identical to the Python and PowerShell twins:
150-
# * LC_ALL=C -- in a UTF-8 locale glibc resolves the a-z *range* through
151-
# collation, so [^a-z0-9] keeps accented lowercase letters that
152-
# re.sub(r"[^a-z0-9]", ...) and .NET's -replace both strip.
149+
# Three details keep this consistent with the Python and PowerShell twins:
150+
# * Unicode classification uses Python: POSIX [:alnum:] differs by platform.
153151
# * `--*` instead of the GNU-only `\+`, which POSIX/BSD sed reads as a literal
154152
# '+', leaving repeated separators uncollapsed on macOS.
155153
# * printf instead of echo, so a name of "-n"/"-e"/"-E" is text, not options.
154+
contains_non_ascii() {
155+
LC_ALL=C grep -q '[^[:print:][:cntrl:]]'
156+
}
157+
158+
UNICODE_LOCALE=""
159+
locale_candidates=(C.UTF-8 C.utf8 en_US.UTF-8 en_US.utf8 "${LC_CTYPE:-${LANG:-}}")
160+
if [ -n "${LC_ALL:-}" ]; then
161+
locale_candidates=("$LC_ALL")
162+
fi
163+
for candidate in "${locale_candidates[@]}"; do
164+
if [ -n "$candidate" ] && [ "$(printf 'é。' | LC_ALL="$candidate" sed 's/[^[:alnum:]]/-/g' 2>/dev/null)" = 'é-' ]; then
165+
UNICODE_LOCALE="$candidate"
166+
break
167+
fi
168+
done
169+
170+
if [ -z "$UNICODE_LOCALE" ]; then
171+
UNICODE_LOCALE=C
172+
if printf '%s' "${SHORT_NAME:-$FEATURE_DESCRIPTION}" | contains_non_ascii; then
173+
if [ -n "${LC_ALL:-}" ]; then
174+
echo "Error: A UTF-8 locale is required to create a Unicode feature name; LC_ALL=$LC_ALL is not usable" >&2
175+
else
176+
echo "Error: A UTF-8 locale is required to create a Unicode feature name" >&2
177+
fi
178+
exit 1
179+
fi
180+
fi
181+
182+
unicode_words() {
183+
local name="${1//$'\n'/ }"
184+
local separator="$2"
185+
if printf '%s' "$name" | contains_non_ascii; then
186+
local -a python_cmd=()
187+
local override="${SPECKIT_PYTHON_EXECUTABLE:-${SPECKIT_PYTHON:-}}"
188+
if [ -n "$override" ] && command -v "$override" >/dev/null 2>&1 &&
189+
"$override" -c 'import sys; raise SystemExit(sys.version_info.major != 3)' >/dev/null 2>&1; then
190+
python_cmd=("$override")
191+
else
192+
local python_line
193+
while IFS= read -r python_line; do
194+
python_cmd+=("$python_line")
195+
done < <(_python3_command)
196+
fi
197+
if [ "${#python_cmd[@]}" -eq 0 ]; then
198+
echo "Error: Python 3 is required to create a Unicode feature name" >&2
199+
return 1
200+
fi
201+
printf '%s' "$name" | "${python_cmd[@]}" -c '
202+
import sys
203+
value = sys.stdin.buffer.read().decode("utf-8")
204+
lower = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
205+
result = "".join(
206+
char if char.isalpha() or char.isdecimal() else sys.argv[1]
207+
for char in value.translate(lower)
208+
)
209+
sys.stdout.buffer.write(result.encode("utf-8"))
210+
' "$separator"
211+
else
212+
printf '%s' "$name" | LC_ALL=C tr '[:upper:]' '[:lower:]' | LC_ALL=C sed "s/[^a-z0-9]/$separator/g"
213+
fi
214+
}
215+
156216
clean_branch_name() {
157217
local name="$1"
158-
local -x LC_ALL=C
159-
printf '%s\n' "$name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//'
218+
local cleaned
219+
cleaned=$(unicode_words "$name" '-') || return 1
220+
printf '%s\n' "$cleaned" | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//'
221+
}
222+
223+
branch_byte_count() {
224+
printf '%s' "$1" | wc -c | tr -d '[:space:]'
160225
}
161226

162227
# Fit a feature prefix and suffix within GitHub's branch-name limit.
@@ -165,11 +230,25 @@ fit_branch_name() {
165230
local branch_suffix="$2"
166231
local branch_name="${feature_num}-${branch_suffix}"
167232

168-
if [ ${#branch_name} -gt $MAX_BRANCH_LENGTH ]; then
233+
if [ "$(branch_byte_count "$branch_name")" -gt "$MAX_BRANCH_LENGTH" ]; then
169234
local prefix_length=$(( ${#feature_num} + 1 ))
170235
local max_suffix_length=$((MAX_BRANCH_LENGTH - prefix_length))
171236
local truncated_suffix
172-
truncated_suffix=$(printf '%s' "$branch_suffix" | cut -c "1-$max_suffix_length" | sed 's/-$//')
237+
local -x LC_ALL="$UNICODE_LOCALE"
238+
local low=0 high=${#branch_suffix} mid
239+
if (( high > max_suffix_length )); then
240+
high=$max_suffix_length
241+
fi
242+
while (( low < high )); do
243+
mid=$(((low + high + 1) / 2))
244+
if [ "$(branch_byte_count "${branch_suffix:0:$mid}")" -le "$max_suffix_length" ]; then
245+
low=$mid
246+
else
247+
high=$((mid - 1))
248+
fi
249+
done
250+
truncated_suffix="${branch_suffix:0:$low}"
251+
truncated_suffix="${truncated_suffix%-}"
173252
branch_name="${feature_num}-${truncated_suffix}"
174253
fi
175254

@@ -209,27 +288,25 @@ generate_branch_name() {
209288
# Common stop words to filter out
210289
local stop_words="^(i|a|an|the|to|for|of|in|on|at|by|with|from|is|are|was|were|be|been|being|have|has|had|do|does|did|will|would|should|could|can|may|might|must|shall|this|that|these|those|my|your|our|their|want|need|add|get|set)$"
211290

212-
# Convert to lowercase and split into words. LC_ALL=C for the same
213-
# collation reason documented on clean_branch_name, and so the `grep -qw`
214-
# acronym probe below uses ASCII word boundaries like the Python twin's
215-
# (?<![0-9A-Za-z_]) lookarounds.
216-
local -x LC_ALL=C
217-
local clean_name=$(printf '%s' "$description" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/ /g')
291+
# Use a UTF-8 locale for character-safe length checks and split words.
292+
local -x LC_ALL="$UNICODE_LOCALE"
293+
local clean_name
294+
clean_name=$(unicode_words "$description" ' ') || return 1
218295

219296
# Filter words: remove stop words and words shorter than 3 chars (unless they're uppercase acronyms in original)
220297
local meaningful_words=()
221298
for word in $clean_name; do
222299
# Skip empty words
223300
[ -z "$word" ] && continue
224301

225-
# Keep words that are NOT stop words AND (length >= 3 OR are potential acronyms)
226-
if ! echo "$word" | grep -qiE "$stop_words"; then
227-
if [ ${#word} -ge 3 ]; then
302+
# Retain non-ASCII words even when shorter than three characters.
303+
if ! printf '%s\n' "$word" | LC_ALL=C grep -qE "$stop_words"; then
304+
if [ ${#word} -ge 3 ] || printf '%s' "$word" | contains_non_ascii; then
228305
meaningful_words+=("$word")
229306
# Keep short words that appear as an uppercase acronym in the original.
230307
# Uppercase via tr and match with grep -w (both portable) rather than
231308
# bash's 4+ "^^" case expansion (breaks on macOS bash 3.2) and \b (non-POSIX).
232-
elif printf '%s' "$description" | grep -qw -- "$(printf '%s' "$word" | tr '[:lower:]' '[:upper:]')"; then
309+
elif printf '%s' "$description" | LC_ALL=C grep -qw -- "$(printf '%s' "$word" | LC_ALL=C tr '[:lower:]' '[:upper:]')"; then
233310
meaningful_words+=("$word")
234311
fi
235312
fi
@@ -266,7 +343,7 @@ else
266343
fi
267344

268345
if [ -z "$BRANCH_SUFFIX" ]; then
269-
echo "[specify] Warning: Feature name is empty after removing unsupported characters. Use --short-name with ASCII letters or digits (for example, user-auth)." >&2
346+
echo "[specify] Warning: Feature name is empty after removing unsupported characters. Use --short-name with letters or digits (for example, user-auth)." >&2
270347
fi
271348

272349
# Warn if --number and --timestamp are both specified
@@ -339,8 +416,8 @@ ORIGINAL_BRANCH_NAME="${FEATURE_NUM}-${BRANCH_SUFFIX}"
339416
BRANCH_NAME=$(fit_branch_name "$FEATURE_NUM" "$BRANCH_SUFFIX")
340417
if [ "$BRANCH_NAME" != "$ORIGINAL_BRANCH_NAME" ]; then
341418
>&2 echo "[specify] Warning: Branch name exceeded GitHub's 244-byte limit"
342-
>&2 echo "[specify] Original: $ORIGINAL_BRANCH_NAME (${#ORIGINAL_BRANCH_NAME} bytes)"
343-
>&2 echo "[specify] Truncated to: $BRANCH_NAME (${#BRANCH_NAME} bytes)"
419+
>&2 echo "[specify] Original: $ORIGINAL_BRANCH_NAME ($(branch_byte_count "$ORIGINAL_BRANCH_NAME") bytes)"
420+
>&2 echo "[specify] Truncated to: $BRANCH_NAME ($(branch_byte_count "$BRANCH_NAME") bytes)"
344421
fi
345422

346423
FEATURE_DIR="$SPECS_DIR/$BRANCH_NAME"

‎scripts/powershell/create-new-feature.ps1‎

Lines changed: 40 additions & 18 deletions

0 commit comments

Comments
 (0)