Fix Tamil nonspacing mark cursor widths by RamasubbuSK · Pull Request #5216 · PowerShell/PSReadLine · GitHub
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions PSReadLine/Changes.txt
47 changes: 39 additions & 8 deletions PSReadLine/Render.Helper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ internal static int LengthInBufferCells(char c)
return Char.IsControl(c) ? 2 : 1;
}

// Tamil anusvara, vowel sign II, and pulli are nonspacing marks.
// Spacing vowel signs still contribute a cell in Terminal's graphemes mode.
if (c is '\u0b82' or '\u0bc0' or '\u0bcd')
{
return 0;
}

// The following is based on http://www.cl.cam.ac.uk/~mgk25/c/wcwidth.c
// which is derived from http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt

Expand All @@ -125,6 +132,15 @@ private static string SubstringByCells(string text, int countOfCells)
return SubstringByCells(text, 0, countOfCells);
}

private static int SkipZeroWidthCharacters(StringBuilder text, int start, int end)
{
while (start < end && LengthInBufferCells(text[start]) == 0)
{
start++;
}
return start;
}

private static string SubstringByCells(string text, int start, int countOfCells)
{
int length = SubstringLengthByCells(text, start, countOfCells);
Expand All @@ -138,6 +154,11 @@ private static int SubstringLengthByCells(string text, int countOfCells)

private static int SubstringLengthByCells(string text, int start, int countOfCells)
{
if (countOfCells <= 0)
{
return 0;
}

int cellLength = 0;
int charLength = 0;

Expand All @@ -151,11 +172,6 @@ private static int SubstringLengthByCells(string text, int start, int countOfCel
}

charLength++;

if (cellLength == countOfCells)
{
return charLength;
}
}

return charLength;
Expand All @@ -168,27 +184,42 @@ private static int SubstringLengthByCellsFromEnd(string text, int countOfCells)

private static int SubstringLengthByCellsFromEnd(string text, int start, int countOfCells)
{
if (countOfCells <= 0)
{
return 0;
}

int cellLength = 0;
int charLength = 0;
int pendingMarks = 0;

for (int i = start; i >= 0; i--)
{
cellLength += LengthInBufferCells(text[i]);
int width = LengthInBufferCells(text[i]);
if (width == 0)
{
pendingMarks++;
continue;
}

cellLength += width;

if (cellLength > countOfCells)
{
return charLength;
}

charLength++;
// Include trailing marks only when their base fits too.
charLength += pendingMarks + 1;
pendingMarks = 0;

if (cellLength == countOfCells)
{
return charLength;
}
}

return charLength;
return charLength + pendingMarks;
}

private static (int newStart, int newEnd) TrimSubstringInPlace(string text, int start, int end)
Expand Down
36 changes: 32 additions & 4 deletions PSReadLine/Render.cs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ public int PhysicalLineCount(int bufferWidth, int initialX, out int lenLastPhysi
}

int size = PSConsoleReadLine.LengthInBufferCells(c);
if (size == 0)
{
continue;
}

if (x == 0 && lenLastPhysicalLine > 0)
{
y++;
Expand Down Expand Up @@ -1395,7 +1400,8 @@ internal Point ConvertOffsetToPoint(int offset, StringBuilder buffer = null)

// If cursor is at column 0 and the next character is newline, let the next loop
// iteration increment y.
if (x != 0 || !(i + 1 < offset && buffer[i + 1] == '\n'))
int next = SkipZeroWidthCharacters(buffer, i + 1, offset);
if (x != 0 || !(next < offset && buffer[next] == '\n'))
{
y += 1;
}
Expand Down Expand Up @@ -1435,7 +1441,7 @@ private int ConvertLineAndColumnToOffset(Point point)
// the correct column
if (point.Y == y && point.X <= x)
{
return offset;
return SkipZeroWidthCharacters(_buffer, offset, _buffer.Length);
}
char c = _buffer[offset];
if (c == '\n')
Expand All @@ -1461,7 +1467,8 @@ private int ConvertLineAndColumnToOffset(Point point)

// If cursor is at column 0 and the next character is newline, let the next loop
// iteration increment y.
if (x != 0 || !(offset + 1 < _buffer.Length && _buffer[offset + 1] == '\n'))
int next = SkipZeroWidthCharacters(_buffer, offset + 1, _buffer.Length);
if (x != 0 || !(next < _buffer.Length && _buffer[next] == '\n'))
{
y += 1;
}
Expand Down Expand Up @@ -1752,7 +1759,28 @@ internal RenderDataOffset ConvertPointToRenderDataOffset(int initialX, int initi
}
else if (point.X == x)
{
// 'x' is pointing to where the next visible character would be rendered.
// Include attached marks before returning the offset
// at this column, including across color sequences.
int next = i + 1;
while (next < line.Length)
{
if (line[next] == 0x1b && next + 1 < line.Length && line[next + 1] == '[')
{
next += 2;
while (next < line.Length && line[next] != 'm')
{
next++;
}
next++;
continue;
}
if (LengthInBufferCells(line[next]) != 0)
{
break;
}
visibleCharIndex++;
next++;
}
return new RenderDataOffset(logicalLineIndex, visibleCharIndex + 1);
}
}
Expand Down
76 changes: 76 additions & 0 deletions test/CellWidthTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the 2-Clause BSD License.

using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using Microsoft.PowerShell;
using Xunit;

namespace Test
{
public class CellWidthTest
{
static CellWidthTest()
{
RuntimeHelpers.RunClassConstructor(typeof(ReadLine).TypeHandle);
}

[Theory]
[InlineData("", 0)]
[InlineData("ABC", 3)]
[InlineData("\u0003", 2)]
[InlineData("\u754c", 2)]
[InlineData("\u2764\ufe0f", 2)]
[InlineData("\u0b95", 1)]
[InlineData("\u0b95\u0b82", 1)]
[InlineData("\u0b95\u0bcd", 1)]
[InlineData("\u0b95\u0bc0", 1)]
[InlineData("\u0b95\u0bbe", 2)]
[InlineData("\u0b95\u0bbf", 2)]
[InlineData("\u0b95\u0bc1", 2)]
[InlineData("\u001b[31m\u0b95\u0bcd\u001b[0m", 1)]
[InlineData("\u0bb5\u0ba3\u0b95\u0bcd\u0b95\u0bae\u0bcd \u0b87\u0baa\u0bcd\u0baa\u0b9f\u0bbf \u0b87\u0bb0\u0bc1\u0b95\u0bcd\u0b95\u0bc0\u0b99\u0bcd\u0b95 ?", 21)]
public void TamilCellWidths(string text, int expected)
{
Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(text));
string padded = "x" + text + "y";
Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(padded, 1, padded.Length - 1));
Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(new StringBuilder(padded), 1, padded.Length - 1));
}

[Theory]
[InlineData("\u0b95\u0bcdX", 0, 1, false, 2)]
[InlineData("\u0b95\u0bc0\u0b82X", 0, 1, false, 3)]
[InlineData("X\u0b95\u0bcdY", 1, 1, false, 2)]
[InlineData("\u0b95\u0bcd", 0, 0, false, 0)]
[InlineData("\u0b95\u0bcd", 1, 0, true, 0)]
[InlineData("\u0b95\u0bcd", 1, 1, true, 2)]
[InlineData("X\u0b95\u0bcdY", 2, 1, true, 2)]
[InlineData("\u754c\u0bcd", 1, 1, true, 0)]
[InlineData("\u754c\u0bcdX", 2, 2, true, 1)]
[InlineData("ABC", 0, 2, false, 2)]
[InlineData("ABC", 2, 2, true, 2)]
public void CellSubstringKeepsAttachedMarks(string text, int start, int cells, bool fromEnd, int expected)
{
string name = fromEnd ? "SubstringLengthByCellsFromEnd" : "SubstringLengthByCells";
var method = typeof(PSConsoleReadLine).GetMethod(
name, BindingFlags.Static | BindingFlags.NonPublic, null,
new[] { typeof(string), typeof(int), typeof(int) }, null);
Assert.NotNull(method);
Assert.Equal(expected, (int)method.Invoke(null, new object[] { text, start, cells }));
}

[Theory]
[InlineData("abc\u0b95\u0bcd", 1, 4)]
[InlineData("abc\u0b95\u0bcdX", 2, 1)]
[InlineData("abc\u0b95\u001b[31m\u0bcd\u001b[0m", 1, 4)]
public void AttachedMarkDoesNotWrapToAnotherPhysicalLine(string text, int lines, int lastLineWidth)
{
var data = new RenderedLineData(text, isFirstLogicalLine: true);
Assert.Equal(lines, data.PhysicalLineCount(4, 0, out int actualLastLineWidth));
Assert.Equal(lastLineWidth, actualLastLineWidth);
}
}
}
17 changes: 17 additions & 0 deletions test/MockConsole.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using Microsoft.PowerShell.Internal;

Expand Down Expand Up @@ -200,6 +201,12 @@ public virtual void SetWindowPosition(int left, int top)
{
}

protected static bool IsTamilNonspacingMark(char c)
{
return c >= '\u0b80' && c <= '\u0bff'
&& char.GetUnicodeCategory(c) == UnicodeCategory.NonSpacingMark;
}

public virtual void SetCursorPosition(int left, int top)
{
if (left != CursorLeft || top != CursorTop)
Expand All @@ -225,6 +232,11 @@ public virtual void Write(string s)
var writePos = CursorTop * BufferWidth + CursorLeft;
for (int i = 0; i < s.Length; i++)
{
if (IsTamilNonspacingMark(s[i]))
{
continue;
}

if (s[i] == (char) 0x1b)
{
// Escape sequence - limited support here, and assumed to be well formed.
Expand Down Expand Up @@ -454,6 +466,11 @@ public override void Write(string s)
var writePos = (_offset + CursorTop) * BufferWidth + CursorLeft;
for (int i = 0; i < s.Length; i++)
{
if (IsTamilNonspacingMark(s[i]))
{
continue;
}

if (s[i] == (char)0x1b)
{
// Escape sequence - limited support here, and assumed to be well formed.
Expand Down
60 changes: 60 additions & 0 deletions test/RenderTest.cs