Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix stringlib against non-ascii chars #184

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+48 −17
Open

Fix stringlib against non-ascii chars #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88be9c4
Fix: string.match ignores upper case of chars
masaedw Feb 15, 2017
19e41ab
Fix: string.match bracket pattern
masaedw Feb 15, 2017
3ec2191
Fix other ignoring upper byte of chars
masaedw Feb 15, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/MoonSharp.Interpreter.Tests/EndToEnd/StringLibTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,39 @@ public void String_Match_1()
TestMatch(s, p, true);
}

[Test]
public void String_Match_2()
{
string s = "糸筍"; // U+7CF8, U+7B4D
string p = "書籍"; // U+66F8, U+7C4D

TestMatch(s, p, false);
}

[Test]
public void String_Match_3()
{
// maxexpand
string s = "書籍籍筍筍筍"; // U+66F8, U+7C4D, U+7B4D, ...
string p = "書籍+"; // U+66F8, U+7C4D

Script S = new Script(CoreModules.String);
S.Globals["s"] = s;
S.Globals["p"] = p;
DynValue res = S.DoString("return string.match(s, p)");

Utils.DynAssert(res, "書籍籍");
}

[Test]
public void String_Match_4()
{
string s = "㍝"; // U+335D
string p = "[Aそ]"; // U+005B, U+0041, U+305D, U+005D

TestMatch(s, p, false);
}

private void TestMatch(string s, string p, bool expected)
{
Script S = new Script(CoreModules.String);
Expand Down
32 changes: 15 additions & 17 deletions src/MoonSharp.Interpreter/CoreLib/StringLib/KopiLua_StrLib.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,10 @@ private static int matchbracketclass(int c, CharPtr p, CharPtr ec)
else if ((p[1] == '-') && (p + 2 < ec))
{
p += 2;
if ((byte)((p[-2])) <= c && (c <= (byte)p[0]))
if (p[-2] <= c && (c <= p[0]))
return sig;
}
else if ((byte)(p[0]) == c) return sig;
else if (p[0] == c) return sig;
}
return (sig == 0) ? 1 : 0;
}
Expand All @@ -219,11 +219,10 @@ private static int singlematch(int c, CharPtr p, CharPtr ep)
case '.': return 1; /* matches any char */
case L_ESC: return match_class((char)c, (char)(p[1]));
case '[': return matchbracketclass(c, p, ep - 1);
default: return ((byte)(p[0]) == c) ? 1 : 0;
default: return (p[0] == c) ? 1 : 0;
}
}


private static CharPtr matchbalance(MatchState ms, CharPtr s,
CharPtr p)
{
Expand All @@ -247,12 +246,11 @@ private static CharPtr matchbalance(MatchState ms, CharPtr s,
return null; /* string ends out of balance */
}


private static CharPtr max_expand(MatchState ms, CharPtr s,
CharPtr p, CharPtr ep)
{
ptrdiff_t i = 0; /* counts maximum expand for item */
while ((s + i < ms.src_end) && (singlematch((byte)(s[i]), p, ep) != 0))
while ((s + i < ms.src_end) && (singlematch(s[i], p, ep) != 0))
i++;
/* keeps trying to match with the maximum repetitions */
while (i >= 0)
Expand All @@ -273,7 +271,7 @@ private static CharPtr min_expand(MatchState ms, CharPtr s,
CharPtr res = match(ms, s, ep + 1);
if (res != null)
return res;
else if ((s < ms.src_end) && (singlematch((byte)(s[0]), p, ep) != 0))
else if ((s < ms.src_end) && (singlematch(s[0], p, ep) != 0))
s = s.next(); /* try with one more repetition */
else return null;
}
Expand Down Expand Up @@ -359,22 +357,22 @@ private static CharPtr match(MatchState ms, CharPtr s, CharPtr p)
LUA_QL("%f") + " in pattern");
ep = classend(ms, p); /* points to what is next */
previous = (s == ms.src_init) ? '\0' : s[-1];
if ((matchbracketclass((byte)(previous), p, ep - 1) != 0) ||
(matchbracketclass((byte)(s[0]), p, ep - 1) == 0)) return null;
if ((matchbracketclass(previous, p, ep - 1) != 0) ||
(matchbracketclass(s[0], p, ep - 1) == 0)) return null;
p = ep; goto init; /* else return match(ms, s, ep); */
}
default:
{
if (isdigit((char)(p[1])))
{ /* capture results (%0-%9)? */
s = match_capture(ms, s, (byte)(p[1]));
s = match_capture(ms, s, p[1]);
if (s == null) return null;
p += 2; goto init; /* else return match(ms, s, p+2) */
}
//ismeretlen hiba miatt lett ide átmásolva
{ /* it is a pattern item */
CharPtr ep = classend(ms, p); /* points to what is next */
int m = (s < ms.src_end) && (singlematch((byte)(s[0]), p, ep) != 0) ? 1 : 0;
int m = (s < ms.src_end) && (singlematch(s[0], p, ep) != 0) ? 1 : 0;
switch (ep[0])
{
case '?':
Expand Down Expand Up @@ -421,7 +419,7 @@ private static CharPtr match(MatchState ms, CharPtr s, CharPtr p)
dflt:
{ /* it is a pattern item */
CharPtr ep = classend(ms, p); /* points to what is next */
int m = (s < ms.src_end) && (singlematch((byte)(s[0]), p, ep) != 0) ? 1 : 0;
int m = (s < ms.src_end) && (singlematch(s[0], p, ep) != 0) ? 1 : 0;
switch (ep[0])
{
case '?':
Expand Down Expand Up @@ -859,15 +857,15 @@ private static CharPtr scanformat(LuaState L, CharPtr strfrmt, CharPtr form)
while (p[0] != '\0' && strchr(FLAGS, p[0]) != null) p = p.next(); /* skip flags */
if ((uint)(p - strfrmt) >= (FLAGS.Length + 1))
LuaLError(L, "invalid format (repeated flags)");
if (isdigit((byte)(p[0]))) p = p.next(); /* skip width */
if (isdigit((byte)(p[0]))) p = p.next(); /* (2 digits at most) */
if (isdigit(p[0])) p = p.next(); /* skip width */
if (isdigit(p[0])) p = p.next(); /* (2 digits at most) */
if (p[0] == '.')
{
p = p.next();
if (isdigit((byte)(p[0]))) p = p.next(); /* skip precision */
if (isdigit((byte)(p[0]))) p = p.next(); /* (2 digits at most) */
if (isdigit(p[0])) p = p.next(); /* skip precision */
if (isdigit(p[0])) p = p.next(); /* (2 digits at most) */
}
if (isdigit((byte)(p[0])))
if (isdigit(p[0]))
LuaLError(L, "invalid format (width or precision too long)");
form[0] = '%';
form = form.next();
Expand Down