Can the official support utf8 variables?
Summary
Can the official support utf8 variables?I have modified the source code to support UTF8 variables and hope that the official support can be provided.
The principle is simple. By expanding the character range, FPC/Lazarus can support UTF8 variables/procedures/functions
modify these 7 files:
scanner.pas、sysstr.inc、parser.inc、options.pas、objcutil.pas、assemble.pas和dfmreader.pp

1.fpcsrc\compiler\scanner.pas:
procedure tscannerfile.readstring;
var
i : longint;
err : boolean;
begin
err:=false;
i:=0;
repeat
case c of
'_',
'0'..'9',#$80..#$FF,
'A'..'Z',
'a'..'z' :
begin
if i<255 then
begin
inc(i);
orgpattern[i]:=c;
if c in ['a'..'z'] then
pattern[i]:=chr(ord(c)-32)
else
pattern[i]:=c;
end
else
{ Save current token position, for EOF its already loaded }
if c<>#26 then
gettokenpos;
{ Check first for a identifier/keyword, this is 20+% faster (PFV) }
if c in ['A'..'Z','a'..'z','_',#$80..#$FF] then
begin
readstring;
token:=_ID;
idtoken:=_ID;
{ keyword or any other known token,
pattern is always uppercased }
'&' :
begin
if [m_fpc,m_delphi] * current_settings.modeswitches <> [] then
begin
readnumber;
if length(pattern)=1 then
begin
{ does really an identifier follow? }
if not (c in ['_','A'..'Z','a'..'z',#$80..#$FF]) then
message2(scan_f_syn_expected,tokeninfo^[_ID].str,c);
readstring;
token:=_ID;
idtoken:=_ID;
end
else
token:=_INTCONST;
goto exit_label;
end
2.fpcsrc\compiler\assemble.pas:
if code<>0 then
internalerror(200702251);
end;
'.','_',
'A'..'Z',
'a'..'z',#$80..#$FF :
begin
pstart:=p;
while not(p^ in [#0,' ','-','+']) do
inc(p);
len:=p-pstart;
if len>255 then
internalerror(200509187);
hs[0]:=chr(len);
move(pstart^,hs[1],len);
sym:=objdata.symbolref(hs);
{ Second symbol? }
3.fpcsrc\compiler\objcutil.pas:
function objcvalidselectorname(value_str: pchar; len: longint): boolean;
var
i : longint;
gotcolon : boolean;
begin
result:=false;
{ empty name is not allowed }
if (len=0) then
exit;
gotcolon:=false;
{ if the first character is a colon, all of them must be colons }
if (value_str[0] = ':') then
begin
for i:=1 to len-1 do
if (value_str[i]<>':') then
exit;
end
else
begin
{ no special characters other than ':'
}
for i:=0 to len-1 do
if (value_str[i] = ':') then
gotcolon:=true
else if not(value_str[i] in ['_','A'..'Z','a'..'z','0'..'9',':',#$80..#$FF]) then
exit;
{ if there is at least one colon, the final character must
also be a colon (in case it's only one character that is
a colon, this was already checked before the above loop)
}
if gotcolon and
(value_str[len-1] <> ':') then
exit;
end;
result:=true;
end;
4.fpcsrc\compiler\options.pas:
function is_identifier(const s: TCmdStr): boolean;
var
i: longint;
begin
result:=false;
if (s='') or not (s[1] in ['A'..'Z','a'..'z','_',#$80..#$FF]) then
exit;
for i:=2 to length(s) do
if not (s[I] in ['A'..'Z','a'..'z','0'..'9','_',#$80..#$FF]) then
exit;
result:=true;
end;
function GetName(var fn:TPathStr):TPathStr;
var
i : longint;
begin
i:=0;
while (i<length(fn)) and (fn[i+1] in ['a'..'z','A'..'Z','0'..'9','_','-',#$80..#$FF]) do
inc(i);
GetName:=Copy(fn,1,i);
Delete(fn,1,i);
end;
5.fpcsrc\rtl\objpas\sysutils\sysstr.inc:
function IsValidIdent(const Ident: string; AllowDots: Boolean = False; StrictDots: Boolean = False): Boolean;
const
Alpha = ['A'..'Z', 'a'..'z', '_',#$80..#$FF];
AlphaNum = Alpha + ['0'..'9'];
Dot = '.';
var
First: Boolean;
I, Len: Integer;
begin
6.fpcsrc\rtl\objpas\classes\parser.inc:
function TParser.IsAlpha: boolean; {$ifdef CLASSESINLINE} inline; {$endif CLASSESINLINE}
begin
Result:=fBuf[fPos] in ['_','A'..'Z','a'..'z',#$80..#$FF];
end;
function TParser.NextToken: Char;
begin
SkipWhiteSpace;
if fEofReached then
HandleEof
else
case fBuf[fPos] of
'_','A'..'Z','a'..'z',#$80..#$FF : HandleAlphaNum;
'$' : HandleHexNumber;
'-' : HandleMinus;
'0'..'9' : HandleNumber;
'''','#' : HandleString;
else
HandleUnknown;
end;
Result:=fToken;
end;
7.fpcsrc\packages\fcl-res\src\dfmreader.pp:
function TDfmResourceReader.IsAlpha: boolean;
begin
Result:=PAnsiChar(fLine)[fLinePos] in ['_','A'..'Z','a'..'z',#$80..#$FF];
end;
Example Project
program project1;
{$mode objfpc}{$H+}
uses
Classes, SysUtils;
var
中文变量测试: string;
カンスウ: string;
begin
中文变量测试:='中文变量测试:Chinese Test!';
カンスウ:='カンスウ Test!';
writeln(utf8toansi(中文变量测试));
writeln(utf8toansi(カンスウ));
readln;
end.
Edited by Ghost User
