diff --git a/_data/stdlib_index.json b/_data/stdlib_index.json index 04f4870..3189300 100644 --- a/_data/stdlib_index.json +++ b/_data/stdlib_index.json @@ -126097,7 +126097,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 61 + "line": 66 }, { "kind": "function", @@ -126113,7 +126113,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 64 + "line": 69 }, { "kind": "function", @@ -126129,7 +126129,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 68 + "line": 73 }, { "kind": "function", @@ -126145,7 +126145,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 71 + "line": 76 }, { "kind": "function", @@ -126161,7 +126161,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 77 + "line": 82 }, { "kind": "function", @@ -126177,7 +126177,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 81 + "line": 86 }, { "kind": "function", @@ -126193,7 +126193,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 85 + "line": 90 }, { "kind": "function", @@ -126209,7 +126209,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 97 + "line": 102 }, { "kind": "function", @@ -126225,7 +126225,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 106 + "line": 111 } ], "enumMembers": [], @@ -126442,6 +126442,37 @@ } ] }, + { + "package": "MultibyteDiagnostics", + "category": "file", + "categoryLabel": "File & Network", + "sourcePath": "wurst/file/MultibyteDiagnostics.wurst", + "githubUrl": "https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/file/MultibyteDiagnostics.wurst", + "summary": "", + "summaryFirstLine": "", + "tags": [ + "file" + ], + "imports": [], + "entities": [ + { + "kind": "function", + "name": "runMultibyteDiagnostics", + "typeParams": "", + "receiver": null, + "signature": "public function runMultibyteDiagnostics()", + "doc": "", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 30 + } + ] + }, { "package": "SaveLoadData", "category": "file", @@ -148478,8 +148509,8 @@ "categoryLabel": "Handle Wrappers", "sourcePath": "wurst/_handles/primitives/String.wurst", "githubUrl": "https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/_handles/primitives/String.wurst", - "summary": "returns the position of the char in the charset\n\ttherefore converting a single char into an int that can\n\tbe converted back to a char with .toCharsetString()", - "summaryFirstLine": "returns the position of the char in the charset", + "summary": "A SubString slice that cuts a multibyte (non-latin) character in half does not\ncontain valid data. On current game versions such invalid slices are collapsed\nto a marker string with a constant string hash. Since this behavior is\nundocumented and has changed between game versions, the marker hash is computed\nat runtime from a known 2-byte character, so a future engine change degrades\ngracefully (multibyte detection turns off) instead of corrupting data.\nThe historically observed hash 1843378377 is kept as an additional fallback.", + "summaryFirstLine": "A SubString slice that cuts a multibyte (non-latin) character in half does not", "tags": [ "handles" ], @@ -148490,7 +148521,7 @@ "name": "ENABLE_MULTIBYTE_SUPPORT", "typeParams": "", "receiver": null, - "signature": "public constant ENABLE_MULTIBYTE_SUPPORT = false", + "signature": "public constant ENABLE_MULTIBYTE_SUPPORT = true", "doc": "", "deprecated": { "flag": false, @@ -148501,6 +148532,86 @@ "enumMembers": [], "line": 7 }, + { + "kind": "constant", + "name": "PARTIAL_CHAR_MARKER", + "typeParams": "", + "receiver": null, + "signature": "public constant PARTIAL_CHAR_MARKER = \"รค\".substring(0, 1)", + "doc": "", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 16 + }, + { + "kind": "constant", + "name": "PARTIAL_CHAR_HASH", + "typeParams": "", + "receiver": null, + "signature": "public constant PARTIAL_CHAR_HASH = PARTIAL_CHAR_MARKER.getHash()", + "doc": "", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 17 + }, + { + "kind": "extension-function", + "name": "isMultibytePartial", + "typeParams": "", + "receiver": "string", + "signature": "public function string.isMultibytePartial() returns boolean", + "doc": "Returns true if this string is an invalid slice of a multibyte character.\n\tOnly meaningful when ENABLE_MULTIBYTE_SUPPORT is enabled.", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 25 + }, + { + "kind": "constant", + "name": "LEAD4_HASH", + "typeParams": "", + "receiver": null, + "signature": "public constant LEAD4_HASH = \"๐Ÿ˜€\".substring(0, 1).getHash()", + "doc": "Hash of a slice containing only the lead byte 0xF0 of a 4-byte character\n\t(emoji and other astral plane chars), which the game does not collapse\n\tto the partial char marker.", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 41 + }, + { + "kind": "extension-function", + "name": "isCharBoundary", + "typeParams": "", + "receiver": "string", + "signature": "public function string.isCharBoundary(int pos) returns boolean", + "doc": "Returns true if this string can be split at the given position without\n\tcutting a multibyte character in half, i.e. the byte at the given position\n\tstarts a new character instead of continuing a preceding one.\n\tPositions at or outside the string's bounds count as boundaries.\n\tOnly meaningful when ENABLE_MULTIBYTE_SUPPORT is enabled.", + "deprecated": { + "flag": false, + "message": null + }, + "configurable": false, + "members": [], + "enumMembers": [], + "line": 52 + }, { "kind": "extension-function", "name": "isNotBlank", @@ -148515,7 +148626,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 16 + "line": 69 }, { "kind": "extension-function", @@ -148523,7 +148634,7 @@ "typeParams": "", "receiver": "string", "signature": "public function string.toCharsetInt() returns int", - "doc": "", + "doc": "returns the position of the char in the charset\n\ttherefore converting a single char into an int that can\n\tbe converted back to a char with .toCharsetString()", "deprecated": { "flag": false, "message": null @@ -148531,7 +148642,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 22 + "line": 75 }, { "kind": "extension-function", @@ -148547,7 +148658,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 32 + "line": 85 }, { "kind": "extension-function", @@ -148563,7 +148674,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 36 + "line": 89 }, { "kind": "extension-function", @@ -148579,7 +148690,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 40 + "line": 93 }, { "kind": "extension-function", @@ -148595,7 +148706,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 44 + "line": 97 }, { "kind": "extension-function", @@ -148611,7 +148722,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 48 + "line": 101 }, { "kind": "extension-function", @@ -148627,7 +148738,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 52 + "line": 105 }, { "kind": "extension-function", @@ -148643,7 +148754,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 56 + "line": 109 }, { "kind": "extension-function", @@ -148659,7 +148770,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 60 + "line": 113 }, { "kind": "extension-function", @@ -148675,7 +148786,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 64 + "line": 117 }, { "kind": "extension-function", @@ -148691,7 +148802,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 73 + "line": 126 }, { "kind": "extension-function", @@ -148707,7 +148818,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 80 + "line": 133 }, { "kind": "extension-function", @@ -148723,7 +148834,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 84 + "line": 137 }, { "kind": "extension-function", @@ -148739,7 +148850,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 88 + "line": 141 }, { "kind": "extension-function", @@ -148755,7 +148866,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 92 + "line": 145 }, { "kind": "extension-function", @@ -148771,7 +148882,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 96 + "line": 149 }, { "kind": "extension-function", @@ -148787,7 +148898,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 106 + "line": 159 }, { "kind": "extension-function", @@ -148803,7 +148914,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 114 + "line": 167 }, { "kind": "extension-function", @@ -148819,7 +148930,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 121 + "line": 174 }, { "kind": "extension-function", @@ -148835,7 +148946,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 128 + "line": 181 }, { "kind": "extension-function", @@ -148851,7 +148962,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 135 + "line": 188 }, { "kind": "extension-function", @@ -148867,7 +148978,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 142 + "line": 195 }, { "kind": "extension-function", @@ -148883,7 +148994,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 154 + "line": 207 }, { "kind": "extension-function", @@ -148899,7 +149010,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 164 + "line": 217 }, { "kind": "extension-function", @@ -148915,7 +149026,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 171 + "line": 224 }, { "kind": "extension-function", @@ -148931,7 +149042,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 183 + "line": 236 }, { "kind": "extension-function", @@ -148947,7 +149058,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 187 + "line": 240 }, { "kind": "extension-function", @@ -148963,7 +149074,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 191 + "line": 244 }, { "kind": "extension-function", @@ -148979,7 +149090,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 197 + "line": 250 }, { "kind": "extension-function", @@ -148995,7 +149106,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 203 + "line": 256 }, { "kind": "function", @@ -149011,7 +149122,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 213 + "line": 266 }, { "kind": "extension-function", @@ -149027,7 +149138,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 230 + "line": 283 }, { "kind": "extension-function", @@ -149043,7 +149154,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 235 + "line": 288 }, { "kind": "extension-function", @@ -149059,7 +149170,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 243 + "line": 296 }, { "kind": "class", @@ -149088,7 +149199,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 251 + "line": 304 }, { "kind": "function", @@ -149104,7 +149215,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 256 + "line": 309 }, { "kind": "function", @@ -149120,7 +149231,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 259 + "line": 312 }, { "kind": "function", @@ -149136,11 +149247,11 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 268 + "line": 331 } ], "enumMembers": [], - "line": 246 + "line": 299 }, { "kind": "class", @@ -149169,7 +149280,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 278 + "line": 341 }, { "kind": "function", @@ -149185,7 +149296,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 284 + "line": 347 }, { "kind": "function", @@ -149201,7 +149312,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 288 + "line": 351 }, { "kind": "function", @@ -149217,7 +149328,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 292 + "line": 355 }, { "kind": "function", @@ -149233,11 +149344,11 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 299 + "line": 362 } ], "enumMembers": [], - "line": 272 + "line": 335 }, { "kind": "extension-function", @@ -149253,7 +149364,7 @@ "configurable": false, "members": [], "enumMembers": [], - "line": 305 + "line": 368 } ] }, diff --git a/_doc/stdlib/ref/_handles/String.md b/_doc/stdlib/ref/_handles/String.md index f984a42..f02868b 100644 --- a/_doc/stdlib/ref/_handles/String.md +++ b/_doc/stdlib/ref/_handles/String.md @@ -9,9 +9,13 @@ source: 'https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/_handles/ generated: true --- -returns the position of the char in the charset - therefore converting a single char into an int that can - be converted back to a char with .toCharsetString() +A SubString slice that cuts a multibyte (non-latin) character in half does not +contain valid data. On current game versions such invalid slices are collapsed +to a marker string with a constant string hash. Since this behavior is +undocumented and has changed between game versions, the marker hash is computed +at runtime from a known 2-byte character, so a future engine change degrades +gracefully (multibyte detection turns off) instead of corrupting data. +The historically observed hash 1843378377 is kept as an additional fallback. **[Source on GitHub](https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/_handles/primitives/String.wurst)** @@ -60,6 +64,27 @@ Returns an integer result based on a lexicographic string comparison of strings ## Extension Functions +### string.isMultibytePartial + +```wurst +public function string.isMultibytePartial() returns boolean +``` + +Returns true if this string is an invalid slice of a multibyte character. + Only meaningful when ENABLE_MULTIBYTE_SUPPORT is enabled. + +### string.isCharBoundary + +```wurst +public function string.isCharBoundary(int pos) returns boolean +``` + +Returns true if this string can be split at the given position without + cutting a multibyte character in half, i.e. the byte at the given position + starts a new character instead of continuing a preceding one. + Positions at or outside the string's bounds count as boundaries. + Only meaningful when ENABLE_MULTIBYTE_SUPPORT is enabled. + ### string.isNotBlank ```wurst @@ -75,6 +100,10 @@ Returns true if this string is non-null and public function string.toCharsetInt() returns int ``` +returns the position of the char in the charset + therefore converting a single char into an int that can + be converted back to a char with .toCharsetString() + ### string.toReal ```wurst @@ -343,7 +372,29 @@ public function string.toLines() returns StringLines ### ENABLE_MULTIBYTE_SUPPORT ```wurst -public constant ENABLE_MULTIBYTE_SUPPORT = false +public constant ENABLE_MULTIBYTE_SUPPORT = true ``` > ๐Ÿ”ง **Configurable.** Override it in your map's config package. + +### PARTIAL_CHAR_MARKER + +```wurst +public constant PARTIAL_CHAR_MARKER = "รค".substring(0, 1) +``` + +### PARTIAL_CHAR_HASH + +```wurst +public constant PARTIAL_CHAR_HASH = PARTIAL_CHAR_MARKER.getHash() +``` + +### LEAD4_HASH + +```wurst +public constant LEAD4_HASH = "๐Ÿ˜€".substring(0, 1).getHash() +``` + +Hash of a slice containing only the lead byte 0xF0 of a 4-byte character + (emoji and other astral plane chars), which the game does not collapse + to the partial char marker. diff --git a/_doc/stdlib/ref/file/MultibyteDiagnostics.md b/_doc/stdlib/ref/file/MultibyteDiagnostics.md new file mode 100644 index 0000000..31e060e --- /dev/null +++ b/_doc/stdlib/ref/file/MultibyteDiagnostics.md @@ -0,0 +1,20 @@ +--- +title: MultibyteDiagnostics +layout: stdlibref +category: file +categoryLabel: File & Network +tags: + - file +source: 'https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/file/MultibyteDiagnostics.wurst' +generated: true +--- + +**[Source on GitHub](https://github.com/wurstscript/WurstStdlib2/blob/master/wurst/file/MultibyteDiagnostics.wurst)** + +## Functions + +### runMultibyteDiagnostics + +```wurst +public function runMultibyteDiagnostics() +``` diff --git a/_doc/stdlib/ref/index.md b/_doc/stdlib/ref/index.md index d93d857..08e1487 100644 --- a/_doc/stdlib/ref/index.md +++ b/_doc/stdlib/ref/index.md @@ -81,6 +81,7 @@ Auto-generated reference for every package in the [Wurst standard library](https - [ChunkedString](/stdlib/ref/file/ChunkedString.html) - [FileIO](/stdlib/ref/file/FileIO.html) - [GameStatus](/stdlib/ref/file/GameStatus.html) +- [MultibyteDiagnostics](/stdlib/ref/file/MultibyteDiagnostics.html) - [SaveLoadData](/stdlib/ref/file/SaveLoadData.html): Asynchronously loads the data from the file of the given player - [Serializable](/stdlib/ref/file/Serializable.html) - [SyncSimple](/stdlib/ref/file/SyncSimple.html) @@ -140,7 +141,7 @@ Auto-generated reference for every package in the [Wurst standard library](https - [Reference](/stdlib/ref/_handles/Reference.html): A simple wrapper class which contains a reference to a value of the given type. - [Region](/stdlib/ref/_handles/Region.html) - [Sound](/stdlib/ref/_handles/Sound.html): Returns sound length in milliseconds. -- [String](/stdlib/ref/_handles/String.html): returns the position of the char in the charset +- [String](/stdlib/ref/_handles/String.html): A SubString slice that cuts a multibyte (non-latin) character in half does not - [Texttag](/stdlib/ref/_handles/Texttag.html) - [Timer](/stdlib/ref/_handles/Timer.html) - [TimerDialog](/stdlib/ref/_handles/TimerDialog.html): Multiplies the timer's tick rate and remaining time, so that the overall elapsed time stays the same.