Version 1.1.0

- split of messages preserves item links, textures, substitutions and raid target markers
- added safety measures to prevent endless replacement loops
- bumped version for Shadowlands
- bumped version for Naxxramas
- split of messages with excessive length no longer causes errors or broken texts
- proper handling of umlauts

.retail.toc

@@ -0,0 +1,3 @@
+## Interface: 90002
+
+## X-Build: Retail

CHANGELOG.md

@@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Version 1.1.0 - 2020-12-08
+### Added
+- split of messages preserves item links, textures, substitutions and raid target markers
+- added safety measures to prevent endless replacement loops
+### Changed
+- bumped version for Shadowlands
+- bumped version for Naxxramas
+### Fixed
+- split of messages with excessive length no longer causes errors or broken texts
+- proper handling of umlauts
+
+## Version 1.0.1 - 2020-10-17
+### Changed
+- bumped version for Shadowlands Pre-Patch
+
 ## Version 1.0.0 - 2020-09-01 [First Release]
 ### Added
 - info section with contact and thanks
@@ -33,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - stop replacements per mapping
 - more ooc recognition patterns
 ### Fixed
-- keep cases of over-long replacements
+- keep cases of replacements with excessive length
 
 ## Version 0.8.0-beta - 2020-06-14 [Feature Complete]
 ### Added

Grichelde.toc

@@ -1,9 +1,9 @@
-## Interface: 11305
+## Interface: 11306
 
 ## Title: Grichelde
 ## Notes: Replaces characters of your chat input line before sending.
-## Notes-de: Ersetzt eingegebene Zeichen in der Chat-Zeile vor dem Versand.
+## Notes-de: Ersetzt eingegebene Zeichen in der Chat-Zeile vor dem Versenden.
-## Version: 1.0.0
+## Version: 1.1.0
 ## Author: Teilzeit-Jedi
 ## eMail: tj@teilzeit-jedi.de
 

GricheldeChat.lua

@@ -2,44 +2,139 @@
 local _G = _G
 local Grichelde = _G.Grichelde or {}
 
-local IsAddOnLoaded, assert, nilOrEmpty, pairs, ipairs, spairs, tContains, tFilter, tInsert, tConcat, tSize, tIsEmpty, find, sub, gsub, gmatch, getNextCharUtf8, isLetter, isUpper, isLower, toUpper, toLower, capitalize, trim, length, lengthUtf8
+local IsAddOnLoaded, assert, nilOrEmpty, pairs, ipairs, spairs, tContains, tFilter, tInsert, tConcat, tSize, tIsEmpty, find, sub, gsub, gmatch, getNextCharUtf8, isLetter, isUpper, isLower, toUpper, toLower, capitalize, bytes2Char, trim, length, lengthUtf8
     = Grichelde.F.IsAddOnLoaded, Grichelde.F.assert, Grichelde.F.nilOrEmpty, Grichelde.F.pairs, Grichelde.F.ipairs, Grichelde.F.spairs, Grichelde.F.tContains, Grichelde.F.tFilter, Grichelde.F.tInsert, Grichelde.F.tConcat, Grichelde.F.tSize, Grichelde.F.tIsEmpty,
-      Grichelde.F.find, Grichelde.F.sub, Grichelde.F.gsub, Grichelde.F.gmatch, Grichelde.F.getNextCharUtf8, Grichelde.F.isLetter, Grichelde.F.isUpper, Grichelde.F.isLower, Grichelde.F.toUpper, Grichelde.F.toLower, Grichelde.F.capitalize, Grichelde.F.trim, Grichelde.F.length, Grichelde.F.lengthUtf8
+      Grichelde.F.find, Grichelde.F.sub, Grichelde.F.gsub, Grichelde.F.gmatch, Grichelde.F.getNextCharUtf8, Grichelde.F.isLetter, Grichelde.F.isUpper, Grichelde.F.isLower, Grichelde.F.toUpper, Grichelde.F.toLower, Grichelde.F.capitalize, Grichelde.F.bytes2Char, Grichelde.F.trim, Grichelde.F.length, Grichelde.F.lengthUtf8
 
 --- Splits a long text in longest possible chunks of <= 255 length, split at last available space
 -- @param text string
--- @return table
+-- @return array of chunks
 function Grichelde:SplitText(text)
     local chunks = {}
-    local splitText = text
+    local leftGuillemet = bytes2Char(194, 171) .. " "
-    local textSize = length(splitText or "")
+    local rightGuillemet = " " .. bytes2Char(194, 187)
+    local chunkSize = Grichelde.INPUT_LIMIT - length(leftGuillemet) - length(rightGuillemet)
 
-    while (textSize > 255) do
+    local function preserveText(newText, chunk, blockText, posEnd)
-        local chunk = sub(splitText, 1, 255)
+        -- link found, block completed
-        local remaining = ""
+        self:TracePrint("SplitText : Found preservable text up to %s", posEnd)
+        local preserved = sub(newText, 1, posEnd)
 
-        -- special case: if space is the start of the next chunk, don't split this chunk
+        if ((length(chunk) > 0) and (length(chunk .. blockText) > chunkSize)) then
-        if (sub(splitText, 256, 256) ~= ' ') then
+            -- block exceeds chunk, chunkify previous blocks
-            -- split at last space, don't assign directly as nil might be returned
+            self:DebugPrint("SplitText : add chunk:", chunk)
-            local left, right = self:SplitOnLastMatch(chunk)
+            tInsert(chunks, chunk .. rightGuillemet)
-            if (left ~= nil) then
+            chunk = leftGuillemet .. trim(blockText)
-                chunk = left
+        else
+            chunk = chunk .. blockText
         end
-            if (right ~= nil) then
+
-                remaining = right
+        if ((length(chunk) > 0) and (length(chunk .. preserved) > chunkSize)) then
+            -- block exceeds chunk, chunkify previous blocks
+            self:DebugPrint("SplitText : add chunk:", chunk)
+            tInsert(chunks, chunk .. rightGuillemet)
+            chunk = leftGuillemet .. trim(preserved)
+        else
+            chunk = chunk .. preserved
+        end
+
+        blockText = ""
+        newText = sub(newText, posEnd + 1)
+
+        return newText, chunk, blockText, posEnd
+    end
+
+    if (length(text or "") <= Grichelde.INPUT_LIMIT) then
+        self:DebugPrint("SplitText : no chunk:", text)
+        tInsert(chunks, text)
+    else
+        local lookAheads = { '|', '*', '<', '%', '{', '(', 'o' }
+
+        local newText = text or ""
+        local chunk, blockText = "", ""
+        local currentChar
+        local escape = 0
+
+        -- must not enforce UTF-8 support here, as the positions are used
+        while ((length(newText) > 0) and (escape < Grichelde.ENDLESS_LOOP_LIMIT)) do
+            escape = escape + 1
+            local previousChar = currentChar
+            local first, textAhead = getNextCharUtf8(newText)
+            currentChar = first
+            self:DebugPrint("SplitText : currentChar, escape: %s, %s", currentChar, escape)
+            self:TracePrint("SplitText : chunk:", chunk)
+            self:TracePrint("SplitText : newText:", newText)
+
+            -- as there is not OR in Luas pattern matching, search for all of the exclude patterns after another is
+            -- cumbersome and inefficient -> look for each char consecutively if it matches the starting pattern only
+            -- and if if matches do full pattern matching
+            if (currentChar == ' ') then
+                self:TracePrint("SplitText : block completed")
+
+                if ((length(chunk) > 0) and (length(chunk .. blockText) > chunkSize)) then
+                    -- block exceeds chunk, chunkify previous blocks
+                    self:DebugPrint("SplitText : add chunk:", chunk)
+                    tInsert(chunks, chunk .. rightGuillemet)
+                    chunk = leftGuillemet .. trim(blockText)
+                else
+                    chunk = chunk .. blockText
+                end
+
+                blockText = currentChar
+                newText = textAhead
+            elseif (tContains(lookAheads, currentChar)) then
+                -- lookahead-check for all preservable patterns (itemLinks, textures, emotes, ooc, etc.)
+
+                -- link detection
+                local linkPosEnd = self:CheckForLink(newText, currentChar)
+                if (linkPosEnd ~= nil) then
+                    -- link found, block completed
+                    newText, chunk, blockText = preserveText(newText, chunk, blockText, linkPosEnd)
+                else
+                    -- substitution detection
+                    local substPosEnd = self:CheckForSubstitutions(newText, currentChar)
+                    if (substPosEnd ~= nil) then
+                        -- substitution found, block completed
+                        newText, chunk, blockText = preserveText(newText, chunk, blockText, substPosEnd)
+                    else
+                        -- raid target marker detection
+                        local rtmPosEnd = self:CheckForRaidTargetMarkers(newText, currentChar)
+                        if (rtmPosEnd ~= nil) then
+                            -- raid target marker found, block completed
+                            newText, chunk, blockText = preserveText(newText, chunk, blockText, rtmPosEnd)
+                        else
+                            blockText = blockText .. currentChar
+                            newText = textAhead
+                        end
+                    end
+                end
+            else
+                blockText = blockText .. currentChar
+                newText = textAhead
             end
         end
 
-        self:DebugPrint("SplitText : chunk:", chunk)
+        self:TracePrint("SplitText : main loop completed")
+        if (length(chunk .. blockText) > 0) then
+            -- catchup remaining text at the end
+            if (length(chunk .. blockText) > chunkSize) then
+                -- block exceeds chunk, chunkify previous blocks
+                if (length(chunk) > 0) then
+                    self:DebugPrint("SplitText : add chunk:", chunk)
+                    tInsert(chunks, chunk .. rightGuillemet)
+                    chunk = leftGuillemet .. trim(blockText)
+                else
+                    chunk = chunk .. blockText
+                end
+            else
+                chunk = chunk .. blockText
+            end
 
+            self:DebugPrint("SplitText : last chunk:", chunk)
+            -- sub(chunk, 1, 255) can result in broken UTF8 chars and error message
             tInsert(chunks, chunk)
-        splitText = remaining .. sub(splitText, 256)
-        textSize = length(splitText)
         end
-
+    end
-    -- pickup remaining text < 255
-    self:DebugPrint("SplitText : last chunk:", splitText)
-    tInsert(chunks, splitText)
 
     return chunks
 end
@@ -63,10 +158,11 @@ function Grichelde:ReplaceCharacters(text, replName, replTable, consolidate, rep
         local ciPattern = ""
         local ignored = {'^', '$', '(', ')', '.'}
         local quantifiers = {'*', '+', '-', '?'}
-        local pos = 1
         local p, patRest = getNextCharUtf8(pattern)
+        local escape = 0
 
-        while (p ~= nil) do
+        while ((p ~= nil) and (escape < Grichelde.ENDLESS_LOOP_LIMIT)) do
+            escape = escape + 1
             if (tContains(ignored, p) or tContains(quantifiers, p)) then
                 -- ignore
                 ciPattern = ciPattern .. p
@@ -429,47 +525,43 @@ function Grichelde:ReplaceAndConsolidate(text, replacements)
     return result
 end
 
---- Checks if the text starts with a preversable ignore pattern, such as itemLinks, textures, raid target icons,
+--- looks for colored items, item links or textures
---- emotes, ooc or %-substitutons and returns the end location of the match, or 0 if no pattern was found
+function Grichelde:CheckForLink(text, currentChar)
--- @param text string
--- @param currentChar string(1) current character (first one) of the text, given for performance reasons
--- @param previousChar string(1) previous character of the text, otherwise unreachable
--- @param preserveEmotes boolean ignore replacements for emotes, for testing purposes
--- @return number
-function Grichelde:CheckForPreversableText(text, currentChar, previousChar, replaceEmotes)
-    self:DebugPrint("CheckForPreversableText : text:", text)
-    local replaceEmotes = replaceEmotes or self.db.profile.channels.emote or false
-
-    -- Calling find on ever pattern might be inefficient but its way less code than marching over every character
     if (currentChar == "|") then
         for _, pattern in ipairs(Grichelde.IGNORE_PATTERNS.LINKS) do
             local pos1, pos2 = find(text, "^" .. pattern)
             if (pos1 == 1) and (pos2 ~= nil) then
-                self:DebugPrint("CheckForPreversableText : Found link or texture pattern \"%s\" at (%d, %d)", pattern, pos1, pos2)
+                local match = sub(text, pos1, pos2)
+                self:DebugPrint("CheckForLink : Found link or texture pattern \"%s\" at (%d, %d)", pattern, pos1, pos2)
                 return pos2
             end
         end
     end
+    return nil
+end
 
-    -- emote detection
+--- looks for emotes
+function Grichelde:CheckForEmote(text, currentChar, replaceEmotes)
     if (currentChar == "*" or currentChar == "<") then
         for _, pattern in ipairs(Grichelde.IGNORE_PATTERNS.EMOTES) do
             local pos1, pos2 = find(text, "^" .. pattern)
             if (pos1 == 1) and (pos2 ~= nil) then
                 local emote = sub(text, pos1, pos2)
                 if (not replaceEmotes) then
-                    self:DebugPrint("CheckForPreversableText : Found emote \"%s\" at (%d, %d), but preserved it", emote, pos1, pos2)
+                    self:DebugPrint("CheckForEmote : Found emote \"%s\" at (%d, %d), but preserved it", emote, pos1, pos2)
                     return pos2
                 else
-                    self:DebugPrint("CheckForPreversableText : ignoring emote \"%s\" at (%d, %d)", emote, pos1, pos2)
+                    self:DebugPrint("CheckForEmote : Processing emote \"%s\" at (%d, %d)", emote, pos1, pos2)
                 end
             end
         end
     end
+    return nil
+end
 
+--- looks for %-substitutions
+function Grichelde:CheckForSubstitutions(text, currentChar)
     local lowerText = toLower(text)
-
-    -- %-substitutions
     if (currentChar == "%") then
         for _, pattern in ipairs(Grichelde.IGNORE_PATTERNS.SUBSTITUTES) do
             local pos1, pos2 = find(lowerText, "^" .. pattern)
@@ -479,8 +571,12 @@ function Grichelde:CheckForPreversableText(text, currentChar, previousChar, repl
             end
         end
     end
+    return nil
+end
 
-    -- raid target markers
+--- looks for general and localized raid target markers
+function Grichelde:CheckForRaidTargetMarkers(text, currentChar)
+    local lowerText = toLower(text)
     if (currentChar == "{") then
         -- rt1-9
         local pattern = Grichelde.IGNORE_PATTERNS.RAID_TARGETS[1]
@@ -501,8 +597,12 @@ function Grichelde:CheckForPreversableText(text, currentChar, previousChar, repl
             end
         end
     end
+    return nil
+end
 
-    -- ooc bracket detection
+--- looks for ooc with brackets
+function Grichelde:CheckForOocBrackets(text, currentChar)
+    local lowerText = toLower(text)
     if (currentChar == "(") then
         for _, pattern in ipairs(Grichelde.IGNORE_PATTERNS.OOC_BRACKETS) do
             local pos1, pos2 = find(lowerText, "^" .. pattern)
@@ -512,15 +612,63 @@ function Grichelde:CheckForPreversableText(text, currentChar, previousChar, repl
             end
         end
     end
+    return nil
+end
 
-    -- ooc without brackets: remaing text is treated as ooc completely!
+--- looks for ooc without brackets
+function Grichelde:CheckForOocNoBrackets(text, currentChar, previousChar)
+    local lowerText = toLower(text)
     if (currentChar == "o") then
         local pattern = Grichelde.IGNORE_PATTERNS.OOC_NO_BRACKETS[1]
         if ((previousChar == nil) or (find(previousChar, "%s") ~= nil)) and (find(lowerText, pattern) ~= nil) then
             self:DebugPrint("CheckForPreversableText : ooc for remaing text")
+            -- remaing text is treated as ooc completely!
             return length(text)
         end
     end
+    return nil
+end
+
+--- Checks if the text starts with a preversable ignore pattern, such as itemLinks, textures, raid target icons,
+--- emotes, ooc or %-substitutons and returns the end location of the match, or 0 if no pattern was found
+-- @param text string
+-- @param currentChar string(1) current character (first one) of the text, given for performance reasons
+-- @param previousChar string(1) previous character of the text, otherwise unreachable
+-- @param preserveEmotes boolean ignore replacements for emotes, for testing purposes
+-- @return number
+function Grichelde:CheckForPreversableText(text, currentChar, previousChar, replaceEmotes)
+    self:DebugPrint("CheckForPreversableText : text:", text)
+    local replaceEmotes = replaceEmotes or self.db.profile.channels.emote or false
+
+    local linkPos = self:CheckForLink(text, currentChar)
+    if (linkPos ~= nil) then
+        return linkPos
+    end
+
+    local emotePos = self:CheckForEmote(text, currentChar, replaceEmotes)
+    if (emotePos ~= nil) then
+        return emotePos
+    end
+
+    local substPos = self:CheckForSubstitutions(text, currentChar)
+    if (substPos ~= nil) then
+        return substPos
+    end
+
+    local rtmPos = self:CheckForRaidTargetMarkers(text, currentChar)
+    if (rtmPos ~= nil) then
+        return rtmPos
+    end
+
+    local oocBracketPos = self:CheckForOocBrackets(text, currentChar)
+    if (oocBracketPos ~= nil) then
+        return oocBracketPos
+    end
+
+    local oocNoBracketPos = self:CheckForOocNoBrackets(text, currentChar, previousChar)
+    if (oocNoBracketPos ~= nil) then
+        return oocNoBracketPos
+    end
 
     self:DebugPrint("CheckForPreversableText : no ignore pattern found")
     return 0
@@ -536,49 +684,49 @@ function Grichelde:ReplaceText(text, replacements, replaceEmotes)
     local newText = text
     local preserveEmotes = replaceEmotes or self.db.profile.channels.emote or false
     local replacements = replacements or self.db.profile.replacements or {}
-    local finalText = ""
+    local finalText, replaceText = "", ""
+    local currentChar
+    local escape = 0
 
-    local currentChar, previousChar
+    -- must not enforce UTF-8 support here, as the positions are used
-    local current = 1
+    while ((length(newText) > 0) and (escape < Grichelde.ENDLESS_LOOP_LIMIT)) do
-    local lastStart = 1
+        escape = escape + 1
-
+        local previousChar = currentChar
-    -- no UTF-8 support required here, as the positions are used
+        local first, textAhead = getNextCharUtf8(newText)
-    while current <= length(newText) do
+        currentChar = first
-        previousChar = currentChar
+        self:TracePrint("ReplaceText : currentChar : %s", currentChar)
-        currentChar = sub(newText, current, current)
-        self:TracePrint("ReplaceText : current/char : %s,%s", current, currentChar)
 
         -- as there is not OR in Luas pattern matching, search for all of the exclude patterns after another is
         -- cumbersome and inefficient -> look for each char consecutively if it matches the starting pattern only
         -- and if if matches do full pattern matching
-        if (not tContains(lookAheads, currentChar)) then
+        if (tContains(lookAheads, currentChar)) then
-            current = current + 1
-        else
             -- lookahead-check for all preservable patterns (itemLinks, textures, emotes, ooc, etc.)
-            local textAhead = sub(newText, current)
+            --local textAhead = sub(newText, current)
-            local posEnd = self:CheckForPreversableText(textAhead, currentChar, previousChar, preserveEmotes)
+            local posEnd = self:CheckForPreversableText(newText, currentChar, previousChar, preserveEmotes)
             if (posEnd > 0) then
                 self:DebugPrint("ReplaceText : Found an ignore pattern")
 
-                -- split text and continue after preserved text
+                -- replace all text up until now
-                local textBefore = sub(newText, lastStart, current - 1)
+                local replacement = self:ReplaceAndConsolidate(replaceText, replacements)
-                local replacement = self:ReplaceAndConsolidate(textBefore, replacements)
+                local preserved = sub(newText, 1, posEnd)
-                local preservedText = sub(textAhead, 1, posEnd)
 
-                finalText = finalText .. replacement .. preservedText
+                finalText = finalText .. replacement .. preserved
-                current = current + posEnd
+                replaceText = ""
-                lastStart = current
+                newText = sub(newText, posEnd + 1)
-                self:DebugPrint("ReplaceText : restarting at", lastStart)
+                self:DebugPrint("ReplaceText : remaining text", newText)
             else
                 -- no corresponding end was found to start pattern, continue loop with next char
-                current = current + 1
+                replaceText = replaceText .. currentChar
+                newText = textAhead
             end
+        else
+            replaceText = replaceText .. currentChar
+            newText = textAhead
         end
     end
 
     -- catchup remaining text to the end
-    local remainingText = sub(newText, lastStart)
+    local replacement = self:ReplaceAndConsolidate(replaceText, replacements)
-    local replacement = self:ReplaceAndConsolidate(remainingText, replacements)
     finalText = finalText .. replacement
 
     self:DebugPrint("ReplaceText : replaced \"%s\"", text)
@@ -592,22 +740,24 @@ function Grichelde:IsOneBigEmote(text)
 
     -- emote detection
     local isEmote = false
-    -- scheme *emote*
+    local firstChar, rest = getNextCharUtf8(firstWord)
-    if (sub(firstWord, 1, 1) == "<") then
+
+    -- scheme <emote>
+    if (firstChar == "<") then
         -- search for emote end
         local _, emoteEnd = find(text, "%>", 2)
-        isEmote = (emoteEnd == length(text))
+        isEmote = (emoteEnd == lengthUtf8(text))
-    end
-    if (not isEmote and (sub(firstWord, 1, 1) == "*")) then
-        -- search for emote end
-        local _, emoteEnd = find(text, "%*", 2)
-        isEmote = (emoteEnd == length(text))
     end
+    if (not isEmote and (firstChar == "*")) then
+        if (getNextCharUtf8(rest) == "*") then
             -- scheme **emote**
-    if (not isEmote and (sub(firstWord, 1, 2) == "**")) then
-        -- search for emote end
             local _, emoteEnd = find(text, "%*%*", 3)
-        isEmote = (emoteEnd == length(text))
+            isEmote = (emoteEnd == lengthUtf8(text))
+        else
+            -- scheme *emote*
+            local _, emoteEnd = find(text, "%*", 2)
+            isEmote = (emoteEnd == lengthUtf8(text))
+        end
     end
 
     -- the whole text is one big emote
@@ -659,7 +809,7 @@ function Grichelde:CheckReplacementAllowed(text, channel)
     assert(firstWord ~= nil, "firstWord is never nil")
 
     -- don't replace slash commands
-    if (sub(firstWord, 1, 1) == "/") then
+    if (getNextCharUtf8(firstWord) == "/") then
         self:DebugPrint("CheckReplacementAllowed : skip other slash commands:", firstWord)
         return false
     end
@@ -691,7 +841,7 @@ function Grichelde:CheckAndExtractMessageTypeTarget(message)
     end
 
     -- first word should be a chat command
-    if (sub(message, 1, 1) == "/") then
+    if (getNextCharUtf8(message) == "/") then
         -- extract chat command
         local chatCmd, targetAndText = self:SplitOnFirstMatch(message)
         assert(chatCmd ~= nil, "chatCmd is never nil")

GricheldeConstants.lua

@@ -5,6 +5,8 @@ local Grichelde = _G.Grichelde or {}
 -- constants and upvalues
 Grichelde.LOG_LEVEL = { DEBUG = 1, TRACE = 2 }
 
+Grichelde.INPUT_LIMIT = 255
+Grichelde.ENDLESS_LOOP_LIMIT = 10000
 Grichelde.MAPPING_OFFSET = 10
 Grichelde.MINIMAP_ENABLED = 1.0
 Grichelde.MINIMAP_DARKENDED = 0.5
@@ -116,6 +118,7 @@ Grichelde.BLIZZ_TYPE_TO_OPTIONS = {
 }
 
 -- do not replace these patterns
+-- combined item links in the chat will look like this: |cff9d9d9d|Hitem:3299::::::::20:257::::::|h[Fractured Canine]|h|r
 Grichelde.IGNORE_PATTERNS = {
     LINKS = {
         "|[Cc]%x%x%x%x%x%x%x%x.-|r", -- colored items (or links)