From c1c8faa46090b00ae804f3aa905e0ae95eaf8e26 Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Thu, 6 Jan 2022 13:48:45 +0100 Subject: [PATCH 1/8] NpcNameFinder: Reduce the Area of intrest by excluding the player and target frame. Also excude the part of the area of the minimap --- SharedLib/NpcFinder/NpcNameFinder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SharedLib/NpcFinder/NpcNameFinder.cs b/SharedLib/NpcFinder/NpcNameFinder.cs index 0a8a72d21..e3808fb1c 100644 --- a/SharedLib/NpcFinder/NpcNameFinder.cs +++ b/SharedLib/NpcFinder/NpcNameFinder.cs @@ -50,7 +50,7 @@ public class NpcNameFinder #region variables - public int topOffset { get; set; } = 30; + public int topOffset { get; set; } = 110; public int npcPosYOffset { get; set; } = 0; public int npcPosYHeightMul { get; set; } = 10; @@ -128,7 +128,7 @@ public void Update() scaleToRefHeight = ScaleHeight(1); Area = new Rectangle(new Point(0, (int)ScaleHeight(topOffset)), - new Size(bitmapProvider.Bitmap.Width, (int)(bitmapProvider.Bitmap.Height * 0.6f))); + new Size((int)(bitmapProvider.Bitmap.Width * 0.87f), (int)(bitmapProvider.Bitmap.Height * 0.6f))); PopulateLinesOfNpcNames(); From 57f7d6af5974740b2c15127e8874b16e785da0ac Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Thu, 6 Jan 2022 13:50:57 +0100 Subject: [PATCH 2/8] NpcNameFinder: Create a new searchMode called 'Fuzzy'. Keep the currently working version called 'Simple'. Fuzzy uses RMSE with colorFuzziness to determine color matching. --- SharedLib/NpcFinder/NpcNameFinder.cs | 139 ++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 13 deletions(-) diff --git a/SharedLib/NpcFinder/NpcNameFinder.cs b/SharedLib/NpcFinder/NpcNameFinder.cs index e3808fb1c..b82c85d64 100644 --- a/SharedLib/NpcFinder/NpcNameFinder.cs +++ b/SharedLib/NpcFinder/NpcNameFinder.cs @@ -19,8 +19,15 @@ public enum NpcNames Corpse = 8 } + public enum SearchMode + { + Simple = 0, + Fuzzy = 1 + } + public class NpcNameFinder { + private SearchMode searchMode = SearchMode.Simple; private NpcNames nameType = NpcNames.Enemy | NpcNames.Neutral; private readonly List npcNameLine = new List(); @@ -45,11 +52,14 @@ public class NpcNameFinder public bool PotentialAddsExist { get; private set; } public DateTime LastPotentialAddsSeen { get; private set; } = default; - public int Sequence { get; private set; } = 0; + private Func colorMatcher; + public int Sequence { get; private set; } = 0; #region variables + public float colorFuzziness { get; set; } = 15f; + public int topOffset { get; set; } = 110; public int npcPosYOffset { get; set; } = 0; @@ -71,11 +81,25 @@ public class NpcNameFinder #endregion + #region Colors + + private readonly Color fEnemy = Color.FromArgb(0, 250, 5, 5); + private readonly Color fFriendly = Color.FromArgb(0, 5, 250, 5); + private readonly Color fNeutrual = Color.FromArgb(0, 250, 250, 5); + private readonly Color fCorpse = Color.FromArgb(0, 128, 128, 128); + + private readonly Color sEnemy = Color.FromArgb(0, 240, 35, 35); + private readonly Color sFriendly = Color.FromArgb(0, 0, 250, 0); + private readonly Color sNeutrual = Color.FromArgb(0, 250, 250, 0); + + #endregion public NpcNameFinder(ILogger logger, IBitmapProvider bitmapProvider) { this.logger = logger; this.bitmapProvider = bitmapProvider; + + UpdateSearchMode(); } private float ScaleWidth(int value) @@ -103,24 +127,113 @@ public void ChangeNpcType(NpcNames type) npcPosYHeightMul = 10; } - logger.LogInformation($"{GetType().Name}.ChangeNpcType = {type}"); + UpdateSearchMode(); + + logger.LogInformation($"{GetType().Name}.ChangeNpcType = {type} | searchMode = {searchMode}"); } } - private bool ColorMatch(Color p) + private void UpdateSearchMode() { - return nameType switch + switch (searchMode) { - NpcNames.Enemy | NpcNames.Neutral => (p.R > 240 && p.G <= 35 && p.B <= 35) || (p.R > 250 && p.G > 250 && p.B == 0), - NpcNames.Friendly | NpcNames.Neutral => (p.R == 0 && p.G > 250 && p.B == 0) || (p.R > 250 && p.G > 250 && p.B == 0), - NpcNames.Enemy => p.R > 240 && p.G <= 35 && p.B <= 35, - NpcNames.Friendly => p.R == 0 && p.G > 250 && p.B == 0, - NpcNames.Neutral => p.R > 250 && p.G > 250 && p.B == 0, - NpcNames.Corpse => p.R == 128 && p.G == 128 && p.B == 128, - _ => false, - }; + case SearchMode.Simple: + BakeSimpleColorMatcher(); + break; + case SearchMode.Fuzzy: + BakeFuzzyColorMatcher(); + break; + } + } + + + #region Simple Color matcher + + private void BakeSimpleColorMatcher() + { + switch (nameType) + { + case NpcNames.Enemy | NpcNames.Neutral: + colorMatcher = (Color c) => SimpleColorEnemy(c) || SimpleColorNeutral(c); + return; + case NpcNames.Friendly | NpcNames.Neutral: + colorMatcher = (Color c) => SimpleColorFriendly(c) || SimpleColorNeutral(c); + return; + case NpcNames.Enemy: + colorMatcher = SimpleColorEnemy; + return; + case NpcNames.Friendly: + colorMatcher = SimpleColorFriendly; + return; + case NpcNames.Neutral: + colorMatcher = SimpleColorNeutral; + return; + case NpcNames.Corpse: + colorMatcher = SimpleColorCorpse; + return; + } + } + + private bool SimpleColorEnemy(Color p) + { + return p.R > sEnemy.R && p.G <= sEnemy.G && p.B <= sEnemy.B; } + private bool SimpleColorFriendly(Color p) + { + return p.R == sFriendly.R && p.G > sFriendly.G && p.B == sFriendly.B; + } + + private bool SimpleColorNeutral(Color p) + { + return p.R > sNeutrual.R && p.G > sNeutrual.G && p.B == sNeutrual.B; + } + + private bool SimpleColorCorpse(Color p) + { + return p.R == fCorpse.R && p.G == fCorpse.G && p.B == fCorpse.B; + } + + #endregion + + + #region Color Fuzziness matcher + + private void BakeFuzzyColorMatcher() + { + switch (nameType) + { + case NpcNames.Enemy | NpcNames.Neutral: + colorMatcher = (Color c) => FuzzyColor(fEnemy, c) || FuzzyColor(fNeutrual, c); + return; + case NpcNames.Friendly | NpcNames.Neutral: + colorMatcher = (Color c) => FuzzyColor(fFriendly, c) || FuzzyColor(fNeutrual, c); + return; + case NpcNames.Enemy: + colorMatcher = (Color c) => FuzzyColor(fEnemy, c); + return; + case NpcNames.Friendly: + colorMatcher = (Color c) => FuzzyColor(fFriendly, c); + return; + case NpcNames.Neutral: + colorMatcher = (Color c) => FuzzyColor(fNeutrual, c); + return; + case NpcNames.Corpse: + colorMatcher = (Color c) => FuzzyColor(fCorpse, c); + return; + } + } + + private bool FuzzyColor(Color target, Color c) + { + return MathF.Sqrt( + ((target.R - c.R) * (target.R - c.R)) + + ((target.G - c.G) * (target.G - c.G)) + + ((target.B - c.B) * (target.B - c.B))) + <= colorFuzziness; + } + + #endregion public void Update() { @@ -230,7 +343,7 @@ private void PopulateLinesOfNpcNames() { int xi = x * bytesPerPixel; - if (ColorMatch(Color.FromArgb(255, currentLine[xi + 2], currentLine[xi + 1], currentLine[xi]))) + if (colorMatcher(Color.FromArgb(255, currentLine[xi + 2], currentLine[xi + 1], currentLine[xi]))) { var isSameSection = lengthStart > -1 && (x - lengthEnd) < minLength; if (isSameSection) From 43267583d42b417782c1ff9731a3cf305a46fed2 Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 01:56:27 +0100 Subject: [PATCH 3/8] NpcNameFinder: remove unused code. --- SharedLib/NpcFinder/NpcNameFinder.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/SharedLib/NpcFinder/NpcNameFinder.cs b/SharedLib/NpcFinder/NpcNameFinder.cs index 611af772f..b24ea87fe 100644 --- a/SharedLib/NpcFinder/NpcNameFinder.cs +++ b/SharedLib/NpcFinder/NpcNameFinder.cs @@ -27,7 +27,7 @@ public enum SearchMode public class NpcNameFinder { - private SearchMode searchMode = SearchMode.Simple; + private readonly SearchMode searchMode = SearchMode.Simple; private NpcNames nameType = NpcNames.Enemy | NpcNames.Neutral; private readonly List npcNameLine = new List(); @@ -55,8 +55,6 @@ public class NpcNameFinder private Func colorMatcher; - public int Sequence { get; private set; } = 0; - #region variables public float colorFuzziness { get; set; } = 15f; From 1851e1cd5d752aef3407d508cb81083906c32ba5 Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 02:57:23 +0100 Subject: [PATCH 4/8] CoreTest: fix compilation --- CoreTests/NpcNameFinder/MockWoWProcess.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CoreTests/NpcNameFinder/MockWoWProcess.cs b/CoreTests/NpcNameFinder/MockWoWProcess.cs index 82ecc15b3..6dc9ba055 100644 --- a/CoreTests/NpcNameFinder/MockWoWProcess.cs +++ b/CoreTests/NpcNameFinder/MockWoWProcess.cs @@ -6,12 +6,12 @@ namespace CoreTests { public class MockWoWProcess : IMouseInput { - public ValueTask RightClickMouse(Point position) + public void RightClickMouse(Point position) { throw new System.NotImplementedException(); } - public ValueTask LeftClickMouse(Point position) + public void LeftClickMouse(Point position) { throw new System.NotImplementedException(); } From ed406e71173e1d3415aa7198095055b8899aa8c1 Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 05:14:55 +0100 Subject: [PATCH 5/8] CoreTest: Show amount of time elapsed for screen capture. --- CoreTests/NpcNameFinder/Test_NpcNameFinderTarget.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CoreTests/NpcNameFinder/Test_NpcNameFinderTarget.cs b/CoreTests/NpcNameFinder/Test_NpcNameFinderTarget.cs index d1d651520..822cab2c4 100644 --- a/CoreTests/NpcNameFinder/Test_NpcNameFinderTarget.cs +++ b/CoreTests/NpcNameFinder/Test_NpcNameFinderTarget.cs @@ -32,10 +32,13 @@ public void Execute() { npcNameFinder.ChangeNpcType(NpcNames.Enemy | NpcNames.Neutral); - capturer.Capture(); - System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); + capturer.Capture(); + stopwatch.Stop(); + logger.LogInformation($"Capture: {stopwatch.ElapsedMilliseconds}ms"); + + stopwatch.Restart(); this.npcNameFinder.Update(); stopwatch.Stop(); logger.LogInformation($"Update: {stopwatch.ElapsedMilliseconds}ms"); From fc4d90c1977cbe6ecf345a7d74d09a6d58ba8aa1 Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 05:16:00 +0100 Subject: [PATCH 6/8] CoreTest: Run multiple times the npcnamefiner test. --- CoreTests/Program.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CoreTests/Program.cs b/CoreTests/Program.cs index 4ba221cb9..aa3eda2c2 100644 --- a/CoreTests/Program.cs +++ b/CoreTests/Program.cs @@ -1,5 +1,6 @@ using Serilog; using Serilog.Extensions.Logging; +using System.Threading; using System.Threading.Tasks; namespace CoreTests @@ -25,7 +26,14 @@ public static void Main() //var test = new Test_NpcNameFinderTarget(logger); var test = new Test_NpcNameFinderLoot(logger); - test.Execute(); + int count = 1; + int i = 0; + while (i < count) + { + test.Execute(); + i++; + Thread.Sleep(150); + } //MainAsync().GetAwaiter().GetResult(); } From 1914e4f9751271e7b6e6915b10d2deec15f04fef Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 05:24:11 +0100 Subject: [PATCH 7/8] Core: Measure the ScreenshotThread times. Screen capture and the NPCNameFinder component. --- BlazorServer/Pages/GoalsComponent.razor | 5 ++- Core/BotController.cs | 44 +++++++++++++++++++++++-- Core/ConfigBotController.cs | 3 ++ Core/IBotController.cs | 3 ++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/BlazorServer/Pages/GoalsComponent.razor b/BlazorServer/Pages/GoalsComponent.razor index 50080be32..e07a6baa3 100644 --- a/BlazorServer/Pages/GoalsComponent.razor +++ b/BlazorServer/Pages/GoalsComponent.razor @@ -6,7 +6,10 @@
Goals -> @addonReader.PlayerReader.MinRange - @addonReader.PlayerReader.MaxRange | Expand - Update: @addonReader.AvgUpdateLatency.ToString("0.00") ms + + Cap: @botController.AvgScreenLatency.ToString("0.0")ms | + Npc: @botController.AvgNPCLatency.ToString("0.0")ms | + Bot: @addonReader.AvgUpdateLatency.ToString("0.0")ms
@if (ShowGoals) { diff --git a/Core/BotController.cs b/Core/BotController.cs index 0f9b5de53..6916be5bc 100644 --- a/Core/BotController.cs +++ b/Core/BotController.cs @@ -15,6 +15,7 @@ using WinAPI; using Microsoft.Extensions.Configuration; using SharedLib.NpcFinder; +using Cyotek.Collections.Generic; namespace Core { @@ -34,7 +35,7 @@ public sealed class BotController : IBotController, IDisposable public Thread? screenshotThread { get; set; } - private const int screenshotTickMs = 150; + private const int screenshotTickMs = 200; private DateTime lastScreenshot; public Thread addonThread { get; set; } @@ -74,6 +75,34 @@ public sealed class BotController : IBotController, IDisposable private readonly AutoResetEvent addonAutoResetEvent = new(false); private readonly AutoResetEvent npcNameFinderAutoResetEvent = new(false); + public double AvgScreenLatency + { + get + { + double avg = 0; + for (int i = 0; i < ScreenLatencys.Size; i++) + { + avg += ScreenLatencys.PeekAt(i); + } + return avg /= ScreenLatencys.Size; + } + } + private readonly CircularBuffer ScreenLatencys; + + public double AvgNPCLatency + { + get + { + double avg = 0; + for (int i = 0; i < NPCLatencys.Size; i++) + { + avg += NPCLatencys.PeekAt(i); + } + return avg /= NPCLatencys.Size; + } + } + private readonly CircularBuffer NPCLatencys; + public BotController(ILogger logger, IPPather pather, DataConfig dataConfig, IConfiguration configuration) { this.logger = logger; @@ -112,6 +141,9 @@ public BotController(ILogger logger, IPPather pather, DataConfig dataConfig, ICo minimapNodeFinder = new MinimapNodeFinder(WowScreen, new PixelClassifier()); MinimapImageFinder = minimapNodeFinder as IImageProvider; + ScreenLatencys = new CircularBuffer(5); + NPCLatencys = new CircularBuffer(5); + addonThread = new Thread(AddonRefreshThread); addonThread.Start(); @@ -157,14 +189,21 @@ public void AddonRefreshThread() public void ScreenshotRefreshThread() { var nodeFound = false; + var stopWatch = new Stopwatch(); while (this.Enabled) { if ((DateTime.UtcNow - lastScreenshot).TotalMilliseconds > screenshotTickMs) { if (this.WowScreen.Enabled) { + stopWatch.Restart(); this.WowScreen.UpdateScreenshot(); + ScreenLatencys.Put(stopWatch.ElapsedMilliseconds); + + stopWatch.Restart(); this.npcNameFinder.Update(); + NPCLatencys.Put(stopWatch.ElapsedMilliseconds); + this.WowScreen.PostProcess(); } else @@ -189,11 +228,10 @@ public void ScreenshotRefreshThread() MapId = this.AddonReader.UIMapId.Value, Spot = this.AddonReader.PlayerReader.PlayerLocation }); - updatePlayerPostion.Reset(); updatePlayerPostion.Restart(); } - Thread.Sleep(10); + Thread.Sleep(5); } this.logger.LogInformation("Screenshot thread stoppped!"); } diff --git a/Core/ConfigBotController.cs b/Core/ConfigBotController.cs index 75585fc8a..38d128faf 100644 --- a/Core/ConfigBotController.cs +++ b/Core/ConfigBotController.cs @@ -37,6 +37,9 @@ public class ConfigBotController : IBotController public event EventHandler? ProfileLoaded; public event EventHandler? StatusChanged; + public double AvgScreenLatency { get => throw new NotImplementedException(); } + public double AvgNPCLatency { get => throw new NotImplementedException(); } + public void Shutdown() { diff --git a/Core/IBotController.cs b/Core/IBotController.cs index f9aed4046..ca39429a5 100644 --- a/Core/IBotController.cs +++ b/Core/IBotController.cs @@ -33,6 +33,9 @@ public interface IBotController event System.EventHandler? ProfileLoaded; event System.EventHandler StatusChanged; + double AvgScreenLatency { get; } + double AvgNPCLatency { get; } + void ToggleBotStatus(); void StopBot(); From 2667e64ce4a266b47336d218055ef734f06d25dc Mon Sep 17 00:00:00 2001 From: Xian55 <367101+Xian55@users.noreply.github.com> Date: Sat, 12 Feb 2022 05:25:09 +0100 Subject: [PATCH 8/8] NpcNameFinder: use Pararell.For instead of raw for loop to speed up the image processing. --- SharedLib/NpcFinder/NpcNameFinder.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/SharedLib/NpcFinder/NpcNameFinder.cs b/SharedLib/NpcFinder/NpcNameFinder.cs index b24ea87fe..8dab6d7fc 100644 --- a/SharedLib/NpcFinder/NpcNameFinder.cs +++ b/SharedLib/NpcFinder/NpcNameFinder.cs @@ -6,6 +6,8 @@ using System.Linq; using System.Drawing.Imaging; using System.Threading; +using System.Diagnostics; +using System.Threading.Tasks; namespace SharedLib.NpcFinder { @@ -336,7 +338,8 @@ private void PopulateLinesOfNpcNames() BitmapData bitmapData = bitmapProvider.Bitmap.LockBits(new Rectangle(0, 0, bitmapProvider.Bitmap.Width, bitmapProvider.Bitmap.Height), ImageLockMode.ReadOnly, bitmapProvider.Bitmap.PixelFormat); int bytesPerPixel = Bitmap.GetPixelFormatSize(bitmapProvider.Bitmap.PixelFormat) / 8; - for (int y = Area.Top; y < Area.Height; y += incY) + //for (int y = Area.Top; y < Area.Height; y += incY) + Parallel.For(Area.Top, Area.Height, y => { bool isEndOfSection; var lengthStart = -1; @@ -373,7 +376,7 @@ private void PopulateLinesOfNpcNames() { npcNameLine.Add(new LineOfNpcName(lengthStart, lengthEnd, y)); } - } + }); bitmapProvider.Bitmap.UnlockBits(bitmapData); }