!7

sbool showToolTipWhenKnownSymbol = true;
sbool doOCR = true;
static double interval = 1.0;
static new LatestAppearance<S> latest;
sS tooltip;
static RectAndMD5 mouse;
static Rect rectShown;
static int maxClipWidth = min(300, screenWidth()/2);
static int maxClipHeight = min(100, screenHeight()/2);
static ImageSurface lastIS;
static Runnable rTeach;
static S hotkey;

p-subst {
  autoRestart(5);
  startBotHere("Screen Segmenter.", #1015121);
  consoleIcon(#1101243);
  
  onGlobalHotKey(hotkey = hotKey_teach(), r { callF(rTeach) });

  onGlobalMousePress(voidfunc(NativeMouseEvent e) {
    lock defaultLock();
    int x = e.getX(), y = e.getY();
    makeToolTip(Pt(x, y));
    S text;
    if (mouse == null)
      text = "User clicked at " + x + "/" + y + " (" + screenWidth() + "x" + screenHeight() + ")";
    else
      text = "User clicked on " + (empty(tooltip) ? "uncommented object of size " + mouse.r.w + "x" + mouse.r.h : multiLineQuote(tooltip)) + " at " + x + "/" + y + " (screen: " + screenWidth() + "x" + screenHeight() + ", rect: " + mouse.r + ", md5: " + mouse.md5 + ")";
    mechAppend_noUniq_inQ("Mouse log", text);
  });
  
  S tooltip = ""; // mouse
  repeat with sleep interval {
    time2 {
      L<S> list = latest.recordAndSort(thingsOnScreen2());
      
      makeToolTip(mousePt());
    }
    print(localDateWithSeconds() + ": Found: " + list);
    consoleTitle(l(list) + " " + joinWithComma(list));
  }
}

svoid makeToolTip(Pt mousePt) {
  lock defaultLock();
  L<RectAndMD5> rects = segmentedScreenFromCacheFile_last;
  mouse = smallestRectAndMD5Containing(rects, mousePt);
  S name = "";
  
  /*if (mouse != null)
    print("Mouse: " + sfu(mouse));*/
  
  if (mouse != null)
    name = unnull(thingsOnScreenDefinitions().get(mouse.md5));
    
  /*if (neq(name, tooltip))
    flatInfoBox_topRightCorner(name, 1.0);*/
  tooltip = name;
  
  bool interpreted = nempty(name);
  final Rect r = mouse == null ? null : mouse.r;
  if (r != null && neq(r, rectShown)
   && r.w <= maxClipWidth && r.h <= maxClipHeight
   && !mouseInWindow(lastIS)) {
    disposeWindow(lastIS);
    double zoom = 1;
    int x = max(r.w, r.h);
    if (x < 30) zoom = 3.0;
    else if (x < 100) zoom = 2.0;
    final BufferedImage clip = shootScreenArea_possiblyBroken(mouse.r);
    final ImageSurface is = zoomedImageSurface(clip, zoom);
    lastIS = is;
    hideWindowOnComponentMouseOver(is);
    fS _name = name;
    final RectAndMD5 _mouse = mouse;
    JLabel lblSimilarity = jcenteredlabel("Interpretation:");
    JLabel lblInterpretation = jcenteredboldlabel(or2(name, "?"));
    final Var<S> interpretation = new(name);
    JComponent panel = withMargin(10,
      jMinWidth(250, vstackWithSpacing(
        jcenteredlabel("Symbol at mouse cursor:"),
        jFullCenter(is),
        lblSimilarity,
        lblInterpretation,
        jbutton((nempty(hotkey) ? "[" + hotkey + "] " : "") + "Teach this symbol...", rTeach = r-thread {
          teach(clip, interpretation!, _mouse);
        }))));
    if (!interpreted) panel = jBackground(Color.red, panel);
    if (showToolTipWhenKnownSymbol || !interpreted) {
      showWindowInTopRightCorner(withLineBorder(Color.green, panel));
      disposeWindowAfter_unlessMouseInIt(3.0, is);
    }
    
    if (!interpreted)
      slowSearch(clip, _mouse, interpretation, lblSimilarity, lblInterpretation);
  }
  rectShown = r;
}

svoid teach(BufferedImage clip, S _name, RectAndMD5 r) {
  fS md5 = ocrMD5OfBufferedImage(clip);
  bool imageChanged = neq(md5, r.md5);
  fS name = unnull(thingsOnScreenDefinitions().get(md5));
  S displayName = or3(name, _name, "\"\"");
  final JTextField tf = jtextfield(displayName);
  if (isQuoted(displayName))
    selectRange(tf, 1, l(displayName)-1);
  biggerSubmitButton("Save", showFormTitled("Teach symbol",
    "MD5:", md5 + (imageChanged ? " (warning: changed)" : ""),
    "Symbol:", imageChooser(clip),
    "Meaning:", tf,
    r {
      S text = getTextTrim(tf);
      addThingOnScreenDefinition(md5, text);
      if (neq(name, text))
        infoBoxTopLeftCorner("Taught new symbol!");
    }));
}

svoid slowSearch(final BufferedImage clip, final RectAndMD5 _mouse, final Var<S> interpretation, final JLabel lblSimilarity, final JLabel lblInterpretation) {
  thread "Slow Image Search" {
    BWImage bw = new BWImage(clip);
    final new Best<S> best;
    
    Runnable update = r {
      interpretation.set(best!);
      setText(lblInterpretation, best!);
      setText(lblSimilarity, "Possible interpretation (" + ratioToIntPercent_avoid100(best.score(), 1) + "%):");
    };
    
    long updateInterval = 100, lastUpdate = sysNow();
    for (File imgFile : listFiles(screenClipsDir())) {
      ping();
      if (mouse != _mouse) break; // new image shown in the mean time
      
      S md5 = md5FromScreenClipFile(imgFile);
      if (md5 == null) continue;
      S def = thingsOnScreenDefinitions().get(md5);
      if (def == null) continue;
      BWImage pat = loadBWImage(imgFile);
      float similarity = bwImageSimilarityResized(bw, pat, (float) best.bestScore());
      if (best.put(def, similarity) && sysNow() >= lastUpdate+updateInterval) {
        lastUpdate = sysNow();
        callF(update);
      }
    }
    
    if (best.has()) callF(update);
    
    if (doOCR)
      topLeftInfoBox(lines(ocr_recognizeMultiLine(bw)));
  }
}

Began life as a copy of #1010713