!7

sbool showToolTipWhenKnownSymbol = true;

sbool doOCR = true;

static double interval = 1.0;

static new LatestAppearance<S> latest;

sS tooltip;

static RectAndMD5 mouse;

static Rect rectShown;

static int maxClipWidth = min(300, screenWidth()/2);

static int maxClipHeight = min(100, screenHeight()/2);

static ImageSurface lastIS;

static Runnable rTeach;

static S hotkey;

p-subst {

  autoRestart(5);

  startBotHere("Screen Segmenter.", #1015121);

  consoleIcon(#1101243);

  onGlobalHotKey(hotkey = hotKey_teach(), r { callF(rTeach) });

  onGlobalMousePress(voidfunc(NativeMouseEvent e) {

    lock defaultLock();

    int x = e.getX(), y = e.getY();

    makeToolTip(Pt(x, y));

    S text;

    if (mouse == null)

      text = "User clicked at " + x + "/" + y + " (" + screenWidth() + "x" + screenHeight() + ")";

    else

      text = "User clicked on " + (empty(tooltip) ? "uncommented object of size " + mouse.r.w + "x" + mouse.r.h : multiLineQuote(tooltip)) + " at " + x + "/" + y + " (screen: " + screenWidth() + "x" + screenHeight() + ", rect: " + mouse.r + ", md5: " + mouse.md5 + ")";

    mechAppend_noUniq_inQ("Mouse log", text);

  });

  S tooltip = ""; // mouse

  repeat with sleep interval {

    time2 {

      L<S> list = latest.recordAndSort(thingsOnScreen2());

      makeToolTip(mousePt());

    }

    print(localDateWithSeconds() + ": Found: " + list);

    consoleTitle(l(list) + " " + joinWithComma(list));

  }

}

svoid makeToolTip(Pt mousePt) {

  lock defaultLock();

  L<RectAndMD5> rects = segmentedScreenFromCacheFile_last;

  mouse = smallestRectAndMD5Containing(rects, mousePt);

  S name = "";

  /*if (mouse != null)

    print("Mouse: " + sfu(mouse));*/

  if (mouse != null)

    name = unnull(thingsOnScreenDefinitions().get(mouse.md5));

  /*if (neq(name, tooltip))

    flatInfoBox_topRightCorner(name, 1.0);*/

  tooltip = name;

  bool interpreted = nempty(name);

  final Rect r = mouse == null ? null : mouse.r;

  if (r != null && neq(r, rectShown)

   && r.w <= maxClipWidth && r.h <= maxClipHeight

   && !mouseInWindow(lastIS)) {

    disposeWindow(lastIS);

    double zoom = 1;

    int x = max(r.w, r.h);

    if (x < 30) zoom = 3.0;

    else if (x < 100) zoom = 2.0;

    final BufferedImage clip = shootScreenArea_possiblyBroken(mouse.r);

    final ImageSurface is = zoomedImageSurface(clip, zoom);

    lastIS = is;

    hideWindowOnComponentMouseOver(is);

    fS _name = name;

    final RectAndMD5 _mouse = mouse;

    JLabel lblSimilarity = jcenteredlabel("Interpretation:");

    JLabel lblInterpretation = jcenteredboldlabel(or2(name, "?"));

    final Var<S> interpretation = new(name);

    JComponent panel = withMargin(10,

      jMinWidth(250, vstackWithSpacing(

        jcenteredlabel("Symbol at mouse cursor:"),

        jFullCenter(is),

        lblSimilarity,

        lblInterpretation,

        jbutton((nempty(hotkey) ? "[" + hotkey + "] " : "") + "Teach this symbol...", rTeach = r-thread {

          teach(clip, interpretation!, _mouse);

        }))));

    if (!interpreted) panel = jBackground(Color.red, panel);

    if (showToolTipWhenKnownSymbol || !interpreted) {

      showWindowInTopRightCorner(withLineBorder(Color.green, panel));

      disposeWindowAfter_unlessMouseInIt(3.0, is);

    }

    if (!interpreted)

      slowSearch(clip, _mouse, interpretation, lblSimilarity, lblInterpretation);

  }

  rectShown = r;

}

svoid teach(BufferedImage clip, S _name, RectAndMD5 r) {

  fS md5 = ocrMD5OfBufferedImage(clip);

  bool imageChanged = neq(md5, r.md5);

  fS name = unnull(thingsOnScreenDefinitions().get(md5));

  S displayName = or3(name, _name, "\"\"");

  final JTextField tf = jtextfield(displayName);

  if (isQuoted(displayName))

    selectRange(tf, 1, l(displayName)-1);

  biggerSubmitButton("Save", showFormTitled("Teach symbol",

    "MD5:", md5 + (imageChanged ? " (warning: changed)" : ""),

    "Symbol:", imageChooser(clip),

    "Meaning:", tf,

    r {

      S text = getTextTrim(tf);

      addThingOnScreenDefinition(md5, text);

      if (neq(name, text))

        infoBoxTopLeftCorner("Taught new symbol!");

    }));

}

svoid slowSearch(final BufferedImage clip, final RectAndMD5 _mouse, final Var<S> interpretation, final JLabel lblSimilarity, final JLabel lblInterpretation) {

  thread "Slow Image Search" {

    BWImage bw = new BWImage(clip);

    final new Best<S> best;

    Runnable update = r {

      interpretation.set(best!);

      setText(lblInterpretation, best!);

      setText(lblSimilarity, "Possible interpretation (" + ratioToIntPercent_avoid100(best.score(), 1) + "%):");

    };

    long updateInterval = 100, lastUpdate = sysNow();

    for (File imgFile : listFiles(screenClipsDir())) {

      ping();

      if (mouse != _mouse) break; // new image shown in the mean time

      S md5 = md5FromScreenClipFile(imgFile);

      if (md5 == null) continue;

      S def = thingsOnScreenDefinitions().get(md5);

      if (def == null) continue;

      BWImage pat = loadBWImage(imgFile);

      float similarity = bwImageSimilarityResized(bw, pat, (float) best.bestScore());

      if (best.put(def, similarity) && sysNow() >= lastUpdate+updateInterval) {

        lastUpdate = sysNow();

        callF(update);

      }

    }

    if (best.has()) callF(update);

    if (doOCR)

      topLeftInfoBox(lines(ocr_recognizeMultiLine(bw)));

  }

}