!7

// Note: Don't use db_mainConcepts(), only the module's concepts field

!include early #1033884 // Compact Module Include Gazelle V [dev version]

module GazelleScreenCam {

  !include early #1025212 // +Enabled without visualization

  int pixelRows = 128, colors = 8;

  S script, testScreenScript;

  bool animate = true;

  bool horizontalLayout; // flat layout

  WatchTarget watchTarget;

  transient ImageSurface isPosterized, isRegions, isTestScreen;

  transient new ScreenCamStream imageStream;

  transient new BWIntegralImageStream integralImages;

  transient new SourceTriggeredStream<BWImage> scaledAndPosterizedImages;

  transient new DoubleFPSCounter fpsCounter;

  transient int fps;

  //transient ScreenSelectorRadioButtons screenSelector;

  transient WatchTargetSelector watchTargetSelector;

  transient new RollingAverage remainingMSPerFrame;

  transient int remainingMS;

  transient new FunctionTimings<S> functionTimings;

  transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript);

  transient JGazelleVScriptRunner scriptRunner;

  transient JGazelleVScriptRunner testScreenScriptRunner;

  transient Animation animation;

  transient BWImage_FastRegions mainScreenRegions;

  transient UIURLSystem uiURLs;

  // set by host

  transient Concepts concepts;

  S uiURL;

  transient FileWatchService fileWatcher;

  transient SimpleCRUD_v2<Label> labelCRUD;

  transient SimpleCRUD_v2<GalleryImage> imageCRUD;

  transient JGallery gallery;

  start {

    dm_onFieldChange horizontalLayout(

      //r dm_revisualize // deh buggy

      r dm_reload

    );

    // non-standalone mode (doesn't work yet)

    if (concepts == null) {

      db();

      concepts = db_mainConcepts();

    }

    // update count in tab when tab isn't selected

    onConceptChanges(concepts, -> {

      labelCRUD?.updateEnclosingTabTitle();

      imageCRUD?.updateEnclosingTabTitle();

    });

    uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL());

    scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me()));

    testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me()));

    imageStream.directlyFeedInto(integralImages);

    integralImages.onNewElement(ii ->

      scaledAndPosterizedImages.newElement(

        scaleAndPosterize(ii, new SnPSettings(pixelRows, colors))));

    integralImages.onNewElement(r {

      if (shouldRunScript()) rstRunScript.go();

    });

    scaledAndPosterizedImages.onNewElement(img -> {

      fpsCounter.inc();

      setField(fps := iround(fpsCounter!));

      // display before analysis (old)

      // isPosterized?.setImage_thisThread(img);

      // find regions

      floodFill(img);

      // display after analysis so we can highlight a region

      if (isPosterized != null) {

        var img2 = highlightRegion(img, isPosterized, mainScreenRegions);

        isPosterized.setImage_thisThread(img2);

      }

    });

    // 20 Hz is enough for everyone

    dm_doEvery(1000/20, r {

      if (enabled) {

        watchTargetSelector?.updateScreenCount();

        Timestamp deadline = tsNowPlusMS(1000/20);

        if (watchTarget cast WatchScreen)

          imageStream.useScreen(watchTarget.screenNr-1);

        else if (watchTarget cast WatchMouse)

          imageStream.area(mouseArea(watchTarget.width, watchTarget.height));

        imageStream.step();

        long remaining = deadline.minus(tsNow());

        remainingMSPerFrame.add(remaining);

        setField(remainingMS := iround(remainingMSPerFrame!));

      }

    });

    startDirWatcher();

    for (f : allImageFiles(galleryDir()))

      addToGallery(f);

  }

  // convert to color & highlight region

  BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) {

    var pixels = image.getRGBPixels();

    var mouse = is.mousePosition;

    if (mouse != null && regions != null) {

      int iHighlightedRegion = regions.regionAt(mouse);

      //int marked = 0;

      if (iHighlightedRegion > 0)

        for i over pixels:

          if (regions.regionAt(i) == iHighlightedRegion) {

            pixels[i] = 0xFF008000;

            //++marked;

          }

      //printVars(+mouse, +iHighlightedRegion, +marked);

    }

    ret bufferedImage(pixels, image.getWidth(), image.getHeight());

  }

  void startDirWatcher {

    fileWatcher = new FileWatchService;

    fileWatcher.addRecursiveListener(picturesDir(), file -> {

      if (!isImageFile(file)) ret;

      addToGallery(file);

    });

  }

  ImageSurface stdImageSurface() {

    ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false);

  }

  visualize {

    gallery = new JGallery;

    var galleryComponent = gallery.visualize();

    new AWTOnConceptChanges(concepts, galleryComponent, -> {

      gallery.setImageFiles(map(list(concepts, GalleryImage), i -> i.path));

    }).install();

    /*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr());

    screenSelector.compactLayout(true);

    screenSelector.hideIfOnlyOne(true);

    screenSelector.screenLabel("");*/

    isPosterized = stdImageSurface();

    isRegions = stdImageSurface();

    isTestScreen = stdImageSurface();

    // when test screen is visible, do the animation

    awtEvery(isTestScreen, 1000/20, r stepAnimation);

    JTextArea taTimings = jTextArea_noUndo();

    awtEveryAndNow(taTimings, .5, r {

      setText(taTimings, renderFunctionTimings())

    });

    var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo()));

    //print("Labels: " + list(concepts, Label));

    labelCRUD = new SimpleCRUD_v2(concepts, Label);

    labelCRUD.hideFields("globalID");

    labelCRUD.addCountToEnclosingTab(true);

    imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage);

    imageCRUD.addCountToEnclosingTab(true);

    imageCRUD.itemToMap_inner2 = img

      -> litorderedmap(

        "File" := fileName(img.path),

        "Folder" := dirPath(img.path));

    // main visual

    var studyPanel = new StudyPanel;

    var tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout,

      "Screen Cam" := withTools(isPosterized),

      WithToolTip("Regions", "Visualize regions detected on screen")

        := jscroll_centered_borderless(isRegions),

      WithToolTip("Study", "Here you can analyze gallery images")

        := withTopMargin(studyPanel.visualize()),

      WithToolTip("Pretty Gallery", "Gallery view with preview images")

        := galleryComponent,

      WithToolTip("Powerful Gallery", "Gallery view with more functions (delete etc)")

        := wrapCRUD(imageCRUD),

      WithToolTip("Labels", "Manage labels (image markers)")

        := wrapCRUD(labelCRUD),

      WithToolTip("Script", "Run a Gazelle V script")

        := scriptRunner.scriptAndResultPanel(),

      "Test Screen" := testScreenPanel(),

      Timings := withRightAlignedButtons(taTimings,

        Reset := r resetTimings)

    ));

    var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled(""));

    var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image",

      jlabel("Screen Cam"));

    tabComponentClickFixer(lblScreenCam);

    var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam);

    replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab);

    // for tab titles

    labelCRUD.update();

    imageCRUD.update();

    watchTargetSelector = new WatchTargetSelector;

    var urlBar = uiURLs.urlBar();

    setToolTip(uiURLs.comboBox, "UI navigation system (currently unused)");

    var vis = northAndCenter(

      withSideAndTopMargin(urlBar),

      centerAndSouthOrEast(horizontalLayout,

        horizontalLayout ? withMargin(tabs) : withSideMargin(tabs),

        /*withMargin*/(borderlessScrollPane(jHigherScrollPane(

        jfullcenter(vstack(

          withLeftAndRightMargin(hstack(

            dm_rcheckBox enabled("Watch"),

            watchTargetSelector.visualize(),

            jlabel(" in "),

            withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)),

            withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)),

            , /* speed info can also go here */

          )),

          verticalStrut(2),

          withRightMargin(jSpeedInfo)

        ))))),

    ));

    setHorizontalMarginForAllButtons(vis, 4);

    ret vis;

  }

  S speedInfo() {

    ret "FPS " + fps + " idle " + remainingMS + " ms";

  }

  S speedInfo_long() {

    ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core.";

  }

  void floodFill(BWImage img) {

    BWImage_FastRegions ff = new(img);

    //ff.tolerance(0);

    ff.collectBounds();

    functionTimings.time("Regions", ff);

    mainScreenRegions = ff;

    if (isRegions != null && isRegions.isShowing_quick()) {

      //print("Showing regions image");

      isRegions.setImage_thisThread(ff.regionsImage());

    }

    setEnclosingTabTitle(isRegions, nRegions(ff.regionCount()));

  }

  bool useErrorHandling() { false; }

  S renderFunctionTimings() {

    ret lines(ciSorted(map(functionTimings!, (f, avg) ->

      firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")")));

  }

  transient long _runScript_idx;

  void _runScript() {

    long n = integralImages.elementCount();

    if (n > _runScript_idx) {

      _runScript_idx = n;

      scriptRunner.parseAndRunOn(integralImages!);

    }

  }

  bool shouldRunScript() {

    ret isShowing(scriptRunner.scpScriptResult);

  }

  void stepAnimation {

    if (!animate) ret;

    if (animation == null) {

      animation = new AnimatedLine;

      animation.start();

    }

    animation.nextFrame();

    var img = whiteImage(animation.w, animation.h);

    animation.setGraphics(createGraphics(img));

    animation.paint();

    isTestScreen?.setImage(img);

    var ii = bwIntegralImage_withMeta(img);

    testScreenScriptRunner.parseAndRunOn(ii);

  }

  JComponent testScreenPanel() {

    ret centerAndSouthWithMargin(

      hsplit(

        northAndCenterWithMargin(centerAndEastWithMargin(

          jlabel("Input"), dm_fieldCheckBox animate()),

        jscroll_centered_borderless(isTestScreen)),

        northAndCenterWithMargin(centerAndEastWithMargin(

          jlabel("Output"), testScreenScriptRunner.lblScore),

        testScreenScriptRunner.scpScriptResult)

      ),

      testScreenScriptRunner.scriptInputField()

    );

  }

  L popDownItems() {

    ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout",

      -> horizontalLayout,

      b -> setField(horizontalLayout := b)));

  }

  void unvisualize {} // don't zero transient component fields

  void resetTimings { functionTimings.reset(); }

  // add tool side bar to image surface

  JComponent withTools(ImageSurface is) {

    new ImageSurface_PositionToolTip(is);

    ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3,

      jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery),

    ));

  }

  class WatchTargetSelector {

    JComboBox<WatchTarget> cb = jComboBox();

    int screenCount;

    visualize {

      updateList();

      print("Selecting watchTarget: " + watchTarget);

      selectItem(cb, watchTarget);

      main onChange(cb, watchTarget -> {

        setField(+watchTarget);

        print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget);

      });

      ret cb;

    }

    void updateScreenCount() {

      if (screenCount != screenCount())

        updateList();

    }

    void updateList() swing {

      setComboBoxItems(cb, makeWatchTargets());

    }

    L<WatchTarget> makeWatchTargets() {

      ret flattenToList(

        countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)),

        new WatchMouse

      );

    }

  }

  class SnPSelector {

    settable new SnPSettings settings;

    event change;

    visualize {

      var colors = jspinner(settings.colors, 2, 256);

      main onChange(colors, -> {

        settings.colors = intFromSpinner(colors);

        change();

      });

      var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows);

      main onChange(pixelRows, -> {

        settings.pixelRows = intFromSpinner(pixelRows);

        change();

      });

      ret hstack(

        colors,

        jlabel(" colors @ "),

        pixelRows,

        jlabel(" p"));

    }

  }

  JComponent wrapCRUD(SimpleCRUD_v2 crud) {

    ret crud == null ?: withTopMargin(jRaisedSection(withMargin(crud.make_dontStartBots())));

  }

  File galleryDir() {

    ret picturesDir(gazelle22_imagesSubDirName());

  }

  void saveScreenshotToGallery enter {

    var img = imageStream!;

    saveImageWithCounter(galleryDir(), "Screenshot", img);

  }

  void addToGallery(File imgFile) {

    if (!isImageFile(imgFile)) ret;

    var img = uniq(concepts, GalleryImage, path := imgFile);

    //printVars("addToGallery", +imgFile, +img);

  }

  BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) {

    ret posterizeBWImage_withMeta(settings.colors,

      scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii));

  }

  class StudyPanel {

    new SnPSelector snpSelector;

    GalleryImage image;

    ImageSurface is = stdImageSurface();

    SingleComponentPanel scp = singleComponentPanel();

    SingleComponentPanel analysisPanel = singleComponentPanel();

    ConceptsComboBox<GalleryImage> cbImage = new(concepts, GalleryImage);

    ImageToRegions itr;

    int iSelectedRegion;

    *() {

      cbImage.sortTheList = l -> sortConceptsByIDDesc(l);

      main onChangeAndNow(cbImage, img -> dm_q(me(), r {

        image = img;

        scp.setComponent(studyImagePanel());

      }));

      is.removeAllTools();

      is.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate));

      imageSurfaceOnLeftMouseDown(is, pt -> dm_q(me(), r { chooseRegionAt(pt) }));

      snpSelector.onChange(r_dm_q(me(), l0 runSnP));

    }

    void chooseRegionAt(Pt p) {

      if (itr == null) ret;

      iSelectedRegion = itr.regions.regionAt(p);

      updateAnalysis();

    }

    // load new image

    JComponent studyImagePanel() { 

      if (image == null) null;

      var i = loadImage2(image.path);

      if (i == null) ret jcenteredlabel("Image not found");

      iSelectedRegion = 0;

      itr = new ImageToRegions(i, snpSelector.settings);

      runSnP();

      ret hsplit(jscroll_centered_borderless(is), analysisPanel);

    }

    void runSnP {

      if (itr == null) ret;

      itr.run();

      regionUpdate();

    }

    void regionUpdate {

      if (itr == null) ret;

      var highlighted = highlightRegion(itr.posterized, is, itr.regions);

      is.setImage(highlighted);

      is.performAutoZoom(); // seems to be necessary for some reason

      updateAnalysis();

    }

    void updateAnalysis {

      new LS lines;

      lines.add(nRegions(itr.regions.regionCount()));

      lines.add("Selected region: " + iSelectedRegion);

      S text = lines_rtrim(lines);

      analysisPanel.setComponent(jscroll(jMultiLineLabel(text)));

    }

    visual

      jRaisedSection(northAndCenterWithMargins(

        centerAndEast(withLabel("Study", cbImage),

          hstack(jlabel(" in "), snpSelector.visualize())),

        scp));

  } // end of StudyPanel

  class ImageToRegions {

    BufferedImage inputImage;

    BWIntegralImage ii;

    SnPSettings snpSettings;

    BWImage posterized;

    BWImage_FastRegions regions;

    *(BufferedImage *inputImage, SnPSettings *snpSettings) {}

    run {

      ii = bwIntegralImage_withMeta(inputImage);

      posterized = scaleAndPosterize(ii, snpSettings);

      regions = new BWImage_FastRegions(posterized);

      regions.collectBounds();

      functionTimings.time("Regions", regions);

    }

  }

} // end of module

concept Label > ConceptWithGlobalID {

  S name;

  //new RefL examples;

  toString { ret "Label " + name; }

}

concept GalleryImage {

  File path;

  toString { ret /*"[" + id + "] " +*/ fileName(path); }

}

concept SavedRegion {

  new Ref image; // e.g. a GalleryImage

  SnPSettings snpSettings;

  Rect bounds;

  BitMatrix bitMatrix;

  //new RefL<Label> labels;

}

concept Example {

  new Ref<Label> label;

  new Ref item; // e.g. a SavedRegion

  double confidence = 1;

}

concept IfThenTheory {

  new Ref if_;

  new Ref then;

}

sclass WatchTarget {}

// screenNr: 1 = screen 1 etc

srecord WatchScreen(int screenNr) > WatchTarget {

  toString { ret "Screen " + screenNr; }

}

srecord WatchMouse(int width, int height) > WatchTarget {

  WatchMouse() {

    height = 256;

    width = iround(height*16.0/9);

  }

  toString { ret "Mouse"; }

}

// SnP = Scale and Posterize

persistable sclass SnPSettings {

  int pixelRows = 128;

  int colors = 8;

  *(int *pixelRows, int *colors) {}

}