!7

// Note: Don't use db_mainConcepts(), only the module's concepts field

!include early #1033884 // Compact Module Include Gazelle V [dev version]

module GazelleScreenCam {
  !include early #1025212 // +Enabled without visualization
  
  int pixelRows = 128, colors = 8;
  S script, testScreenScript;
  bool animate = true;
  bool horizontalLayout; // flat layout
  
  WatchTarget watchTarget;
  
  transient ImageSurface isPosterized, isRegions, isTestScreen;
  transient new ScreenCamStream imageStream;
  transient new BWIntegralImageStream integralImages;
  transient new SourceTriggeredStream<BWImage> scaledAndPosterizedImages;
  transient new DoubleFPSCounter fpsCounter;
  transient int fps;
  //transient ScreenSelectorRadioButtons screenSelector;
  transient WatchTargetSelector watchTargetSelector;
  transient new RollingAverage remainingMSPerFrame;
  transient int remainingMS;
  
  transient new FunctionTimings<S> functionTimings;
  
  transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript);
  
  transient JGazelleVScriptRunner scriptRunner;
  transient JGazelleVScriptRunner testScreenScriptRunner;
  
  transient Animation animation;
  
  transient BWImage_FastRegions mainScreenRegions;
  
  transient UIURLSystem uiURLs;
  
  // set by host
  transient Concepts concepts;
  
  S uiURL;
  
  transient FileWatchService fileWatcher;
  
  transient SimpleCRUD_v2<Label> labelCRUD;
  transient SimpleCRUD_v2<GalleryImage> imageCRUD;

  transient JGallery gallery;

  start {
    dm_onFieldChange horizontalLayout(
      //r dm_revisualize // deh buggy
      r dm_reload
    );
    
    // non-standalone mode (doesn't work yet)
    if (concepts == null) {
      db();
      concepts = db_mainConcepts();
    }
    
    // update count in tab when tab isn't selected
    onConceptChanges(concepts, -> {
      labelCRUD?.updateEnclosingTabTitle();
      imageCRUD?.updateEnclosingTabTitle();
    });
    
    uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL());
    
    scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me()));
    testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me()));

    imageStream.directlyFeedInto(integralImages);
    
    integralImages.onNewElement(ii ->
      scaledAndPosterizedImages.newElement(
        scaleAndPosterize(ii, new SnPSettings(pixelRows, colors))));
          
    integralImages.onNewElement(r {
      if (shouldRunScript()) rstRunScript.go();
    });
          
    scaledAndPosterizedImages.onNewElement(img -> {
      fpsCounter.inc();
      setField(fps := iround(fpsCounter!));
      
      // display before analysis (old)
      // isPosterized?.setImage_thisThread(img);
      
      // find regions
      floodFill(img);
      
      // display after analysis so we can highlight a region
      if (isPosterized != null) {
        var img2 = highlightRegion(img, isPosterized, mainScreenRegions);
        isPosterized.setImage_thisThread(img2);
      }
    });
    
    // 20 Hz is enough for everyone
    dm_doEvery(1000/20, r {
      if (enabled) {
        watchTargetSelector?.updateScreenCount();
        Timestamp deadline = tsNowPlusMS(1000/20);
        if (watchTarget cast WatchScreen)
          imageStream.useScreen(watchTarget.screenNr-1);
        else if (watchTarget cast WatchMouse)
          imageStream.area(mouseArea(watchTarget.width, watchTarget.height));
        imageStream.step();
        long remaining = deadline.minus(tsNow());
        remainingMSPerFrame.add(remaining);
        setField(remainingMS := iround(remainingMSPerFrame!));
      }
    });
    
    startDirWatcher();
    for (f : allImageFiles(galleryDir()))
      addToGallery(f);
  }
  
  // convert to color & highlight region
  BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) {
    var pixels = image.getRGBPixels();
    
    var mouse = is.mousePosition;
    if (mouse != null && regions != null) {
      int iHighlightedRegion = regions.regionAt(mouse);
      //int marked = 0;
      if (iHighlightedRegion > 0)
        for i over pixels:
          if (regions.regionAt(i) == iHighlightedRegion) {
            pixels[i] = 0xFF008000;
            //++marked;
          }
      //printVars(+mouse, +iHighlightedRegion, +marked);
    }
    
    ret bufferedImage(pixels, image.getWidth(), image.getHeight());
  }
  
  void startDirWatcher {
    fileWatcher = new FileWatchService;
    fileWatcher.addRecursiveListener(picturesDir(), file -> {
      if (!isImageFile(file)) ret;
      addToGallery(file);
    });
  }
  
  ImageSurface stdImageSurface() {
    ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false);
  }
  
  visualize {
    gallery = new JGallery;
    var galleryComponent = gallery.visualize();

    new AWTOnConceptChanges(concepts, galleryComponent, -> {
      gallery.setImageFiles(map(list(concepts, GalleryImage), i -> i.path));
    }).install();
    
    /*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr());
    screenSelector.compactLayout(true);
    screenSelector.hideIfOnlyOne(true);
    screenSelector.screenLabel("");*/
    isPosterized = stdImageSurface();
    
    isRegions = stdImageSurface();
    isTestScreen = stdImageSurface();
    
    // when test screen is visible, do the animation
    awtEvery(isTestScreen, 1000/20, r stepAnimation);
    
    JTextArea taTimings = jTextArea_noUndo();
    awtEveryAndNow(taTimings, .5, r {
      setText(taTimings, renderFunctionTimings())
    });
    
    var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo()));
    
    //print("Labels: " + list(concepts, Label));
    labelCRUD = new SimpleCRUD_v2(concepts, Label);
    labelCRUD.hideFields("globalID");
    labelCRUD.addCountToEnclosingTab(true);
    
    imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage);
    imageCRUD.addCountToEnclosingTab(true);
    imageCRUD.itemToMap_inner2 = img
      -> litorderedmap(
        "File" := fileName(img.path),
        "Folder" := dirPath(img.path));
    
    // main visual

    var studyPanel = new StudyPanel;

    var tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout,
      "Screen Cam" := withTools(isPosterized),
      WithToolTip("Regions", "Visualize regions detected on screen")
        := jscroll_centered_borderless(isRegions),
      WithToolTip("Study", "Here you can analyze gallery images")
        := withTopMargin(studyPanel.visualize()),
      WithToolTip("Pretty Gallery", "Gallery view with preview images")
        := galleryComponent,
      WithToolTip("Powerful Gallery", "Gallery view with more functions (delete etc)")
        := wrapCRUD(imageCRUD),
      WithToolTip("Labels", "Manage labels (image markers)")
        := wrapCRUD(labelCRUD),
      WithToolTip("Script", "Run a Gazelle V script")
        := scriptRunner.scriptAndResultPanel(),
      "Test Screen" := testScreenPanel(),
      Timings := withRightAlignedButtons(taTimings,
        Reset := r resetTimings)
    ));

    var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled(""));
    var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image",
      jlabel("Screen Cam"));
    tabComponentClickFixer(lblScreenCam);
    var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam);
    
    replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab);
    
    // for tab titles
    labelCRUD.update();
    imageCRUD.update();
    
    watchTargetSelector = new WatchTargetSelector;

    var urlBar = uiURLs.urlBar();
    setToolTip(uiURLs.comboBox, "UI navigation system (currently unused)");
    
    var vis = northAndCenter(
      withSideAndTopMargin(urlBar),
      centerAndSouthOrEast(horizontalLayout,
        horizontalLayout ? withMargin(tabs) : withSideMargin(tabs),
        /*withMargin*/(borderlessScrollPane(jHigherScrollPane(
        jfullcenter(vstack(
          withLeftAndRightMargin(hstack(
            dm_rcheckBox enabled("Watch"),
            watchTargetSelector.visualize(),
            jlabel(" in "),
            withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)),
            withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)),
            , /* speed info can also go here */
          )),
          verticalStrut(2),
          withRightMargin(jSpeedInfo)
        ))))),
    ));
    setHorizontalMarginForAllButtons(vis, 4);
    ret vis;
  }
  
  S speedInfo() {
    ret "FPS " + fps + " idle " + remainingMS + " ms";
  }
  
  S speedInfo_long() {
    ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core.";
  }
  
  void floodFill(BWImage img) {
    BWImage_FastRegions ff = new(img);
    //ff.tolerance(0);
    ff.collectBounds();
    functionTimings.time("Regions", ff);
    mainScreenRegions = ff;
    if (isRegions != null && isRegions.isShowing_quick()) {
      //print("Showing regions image");
      isRegions.setImage_thisThread(ff.regionsImage());
    }
    setEnclosingTabTitle(isRegions, nRegions(ff.regionCount()));
  }
  
  bool useErrorHandling() { false; }
  
  S renderFunctionTimings() {
    ret lines(ciSorted(map(functionTimings!, (f, avg) ->
      firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")")));
  }
  
  transient long _runScript_idx;

  void _runScript() {
    long n = integralImages.elementCount();
    if (n > _runScript_idx) {
      _runScript_idx = n;

      scriptRunner.parseAndRunOn(integralImages!);
    }
  }
  
  bool shouldRunScript() {
    ret isShowing(scriptRunner.scpScriptResult);
  }
  
  void stepAnimation {
    if (!animate) ret;
    if (animation == null) {
      animation = new AnimatedLine;
      animation.start();
    }
    animation.nextFrame();
    var img = whiteImage(animation.w, animation.h);
    animation.setGraphics(createGraphics(img));
    animation.paint();
    isTestScreen?.setImage(img);
    
    var ii = bwIntegralImage_withMeta(img);
    testScreenScriptRunner.parseAndRunOn(ii);
  }
  
  JComponent testScreenPanel() {
    ret centerAndSouthWithMargin(
      hsplit(
        northAndCenterWithMargin(centerAndEastWithMargin(
          jlabel("Input"), dm_fieldCheckBox animate()),
        jscroll_centered_borderless(isTestScreen)),
        
        northAndCenterWithMargin(centerAndEastWithMargin(
          jlabel("Output"), testScreenScriptRunner.lblScore),
        testScreenScriptRunner.scpScriptResult)
      ),
        
      testScreenScriptRunner.scriptInputField()
    );
  }
  
  L popDownItems() {
    ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout",
      -> horizontalLayout,
      b -> setField(horizontalLayout := b)));
  }
  
  void unvisualize {} // don't zero transient component fields
  
  void resetTimings { functionTimings.reset(); }
  
  // add tool side bar to image surface
  JComponent withTools(ImageSurface is) {
    new ImageSurface_PositionToolTip(is);
    ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3,
      jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery),
    ));
  }
  
  class WatchTargetSelector {
    JComboBox<WatchTarget> cb = jComboBox();
    int screenCount;
    
    visualize {
      updateList();
      print("Selecting watchTarget: " + watchTarget);
      selectItem(cb, watchTarget);
      main onChange(cb, watchTarget -> {
        setField(+watchTarget);
        print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget);
      });
      ret cb;
    }
    
    void updateScreenCount() {
      if (screenCount != screenCount())
        updateList();
    }
    
    void updateList() swing {
      setComboBoxItems(cb, makeWatchTargets());
    }
    
    L<WatchTarget> makeWatchTargets() {
      ret flattenToList(
        countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)),
        new WatchMouse
      );
    }
  }

  class SnPSelector {
    settable new SnPSettings settings;
    
    event change;

    visualize {
      var colors = jspinner(settings.colors, 2, 256);
      main onChange(colors, -> {
        settings.colors = intFromSpinner(colors);
        change();
      });
      
      var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows);
      main onChange(pixelRows, -> {
        settings.pixelRows = intFromSpinner(pixelRows);
        change();
      });
      
      ret hstack(
        colors,
        jlabel(" colors @ "),
        pixelRows,
        jlabel(" p"));
    }
  }
  
  JComponent wrapCRUD(SimpleCRUD_v2 crud) {
    ret crud == null ?: withTopMargin(jRaisedSection(withMargin(crud.make_dontStartBots())));
  }
  
  File galleryDir() {
    ret picturesDir(gazelle22_imagesSubDirName());
  }
  
  void saveScreenshotToGallery enter {
    var img = imageStream!;
    saveImageWithCounter(galleryDir(), "Screenshot", img);
  }
  
  void addToGallery(File imgFile) {
    if (!isImageFile(imgFile)) ret;
    var img = uniq(concepts, GalleryImage, path := imgFile);
    //printVars("addToGallery", +imgFile, +img);
  }

  BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) {
    ret posterizeBWImage_withMeta(settings.colors,
      scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii));
  }

  class StudyPanel {
    new SnPSelector snpSelector;
    GalleryImage image;
    ImageSurface is = stdImageSurface();
    SingleComponentPanel scp = singleComponentPanel();
    SingleComponentPanel analysisPanel = singleComponentPanel();
    ConceptsComboBox<GalleryImage> cbImage = new(concepts, GalleryImage);
    ImageToRegions itr;
    int iSelectedRegion;

    *() {
      cbImage.sortTheList = l -> sortConceptsByIDDesc(l);

      main onChangeAndNow(cbImage, img -> dm_q(me(), r {
        image = img;
        scp.setComponent(studyImagePanel());
      }));

      is.removeAllTools();
      is.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate));

      imageSurfaceOnLeftMouseDown(is, pt -> dm_q(me(), r { chooseRegionAt(pt) }));

      snpSelector.onChange(r_dm_q(me(), l0 runSnP));
    }

    void chooseRegionAt(Pt p) {
      if (itr == null) ret;
      iSelectedRegion = itr.regions.regionAt(p);
      updateAnalysis();
    }

    // load new image
    JComponent studyImagePanel() { 
      if (image == null) null;
      var i = loadImage2(image.path);
      if (i == null) ret jcenteredlabel("Image not found");

      iSelectedRegion = 0;
      itr = new ImageToRegions(i, snpSelector.settings);
      runSnP();
      ret hsplit(jscroll_centered_borderless(is), analysisPanel);
    }

    void runSnP {
      if (itr == null) ret;
      itr.run();
      regionUpdate();
    }

    void regionUpdate {
      if (itr == null) ret;
      var highlighted = highlightRegion(itr.posterized, is, itr.regions);
      is.setImage(highlighted);
      is.performAutoZoom(); // seems to be necessary for some reason

      updateAnalysis();
    }

    void updateAnalysis {
      new LS lines;
      lines.add(nRegions(itr.regions.regionCount()));
      lines.add("Selected region: " + iSelectedRegion);
      S text = lines_rtrim(lines);
      analysisPanel.setComponent(jscroll(jMultiLineLabel(text)));
    }

    visual
      jRaisedSection(northAndCenterWithMargins(
        centerAndEast(withLabel("Study", cbImage),
          hstack(jlabel(" in "), snpSelector.visualize())),
        scp));
  } // end of StudyPanel

  class ImageToRegions {
    BufferedImage inputImage;
    BWIntegralImage ii;
    SnPSettings snpSettings;
    BWImage posterized;
    BWImage_FastRegions regions;

    *(BufferedImage *inputImage, SnPSettings *snpSettings) {}

    run {
      ii = bwIntegralImage_withMeta(inputImage);
      posterized = scaleAndPosterize(ii, snpSettings);
      regions = new BWImage_FastRegions(posterized);
      regions.collectBounds();
      functionTimings.time("Regions", regions);
    }
  }
} // end of module

concept Label > ConceptWithGlobalID {
  S name;
  //new RefL examples;
  
  toString { ret "Label " + name; }
}

concept GalleryImage {
  File path;

  toString { ret /*"[" + id + "] " +*/ fileName(path); }
}

concept SavedRegion {
  new Ref image; // e.g. a GalleryImage
  SnPSettings snpSettings;
  Rect bounds;
  BitMatrix bitMatrix;
  //new RefL<Label> labels;
}

concept Example {
  new Ref<Label> label;
  new Ref item; // e.g. a SavedRegion
  double confidence = 1;
}

concept IfThenTheory {
  new Ref if_;
  new Ref then;
}

sclass WatchTarget {}

// screenNr: 1 = screen 1 etc
srecord WatchScreen(int screenNr) > WatchTarget {
  toString { ret "Screen " + screenNr; }
}

srecord WatchMouse(int width, int height) > WatchTarget {
  WatchMouse() {
    height = 256;
    width = iround(height*16.0/9);
  }
  
  toString { ret "Mouse"; }
}

// SnP = Scale and Posterize
persistable sclass SnPSettings {
  int pixelRows = 128;
  int colors = 8;

  *(int *pixelRows, int *colors) {}
}

Began life as a copy of #1033862