Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

830
LINES

< > BotCompany Repo | #1034012 // Gazelle Screen Cam / Gazelle 22 Module [backup]

JavaX source code (Dynamic Module) - run with: Stefan's OS

1  
!7
2  
3  
// Note: It's ok again to use db_mainConcepts()
4  
5  
!include early #1033884 // Compact Module Include Gazelle V [dev version]
6  
7  
module GazelleScreenCam {
8  
  !include early #1025212 // +Enabled without visualization
9  
  
10  
  int pixelRows = 128, colors = 8;
11  
  S script = "64p 8c gradientImage";
12  
  S testScreenScript;
13  
  S newScript;
14  
  bool animate = true;
15  
  bool horizontalLayout; // flat layout
16  
  int fpsTarget = 20;
17  
  
18  
  WatchTarget watchTarget;
19  
20  
  transient NewScriptCompileResult newScriptCompileResult;
21  
  transient ImageSurface isPosterized, isRegions, isTestScreen;
22  
  transient new ScreenCamStream imageStream;
23  
  transient new BWIntegralImageStream integralImages;
24  
  transient new SourceTriggeredStream<BWImage> scaledAndPosterizedImages;
25  
  transient new DoubleFPSCounter fpsCounter;
26  
  transient int fps;
27  
  //transient ScreenSelectorRadioButtons screenSelector;
28  
  transient WatchTargetSelector watchTargetSelector;
29  
  transient new RollingAverage remainingMSPerFrame;
30  
  transient int remainingMS;
31  
  
32  
  transient new FunctionTimings<S> functionTimings;
33  
  
34  
  transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript);
35  
  
36  
  transient JGazelleVScriptRunner scriptRunner;
37  
  transient JGazelleVScriptRunner testScreenScriptRunner;
38  
  
39  
  transient Animation animation;
40  
  
41  
  transient BWImage_FastRegions mainScreenRegions;
42  
  
43  
  transient UIURLSystem uiURLs;
44  
  
45  
  // set by host
46  
  transient Concepts concepts;
47  
  
48  
  S uiURL = "Main Tabs";
49  
  
50  
  transient FileWatchService fileWatcher;
51  
  
52  
  transient SimpleCRUD_v2<Label> labelCRUD;
53  
  transient SimpleCRUD_v2<GalleryImage> imageCRUD;
54  
55  
  transient JGallery gallery;
56  
  transient JComponent galleryComponent;
57  
58  
  transient FlexibleRateTimer screenCamTimer;
59  
  transient SingleComponentPanel scpMain;
60  
  transient JTabbedPane tabs;
61  
  transient bool showRegionsAsOutline = true;
62  
  transient JComponent watchScreenPane;
63  
  transient S screenCamRecognitionOutput;
64  
65  
  start {
66  
    dm_onFieldChange horizontalLayout(
67  
      //r dm_revisualize // deh buggy
68  
      r dm_reload
69  
    );
70  
    
71  
    // non-standalone mode (doesn't work yet)
72  
    if (concepts == null) {
73  
      printWithMS("Starting DB");
74  
      db();
75  
      concepts = db_mainConcepts();
76  
    } else
77  
      assertSame(concepts, db_mainConcepts());
78  
79  
    // make indexes
80  
    indexConceptField(concepts, GalleryImage, "path");
81  
    indexConceptField(concepts, Example, "item");
82  
    
83  
    // update count in tab when tab isn't selected
84  
    onConceptChanges(concepts, -> {
85  
      labelCRUD?.updateEnclosingTabTitle();
86  
      //imageCRUD?.updateEnclosingTabTitle();
87  
      updateEnclosingTabTitleWithCount(galleryComponent, countConcepts(GalleryImage));
88  
    });
89  
    
90  
    uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL());
91  
    
92  
    scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me()));
93  
    testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me()));
94  
95  
    printWithMS("Making image streams");
96  
97  
    imageStream.directlyFeedInto(integralImages);
98  
    
99  
    integralImages.onNewElement(ii ->
100  
      scaledAndPosterizedImages.newElement(
101  
        scaleAndPosterize(ii, new SnPSettings(pixelRows, colors))));
102  
          
103  
    integralImages.onNewElement(r {
104  
      if (shouldRunScript()) rstRunScript.go();
105  
    });
106  
107  
    printWithMS("Almost done");    
108  
          
109  
    scaledAndPosterizedImages.onNewElement(img -> {
110  
      fpsCounter.inc();
111  
      setField(fps := iround(fpsCounter!));
112  
      
113  
      // display before analysis (old)
114  
      // isPosterized?.setImage_thisThread(img);
115  
      
116  
      // find regions
117  
      floodFill(img);
118  
      
119  
      // display after analysis so we can highlight a region
120  
      if (isPosterized != null) {
121  
        var img2 = highlightRegion(img, isPosterized, mainScreenRegions);
122  
        isPosterized.setImage_thisThread(img2);
123  
      }
124  
    });
125  
126  
    printWithMS("Starting screen cam");
127  
    
128  
    ownResource(screenCamTimer = new FlexibleRateTimer(fpsTarget, rEnter {
129  
      if (!enabled) ret;
130  
      watchTargetSelector?.updateScreenCount();
131  
      Timestamp deadline = tsNowPlusMS(1000/fpsTarget);
132  
      if (watchTarget cast WatchScreen)
133  
        imageStream.useScreen(watchTarget.screenNr-1);
134  
      else if (watchTarget cast WatchMouse)
135  
        imageStream.area(mouseArea(watchTarget.width, watchTarget.height));
136  
      imageStream.step();
137  
      long remaining = deadline.minus(tsNow());
138  
      remainingMSPerFrame.add(remaining);
139  
      setField(remainingMS := iround(remainingMSPerFrame!));
140  
    }));
141  
    screenCamTimer.start();
142  
    dm_onFieldChange fpsTarget(-> screenCamTimer.setFrequencyImmediately(fpsTarget));
143  
144  
    printWithMS("Starting dir watcher");
145  
    
146  
    startDirWatcher();
147  
    
148  
    printWithMS("Gathering images from disk");
149  
    for (f : allImageFiles(galleryDir()))
150  
      addToGallery(f);
151  
    printWithMS("Got dem images");
152  
  }
153  
  
154  
  // convert to color & highlight region
155  
  BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) {
156  
    var pixels = image.getRGBPixels();
157  
    highlightRegion(pixels, is, regions);
158  
    ret bufferedImage(pixels, image.getWidth(), image.getHeight());
159  
  }
160  
  
161  
  // convert to color & highlight region
162  
  void highlightRegion(int[] pixels, ImageSurface is, BWImage_FastRegions regions) {
163  
    var mouse = is.mousePosition;
164  
    int color = 0xFF008000, holeColor = 0xFFFFB266;
165  
    if (mouse != null && regions != null) {
166  
      int iHighlightedRegion = regions.regionAt(mouse);
167  
168  
      if (showRegionsAsOutline) {
169  
        RegionBorder_innerPoints x = new(regions, iHighlightedRegion);
170  
        new BoolVar hole;
171  
        x.onNewTrace(isHole -> hole.set(isHole));
172  
        x.onFoundPoint(p ->
173  
          pixels[p.y*regions.w+p.x] = hole! ? holeColor : color);
174  
        x.run();
175  
      } else
176  
        regions.markRegionInPixelArray(pixels, iHighlightedRegion, color);
177  
    }
178  
  }
179  
  
180  
  void startDirWatcher {
181  
    fileWatcher = new FileWatchService;
182  
    fileWatcher.addRecursiveListener(picturesDir(), file -> {
183  
      if (!isImageFile(file)) ret;
184  
      addToGallery(file);
185  
    });
186  
  }
187  
  
188  
  ImageSurface stdImageSurface() {
189  
    ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false);
190  
  }
191  
  
192  
  visualize {
193  
    gallery = new JGallery;
194  
    galleryComponent = gallery.visualize();
195  
196  
    new AWTOnConceptChanges(concepts, galleryComponent, -> {
197  
      gallery.setImageFiles(map(list(GalleryImage), i -> i.path));
198  
    }).install();
199  
    
200  
    /*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr());
201  
    screenSelector.compactLayout(true);
202  
    screenSelector.hideIfOnlyOne(true);
203  
    screenSelector.screenLabel("");*/
204  
    isPosterized = stdImageSurface();
205  
    
206  
    //isRegions = stdImageSurface();
207  
    isTestScreen = stdImageSurface();
208  
    
209  
    // when test screen is visible, do the animation
210  
    awtEvery(isTestScreen, 1000/20, r stepAnimation);
211  
    
212  
    var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo()));
213  
    
214  
    //print("Labels: " + list(concepts, Label));
215  
    labelCRUD = new SimpleCRUD_v2(concepts, Label);
216  
    labelCRUD.hideFields("globalID");
217  
    labelCRUD.addCountToEnclosingTab(true);
218  
    labelCRUD.itemToMap_inner2 = l
219  
      -> litorderedmap(
220  
        "Name" := l.name,
221  
        "# Examples" := n2(countConcepts(concepts, Example, label := l)));
222  
    
223  
    imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage);
224  
    imageCRUD.addCountToEnclosingTab(true);
225  
    imageCRUD.itemToMap_inner2 = img
226  
      -> litorderedmap(
227  
        "File" := fileName(img.path),
228  
        "Folder" := dirPath(img.path));
229  
    
230  
    // main visual
231  
232  
    var studyPanel = new StudyPanel;
233  
234  
    watchTargetSelector = new WatchTargetSelector;
235  
236  
    watchScreenPane = borderlessScrollPane(jHigherScrollPane(
237  
      jfullcenter(vstack(
238  
        withLeftAndRightMargin(hstack(
239  
          dm_rcheckBox enabled("Watch"),
240  
          watchTargetSelector.visualize(),
241  
          jlabel(" in "),
242  
          withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)),
243  
          withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)),
244  
          , /* speed info can also go here */
245  
        )),
246  
        verticalStrut(2),
247  
        withSideMargins(centerAndEastWithMargin(
248  
          dm_fieldLabel screenCamRecognitionOutput(),
249  
          jSpeedInfo)),
250  
      ))));
251  
252  
    tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout,
253  
      "Screen Cam" := screenCamPanel(),
254  
      /*WithToolTip("Regions", "Visualize regions detected on screen")
255  
        := jscroll_centered_borderless(isRegions),*/
256  
      WithToolTip("Study", "Here you can analyze gallery images")
257  
        := withTopAndBottomMargin(studyPanel.visualize()),
258  
      WithToolTip("Simple Script", "Run a simple (linear) Gazelle V script")
259  
        := withBottomMargin(scriptRunner.scriptAndResultPanel()),
260  
      WithToolTip("Full Script", "Run a \"left-arrow script\"")
261  
        := withBottomMargin(newScriptPanel()),
262  
      WithToolTip("Gallery", "Gallery view with preview images")
263  
        := withBottomMargin(galleryComponent),
264  
      WithToolTip("Gallery 2", "Gallery view with more functions (delete etc)")
265  
        := wrapCRUD(imageCRUD),
266  
      WithToolTip("Labels", "Manage labels (image markers)")
267  
        := wrapCRUD(labelCRUD),
268  
    ));
269  
270  
    var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled(""));
271  
    var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image",
272  
      jlabel("Screen Cam"));
273  
    tabComponentClickFixer(lblScreenCam);
274  
    var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam);
275  
    
276  
    replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab);
277  
    
278  
    // for tab titles
279  
    labelCRUD.update();
280  
    imageCRUD.update();
281  
    
282  
    initUIURLs();
283  
284  
    var urlBar = uiURLs.urlBar();
285  
    setToolTip(uiURLs.comboBox, "UI navigation system (partially populated)");
286  
287  
    scpMain = singleComponentPanel();
288  
    uiURLs.scp(scpMain);
289  
290  
    uiURLs.showUIURL(uiURL);
291  
              
292  
    var vis = northAndCenter(
293  
      withSideAndTopMargin(urlBar),
294  
      //centerAndSouthOrEast(horizontalLayout,
295  
        /*withBottomMargin*/(scpMain),
296  
      /*  watchScreenPane
297  
      )*/
298  
    );
299  
    setHorizontalMarginForAllButtons(vis, 4);
300  
    ret vis;
301  
  }
302  
303  
  JComponent screenCamPanel() {
304  
    ret centerAndSouthOrEast(horizontalLayout,
305  
      withTools(isPosterized),
306  
      watchScreenPane);
307  
  }
308  
  
309  
  S speedInfo() {
310  
    ret "FPS " + fps + " idle " + remainingMS + " ms";
311  
  }
312  
  
313  
  S speedInfo_long() {
314  
    ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core"
315  
      + " (of targeted " + fpsTarget + " FPS)";
316  
  }
317  
  
318  
  void floodFill(BWImage img) {
319  
    BWImage_FastRegions ff = new(img);
320  
    //ff.tolerance(0);
321  
    ff.collectBounds();
322  
    functionTimings.time("Regions", ff);
323  
    mainScreenRegions = ff;
324  
    if (isRegions != null && isRegions.isShowing_quick()) {
325  
      //print("Showing regions image");
326  
      isRegions.setImage_thisThread(ff.regionsImage());
327  
    }
328  
    S text = nRegions(ff.regionCount());
329  
    setField(screenCamRecognitionOutput := text);
330  
    setEnclosingTabTitle(isRegions, text);
331  
  }
332  
  
333  
  bool useErrorHandling() { false; }
334  
  
335  
  S renderFunctionTimings() {
336  
    ret lines(ciSorted(map(functionTimings!, (f, avg) ->
337  
      firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")")));
338  
  }
339  
  
340  
  transient long _runScript_idx;
341  
342  
  void _runScript() {
343  
    long n = integralImages.elementCount();
344  
    if (n > _runScript_idx) {
345  
      _runScript_idx = n;
346  
347  
      scriptRunner.parseAndRunOn(integralImages!);
348  
    }
349  
  }
350  
  
351  
  bool shouldRunScript() {
352  
    ret isShowing(scriptRunner.scpScriptResult);
353  
  }
354  
  
355  
  void stepAnimation {
356  
    if (!animate) ret;
357  
    if (animation == null) {
358  
      animation = new AnimatedLine;
359  
      animation.start();
360  
    }
361  
    animation.nextFrame();
362  
    var img = whiteImage(animation.w, animation.h);
363  
    animation.setGraphics(createGraphics(img));
364  
    animation.paint();
365  
    isTestScreen?.setImage(img);
366  
    
367  
    var ii = bwIntegralImage_withMeta(img);
368  
    testScreenScriptRunner.parseAndRunOn(ii);
369  
  }
370  
  
371  
  JComponent testScreenPanel() {
372  
    ret centerAndSouthWithMargin(
373  
      hsplit(
374  
        northAndCenterWithMargin(centerAndEastWithMargin(
375  
          jlabel("Input"), dm_fieldCheckBox animate()),
376  
        jscroll_centered_borderless(isTestScreen)),
377  
        
378  
        northAndCenterWithMargin(centerAndEastWithMargin(
379  
          jlabel("Output"), testScreenScriptRunner.lblScore),
380  
        testScreenScriptRunner.scpScriptResult)
381  
      ),
382  
        
383  
      testScreenScriptRunner.scriptInputField()
384  
    );
385  
  }
386  
  
387  
  L popDownItems() {
388  
    ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout",
389  
      -> horizontalLayout,
390  
      b -> setField(horizontalLayout := b)));
391  
  }
392  
  
393  
  void unvisualize {} // don't zero transient component fields
394  
  
395  
  void resetTimings { functionTimings.reset(); }
396  
  
397  
  // add tool side bar to image surface
398  
  JComponent withTools(ImageSurface is) {
399  
    new ImageSurface_PositionToolTip(is);
400  
    ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3,
401  
      jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery),
402  
    ));
403  
  }
404  
  
405  
  class WatchTargetSelector {
406  
    JComboBox<WatchTarget> cb = jComboBox();
407  
    int screenCount;
408  
    
409  
    visualize {
410  
      updateList();
411  
      print("Selecting watchTarget: " + watchTarget);
412  
      selectItem(cb, watchTarget);
413  
      main onChange(cb, watchTarget -> {
414  
        setField(+watchTarget);
415  
        print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget);
416  
      });
417  
      ret cb;
418  
    }
419  
    
420  
    void updateScreenCount() {
421  
      if (screenCount != screenCount())
422  
        updateList();
423  
    }
424  
    
425  
    void updateList() swing {
426  
      setComboBoxItems(cb, makeWatchTargets());
427  
    }
428  
    
429  
    L<WatchTarget> makeWatchTargets() {
430  
      ret flattenToList(
431  
        countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)),
432  
        new WatchMouse
433  
      );
434  
    }
435  
  }
436  
437  
  class SnPSelector {
438  
    settable new SnPSettings settings;
439  
    
440  
    event change;
441  
442  
    visualize {
443  
      var colors = jspinner(settings.colors, 2, 256);
444  
      main onChange(colors, -> {
445  
        settings.colors = intFromSpinner(colors);
446  
        change();
447  
      });
448  
      
449  
      var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows);
450  
      main onChange(pixelRows, -> {
451  
        settings.pixelRows = intFromSpinner(pixelRows);
452  
        change();
453  
      });
454  
      
455  
      ret hstack(
456  
        colors,
457  
        jlabel(" colors @ "),
458  
        pixelRows,
459  
        jlabel(" p"));
460  
    }
461  
  }
462  
  
463  
  JComponent wrapCRUD(SimpleCRUD_v2 crud) {
464  
    ret crud == null ?: withTopAndBottomMargin(jRaisedSection(withMargin(crud.make_dontStartBots())));
465  
  }
466  
  
467  
  File galleryDir() {
468  
    ret picturesDir(gazelle22_imagesSubDirName());
469  
  }
470  
  
471  
  void saveScreenshotToGallery enter {
472  
    var img = imageStream!;
473  
    saveImageWithCounter(galleryDir(), "Screenshot", img);
474  
  }
475  
  
476  
  GalleryImage addToGallery(File imgFile) {
477  
    if (!isImageFile(imgFile)) null;
478  
    var img = uniq(concepts, GalleryImage, path := imgFile);
479  
    //printVars("addToGallery", +imgFile, +img);
480  
    ret img;
481  
  }
482  
483  
  BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) {
484  
    ret posterizeBWImage_withMeta(settings.colors,
485  
      scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii));
486  
  }
487  
488  
  class StudyPanel {
489  
    new SnPSelector snpSelector;
490  
    GalleryImage image;
491  
    BufferedImage originalImage;
492  
    ImageSurface isOriginal = stdImageSurface();
493  
    ImageSurface isOriginalWithRegions = stdImageSurface();
494  
    ImageSurface isPosterized = stdImageSurface();
495  
    SingleComponentPanel scp = singleComponentPanel();
496  
    SingleComponentPanel analysisPanel = singleComponentPanel();
497  
    SingleComponentPanel scpExampleCRUD = singleComponentPanel();
498  
    ConceptsComboBox<GalleryImage> cbImage = new(concepts, GalleryImage);
499  
    ImageToRegions itr;
500  
    int iSelectedRegion;
501  
    SimpleCRUD_v2<SavedRegion> regionCRUD;
502  
    SimpleCRUD_v2<Example> exampleCRUD;
503  
504  
    *() {
505  
      //cbImage.sortTheList = l -> sortConceptsByIDDesc(l);
506  
      cbImage.sortTheList = l -> sortedByComparator(l, (a, b)
507  
        -> cmpAlphanumIC(fileName(a.path), fileName(b.path)));
508  
509  
      main onChangeAndNow(cbImage, img -> dm_q(me(), r {
510  
        image = img;
511  
        scp.setComponent(studyImagePanel());
512  
      }));
513  
514  
      isPosterized.removeAllTools();
515  
      isPosterized.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate));
516  
517  
      imageSurfaceOnLeftMouseDown(isPosterized, pt -> dm_q(me(), r { chooseRegionAt(pt) }));
518  
519  
      snpSelector.onChange(r_dm_q(me(), l0 runSnP));
520  
    }
521  
522  
    void chooseRegionAt(Pt p) {
523  
      if (itr == null) ret;
524  
      iSelectedRegion = itr.regions.regionAt(p);
525  
526  
      if (iSelectedRegion > 0) {
527  
        var savedRegion = uniq_returnIfNew(concepts, SavedRegion,
528  
          +image,
529  
          snpSettings := snpSelector.settings.cloneMe(),
530  
          regionIndex := iSelectedRegion);
531  
532  
        // it's new, add values
533  
        if (savedRegion != null) {
534  
          print("Saved new region!");
535  
          var bitMatrix = toScanlineBitMatrix(itr.regions.regionBitMatrix(iSelectedRegion));
536  
          cset(savedRegion, +bitMatrix);
537  
        }
538  
      }
539  
      
540  
      regionUpdate();
541  
    }
542  
543  
    // load new image
544  
    JComponent studyImagePanel() { 
545  
      if (image == null) null;
546  
      originalImage = loadImage2(image.path);
547  
      if (originalImage == null) ret jcenteredlabel("Image not found");
548  
549  
      isOriginal.setImage(originalImage);
550  
      isOriginalWithRegions.setImage(originalImage);
551  
552  
      iSelectedRegion = 0;
553  
      itr = new ImageToRegions(originalImage, snpSelector.settings);
554  
      runSnP();
555  
556  
      regionCRUD = new SimpleCRUD_v2<SavedRegion>(concepts, SavedRegion);
557  
      regionCRUD.addFilter(+image);
558  
      regionCRUD
559  
        .showSearchBar(false)
560  
        .showAddButton(false)
561  
        .showEditButton(false);
562  
        
563  
      regionCRUD.itemToMap_inner2 = region -> {
564  
        var examples = conceptsWhere(concepts, Example, item := region);
565  
        
566  
        ret litorderedmap(
567  
          "Pixels" := region.bitMatrix == null ? "-" : n2(region.bitMatrix.pixelCount()),
568  
          //"Shape" := "TODO",
569  
          "Labels" := joinWithComma(map(examples, e -> e.label)),
570  
571  
          // TODO: scale using snpsettings
572  
          "Position" := region.bitMatrix == null ? "-" : region.bitMatrix.boundingBoxOfTrueBits());
573  
      };
574  
575  
      var regionCRUDComponent = regionCRUD.make_dontStartBots();
576  
      regionCRUD.onSelectionChanged(l0 updateExampleCRUD);
577  
578  
      ret hsplit(
579  
        jtabs(
580  
          "Image" := jscroll_centered_borderless(isOriginal),
581  
          "Image + regions" := jscroll_centered_borderless(isOriginalWithRegions),
582  
          "Posterized" := jscroll_centered_borderless(isPosterized),
583  
        ),
584  
        vsplit(
585  
          analysisPanel,
586  
          hsplit(
587  
            jCenteredSection("Saved regions", regionCRUDComponent),
588  
            scpExampleCRUD)
589  
          ));
590  
    }
591  
592  
    void runSnP {
593  
      if (itr == null) ret;
594  
      itr.run();
595  
      regionUpdate();
596  
    }
597  
598  
    void regionUpdate {
599  
      if (itr == null) ret;
600  
601  
      var pixels = itr.posterized.getRGBPixels();
602  
      
603  
      // hovering region marked green
604  
      highlightRegion(pixels, isPosterized, itr.regions);
605  
606  
      // selected region marked blue
607  
      itr.regions.markRegionInPixelArray(pixels, iSelectedRegion, 0xFFADD8E6);
608  
609  
      var highlighted = bufferedImage(pixels, itr.posterized.getWidth(), itr.posterized.getHeight());
610  
611  
      isPosterized.setImage(highlighted);
612  
      isPosterized.performAutoZoom(); // seems to be necessary for some reason
613  
614  
      updateAnalysis();
615  
    }
616  
617  
    void updateAnalysis {
618  
      new LS lines;
619  
      lines.add(nRegions(itr.regions.regionCount()));
620  
      lines.add("Selected region: " + iSelectedRegion);
621  
      S text = lines_rtrim(lines);
622  
      analysisPanel.setComponent(jscroll(jMultiLineLabel(text)));
623  
    }
624  
625  
    void updateExampleCRUD {
626  
      var region = regionCRUD.selected();
627  
      if (region == null) ret with scpExampleCRUD.set(null);
628  
      
629  
      exampleCRUD = new SimpleCRUD_v2<Example>(concepts, Example);
630  
      exampleCRUD.entityName = -> "label for region";
631  
      exampleCRUD.addFilter(item := region);
632  
      exampleCRUD.showSearchBar(false);
633  
      scpExampleCRUD.set(jCenteredSection("Labels for region",
634  
        exampleCRUD.make_dontStartBots()));
635  
    }
636  
637  
    void importImage {
638  
      new JFileChooser fc;
639  
      if (fc.showOpenDialog(cbImage) == JFileChooser.APPROVE_OPTION) {
640  
        File file = fc.getSelectedFile().getAbsoluteFile();
641  
        if (!isImageFile(file)) ret with infoMessage("Not an image file");
642  
        var img = addToGallery(file);
643  
        waitUntil(250, 5.0, -> comboBoxContainsItem(cbImage, img));
644  
        setSelectedItem(cbImage, img);
645  
      }
646  
    }
647  
648  
    visual
649  
      jRaisedSection(northAndCenterWithMargins(
650  
        centerAndEast(withLabel("Study", cbImage),
651  
          hstack(
652  
            jlabel(" in "),
653  
            snpSelector.visualize(),
654  
            horizontalStrut(10),
655  
            jPopDownButton_noText(
656  
              "Import image...", rThreadEnter importImage,
657  
            ),
658  
            )),
659  
        scp));
660  
  } // end of StudyPanel
661  
662  
  // S&P, then regions
663  
  class ImageToRegions {
664  
    BufferedImage inputImage;
665  
    BWIntegralImage ii;
666  
    SnPSettings snpSettings;
667  
    BWImage posterized;
668  
    BWImage_FastRegions regions;
669  
670  
    *(BufferedImage *inputImage, SnPSettings *snpSettings) {}
671  
672  
    run {
673  
      ii = bwIntegralImage_withMeta(inputImage);
674  
      posterized = scaleAndPosterize(ii, snpSettings);
675  
      regions = new BWImage_FastRegions(posterized);
676  
      regions.collectBounds();
677  
      functionTimings.time("Regions", regions);
678  
    }
679  
  }
680  
681  
  void initUIURLs {
682  
    uiURLs.put("Main Tabs", ->
683  
      horizontalLayout ? withMargin(tabs) : withSideMargin(tabs));
684  
      
685  
    uiURLs.put("Screen Cam FPS", -> jFullCenter(
686  
      vstackWithSpacing(
687  
        jCenteredLabel("FPS target (frames per second) for screen cam:"),
688  
        jFullCenter(dm_spinner fpsTarget(1, 60)),
689  
        jCenteredLabel("(You can lower this value if you have a slower computer)"))));
690  
691  
    uiURLs.put("Timings", -> {
692  
      JTextArea taTimings = jTextArea_noUndo();
693  
      awtEveryAndNow(taTimings, .5, r {
694  
        setText(taTimings, renderFunctionTimings())
695  
      });
696  
      ret withRightAlignedButtons(taTimings,
697  
        Reset := r resetTimings);
698  
    });
699  
700  
    uiURLs.put("Test Screen" := -> testScreenPanel());
701  
702  
    //uiURLs.put("New Script" := -> newScriptPanel());
703  
      
704  
    // add more ui urls here
705  
  }
706  
707  
  JComponent newScriptPanel() {
708  
    //var ta = dm_textArea newScript();
709  
    var ta = dm_syntaxTextArea newScript();
710  
    
711  
    awtCalcEvery(ta.textArea(), 1.0, r_dm_q(r compileNewScript));
712  
    ret jCenteredSection("Left-arrow style script",
713  
      centerAndSouthWithMargin(
714  
        ta.visualize(),
715  
        centerAndEastWithMargin(
716  
          dm_label newScriptCompileResult(),
717  
          jbutton("Run" := rThreadEnter runNewScript))));
718  
  }
719  
720  
  void compileNewScript {
721  
    var newScript = this.newScript;
722  
    var result = newScriptCompileResult;
723  
    if (result == null || !eq(result.script, newScript)) {
724  
      try {
725  
        result = new NewScriptCompileResult;
726  
        result.script = newScript;
727  
        result.parser = new GazelleV_LeftArrowScriptParser;
728  
        result.parser.allowTheWorld(mc(), utils.class);
729  
        result.parsedScript = result.parser.parse(result.script);
730  
        print(result.parsedScript);
731  
      } catch print e {
732  
        result.compileError = e;
733  
      }
734  
      setField(newScriptCompileResult := result);
735  
    }
736  
  }
737  
738  
  void runNewScript {
739  
    //showText_fast_noWrap("Script Error", renderStackTrace(e));
740  
    dm_runInQAndWait(r compileNewScript);
741  
    var result = newScriptCompileResult;
742  
    if (result.parsedScript != null)
743  
      result.result = okOrError(-> result.parsedScript!);
744  
  }
745  
746  
  class NewScriptCompileResult {
747  
    S script;
748  
    GazelleV_LeftArrowScriptParser parser;
749  
    Throwable compileError;
750  
    GazelleV_LeftArrowScript.Script parsedScript;
751  
    OKOrError result;
752  
753  
    toString {
754  
      ret compileError != null ? exceptionToStringShorter(compileError) : "Compiled OK";
755  
    }
756  
  }
757  
} // end of module
758  
759  
concept Label > ConceptWithGlobalID {
760  
  S name;
761  
  //new RefL examples;
762  
  
763  
  toString { ret /*"Label " +*/ name; }
764  
}
765  
766  
concept GalleryImage {
767  
  File path;
768  
769  
  toString { ret /*"[" + id + "] " +*/ fileName(path); }
770  
}
771  
772  
concept SavedRegion {
773  
  new Ref image; // e.g. a GalleryImage
774  
  SnPSettings snpSettings;
775  
  int regionIndex; // region number relative to snpSettings
776  
  //Rect bounds; // get it from bitMatrix instead
777  
  ScanlineBitMatrix bitMatrix;
778  
  //new RefL<Label> labels;
779  
}
780  
781  
concept Example {
782  
  new Ref<Label> label;
783  
  new Ref item; // e.g. a SavedRegion
784  
  double confidence = 1;
785  
}
786  
787  
concept IfThenTheory {
788  
  new Ref if_;
789  
  new Ref then;
790  
}
791  
792  
sclass WatchTarget {}
793  
794  
// screenNr: 1 = screen 1 etc
795  
srecord WatchScreen(int screenNr) > WatchTarget {
796  
  toString { ret "Screen " + screenNr; }
797  
}
798  
799  
srecord WatchMouse(int width, int height) > WatchTarget {
800  
  WatchMouse() {
801  
    height = 256;
802  
    width = iround(height*16.0/9);
803  
  }
804  
  
805  
  toString { ret "Mouse"; }
806  
}
807  
808  
// SnP = Scale and Posterize
809  
srecord SnPSettings(int pixelRows, int colors) {
810  
  SnPSettings() { pixelRows = 128; colors = 8; }
811  
812  
  SnPSettings cloneMe() { ret shallowClone(this); }
813  
}
814  
815  
/*asclass RegionPaintMode {
816  
}
817  
818  
sclass RegionPaintMode_Normal > RegionPaintMode {
819  
}
820  
821  
sclass RegionPaintMode_Outline > RegionPaintMode {
822  
}*/
823  
824  
// include functions that scripts may want to use
825  
826  
please include function leftScreenBounds.
827  
please include function rightScreenBounds.
828  
please include class ScreenOverlay.
829  
please include function mergeRects.
830  
please include class TranslucentWindowTest.

Author comment

Began life as a copy of #1033862

download  show line numbers  debug dex  old transpilations   

Travelled to 2 computer(s): bhatertpkbcr, mqqgnosmbjvj

No comments. add comment

Snippet ID: #1034012
Snippet name: Gazelle Screen Cam / Gazelle 22 Module [backup]
Eternal ID of this version: #1034012/1
Text MD5: b8a0771ca5647560ebb13c07c14b303f
Author: stefan
Category: javax / gazelle v
Type: JavaX source code (Dynamic Module)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2022-01-17 00:16:47
Source code size: 26413 bytes / 830 lines
Pitched / IR pitched: No / No
Views / Downloads: 138 / 151
Referenced in: [show references]