| 1 | !7 | 
| 2 | |
| 3 | // Note: It's ok again to use db_mainConcepts() | 
| 4 | |
| 5 | !include early #1033884 // Compact Module Include Gazelle V [dev version] | 
| 6 | |
| 7 | module GazelleScreenCam {
 | 
| 8 | !include early #1025212 // +Enabled without visualization | 
| 9 | |
| 10 | int pixelRows = 128, colors = 8; | 
| 11 | S script = "64p 8c gradientImage"; | 
| 12 | S testScreenScript; | 
| 13 | S newScript; | 
| 14 | bool animate = true; | 
| 15 | bool horizontalLayout; // flat layout | 
| 16 | int fpsTarget = 20; | 
| 17 | |
| 18 | WatchTarget watchTarget; | 
| 19 | |
| 20 | transient NewScriptCompileResult newScriptCompileResult; | 
| 21 | transient ImageSurface isPosterized, isRegions, isTestScreen; | 
| 22 | transient new ScreenCamStream imageStream; | 
| 23 | transient new BWIntegralImageStream integralImages; | 
| 24 | transient new SourceTriggeredStream<BWImage> scaledAndPosterizedImages; | 
| 25 | transient new DoubleFPSCounter fpsCounter; | 
| 26 | transient int fps; | 
| 27 | //transient ScreenSelectorRadioButtons screenSelector; | 
| 28 | transient WatchTargetSelector watchTargetSelector; | 
| 29 | transient new RollingAverage remainingMSPerFrame; | 
| 30 | transient int remainingMS; | 
| 31 | |
| 32 | transient new FunctionTimings<S> functionTimings; | 
| 33 | |
| 34 | transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript); | 
| 35 | |
| 36 | transient JGazelleVScriptRunner scriptRunner; | 
| 37 | transient JGazelleVScriptRunner testScreenScriptRunner; | 
| 38 | |
| 39 | transient Animation animation; | 
| 40 | |
| 41 | transient BWImage_FastRegions mainScreenRegions; | 
| 42 | |
| 43 | transient UIURLSystem uiURLs; | 
| 44 | |
| 45 | // set by host | 
| 46 | transient Concepts concepts; | 
| 47 | |
| 48 | S uiURL = "Main Tabs"; | 
| 49 | |
| 50 | transient FileWatchService fileWatcher; | 
| 51 | |
| 52 | transient SimpleCRUD_v2<Label> labelCRUD; | 
| 53 | transient SimpleCRUD_v2<GalleryImage> imageCRUD; | 
| 54 | |
| 55 | transient JGallery gallery; | 
| 56 | transient JComponent galleryComponent; | 
| 57 | |
| 58 | transient FlexibleRateTimer screenCamTimer; | 
| 59 | transient SingleComponentPanel scpMain; | 
| 60 | transient JTabbedPane tabs; | 
| 61 | transient bool showRegionsAsOutline = true; | 
| 62 | transient JComponent watchScreenPane; | 
| 63 | transient S screenCamRecognitionOutput; | 
| 64 | |
| 65 |   start {
 | 
| 66 | dm_onFieldChange horizontalLayout( | 
| 67 | //r dm_revisualize // deh buggy | 
| 68 | r dm_reload | 
| 69 | ); | 
| 70 | |
| 71 | // non-standalone mode (doesn't work yet) | 
| 72 |     if (concepts == null) {
 | 
| 73 |       printWithMS("Starting DB");
 | 
| 74 | db(); | 
| 75 | concepts = db_mainConcepts(); | 
| 76 | } else | 
| 77 | assertSame(concepts, db_mainConcepts()); | 
| 78 | |
| 79 | // make indexes | 
| 80 | indexConceptField(concepts, GalleryImage, "path"); | 
| 81 | indexConceptField(concepts, Example, "item"); | 
| 82 | |
| 83 | // update count in tab when tab isn't selected | 
| 84 |     onConceptChanges(concepts, -> {
 | 
| 85 | labelCRUD?.updateEnclosingTabTitle(); | 
| 86 | //imageCRUD?.updateEnclosingTabTitle(); | 
| 87 | updateEnclosingTabTitleWithCount(galleryComponent, countConcepts(GalleryImage)); | 
| 88 | }); | 
| 89 | |
| 90 | uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL()); | 
| 91 | |
| 92 | scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me())); | 
| 93 | testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me())); | 
| 94 | |
| 95 |     printWithMS("Making image streams");
 | 
| 96 | |
| 97 | imageStream.directlyFeedInto(integralImages); | 
| 98 | |
| 99 | integralImages.onNewElement(ii -> | 
| 100 | scaledAndPosterizedImages.newElement( | 
| 101 | scaleAndPosterize(ii, new SnPSettings(pixelRows, colors)))); | 
| 102 | |
| 103 |     integralImages.onNewElement(r {
 | 
| 104 | if (shouldRunScript()) rstRunScript.go(); | 
| 105 | }); | 
| 106 | |
| 107 |     printWithMS("Almost done");    
 | 
| 108 | |
| 109 |     scaledAndPosterizedImages.onNewElement(img -> {
 | 
| 110 | fpsCounter.inc(); | 
| 111 | setField(fps := iround(fpsCounter!)); | 
| 112 | |
| 113 | // display before analysis (old) | 
| 114 | // isPosterized?.setImage_thisThread(img); | 
| 115 | |
| 116 | // find regions | 
| 117 | floodFill(img); | 
| 118 | |
| 119 | // display after analysis so we can highlight a region | 
| 120 |       if (isPosterized != null) {
 | 
| 121 | var img2 = highlightRegion(img, isPosterized, mainScreenRegions); | 
| 122 | isPosterized.setImage_thisThread(img2); | 
| 123 | } | 
| 124 | }); | 
| 125 | |
| 126 |     printWithMS("Starting screen cam");
 | 
| 127 | |
| 128 |     ownResource(screenCamTimer = new FlexibleRateTimer(fpsTarget, rEnter {
 | 
| 129 | if (!enabled) ret; | 
| 130 | watchTargetSelector?.updateScreenCount(); | 
| 131 | Timestamp deadline = tsNowPlusMS(1000/fpsTarget); | 
| 132 | if (watchTarget cast WatchScreen) | 
| 133 | imageStream.useScreen(watchTarget.screenNr-1); | 
| 134 | else if (watchTarget cast WatchMouse) | 
| 135 | imageStream.area(mouseArea(watchTarget.width, watchTarget.height)); | 
| 136 | imageStream.step(); | 
| 137 | long remaining = deadline.minus(tsNow()); | 
| 138 | remainingMSPerFrame.add(remaining); | 
| 139 | setField(remainingMS := iround(remainingMSPerFrame!)); | 
| 140 | })); | 
| 141 | screenCamTimer.start(); | 
| 142 | dm_onFieldChange fpsTarget(-> screenCamTimer.setFrequencyImmediately(fpsTarget)); | 
| 143 | |
| 144 |     printWithMS("Starting dir watcher");
 | 
| 145 | |
| 146 | startDirWatcher(); | 
| 147 | |
| 148 |     printWithMS("Gathering images from disk");
 | 
| 149 | for (f : allImageFiles(galleryDir())) | 
| 150 | addToGallery(f); | 
| 151 |     printWithMS("Got dem images");
 | 
| 152 | } | 
| 153 | |
| 154 | // convert to color & highlight region | 
| 155 |   BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) {
 | 
| 156 | var pixels = image.getRGBPixels(); | 
| 157 | highlightRegion(pixels, is, regions); | 
| 158 | ret bufferedImage(pixels, image.getWidth(), image.getHeight()); | 
| 159 | } | 
| 160 | |
| 161 | // convert to color & highlight region | 
| 162 |   void highlightRegion(int[] pixels, ImageSurface is, BWImage_FastRegions regions) {
 | 
| 163 | var mouse = is.mousePosition; | 
| 164 | int color = 0xFF008000, holeColor = 0xFFFFB266; | 
| 165 |     if (mouse != null && regions != null) {
 | 
| 166 | int iHighlightedRegion = regions.regionAt(mouse); | 
| 167 | |
| 168 |       if (showRegionsAsOutline) {
 | 
| 169 | RegionBorder_innerPoints x = new(regions, iHighlightedRegion); | 
| 170 | new BoolVar hole; | 
| 171 | x.onNewTrace(isHole -> hole.set(isHole)); | 
| 172 | x.onFoundPoint(p -> | 
| 173 | pixels[p.y*regions.w+p.x] = hole! ? holeColor : color); | 
| 174 | x.run(); | 
| 175 | } else | 
| 176 | regions.markRegionInPixelArray(pixels, iHighlightedRegion, color); | 
| 177 | } | 
| 178 | } | 
| 179 | |
| 180 |   void startDirWatcher {
 | 
| 181 | fileWatcher = new FileWatchService; | 
| 182 |     fileWatcher.addRecursiveListener(picturesDir(), file -> {
 | 
| 183 | if (!isImageFile(file)) ret; | 
| 184 | addToGallery(file); | 
| 185 | }); | 
| 186 | } | 
| 187 | |
| 188 |   ImageSurface stdImageSurface() {
 | 
| 189 | ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false); | 
| 190 | } | 
| 191 | |
| 192 |   visualize {
 | 
| 193 | gallery = new JGallery; | 
| 194 | galleryComponent = gallery.visualize(); | 
| 195 | |
| 196 |     new AWTOnConceptChanges(concepts, galleryComponent, -> {
 | 
| 197 | gallery.setImageFiles(map(list(GalleryImage), i -> i.path)); | 
| 198 | }).install(); | 
| 199 | |
| 200 | /*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr()); | 
| 201 | screenSelector.compactLayout(true); | 
| 202 | screenSelector.hideIfOnlyOne(true); | 
| 203 |     screenSelector.screenLabel("");*/
 | 
| 204 | isPosterized = stdImageSurface(); | 
| 205 | |
| 206 | //isRegions = stdImageSurface(); | 
| 207 | isTestScreen = stdImageSurface(); | 
| 208 | |
| 209 | // when test screen is visible, do the animation | 
| 210 | awtEvery(isTestScreen, 1000/20, r stepAnimation); | 
| 211 | |
| 212 | var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo())); | 
| 213 | |
| 214 |     //print("Labels: " + list(concepts, Label));
 | 
| 215 | labelCRUD = new SimpleCRUD_v2(concepts, Label); | 
| 216 |     labelCRUD.hideFields("globalID");
 | 
| 217 | labelCRUD.addCountToEnclosingTab(true); | 
| 218 | labelCRUD.itemToMap_inner2 = l | 
| 219 | -> litorderedmap( | 
| 220 | "Name" := l.name, | 
| 221 | "# Examples" := n2(countConcepts(concepts, Example, label := l))); | 
| 222 | |
| 223 | imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage); | 
| 224 | imageCRUD.addCountToEnclosingTab(true); | 
| 225 | imageCRUD.itemToMap_inner2 = img | 
| 226 | -> litorderedmap( | 
| 227 | "File" := fileName(img.path), | 
| 228 | "Folder" := dirPath(img.path)); | 
| 229 | |
| 230 | // main visual | 
| 231 | |
| 232 | var studyPanel = new StudyPanel; | 
| 233 | |
| 234 | watchTargetSelector = new WatchTargetSelector; | 
| 235 | |
| 236 | watchScreenPane = borderlessScrollPane(jHigherScrollPane( | 
| 237 | jfullcenter(vstack( | 
| 238 | withLeftAndRightMargin(hstack( | 
| 239 |           dm_rcheckBox enabled("Watch"),
 | 
| 240 | watchTargetSelector.visualize(), | 
| 241 |           jlabel(" in "),
 | 
| 242 |           withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)),
 | 
| 243 |           withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)),
 | 
| 244 | , /* speed info can also go here */ | 
| 245 | )), | 
| 246 | verticalStrut(2), | 
| 247 | withSideMargins(centerAndEastWithMargin( | 
| 248 | dm_fieldLabel screenCamRecognitionOutput(), | 
| 249 | jSpeedInfo)), | 
| 250 | )))); | 
| 251 | |
| 252 | tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout, | 
| 253 | "Screen Cam" := screenCamPanel(), | 
| 254 |       /*WithToolTip("Regions", "Visualize regions detected on screen")
 | 
| 255 | := jscroll_centered_borderless(isRegions),*/ | 
| 256 |       WithToolTip("Study", "Here you can analyze gallery images")
 | 
| 257 | := withTopAndBottomMargin(studyPanel.visualize()), | 
| 258 |       WithToolTip("Simple Script", "Run a simple (linear) Gazelle V script")
 | 
| 259 | := withBottomMargin(scriptRunner.scriptAndResultPanel()), | 
| 260 |       WithToolTip("Full Script", "Run a \"left-arrow script\"")
 | 
| 261 | := withBottomMargin(newScriptPanel()), | 
| 262 |       WithToolTip("Gallery", "Gallery view with preview images")
 | 
| 263 | := withBottomMargin(galleryComponent), | 
| 264 |       WithToolTip("Gallery 2", "Gallery view with more functions (delete etc)")
 | 
| 265 | := wrapCRUD(imageCRUD), | 
| 266 |       WithToolTip("Labels", "Manage labels (image markers)")
 | 
| 267 | := wrapCRUD(labelCRUD), | 
| 268 | )); | 
| 269 | |
| 270 |     var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled(""));
 | 
| 271 |     var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image",
 | 
| 272 |       jlabel("Screen Cam"));
 | 
| 273 | tabComponentClickFixer(lblScreenCam); | 
| 274 | var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam); | 
| 275 | |
| 276 | replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab); | 
| 277 | |
| 278 | // for tab titles | 
| 279 | labelCRUD.update(); | 
| 280 | imageCRUD.update(); | 
| 281 | |
| 282 | initUIURLs(); | 
| 283 | |
| 284 | var urlBar = uiURLs.urlBar(); | 
| 285 | setToolTip(uiURLs.comboBox, "UI navigation system (partially populated)"); | 
| 286 | |
| 287 | scpMain = singleComponentPanel(); | 
| 288 | uiURLs.scp(scpMain); | 
| 289 | |
| 290 | uiURLs.showUIURL(uiURL); | 
| 291 | |
| 292 | var vis = northAndCenter( | 
| 293 | withSideAndTopMargin(urlBar), | 
| 294 | //centerAndSouthOrEast(horizontalLayout, | 
| 295 | /*withBottomMargin*/(scpMain), | 
| 296 | /* watchScreenPane | 
| 297 | )*/ | 
| 298 | ); | 
| 299 | setHorizontalMarginForAllButtons(vis, 4); | 
| 300 | ret vis; | 
| 301 | } | 
| 302 | |
| 303 |   JComponent screenCamPanel() {
 | 
| 304 | ret centerAndSouthOrEast(horizontalLayout, | 
| 305 | withTools(isPosterized), | 
| 306 | watchScreenPane); | 
| 307 | } | 
| 308 | |
| 309 |   S speedInfo() {
 | 
| 310 | ret "FPS " + fps + " idle " + remainingMS + " ms"; | 
| 311 | } | 
| 312 | |
| 313 |   S speedInfo_long() {
 | 
| 314 | ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core" | 
| 315 | + " (of targeted " + fpsTarget + " FPS)"; | 
| 316 | } | 
| 317 | |
| 318 |   void floodFill(BWImage img) {
 | 
| 319 | BWImage_FastRegions ff = new(img); | 
| 320 | //ff.tolerance(0); | 
| 321 | ff.collectBounds(); | 
| 322 |     functionTimings.time("Regions", ff);
 | 
| 323 | mainScreenRegions = ff; | 
| 324 |     if (isRegions != null && isRegions.isShowing_quick()) {
 | 
| 325 |       //print("Showing regions image");
 | 
| 326 | isRegions.setImage_thisThread(ff.regionsImage()); | 
| 327 | } | 
| 328 | S text = nRegions(ff.regionCount()); | 
| 329 | setField(screenCamRecognitionOutput := text); | 
| 330 | setEnclosingTabTitle(isRegions, text); | 
| 331 | } | 
| 332 | |
| 333 |   bool useErrorHandling() { false; }
 | 
| 334 | |
| 335 |   S renderFunctionTimings() {
 | 
| 336 | ret lines(ciSorted(map(functionTimings!, (f, avg) -> | 
| 337 |       firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")")));
 | 
| 338 | } | 
| 339 | |
| 340 | transient long _runScript_idx; | 
| 341 | |
| 342 |   void _runScript() {
 | 
| 343 | long n = integralImages.elementCount(); | 
| 344 |     if (n > _runScript_idx) {
 | 
| 345 | _runScript_idx = n; | 
| 346 | |
| 347 | scriptRunner.parseAndRunOn(integralImages!); | 
| 348 | } | 
| 349 | } | 
| 350 | |
| 351 |   bool shouldRunScript() {
 | 
| 352 | ret isShowing(scriptRunner.scpScriptResult); | 
| 353 | } | 
| 354 | |
| 355 |   void stepAnimation {
 | 
| 356 | if (!animate) ret; | 
| 357 |     if (animation == null) {
 | 
| 358 | animation = new AnimatedLine; | 
| 359 | animation.start(); | 
| 360 | } | 
| 361 | animation.nextFrame(); | 
| 362 | var img = whiteImage(animation.w, animation.h); | 
| 363 | animation.setGraphics(createGraphics(img)); | 
| 364 | animation.paint(); | 
| 365 | isTestScreen?.setImage(img); | 
| 366 | |
| 367 | var ii = bwIntegralImage_withMeta(img); | 
| 368 | testScreenScriptRunner.parseAndRunOn(ii); | 
| 369 | } | 
| 370 | |
| 371 |   JComponent testScreenPanel() {
 | 
| 372 | ret centerAndSouthWithMargin( | 
| 373 | hsplit( | 
| 374 | northAndCenterWithMargin(centerAndEastWithMargin( | 
| 375 |           jlabel("Input"), dm_fieldCheckBox animate()),
 | 
| 376 | jscroll_centered_borderless(isTestScreen)), | 
| 377 | |
| 378 | northAndCenterWithMargin(centerAndEastWithMargin( | 
| 379 |           jlabel("Output"), testScreenScriptRunner.lblScore),
 | 
| 380 | testScreenScriptRunner.scpScriptResult) | 
| 381 | ), | 
| 382 | |
| 383 | testScreenScriptRunner.scriptInputField() | 
| 384 | ); | 
| 385 | } | 
| 386 | |
| 387 |   L popDownItems() {
 | 
| 388 |     ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout",
 | 
| 389 | -> horizontalLayout, | 
| 390 | b -> setField(horizontalLayout := b))); | 
| 391 | } | 
| 392 | |
| 393 |   void unvisualize {} // don't zero transient component fields
 | 
| 394 | |
| 395 |   void resetTimings { functionTimings.reset(); }
 | 
| 396 | |
| 397 | // add tool side bar to image surface | 
| 398 |   JComponent withTools(ImageSurface is) {
 | 
| 399 | new ImageSurface_PositionToolTip(is); | 
| 400 | ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3, | 
| 401 | jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery), | 
| 402 | )); | 
| 403 | } | 
| 404 | |
| 405 |   class WatchTargetSelector {
 | 
| 406 | JComboBox<WatchTarget> cb = jComboBox(); | 
| 407 | int screenCount; | 
| 408 | |
| 409 |     visualize {
 | 
| 410 | updateList(); | 
| 411 |       print("Selecting watchTarget: " + watchTarget);
 | 
| 412 | selectItem(cb, watchTarget); | 
| 413 |       main onChange(cb, watchTarget -> {
 | 
| 414 | setField(+watchTarget); | 
| 415 |         print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget);
 | 
| 416 | }); | 
| 417 | ret cb; | 
| 418 | } | 
| 419 | |
| 420 |     void updateScreenCount() {
 | 
| 421 | if (screenCount != screenCount()) | 
| 422 | updateList(); | 
| 423 | } | 
| 424 | |
| 425 |     void updateList() swing {
 | 
| 426 | setComboBoxItems(cb, makeWatchTargets()); | 
| 427 | } | 
| 428 | |
| 429 |     L<WatchTarget> makeWatchTargets() {
 | 
| 430 | ret flattenToList( | 
| 431 | countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)), | 
| 432 | new WatchMouse | 
| 433 | ); | 
| 434 | } | 
| 435 | } | 
| 436 | |
| 437 |   class SnPSelector {
 | 
| 438 | settable new SnPSettings settings; | 
| 439 | |
| 440 | event change; | 
| 441 | |
| 442 |     visualize {
 | 
| 443 | var colors = jspinner(settings.colors, 2, 256); | 
| 444 |       main onChange(colors, -> {
 | 
| 445 | settings.colors = intFromSpinner(colors); | 
| 446 | change(); | 
| 447 | }); | 
| 448 | |
| 449 | var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows); | 
| 450 |       main onChange(pixelRows, -> {
 | 
| 451 | settings.pixelRows = intFromSpinner(pixelRows); | 
| 452 | change(); | 
| 453 | }); | 
| 454 | |
| 455 | ret hstack( | 
| 456 | colors, | 
| 457 |         jlabel(" colors @ "),
 | 
| 458 | pixelRows, | 
| 459 |         jlabel(" p"));
 | 
| 460 | } | 
| 461 | } | 
| 462 | |
| 463 |   JComponent wrapCRUD(SimpleCRUD_v2 crud) {
 | 
| 464 | ret crud == null ?: withTopAndBottomMargin(jRaisedSection(withMargin(crud.make_dontStartBots()))); | 
| 465 | } | 
| 466 | |
| 467 |   File galleryDir() {
 | 
| 468 | ret picturesDir(gazelle22_imagesSubDirName()); | 
| 469 | } | 
| 470 | |
| 471 |   void saveScreenshotToGallery enter {
 | 
| 472 | var img = imageStream!; | 
| 473 | saveImageWithCounter(galleryDir(), "Screenshot", img); | 
| 474 | } | 
| 475 | |
| 476 |   GalleryImage addToGallery(File imgFile) {
 | 
| 477 | if (!isImageFile(imgFile)) null; | 
| 478 | var img = uniq(concepts, GalleryImage, path := imgFile); | 
| 479 |     //printVars("addToGallery", +imgFile, +img);
 | 
| 480 | ret img; | 
| 481 | } | 
| 482 | |
| 483 |   BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) {
 | 
| 484 | ret posterizeBWImage_withMeta(settings.colors, | 
| 485 | scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii)); | 
| 486 | } | 
| 487 | |
| 488 |   class StudyPanel {
 | 
| 489 | new SnPSelector snpSelector; | 
| 490 | GalleryImage image; | 
| 491 | BufferedImage originalImage; | 
| 492 | ImageSurface isOriginal = stdImageSurface(); | 
| 493 | ImageSurface isOriginalWithRegions = stdImageSurface(); | 
| 494 | ImageSurface isPosterized = stdImageSurface(); | 
| 495 | SingleComponentPanel scp = singleComponentPanel(); | 
| 496 | SingleComponentPanel analysisPanel = singleComponentPanel(); | 
| 497 | SingleComponentPanel scpExampleCRUD = singleComponentPanel(); | 
| 498 | ConceptsComboBox<GalleryImage> cbImage = new(concepts, GalleryImage); | 
| 499 | ImageToRegions itr; | 
| 500 | int iSelectedRegion; | 
| 501 | SimpleCRUD_v2<SavedRegion> regionCRUD; | 
| 502 | SimpleCRUD_v2<Example> exampleCRUD; | 
| 503 | |
| 504 |     *() {
 | 
| 505 | //cbImage.sortTheList = l -> sortConceptsByIDDesc(l); | 
| 506 | cbImage.sortTheList = l -> sortedByComparator(l, (a, b) | 
| 507 | -> cmpAlphanumIC(fileName(a.path), fileName(b.path))); | 
| 508 | |
| 509 |       main onChangeAndNow(cbImage, img -> dm_q(me(), r {
 | 
| 510 | image = img; | 
| 511 | scp.setComponent(studyImagePanel()); | 
| 512 | })); | 
| 513 | |
| 514 | isPosterized.removeAllTools(); | 
| 515 | isPosterized.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate)); | 
| 516 | |
| 517 |       imageSurfaceOnLeftMouseDown(isPosterized, pt -> dm_q(me(), r { chooseRegionAt(pt) }));
 | 
| 518 | |
| 519 | snpSelector.onChange(r_dm_q(me(), l0 runSnP)); | 
| 520 | } | 
| 521 | |
| 522 |     void chooseRegionAt(Pt p) {
 | 
| 523 | if (itr == null) ret; | 
| 524 | iSelectedRegion = itr.regions.regionAt(p); | 
| 525 | |
| 526 |       if (iSelectedRegion > 0) {
 | 
| 527 | var savedRegion = uniq_returnIfNew(concepts, SavedRegion, | 
| 528 | +image, | 
| 529 | snpSettings := snpSelector.settings.cloneMe(), | 
| 530 | regionIndex := iSelectedRegion); | 
| 531 | |
| 532 | // it's new, add values | 
| 533 |         if (savedRegion != null) {
 | 
| 534 |           print("Saved new region!");
 | 
| 535 | var bitMatrix = toScanlineBitMatrix(itr.regions.regionBitMatrix(iSelectedRegion)); | 
| 536 | cset(savedRegion, +bitMatrix); | 
| 537 | } | 
| 538 | } | 
| 539 | |
| 540 | regionUpdate(); | 
| 541 | } | 
| 542 | |
| 543 | // load new image | 
| 544 |     JComponent studyImagePanel() { 
 | 
| 545 | if (image == null) null; | 
| 546 | originalImage = loadImage2(image.path); | 
| 547 |       if (originalImage == null) ret jcenteredlabel("Image not found");
 | 
| 548 | |
| 549 | isOriginal.setImage(originalImage); | 
| 550 | isOriginalWithRegions.setImage(originalImage); | 
| 551 | |
| 552 | iSelectedRegion = 0; | 
| 553 | itr = new ImageToRegions(originalImage, snpSelector.settings); | 
| 554 | runSnP(); | 
| 555 | |
| 556 | regionCRUD = new SimpleCRUD_v2<SavedRegion>(concepts, SavedRegion); | 
| 557 | regionCRUD.addFilter(+image); | 
| 558 | regionCRUD | 
| 559 | .showSearchBar(false) | 
| 560 | .showAddButton(false) | 
| 561 | .showEditButton(false); | 
| 562 | |
| 563 |       regionCRUD.itemToMap_inner2 = region -> {
 | 
| 564 | var examples = conceptsWhere(concepts, Example, item := region); | 
| 565 | |
| 566 | ret litorderedmap( | 
| 567 | "Pixels" := region.bitMatrix == null ? "-" : n2(region.bitMatrix.pixelCount()), | 
| 568 | //"Shape" := "TODO", | 
| 569 | "Labels" := joinWithComma(map(examples, e -> e.label)), | 
| 570 | |
| 571 | // TODO: scale using snpsettings | 
| 572 | "Position" := region.bitMatrix == null ? "-" : region.bitMatrix.boundingBoxOfTrueBits()); | 
| 573 | }; | 
| 574 | |
| 575 | var regionCRUDComponent = regionCRUD.make_dontStartBots(); | 
| 576 | regionCRUD.onSelectionChanged(l0 updateExampleCRUD); | 
| 577 | |
| 578 | ret hsplit( | 
| 579 | jtabs( | 
| 580 | "Image" := jscroll_centered_borderless(isOriginal), | 
| 581 | "Image + regions" := jscroll_centered_borderless(isOriginalWithRegions), | 
| 582 | "Posterized" := jscroll_centered_borderless(isPosterized), | 
| 583 | ), | 
| 584 | vsplit( | 
| 585 | analysisPanel, | 
| 586 | hsplit( | 
| 587 |             jCenteredSection("Saved regions", regionCRUDComponent),
 | 
| 588 | scpExampleCRUD) | 
| 589 | )); | 
| 590 | } | 
| 591 | |
| 592 |     void runSnP {
 | 
| 593 | if (itr == null) ret; | 
| 594 | itr.run(); | 
| 595 | regionUpdate(); | 
| 596 | } | 
| 597 | |
| 598 |     void regionUpdate {
 | 
| 599 | if (itr == null) ret; | 
| 600 | |
| 601 | var pixels = itr.posterized.getRGBPixels(); | 
| 602 | |
| 603 | // hovering region marked green | 
| 604 | highlightRegion(pixels, isPosterized, itr.regions); | 
| 605 | |
| 606 | // selected region marked blue | 
| 607 | itr.regions.markRegionInPixelArray(pixels, iSelectedRegion, 0xFFADD8E6); | 
| 608 | |
| 609 | var highlighted = bufferedImage(pixels, itr.posterized.getWidth(), itr.posterized.getHeight()); | 
| 610 | |
| 611 | isPosterized.setImage(highlighted); | 
| 612 | isPosterized.performAutoZoom(); // seems to be necessary for some reason | 
| 613 | |
| 614 | updateAnalysis(); | 
| 615 | } | 
| 616 | |
| 617 |     void updateAnalysis {
 | 
| 618 | new LS lines; | 
| 619 | lines.add(nRegions(itr.regions.regionCount())); | 
| 620 |       lines.add("Selected region: " + iSelectedRegion);
 | 
| 621 | S text = lines_rtrim(lines); | 
| 622 | analysisPanel.setComponent(jscroll(jMultiLineLabel(text))); | 
| 623 | } | 
| 624 | |
| 625 |     void updateExampleCRUD {
 | 
| 626 | var region = regionCRUD.selected(); | 
| 627 | if (region == null) ret with scpExampleCRUD.set(null); | 
| 628 | |
| 629 | exampleCRUD = new SimpleCRUD_v2<Example>(concepts, Example); | 
| 630 | exampleCRUD.entityName = -> "label for region"; | 
| 631 | exampleCRUD.addFilter(item := region); | 
| 632 | exampleCRUD.showSearchBar(false); | 
| 633 |       scpExampleCRUD.set(jCenteredSection("Labels for region",
 | 
| 634 | exampleCRUD.make_dontStartBots())); | 
| 635 | } | 
| 636 | |
| 637 |     void importImage {
 | 
| 638 | new JFileChooser fc; | 
| 639 |       if (fc.showOpenDialog(cbImage) == JFileChooser.APPROVE_OPTION) {
 | 
| 640 | File file = fc.getSelectedFile().getAbsoluteFile(); | 
| 641 |         if (!isImageFile(file)) ret with infoMessage("Not an image file");
 | 
| 642 | var img = addToGallery(file); | 
| 643 | waitUntil(250, 5.0, -> comboBoxContainsItem(cbImage, img)); | 
| 644 | setSelectedItem(cbImage, img); | 
| 645 | } | 
| 646 | } | 
| 647 | |
| 648 | visual | 
| 649 | jRaisedSection(northAndCenterWithMargins( | 
| 650 |         centerAndEast(withLabel("Study", cbImage),
 | 
| 651 | hstack( | 
| 652 |             jlabel(" in "),
 | 
| 653 | snpSelector.visualize(), | 
| 654 | horizontalStrut(10), | 
| 655 | jPopDownButton_noText( | 
| 656 | "Import image...", rThreadEnter importImage, | 
| 657 | ), | 
| 658 | )), | 
| 659 | scp)); | 
| 660 | } // end of StudyPanel | 
| 661 | |
| 662 | // S&P, then regions | 
| 663 |   class ImageToRegions {
 | 
| 664 | BufferedImage inputImage; | 
| 665 | BWIntegralImage ii; | 
| 666 | SnPSettings snpSettings; | 
| 667 | BWImage posterized; | 
| 668 | BWImage_FastRegions regions; | 
| 669 | |
| 670 |     *(BufferedImage *inputImage, SnPSettings *snpSettings) {}
 | 
| 671 | |
| 672 |     run {
 | 
| 673 | ii = bwIntegralImage_withMeta(inputImage); | 
| 674 | posterized = scaleAndPosterize(ii, snpSettings); | 
| 675 | regions = new BWImage_FastRegions(posterized); | 
| 676 | regions.collectBounds(); | 
| 677 |       functionTimings.time("Regions", regions);
 | 
| 678 | } | 
| 679 | } | 
| 680 | |
| 681 |   void initUIURLs {
 | 
| 682 |     uiURLs.put("Main Tabs", ->
 | 
| 683 | horizontalLayout ? withMargin(tabs) : withSideMargin(tabs)); | 
| 684 | |
| 685 |     uiURLs.put("Screen Cam FPS", -> jFullCenter(
 | 
| 686 | vstackWithSpacing( | 
| 687 |         jCenteredLabel("FPS target (frames per second) for screen cam:"),
 | 
| 688 | jFullCenter(dm_spinner fpsTarget(1, 60)), | 
| 689 |         jCenteredLabel("(You can lower this value if you have a slower computer)"))));
 | 
| 690 | |
| 691 |     uiURLs.put("Timings", -> {
 | 
| 692 | JTextArea taTimings = jTextArea_noUndo(); | 
| 693 |       awtEveryAndNow(taTimings, .5, r {
 | 
| 694 | setText(taTimings, renderFunctionTimings()) | 
| 695 | }); | 
| 696 | ret withRightAlignedButtons(taTimings, | 
| 697 | Reset := r resetTimings); | 
| 698 | }); | 
| 699 | |
| 700 |     uiURLs.put("Test Screen" := -> testScreenPanel());
 | 
| 701 | |
| 702 |     //uiURLs.put("New Script" := -> newScriptPanel());
 | 
| 703 | |
| 704 | // add more ui urls here | 
| 705 | } | 
| 706 | |
| 707 |   JComponent newScriptPanel() {
 | 
| 708 | //var ta = dm_textArea newScript(); | 
| 709 | var ta = dm_syntaxTextArea newScript(); | 
| 710 | |
| 711 | awtCalcEvery(ta.textArea(), 1.0, r_dm_q(r compileNewScript)); | 
| 712 |     ret jCenteredSection("Left-arrow style script",
 | 
| 713 | centerAndSouthWithMargin( | 
| 714 | ta.visualize(), | 
| 715 | centerAndEastWithMargin( | 
| 716 | dm_label newScriptCompileResult(), | 
| 717 |           jbutton("Run" := rThreadEnter runNewScript))));
 | 
| 718 | } | 
| 719 | |
| 720 |   void compileNewScript {
 | 
| 721 | var newScript = this.newScript; | 
| 722 | var result = newScriptCompileResult; | 
| 723 |     if (result == null || !eq(result.script, newScript)) {
 | 
| 724 |       try {
 | 
| 725 | result = new NewScriptCompileResult; | 
| 726 | result.script = newScript; | 
| 727 | result.parser = new GazelleV_LeftArrowScriptParser; | 
| 728 | result.parser.allowTheWorld(mc(), utils.class); | 
| 729 | result.parsedScript = result.parser.parse(result.script); | 
| 730 | print(result.parsedScript); | 
| 731 |       } catch print e {
 | 
| 732 | result.compileError = e; | 
| 733 | } | 
| 734 | setField(newScriptCompileResult := result); | 
| 735 | } | 
| 736 | } | 
| 737 | |
| 738 |   void runNewScript {
 | 
| 739 |     //showText_fast_noWrap("Script Error", renderStackTrace(e));
 | 
| 740 | dm_runInQAndWait(r compileNewScript); | 
| 741 | var result = newScriptCompileResult; | 
| 742 | if (result.parsedScript != null) | 
| 743 | result.result = okOrError(-> result.parsedScript!); | 
| 744 | } | 
| 745 | |
| 746 |   class NewScriptCompileResult {
 | 
| 747 | S script; | 
| 748 | GazelleV_LeftArrowScriptParser parser; | 
| 749 | Throwable compileError; | 
| 750 | GazelleV_LeftArrowScript.Script parsedScript; | 
| 751 | OKOrError result; | 
| 752 | |
| 753 |     toString {
 | 
| 754 | ret compileError != null ? exceptionToStringShorter(compileError) : "Compiled OK"; | 
| 755 | } | 
| 756 | } | 
| 757 | } // end of module | 
| 758 | |
| 759 | concept Label > ConceptWithGlobalID {
 | 
| 760 | S name; | 
| 761 | //new RefL examples; | 
| 762 | |
| 763 |   toString { ret /*"Label " +*/ name; }
 | 
| 764 | } | 
| 765 | |
| 766 | concept GalleryImage {
 | 
| 767 | File path; | 
| 768 | |
| 769 |   toString { ret /*"[" + id + "] " +*/ fileName(path); }
 | 
| 770 | } | 
| 771 | |
| 772 | concept SavedRegion {
 | 
| 773 | new Ref image; // e.g. a GalleryImage | 
| 774 | SnPSettings snpSettings; | 
| 775 | int regionIndex; // region number relative to snpSettings | 
| 776 | //Rect bounds; // get it from bitMatrix instead | 
| 777 | ScanlineBitMatrix bitMatrix; | 
| 778 | //new RefL<Label> labels; | 
| 779 | } | 
| 780 | |
| 781 | concept Example {
 | 
| 782 | new Ref<Label> label; | 
| 783 | new Ref item; // e.g. a SavedRegion | 
| 784 | double confidence = 1; | 
| 785 | } | 
| 786 | |
| 787 | concept IfThenTheory {
 | 
| 788 | new Ref if_; | 
| 789 | new Ref then; | 
| 790 | } | 
| 791 | |
| 792 | sclass WatchTarget {}
 | 
| 793 | |
| 794 | // screenNr: 1 = screen 1 etc | 
| 795 | srecord WatchScreen(int screenNr) > WatchTarget {
 | 
| 796 |   toString { ret "Screen " + screenNr; }
 | 
| 797 | } | 
| 798 | |
| 799 | srecord WatchMouse(int width, int height) > WatchTarget {
 | 
| 800 |   WatchMouse() {
 | 
| 801 | height = 256; | 
| 802 | width = iround(height*16.0/9); | 
| 803 | } | 
| 804 | |
| 805 |   toString { ret "Mouse"; }
 | 
| 806 | } | 
| 807 | |
| 808 | // SnP = Scale and Posterize | 
| 809 | srecord SnPSettings(int pixelRows, int colors) {
 | 
| 810 |   SnPSettings() { pixelRows = 128; colors = 8; }
 | 
| 811 | |
| 812 |   SnPSettings cloneMe() { ret shallowClone(this); }
 | 
| 813 | } | 
| 814 | |
| 815 | /*asclass RegionPaintMode {
 | 
| 816 | } | 
| 817 | |
| 818 | sclass RegionPaintMode_Normal > RegionPaintMode {
 | 
| 819 | } | 
| 820 | |
| 821 | sclass RegionPaintMode_Outline > RegionPaintMode {
 | 
| 822 | }*/ | 
| 823 | |
| 824 | // include functions that scripts may want to use | 
| 825 | |
| 826 | please include function leftScreenBounds. | 
| 827 | please include function rightScreenBounds. | 
| 828 | please include class ScreenOverlay. | 
| 829 | please include function mergeRects. | 
| 830 | please include class TranslucentWindowTest. | 
Began life as a copy of #1033862
download show line numbers debug dex old transpilations
Travelled to 2 computer(s): bhatertpkbcr, mqqgnosmbjvj
No comments. add comment
| Snippet ID: | #1034012 | 
| Snippet name: | Gazelle Screen Cam / Gazelle 22 Module [backup] | 
| Eternal ID of this version: | #1034012/1 | 
| Text MD5: | b8a0771ca5647560ebb13c07c14b303f | 
| Author: | stefan | 
| Category: | javax / gazelle v | 
| Type: | JavaX source code (Dynamic Module) | 
| Public (visible to everyone): | Yes | 
| Archived (hidden from active list): | No | 
| Created/modified: | 2022-01-17 00:16:47 | 
| Source code size: | 26413 bytes / 830 lines | 
| Pitched / IR pitched: | No / No | 
| Views / Downloads: | 357 / 383 | 
| Referenced in: | [show references] |