Uses 1405K of libraries. Click here for Pure Java version (21004L/117K).
!7 // Note: Don't use db_mainConcepts(), only the module's concepts field !include early #1033884 // Compact Module Include Gazelle V [dev version] module GazelleScreenCam { !include early #1025212 // +Enabled without visualization int pixelRows = 128, colors = 8; S script, testScreenScript; bool animate = true; bool horizontalLayout; // flat layout WatchTarget watchTarget; transient ImageSurface isPosterized, isRegions, isTestScreen; transient new ScreenCamStream imageStream; transient new BWIntegralImageStream integralImages; transient new SourceTriggeredStream<BWImage> scaledAndPosterizedImages; transient new DoubleFPSCounter fpsCounter; transient int fps; //transient ScreenSelectorRadioButtons screenSelector; transient WatchTargetSelector watchTargetSelector; transient new RollingAverage remainingMSPerFrame; transient int remainingMS; transient new FunctionTimings<S> functionTimings; transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript); transient JGazelleVScriptRunner scriptRunner; transient JGazelleVScriptRunner testScreenScriptRunner; transient Animation animation; transient BWImage_FastRegions mainScreenRegions; transient UIURLSystem uiURLs; // set by host transient Concepts concepts; S uiURL; transient FileWatchService fileWatcher; transient SimpleCRUD_v2<Label> labelCRUD; transient SimpleCRUD_v2<GalleryImage> imageCRUD; transient JGallery gallery; start { dm_onFieldChange horizontalLayout( //r dm_revisualize // deh buggy r dm_reload ); // non-standalone mode (doesn't work yet) if (concepts == null) { db(); concepts = db_mainConcepts(); } // update count in tab when tab isn't selected onConceptChanges(concepts, -> { labelCRUD?.updateEnclosingTabTitle(); imageCRUD?.updateEnclosingTabTitle(); }); uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL()); scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me())); testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me())); imageStream.directlyFeedInto(integralImages); integralImages.onNewElement(ii -> scaledAndPosterizedImages.newElement( scaleAndPosterize(ii, new SnPSettings(pixelRows, colors)))); integralImages.onNewElement(r { if (shouldRunScript()) rstRunScript.go(); }); scaledAndPosterizedImages.onNewElement(img -> { fpsCounter.inc(); setField(fps := iround(fpsCounter!)); // display before analysis (old) // isPosterized?.setImage_thisThread(img); // find regions floodFill(img); // display after analysis so we can highlight a region if (isPosterized != null) { var img2 = highlightRegion(img, isPosterized, mainScreenRegions); isPosterized.setImage_thisThread(img2); } }); // 20 Hz is enough for everyone dm_doEvery(1000/20, r { if (enabled) { watchTargetSelector?.updateScreenCount(); Timestamp deadline = tsNowPlusMS(1000/20); if (watchTarget cast WatchScreen) imageStream.useScreen(watchTarget.screenNr-1); else if (watchTarget cast WatchMouse) imageStream.area(mouseArea(watchTarget.width, watchTarget.height)); imageStream.step(); long remaining = deadline.minus(tsNow()); remainingMSPerFrame.add(remaining); setField(remainingMS := iround(remainingMSPerFrame!)); } }); startDirWatcher(); for (f : allImageFiles(galleryDir())) addToGallery(f); } // convert to color & highlight region BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) { var pixels = image.getRGBPixels(); var mouse = is.mousePosition; if (mouse != null && regions != null) { int iHighlightedRegion = regions.regionAt(mouse); //int marked = 0; if (iHighlightedRegion > 0) for i over pixels: if (regions.regionAt(i) == iHighlightedRegion) { pixels[i] = 0xFF008000; //++marked; } //printVars(+mouse, +iHighlightedRegion, +marked); } ret bufferedImage(pixels, image.getWidth(), image.getHeight()); } void startDirWatcher { fileWatcher = new FileWatchService; fileWatcher.addRecursiveListener(picturesDir(), file -> { if (!isImageFile(file)) ret; addToGallery(file); }); } ImageSurface stdImageSurface() { ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false); } visualize { gallery = new JGallery; var galleryComponent = gallery.visualize(); new AWTOnConceptChanges(concepts, galleryComponent, -> { gallery.setImageFiles(map(list(concepts, GalleryImage), i -> i.path)); }).install(); /*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr()); screenSelector.compactLayout(true); screenSelector.hideIfOnlyOne(true); screenSelector.screenLabel("");*/ isPosterized = stdImageSurface(); isRegions = stdImageSurface(); isTestScreen = stdImageSurface(); // when test screen is visible, do the animation awtEvery(isTestScreen, 1000/20, r stepAnimation); JTextArea taTimings = jTextArea_noUndo(); awtEveryAndNow(taTimings, .5, r { setText(taTimings, renderFunctionTimings()) }); var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo())); //print("Labels: " + list(concepts, Label)); labelCRUD = new SimpleCRUD_v2(concepts, Label); labelCRUD.hideFields("globalID"); labelCRUD.addCountToEnclosingTab(true); imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage); imageCRUD.addCountToEnclosingTab(true); imageCRUD.itemToMap_inner2 = img -> litorderedmap( "File" := fileName(img.path), "Folder" := dirPath(img.path)); // main visual var studyPanel = new StudyPanel; var tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout, "Screen Cam" := withTools(isPosterized), WithToolTip("Regions", "Visualize regions detected on screen") := jscroll_centered_borderless(isRegions), WithToolTip("Study", "Here you can analyze gallery images") := withTopMargin(studyPanel.visualize()), WithToolTip("Pretty Gallery", "Gallery view with preview images") := galleryComponent, WithToolTip("Powerful Gallery", "Gallery view with more functions (delete etc)") := wrapCRUD(imageCRUD), WithToolTip("Labels", "Manage labels (image markers)") := wrapCRUD(labelCRUD), WithToolTip("Script", "Run a Gazelle V script") := scriptRunner.scriptAndResultPanel(), "Test Screen" := testScreenPanel(), Timings := withRightAlignedButtons(taTimings, Reset := r resetTimings) )); var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled("")); var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image", jlabel("Screen Cam")); tabComponentClickFixer(lblScreenCam); var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam); replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab); // for tab titles labelCRUD.update(); imageCRUD.update(); watchTargetSelector = new WatchTargetSelector; var urlBar = uiURLs.urlBar(); setToolTip(uiURLs.comboBox, "UI navigation system (currently unused)"); var vis = northAndCenter( withSideAndTopMargin(urlBar), centerAndSouthOrEast(horizontalLayout, horizontalLayout ? withMargin(tabs) : withSideMargin(tabs), /*withMargin*/(borderlessScrollPane(jHigherScrollPane( jfullcenter(vstack( withLeftAndRightMargin(hstack( dm_rcheckBox enabled("Watch"), watchTargetSelector.visualize(), jlabel(" in "), withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)), withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)), , /* speed info can also go here */ )), verticalStrut(2), withRightMargin(jSpeedInfo) ))))), )); setHorizontalMarginForAllButtons(vis, 4); ret vis; } S speedInfo() { ret "FPS " + fps + " idle " + remainingMS + " ms"; } S speedInfo_long() { ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core."; } void floodFill(BWImage img) { BWImage_FastRegions ff = new(img); //ff.tolerance(0); ff.collectBounds(); functionTimings.time("Regions", ff); mainScreenRegions = ff; if (isRegions != null && isRegions.isShowing_quick()) { //print("Showing regions image"); isRegions.setImage_thisThread(ff.regionsImage()); } setEnclosingTabTitle(isRegions, nRegions(ff.regionCount())); } bool useErrorHandling() { false; } S renderFunctionTimings() { ret lines(ciSorted(map(functionTimings!, (f, avg) -> firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")"))); } transient long _runScript_idx; void _runScript() { long n = integralImages.elementCount(); if (n > _runScript_idx) { _runScript_idx = n; scriptRunner.parseAndRunOn(integralImages!); } } bool shouldRunScript() { ret isShowing(scriptRunner.scpScriptResult); } void stepAnimation { if (!animate) ret; if (animation == null) { animation = new AnimatedLine; animation.start(); } animation.nextFrame(); var img = whiteImage(animation.w, animation.h); animation.setGraphics(createGraphics(img)); animation.paint(); isTestScreen?.setImage(img); var ii = bwIntegralImage_withMeta(img); testScreenScriptRunner.parseAndRunOn(ii); } JComponent testScreenPanel() { ret centerAndSouthWithMargin( hsplit( northAndCenterWithMargin(centerAndEastWithMargin( jlabel("Input"), dm_fieldCheckBox animate()), jscroll_centered_borderless(isTestScreen)), northAndCenterWithMargin(centerAndEastWithMargin( jlabel("Output"), testScreenScriptRunner.lblScore), testScreenScriptRunner.scpScriptResult) ), testScreenScriptRunner.scriptInputField() ); } L popDownItems() { ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout", -> horizontalLayout, b -> setField(horizontalLayout := b))); } void unvisualize {} // don't zero transient component fields void resetTimings { functionTimings.reset(); } // add tool side bar to image surface JComponent withTools(ImageSurface is) { new ImageSurface_PositionToolTip(is); ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3, jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery), )); } class WatchTargetSelector { JComboBox<WatchTarget> cb = jComboBox(); int screenCount; visualize { updateList(); print("Selecting watchTarget: " + watchTarget); selectItem(cb, watchTarget); main onChange(cb, watchTarget -> { setField(+watchTarget); print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget); }); ret cb; } void updateScreenCount() { if (screenCount != screenCount()) updateList(); } void updateList() swing { setComboBoxItems(cb, makeWatchTargets()); } L<WatchTarget> makeWatchTargets() { ret flattenToList( countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)), new WatchMouse ); } } class SnPSelector { settable new SnPSettings settings; event change; visualize { var colors = jspinner(settings.colors, 2, 256); main onChange(colors, -> { settings.colors = intFromSpinner(colors); change(); }); var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows); main onChange(pixelRows, -> { settings.pixelRows = intFromSpinner(pixelRows); change(); }); ret hstack( colors, jlabel(" colors @ "), pixelRows, jlabel(" p")); } } JComponent wrapCRUD(SimpleCRUD_v2 crud) { ret crud == null ?: withTopMargin(jRaisedSection(withMargin(crud.make_dontStartBots()))); } File galleryDir() { ret picturesDir(gazelle22_imagesSubDirName()); } void saveScreenshotToGallery enter { var img = imageStream!; saveImageWithCounter(galleryDir(), "Screenshot", img); } void addToGallery(File imgFile) { if (!isImageFile(imgFile)) ret; var img = uniq(concepts, GalleryImage, path := imgFile); //printVars("addToGallery", +imgFile, +img); } BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) { ret posterizeBWImage_withMeta(settings.colors, scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii)); } class StudyPanel { new SnPSelector snpSelector; GalleryImage image; ImageSurface is = stdImageSurface(); SingleComponentPanel scp = singleComponentPanel(); SingleComponentPanel analysisPanel = singleComponentPanel(); ConceptsComboBox<GalleryImage> cbImage = new(concepts, GalleryImage); ImageToRegions itr; int iSelectedRegion; *() { cbImage.sortTheList = l -> sortConceptsByIDDesc(l); main onChangeAndNow(cbImage, img -> dm_q(me(), r { image = img; scp.setComponent(studyImagePanel()); })); is.removeAllTools(); is.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate)); imageSurfaceOnLeftMouseDown(is, pt -> dm_q(me(), r { chooseRegionAt(pt) })); snpSelector.onChange(r_dm_q(me(), l0 runSnP)); } void chooseRegionAt(Pt p) { if (itr == null) ret; iSelectedRegion = itr.regions.regionAt(p); updateAnalysis(); } // load new image JComponent studyImagePanel() { if (image == null) null; var i = loadImage2(image.path); if (i == null) ret jcenteredlabel("Image not found"); iSelectedRegion = 0; itr = new ImageToRegions(i, snpSelector.settings); runSnP(); ret hsplit(jscroll_centered_borderless(is), analysisPanel); } void runSnP { if (itr == null) ret; itr.run(); regionUpdate(); } void regionUpdate { if (itr == null) ret; var highlighted = highlightRegion(itr.posterized, is, itr.regions); is.setImage(highlighted); is.performAutoZoom(); // seems to be necessary for some reason updateAnalysis(); } void updateAnalysis { new LS lines; lines.add(nRegions(itr.regions.regionCount())); lines.add("Selected region: " + iSelectedRegion); S text = lines_rtrim(lines); analysisPanel.setComponent(jscroll(jMultiLineLabel(text))); } visual jRaisedSection(northAndCenterWithMargins( centerAndEast(withLabel("Study", cbImage), hstack(jlabel(" in "), snpSelector.visualize())), scp)); } // end of StudyPanel class ImageToRegions { BufferedImage inputImage; BWIntegralImage ii; SnPSettings snpSettings; BWImage posterized; BWImage_FastRegions regions; *(BufferedImage *inputImage, SnPSettings *snpSettings) {} run { ii = bwIntegralImage_withMeta(inputImage); posterized = scaleAndPosterize(ii, snpSettings); regions = new BWImage_FastRegions(posterized); regions.collectBounds(); functionTimings.time("Regions", regions); } } } // end of module concept Label > ConceptWithGlobalID { S name; //new RefL examples; toString { ret "Label " + name; } } concept GalleryImage { File path; toString { ret /*"[" + id + "] " +*/ fileName(path); } } concept SavedRegion { new Ref image; // e.g. a GalleryImage SnPSettings snpSettings; Rect bounds; BitMatrix bitMatrix; //new RefL<Label> labels; } concept Example { new Ref<Label> label; new Ref item; // e.g. a SavedRegion double confidence = 1; } concept IfThenTheory { new Ref if_; new Ref then; } sclass WatchTarget {} // screenNr: 1 = screen 1 etc srecord WatchScreen(int screenNr) > WatchTarget { toString { ret "Screen " + screenNr; } } srecord WatchMouse(int width, int height) > WatchTarget { WatchMouse() { height = 256; width = iround(height*16.0/9); } toString { ret "Mouse"; } } // SnP = Scale and Posterize persistable sclass SnPSettings { int pixelRows = 128; int colors = 8; *(int *pixelRows, int *colors) {} }
Began life as a copy of #1033862
download show line numbers debug dex old transpilations
Travelled to 2 computer(s): bhatertpkbcr, mqqgnosmbjvj
No comments. add comment
Snippet ID: | #1033924 |
Snippet name: | Gazelle Screen Cam / Gazelle 22 Module [backup before FlexibleRateTimer] |
Eternal ID of this version: | #1033924/1 |
Text MD5: | 9db08d41b57d7d33177c3c184ab40fbd |
Transpilation MD5: | 588eea842591e146471f700b367242cc |
Author: | stefan |
Category: | javax / gazelle v |
Type: | JavaX source code (Dynamic Module) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2022-01-12 16:22:52 |
Source code size: | 17597 bytes / 579 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 113 / 164 |
Referenced in: | -