Warning : session_start(): open(/var/lib/php/sessions/sess_lr5lgade3rs8v3ude2q8k7ncbe, O_RDWR) failed: No space left on device (28) in /var/www/tb-usercake/models/config.php on line 51
Warning : session_start(): Failed to read session data: files (path: /var/lib/php/sessions) in /var/www/tb-usercake/models/config.php on line 51
!7
// Note: Don't use db_mainConcepts(), only the module's concepts field
!include early #1033884 // Compact Module Include Gazelle V [dev version]
module GazelleScreenCam {
!include early #1025212 // +Enabled without visualization
int pixelRows = 128, colors = 8;
S script, testScreenScript;
bool animate = true;
bool horizontalLayout; // flat layout
int fpsTarget = 20;
WatchTarget watchTarget;
transient ImageSurface isPosterized, isRegions, isTestScreen;
transient new ScreenCamStream imageStream;
transient new BWIntegralImageStream integralImages;
transient new SourceTriggeredStream scaledAndPosterizedImages;
transient new DoubleFPSCounter fpsCounter;
transient int fps;
//transient ScreenSelectorRadioButtons screenSelector;
transient WatchTargetSelector watchTargetSelector;
transient new RollingAverage remainingMSPerFrame;
transient int remainingMS;
transient new FunctionTimings functionTimings;
transient ReliableSingleThread rstRunScript = dm_rst(me(), r _runScript);
transient JGazelleVScriptRunner scriptRunner;
transient JGazelleVScriptRunner testScreenScriptRunner;
transient Animation animation;
transient BWImage_FastRegions mainScreenRegions;
transient UIURLSystem uiURLs;
// set by host
transient Concepts concepts;
S uiURL;
transient FileWatchService fileWatcher;
transient SimpleCRUD_v2 labelCRUD;
transient SimpleCRUD_v2 imageCRUD;
transient JGallery gallery;
transient FlexibleRateTimer screenCamTimer;
transient SingleComponentPanel scpMain;
transient JTabbedPane tabs;
transient bool showRegionsAsOutline = true;
transient JComponent watchScreenPane;
transient S screenCamRecognitionOutput;
start {
dm_onFieldChange horizontalLayout(
//r dm_revisualize // deh buggy
r dm_reload
);
// non-standalone mode (doesn't work yet)
if (concepts == null) {
printWithMS("Starting DB");
db();
concepts = db_mainConcepts();
} else
assertSame(concepts, db_mainConcepts());
// make indexes
indexConceptField(concepts, GalleryImage, "path");
indexConceptField(concepts, Example, "item");
// update count in tab when tab isn't selected
onConceptChanges(concepts, -> {
labelCRUD?.updateEnclosingTabTitle();
imageCRUD?.updateEnclosingTabTitle();
});
uiURLs = new UIURLSystem(me(), dm_fieldLiveValue uiURL());
scriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue script(me()));
testScreenScriptRunner = new JGazelleVScriptRunner(dm_fieldLiveValue testScreenScript(me()));
printWithMS("Making image streams");
imageStream.directlyFeedInto(integralImages);
integralImages.onNewElement(ii ->
scaledAndPosterizedImages.newElement(
scaleAndPosterize(ii, new SnPSettings(pixelRows, colors))));
integralImages.onNewElement(r {
if (shouldRunScript()) rstRunScript.go();
});
printWithMS("Almost done");
scaledAndPosterizedImages.onNewElement(img -> {
fpsCounter.inc();
setField(fps := iround(fpsCounter!));
// display before analysis (old)
// isPosterized?.setImage_thisThread(img);
// find regions
floodFill(img);
// display after analysis so we can highlight a region
if (isPosterized != null) {
var img2 = highlightRegion(img, isPosterized, mainScreenRegions);
isPosterized.setImage_thisThread(img2);
}
});
printWithMS("Starting screen cam");
ownResource(screenCamTimer = new FlexibleRateTimer(fpsTarget, rEnter {
if (!enabled) ret;
watchTargetSelector?.updateScreenCount();
Timestamp deadline = tsNowPlusMS(1000/fpsTarget);
if (watchTarget cast WatchScreen)
imageStream.useScreen(watchTarget.screenNr-1);
else if (watchTarget cast WatchMouse)
imageStream.area(mouseArea(watchTarget.width, watchTarget.height));
imageStream.step();
long remaining = deadline.minus(tsNow());
remainingMSPerFrame.add(remaining);
setField(remainingMS := iround(remainingMSPerFrame!));
}));
screenCamTimer.start();
dm_onFieldChange fpsTarget(-> screenCamTimer.setFrequencyImmediately(fpsTarget));
printWithMS("Starting dir watcher");
startDirWatcher();
printWithMS("Gathering images from disk");
for (f : allImageFiles(galleryDir()))
addToGallery(f);
printWithMS("Got dem images");
}
// convert to color & highlight region
BufferedImage highlightRegion(BWImage image, ImageSurface is, BWImage_FastRegions regions) {
var pixels = image.getRGBPixels();
highlightRegion(pixels, is, regions);
ret bufferedImage(pixels, image.getWidth(), image.getHeight());
}
// convert to color & highlight region
void highlightRegion(int[] pixels, ImageSurface is, BWImage_FastRegions regions) {
var mouse = is.mousePosition;
int color = 0xFF008000, holeColor = 0xFFFFB266;
if (mouse != null && regions != null) {
int iHighlightedRegion = regions.regionAt(mouse);
if (showRegionsAsOutline) {
RegionBorder_innerPoints x = new(regions, iHighlightedRegion);
new BoolVar hole;
x.onNewTrace(isHole -> hole.set(isHole));
x.onFoundPoint(p ->
pixels[p.y*regions.w+p.x] = hole! ? holeColor : color);
x.run();
} else
regions.markRegionInPixelArray(pixels, iHighlightedRegion, color);
}
}
void startDirWatcher {
fileWatcher = new FileWatchService;
fileWatcher.addRecursiveListener(picturesDir(), file -> {
if (!isImageFile(file)) ret;
addToGallery(file);
});
}
ImageSurface stdImageSurface() {
ret pixelatedImageSurface().setAutoZoomToDisplay(true).repaintInThread(false);
}
visualize {
gallery = new JGallery;
var galleryComponent = gallery.visualize();
new AWTOnConceptChanges(concepts, galleryComponent, -> {
gallery.setImageFiles(map(list(concepts, GalleryImage), i -> i.path));
}).install();
/*screenSelector = new ScreenSelectorRadioButtons(dm_fieldLiveValue screenNr());
screenSelector.compactLayout(true);
screenSelector.hideIfOnlyOne(true);
screenSelector.screenLabel("");*/
isPosterized = stdImageSurface();
//isRegions = stdImageSurface();
isTestScreen = stdImageSurface();
// when test screen is visible, do the animation
awtEvery(isTestScreen, 1000/20, r stepAnimation);
var jSpeedInfo = dm_transientCalculatedToolTip speedInfo_long(rightAlignLabel(dm_transientCalculatedLabel speedInfo()));
//print("Labels: " + list(concepts, Label));
labelCRUD = new SimpleCRUD_v2(concepts, Label);
labelCRUD.hideFields("globalID");
labelCRUD.addCountToEnclosingTab(true);
labelCRUD.itemToMap_inner2 = l
-> litorderedmap(
"Name" := l.name,
"# Examples" := n2(countConcepts(concepts, Example, label := l)));
imageCRUD = new SimpleCRUD_v2(concepts, GalleryImage);
imageCRUD.addCountToEnclosingTab(true);
imageCRUD.itemToMap_inner2 = img
-> litorderedmap(
"File" := fileName(img.path),
"Folder" := dirPath(img.path));
// main visual
var studyPanel = new StudyPanel;
watchTargetSelector = new WatchTargetSelector;
watchScreenPane = borderlessScrollPane(jHigherScrollPane(
jfullcenter(vstack(
withLeftAndRightMargin(hstack(
dm_rcheckBox enabled("Watch"),
watchTargetSelector.visualize(),
jlabel(" in "),
withLabelToTheRight("colors @ ", dm_spinner colors(2, 256)),
withLabelToTheRight("p", dm_powersOfTwoSpinner pixelRows(512)),
, /* speed info can also go here */
)),
verticalStrut(2),
withSideMargins(centerAndEastWithMargin(
dm_fieldLabel screenCamRecognitionOutput(),
jSpeedInfo)),
))));
tabs = scrollingTabs(jTopOrLeftTabs(horizontalLayout,
"Screen Cam" := screenCamPanel(),
/*WithToolTip("Regions", "Visualize regions detected on screen")
:= jscroll_centered_borderless(isRegions),*/
WithToolTip("Study", "Here you can analyze gallery images")
:= withTopAndBottomMargin(studyPanel.visualize()),
WithToolTip("Pretty Gallery", "Gallery view with preview images")
:= withBottomMargin(galleryComponent),
WithToolTip("Powerful Gallery", "Gallery view with more functions (delete etc)")
:= wrapCRUD(imageCRUD),
WithToolTip("Labels", "Manage labels (image markers)")
:= wrapCRUD(labelCRUD),
WithToolTip("Script", "Run a Gazelle V script")
:= withBottomMargin(scriptRunner.scriptAndResultPanel()),
));
var cbEnabled = toolTip("Switch screen cam on or off", dm_checkBox enabled(""));
var lblScreenCam = setToolTip("Show scaled down and color-reduced screen image",
jlabel("Screen Cam"));
tabComponentClickFixer(lblScreenCam);
var screenCamTab = hstackWithSpacing(cbEnabled, lblScreenCam);
replaceTabTitleComponent(tabs, "Screen Cam", screenCamTab);
// for tab titles
labelCRUD.update();
imageCRUD.update();
initUIURLs();
var urlBar = uiURLs.urlBar();
setToolTip(uiURLs.comboBox, "UI navigation system (partially populated)");
scpMain = singleComponentPanel();
uiURLs.scp(scpMain);
uiURLs.showUIURL("Main Tabs");
var vis = northAndCenter(
withSideAndTopMargin(urlBar),
//centerAndSouthOrEast(horizontalLayout,
/*withBottomMargin*/(scpMain),
/* watchScreenPane
)*/
);
setHorizontalMarginForAllButtons(vis, 4);
ret vis;
}
JComponent screenCamPanel() {
ret centerAndSouthOrEast(horizontalLayout,
withTools(isPosterized),
watchScreenPane);
}
S speedInfo() {
ret "FPS " + fps + " idle " + remainingMS + " ms";
}
S speedInfo_long() {
ret "Screen cam running at " + nFrames(fps) + "/second. " + n2(remainingMS) + " ms remaining per frame in first core"
+ " (of targeted " + fpsTarget + " FPS)";
}
void floodFill(BWImage img) {
BWImage_FastRegions ff = new(img);
//ff.tolerance(0);
ff.collectBounds();
functionTimings.time("Regions", ff);
mainScreenRegions = ff;
if (isRegions != null && isRegions.isShowing_quick()) {
//print("Showing regions image");
isRegions.setImage_thisThread(ff.regionsImage());
}
S text = nRegions(ff.regionCount());
setField(screenCamRecognitionOutput := text);
setEnclosingTabTitle(isRegions, text);
}
bool useErrorHandling() { false; }
S renderFunctionTimings() {
ret lines(ciSorted(map(functionTimings!, (f, avg) ->
firstToUpper(f) + ": " + n2(iround(nsToMicroseconds(avg!))) + " " + microSymbol() + "s (" + n2(iround(avg.n())) + ")")));
}
transient long _runScript_idx;
void _runScript() {
long n = integralImages.elementCount();
if (n > _runScript_idx) {
_runScript_idx = n;
scriptRunner.parseAndRunOn(integralImages!);
}
}
bool shouldRunScript() {
ret isShowing(scriptRunner.scpScriptResult);
}
void stepAnimation {
if (!animate) ret;
if (animation == null) {
animation = new AnimatedLine;
animation.start();
}
animation.nextFrame();
var img = whiteImage(animation.w, animation.h);
animation.setGraphics(createGraphics(img));
animation.paint();
isTestScreen?.setImage(img);
var ii = bwIntegralImage_withMeta(img);
testScreenScriptRunner.parseAndRunOn(ii);
}
JComponent testScreenPanel() {
ret centerAndSouthWithMargin(
hsplit(
northAndCenterWithMargin(centerAndEastWithMargin(
jlabel("Input"), dm_fieldCheckBox animate()),
jscroll_centered_borderless(isTestScreen)),
northAndCenterWithMargin(centerAndEastWithMargin(
jlabel("Output"), testScreenScriptRunner.lblScore),
testScreenScriptRunner.scpScriptResult)
),
testScreenScriptRunner.scriptInputField()
);
}
L popDownItems() {
ret ll(jCheckBoxMenuItem_dyn("Horizontal Layout",
-> horizontalLayout,
b -> setField(horizontalLayout := b)));
}
void unvisualize {} // don't zero transient component fields
void resetTimings { functionTimings.reset(); }
// add tool side bar to image surface
JComponent withTools(ImageSurface is) {
new ImageSurface_PositionToolTip(is);
ret centerAndEastWithMargin(jscroll_centered_borderless(is), vstackWithSpacing(3,
jimageButtonScaledToWidth(16, #1103054, "Save screenshot in gallery", rThread saveScreenshotToGallery),
));
}
class WatchTargetSelector {
JComboBox cb = jComboBox();
int screenCount;
visualize {
updateList();
print("Selecting watchTarget: " + watchTarget);
selectItem(cb, watchTarget);
main onChange(cb, watchTarget -> {
setField(+watchTarget);
print("Chose watchTarget: " + GazelleScreenCam.this.watchTarget);
});
ret cb;
}
void updateScreenCount() {
if (screenCount != screenCount())
updateList();
}
void updateList() swing {
setComboBoxItems(cb, makeWatchTargets());
}
L makeWatchTargets() {
ret flattenToList(
countIteratorAsList_incl(1, screenCount = screenCount(), i -> WatchScreen(i)),
new WatchMouse
);
}
}
class SnPSelector {
settable new SnPSettings settings;
event change;
visualize {
var colors = jspinner(settings.colors, 2, 256);
main onChange(colors, -> {
settings.colors = intFromSpinner(colors);
change();
});
var pixelRows = jPowersOfTwoSpinner(512, settings.pixelRows);
main onChange(pixelRows, -> {
settings.pixelRows = intFromSpinner(pixelRows);
change();
});
ret hstack(
colors,
jlabel(" colors @ "),
pixelRows,
jlabel(" p"));
}
}
JComponent wrapCRUD(SimpleCRUD_v2 crud) {
ret crud == null ?: withTopAndBottomMargin(jRaisedSection(withMargin(crud.make_dontStartBots())));
}
File galleryDir() {
ret picturesDir(gazelle22_imagesSubDirName());
}
void saveScreenshotToGallery enter {
var img = imageStream!;
saveImageWithCounter(galleryDir(), "Screenshot", img);
}
GalleryImage addToGallery(File imgFile) {
if (!isImageFile(imgFile)) null;
var img = uniq(concepts, GalleryImage, path := imgFile);
//printVars("addToGallery", +imgFile, +img);
ret img;
}
BWImage scaleAndPosterize(IBWIntegralImage ii, SnPSettings settings) {
ret posterizeBWImage_withMeta(settings.colors,
scaledBWImageFromBWIntegralImage_withMeta_height(settings.pixelRows, ii));
}
class StudyPanel {
new SnPSelector snpSelector;
GalleryImage image;
BufferedImage originalImage;
ImageSurface isOriginal = stdImageSurface();
ImageSurface isOriginalWithRegions = stdImageSurface();
ImageSurface isPosterized = stdImageSurface();
SingleComponentPanel scp = singleComponentPanel();
SingleComponentPanel analysisPanel = singleComponentPanel();
SingleComponentPanel scpExampleCRUD = singleComponentPanel();
ConceptsComboBox cbImage = new(concepts, GalleryImage);
ImageToRegions itr;
int iSelectedRegion;
SimpleCRUD_v2 regionCRUD;
SimpleCRUD_v2 exampleCRUD;
*() {
//cbImage.sortTheList = l -> sortConceptsByIDDesc(l);
cbImage.sortTheList = l -> sortedByComparator(l, (a, b)
-> cmpAlphanumIC(fileName(a.path), fileName(b.path)));
main onChangeAndNow(cbImage, img -> dm_q(me(), r {
image = img;
scp.setComponent(studyImagePanel());
}));
isPosterized.removeAllTools();
isPosterized.onMousePositionChanged(r_dm_q(me(), l0 regionUpdate));
imageSurfaceOnLeftMouseDown(isPosterized, pt -> dm_q(me(), r { chooseRegionAt(pt) }));
snpSelector.onChange(r_dm_q(me(), l0 runSnP));
}
void chooseRegionAt(Pt p) {
if (itr == null) ret;
iSelectedRegion = itr.regions.regionAt(p);
if (iSelectedRegion > 0) {
var savedRegion = uniq_returnIfNew(concepts, SavedRegion,
+image,
snpSettings := snpSelector.settings.cloneMe(),
regionIndex := iSelectedRegion);
// it's new, add values
if (savedRegion != null) {
print("Saved new region!");
var bitMatrix = toScanlineBitMatrix(itr.regions.regionBitMatrix(iSelectedRegion));
cset(savedRegion, +bitMatrix);
}
}
regionUpdate();
}
// load new image
JComponent studyImagePanel() {
if (image == null) null;
originalImage = loadImage2(image.path);
if (originalImage == null) ret jcenteredlabel("Image not found");
isOriginal.setImage(originalImage);
isOriginalWithRegions.setImage(originalImage);
iSelectedRegion = 0;
itr = new ImageToRegions(originalImage, snpSelector.settings);
runSnP();
regionCRUD = new SimpleCRUD_v2(concepts, SavedRegion);
regionCRUD.addFilter(+image);
regionCRUD
.showSearchBar(false)
.showAddButton(false)
.showEditButton(false);
regionCRUD.itemToMap_inner2 = region -> {
var examples = conceptsWhere(concepts, Example, item := region);
ret litorderedmap(
"Pixels" := region.bitMatrix == null ? "-" : n2(region.bitMatrix.pixelCount()),
//"Shape" := "TODO",
"Labels" := joinWithComma(map(examples, e -> e.label)),
// TODO: scale using snpsettings
"Position" := region.bitMatrix == null ? "-" : region.bitMatrix.boundingBoxOfTrueBits());
};
var regionCRUDComponent = regionCRUD.make_dontStartBots();
regionCRUD.onSelectionChanged(l0 updateExampleCRUD);
ret hsplit(
jtabs(
"Image" := jscroll_centered_borderless(isOriginal),
"Image + regions" := jscroll_centered_borderless(isOriginalWithRegions),
"Posterized" := jscroll_centered_borderless(isPosterized),
),
vsplit(
analysisPanel,
hsplit(
jCenteredSection("Saved regions", regionCRUDComponent),
scpExampleCRUD)
));
}
void runSnP {
if (itr == null) ret;
itr.run();
regionUpdate();
}
void regionUpdate {
if (itr == null) ret;
var pixels = itr.posterized.getRGBPixels();
// hovering region marked green
highlightRegion(pixels, isPosterized, itr.regions);
// selected region marked blue
itr.regions.markRegionInPixelArray(pixels, iSelectedRegion, 0xFFADD8E6);
var highlighted = bufferedImage(pixels, itr.posterized.getWidth(), itr.posterized.getHeight());
isPosterized.setImage(highlighted);
isPosterized.performAutoZoom(); // seems to be necessary for some reason
updateAnalysis();
}
void updateAnalysis {
new LS lines;
lines.add(nRegions(itr.regions.regionCount()));
lines.add("Selected region: " + iSelectedRegion);
S text = lines_rtrim(lines);
analysisPanel.setComponent(jscroll(jMultiLineLabel(text)));
}
void updateExampleCRUD {
var region = regionCRUD.selected();
if (region == null) ret with scpExampleCRUD.set(null);
exampleCRUD = new SimpleCRUD_v2(concepts, Example);
exampleCRUD.entityName = -> "label for region";
exampleCRUD.addFilter(item := region);
exampleCRUD.showSearchBar(false);
scpExampleCRUD.set(jCenteredSection("Labels for region",
exampleCRUD.make_dontStartBots()));
}
void importImage {
new JFileChooser fc;
if (fc.showOpenDialog(cbImage) == JFileChooser.APPROVE_OPTION) {
File file = fc.getSelectedFile().getAbsoluteFile();
if (!isImageFile(file)) ret with infoMessage("Not an image file");
var img = addToGallery(file);
waitUntil(250, 5.0, -> comboBoxContainsItem(cbImage, img));
setSelectedItem(cbImage, img);
}
}
visual
jRaisedSection(northAndCenterWithMargins(
centerAndEast(withLabel("Study", cbImage),
hstack(
jlabel(" in "),
snpSelector.visualize(),
horizontalStrut(10),
jPopDownButton_noText(
"Import image...", rThreadEnter importImage,
),
)),
scp));
} // end of StudyPanel
// S&P, then regions
class ImageToRegions {
BufferedImage inputImage;
BWIntegralImage ii;
SnPSettings snpSettings;
BWImage posterized;
BWImage_FastRegions regions;
*(BufferedImage *inputImage, SnPSettings *snpSettings) {}
run {
ii = bwIntegralImage_withMeta(inputImage);
posterized = scaleAndPosterize(ii, snpSettings);
regions = new BWImage_FastRegions(posterized);
regions.collectBounds();
functionTimings.time("Regions", regions);
}
}
void initUIURLs {
uiURLs.put("Main Tabs", ->
horizontalLayout ? withMargin(tabs) : withSideMargin(tabs));
uiURLs.put("Screen Cam FPS", -> jFullCenter(
vstackWithSpacing(
jCenteredLabel("FPS target (frames per second) for screen cam:"),
jFullCenter(dm_spinner fpsTarget(1, 60)),
jCenteredLabel("(You can lower this value if you have a slower computer)"))));
uiURLs.put("Timings", -> {
JTextArea taTimings = jTextArea_noUndo();
awtEveryAndNow(taTimings, .5, r {
setText(taTimings, renderFunctionTimings())
});
ret withRightAlignedButtons(taTimings,
Reset := r resetTimings);
});
uiURLs.put("Test Screen" := -> testScreenPanel());
// add more ui urls here
}
} // end of module
concept Label > ConceptWithGlobalID {
S name;
//new RefL examples;
toString { ret /*"Label " +*/ name; }
}
concept GalleryImage {
File path;
toString { ret /*"[" + id + "] " +*/ fileName(path); }
}
concept SavedRegion {
new Ref image; // e.g. a GalleryImage
SnPSettings snpSettings;
int regionIndex; // region number relative to snpSettings
//Rect bounds; // get it from bitMatrix instead
ScanlineBitMatrix bitMatrix;
//new RefL labels;
}
concept Example {
new Ref label;
new Ref item; // e.g. a SavedRegion
double confidence = 1;
}
concept IfThenTheory {
new Ref if_;
new Ref then;
}
sclass WatchTarget {}
// screenNr: 1 = screen 1 etc
srecord WatchScreen(int screenNr) > WatchTarget {
toString { ret "Screen " + screenNr; }
}
srecord WatchMouse(int width, int height) > WatchTarget {
WatchMouse() {
height = 256;
width = iround(height*16.0/9);
}
toString { ret "Mouse"; }
}
// SnP = Scale and Posterize
srecord SnPSettings(int pixelRows, int colors) {
SnPSettings() { pixelRows = 128; colors = 8; }
SnPSettings cloneMe() { ret shallowClone(this); }
}
/*asclass RegionPaintMode {
}
sclass RegionPaintMode_Normal > RegionPaintMode {
}
sclass RegionPaintMode_Outline > RegionPaintMode {
}*/