1 | abstract sclass ChessOCR_DynPipelinedRecognizer extends DynModule { |
2 | S segmenterPreset; |
3 | transient Iterator<ParameterizedSegmenter> segmenters; |
4 | transient ChessOCR_Pipeline pipeline; |
5 | |
6 | Rect boardLocation; |
7 | double recognitionScore; |
8 | S status; |
9 | S selectedSquare; |
10 | S piece; |
11 | LPair<S, Double> recognized; |
12 | S fen; |
13 | ChessOCR_BoardsFound boardsFound; |
14 | |
15 | switchable bool squareHeightAtBottom = true; // for YouTube videos |
16 | transient int recreatedSquareSize = 32; |
17 | transient ReliableSingleThread rst = dm_rst(module(), r recognize); |
18 | transient BufferedImage img, board, square, recreated; |
19 | transient LL<BufferedImage> squareImages; |
20 | transient ImageSurface isInput, isBoard, isSquare, isRecreated; |
21 | |
22 | transient LS pieces = ai_chessPieces(); |
23 | transient Map<Bool, Map<S, BufferedImage>> recreateStock; // false = dark, true = light |
24 | |
25 | visualize { |
26 | isBoard = jImageSurface(); |
27 | imageSurfaceOnHover(isBoard, voidfunc(Pt p) { |
28 | if (board == null || p == null) ret with noToolTip(isBoard); |
29 | setToolTip(isBoard, toolTipForSquare(boardPointToSquare(p)); |
30 | }); |
31 | imageSurfaceOnLeftClick(isBoard, voidfunc(Pt p) { |
32 | selectSquare(boardPointToSquare(p)) |
33 | }); |
34 | |
35 | isRecreated = jImageSurface(recreated); |
36 | imageSurfaceOnHover(isRecreated, voidfunc(Pt p) { |
37 | if (recreated == null || p == null) ret with noToolTip(isRecreated); |
38 | setToolTip(isRecreated, toolTipForSquare(recreatedPointToSquare(p)); |
39 | }); |
40 | imageSurfaceOnLeftClick(isRecreated, voidfunc(Pt p) { |
41 | selectSquare(recreatedPointToSquare(p)) |
42 | }); |
43 | |
44 | isInput = jImageSurface(or_func(img, () -> whiteImage(100))); |
45 | isInput.onNewImage = rst; |
46 | |
47 | ret withCalc(rst, jvsplit( |
48 | jCenteredSection("Input image", jscroll_center(isInput)), |
49 | northCenterAndSouthWithMargins( |
50 | centerAndEastWithMargin(segmenterSection(), |
51 | hstackWithSpacing( |
52 | jbutton("Recognize", rst), |
53 | jPopDownButton_noText( |
54 | "Show possible chess boards found", rThread showSegments, |
55 | "Publish training image...", rThread publishImage, |
56 | "---", null, |
57 | "Screenshot (hiding OS)", rThread { |
58 | isInput.setImageAndZoomToDisplay(dm_shootScreenHidingOS()); |
59 | rst.trigger(); |
60 | }) |
61 | )), |
62 | hgridWithSpacing( |
63 | jCenteredSection("Recreated", jscroll_center(isRecreated)), |
64 | jCenteredSection("Chess board in image", |
65 | centerAndSouthWithMargin(jscroll_center(isBoard), |
66 | dm_centeredCalculatedLabel(() -> "Recognition score: " + formatDouble(recognitionScore, 2)) |
67 | )), |
68 | jCenteredLiveValueSection(dm_calculatedLiveValue(S, () -> "Selected square" + (empty(selectedSquare) ? "" : " (" + selectedSquare + ")")), centerAndSouthWithMargin( |
69 | jscroll_center(isSquare = jImageSurface(square)), |
70 | rightAlignedLine( |
71 | withLabel("Piece: ", dm_comboBox('piece, pieces)), |
72 | jbutton("Save & upload", rThread saveAndUploadPiece)) |
73 | ))), |
74 | vstackWithSpacing( |
75 | dm_label('status), |
76 | withLabel("FEN:", dm_label('fen)) |
77 | ) |
78 | ))); |
79 | } |
80 | |
81 | void recognize { |
82 | if (pipeline != null) |
83 | pipeline.cancel(); |
84 | status("Recognizing..."); |
85 | segmenters = makeSegmenters(); |
86 | if (empty(segmenters)) ret with infoBox(status("No segmenter")); |
87 | //img = dm_shootScreenHidingOS(); |
88 | if (isInput != null) img = isInput.getImage(); |
89 | |
90 | ChessPieceRecognizer pieceRecognizer = chessOCR_pieceRecognizer(); |
91 | |
92 | ChessOCR_TwoStageRecognizer rec = new(pieceRecognizer, img, segmenters); |
93 | rec.verbose(true); |
94 | |
95 | clearImageSurfaces(isBoard, isRecreated); |
96 | setField(fen := ""); |
97 | |
98 | bool done = false; |
99 | new Var<Rect> lastLocation; |
100 | while (!done && !rst.triggered()) { |
101 | done = !rec.step(); |
102 | |
103 | Rect r = rec.bestBoardLocation(); |
104 | if (r != null) |
105 | rec.tryRect(print("wiggling", wiggleRect(r))); |
106 | |
107 | r = rec.bestBoardLocation(); |
108 | if (r != null && setVar_trueIfChanged(lastLocation, r)) |
109 | newBestLocation(r, pieceRecognizer); |
110 | } |
111 | |
112 | if (done && !lastLocation.has()) |
113 | status("No chess board found"); |
114 | } |
115 | |
116 | S status(S status) { setField(+status); ret status; } |
117 | |
118 | O _getReloadData() { ret img; } |
119 | void _setReloadData(BufferedImage img) { this.img = img; } |
120 | |
121 | void setImage(BufferedImage img) { |
122 | this.img = img; |
123 | if (isInput != null) isInput.setImageAndZoomToDisplay(img); |
124 | rst.trigger(); |
125 | } |
126 | |
127 | S boardPointToSquare(Pt p) { |
128 | double space = imageMergeSpacing(); |
129 | double squareW = board.getWidth()/8.0+space; |
130 | double squareH = board.getHeight()/8.0+space; |
131 | int x = clamp(iratio_floor(p.x+space/2, squareW), 0, 7); |
132 | int y = clamp(iratio_floor(p.y+space/2, squareH), 0, 7); |
133 | ret strCharPlus('A', x) + (8-y); |
134 | } |
135 | |
136 | S recreatedPointToSquare(Pt p) { |
137 | int x = clamp(iratio_floor(p.x, recreatedSquareSize), 0, 7); |
138 | int y = clamp(iratio_floor(p.y, recreatedSquareSize), 0, 7); |
139 | ret strCharPlus('A', x) + (8-y); |
140 | } |
141 | |
142 | void saveAndUploadPiece enter { |
143 | if (square == null || empty(piece)) ret; |
144 | chessOCR_uploadPieceImage(selectedSquare, piece, square); |
145 | if (mapGet(mapGet(recreateStock, chess_isLightSquare(selectedSquare)), piece) == null) recreateStock = null; |
146 | rst.trigger(); |
147 | } |
148 | |
149 | void recreate { |
150 | if (recognized == null) ret; |
151 | if (recreateStock == null) |
152 | recreateStock = mapToValues(ll(false, true), light -> |
153 | mapValues(pairsToMap(reversePairs(chessPieceImagesFromAGIBlue(light))), imageID -> resizeImage(loadImage2(imageID), recreatedSquareSize, recreatedSquareSize))); |
154 | |
155 | new L<BufferedImage> out; |
156 | int i = 0; |
157 | for (S piece : pairsA(recognized)) { |
158 | bool light = even(i/8+i%8); |
159 | out.add(or_func(mapGet(mapGet(recreateStock, light), piece), () -> whiteImage(recreatedSquareSize, recreatedSquareSize))); |
160 | ++i; |
161 | } |
162 | |
163 | new L<BufferedImage> rows; |
164 | for (L<BufferedImage> row : listToChunks(out, 8)) |
165 | rows.add(mergeBufferedImagesHorizontally(row, spacing := 0)); |
166 | recreated = mergeBufferedImagesVertically(rows, spacing := 0); |
167 | if (isRecreated != null) isRecreated.setImageAndZoomToDisplay(recreated); |
168 | } |
169 | |
170 | Pair<S, Double> recognitionForSquare(S square) { |
171 | int x, y = unpair chess_squareToPos(square); |
172 | ret _get(recognized, y*8+x); |
173 | } |
174 | |
175 | S toolTipForSquare(S square) { |
176 | Pair<S, Double> rec = recognitionForSquare(square); |
177 | ret square + " - " + (rec == null ? "?" : rec.a + " [" + iround(clamp(rec.b, 0, 100)) + "%]"); |
178 | } |
179 | |
180 | void selectSquare(S _square) { |
181 | setField(selectedSquare := _square); |
182 | S rec = pairA(recognitionForSquare(selectedSquare)); |
183 | if (rec != null) setField(piece := rec); |
184 | int x, y = unpair chess_squareToPos(selectedSquare); |
185 | square = _get(_get(squareImages, y), x); |
186 | |
187 | ChessPieceProfile1 profile = chessOCR_pieceProfileFromRawImage_1(square, withPreprocessedImages := true); |
188 | isSquare.setImage(mergeBufferedImagesHorizontally( |
189 | itemPlusList(square, map toBufferedImage(profile.preprocessedImages)))); |
190 | //printVars_str("leftClick", +p, +selectedSquare, +x, +y, +square); |
191 | } |
192 | |
193 | void showSegments { |
194 | L<Rect> segments = this.segments; |
195 | showImage_centered(n2(segments, "segment") + " found", |
196 | mergeBufferedImagesVertically(map(segments, r -> clipBufferedImage(img, r)))); |
197 | } |
198 | |
199 | void publishImage enter { |
200 | BufferedImage img = this.img; |
201 | S segmenterPreset = this.segmenterPreset; |
202 | |
203 | JCheckBox cbFound = jcheckbox(true), cbHasBoard = jcheckbox(true); |
204 | JTextField tfFEN = jtextfield(fen), tfComments = jtextfield(), |
205 | tfImageCredits = jtextfield(); |
206 | |
207 | showFormTitled("Publish training image", |
208 | "Image", jImageSurface(scaleImageToWidth(img, 500)), |
209 | "Segmenter used", jlabel(segmenterPreset), |
210 | "Is there a chess board in the image?", cbHasBoard, |
211 | "FEN (empty if no board in image)", tfFEN, |
212 | "Was the chess board found?", cbFound, // TODO: make clearer what this means if there is no board in image |
213 | "Image credits (optional)", tfImageCredits, |
214 | "Random comments", tfComments, |
215 | "NOTE:", jMultiLineLabel("By clicking 'publish', you PUBLISH this image (fair use/public domain)."), |
216 | "", jThreadedButton("Publish", r { |
217 | temp tempInfoBox_noHide("Uploading training image..."); |
218 | S imageURL = uploadToImageServer("Chess board recognition training image", img); |
219 | S desc = "Chess board recognition training image " + assertNempty("Parseable image URL", parseSnippetImageURL(imageURL)); |
220 | new LinkedHashMap<S> map; |
221 | map.put("Input image", imageURL); |
222 | mapPutIfNemptyValue(map, "Image credits", gtt(tfImageCredits)); |
223 | S fen = gtt(tfFEN); |
224 | map.put("Image has chess board", yesNo_short_firstUpper(isChecked(cbHasBoard) || nempty(fen))); |
225 | mapPutIfNemptyValue(map, "FEN", fen); |
226 | map.put("Segmenter used", segmenterPreset); |
227 | map.put("Chess board found", yesNo_short_firstUpper(isChecked(cbFound))); |
228 | if (isChecked(cbFound)) |
229 | map.put("Board location", struct(boardLocation)); |
230 | S sliceID = agiBlue_chessBoardRecognitionTrainingImagesSliceID(); |
231 | new L<Map> toPost; |
232 | S name = agiBlue_createUnusedNumberedPage(sliceID, desc + " #"); |
233 | for (S key, value : map) |
234 | toPost.add(litmap(q := name, +key, +value)); |
235 | agiBot_postMulti(keyPairForProgram(), toPost, slice := sliceID); |
236 | infoBox("Training image uploaded!"); |
237 | openURLInBrowser(agiBlue_linkForPhrase(name, slice := sliceID)); |
238 | disposeWindow(heldButton()); |
239 | }) |
240 | ); |
241 | } |
242 | |
243 | S makeFEN() { |
244 | ret chess_makeFEN(listToChunks(pairsA(recognized), 8)); |
245 | } |
246 | |
247 | void newBestLocation(Rect r, ChessPieceRecognizer pieceRecognizer) { |
248 | setField(segments := singletonUnlessNull(r)); |
249 | bool corrected = squareHeightAtBottom && abs(r.w-r.h) > 3; |
250 | if (corrected) r.h = r.w; |
251 | new LS comments; |
252 | if (corrected) comments.add("height-corrected"); |
253 | status("Chess board found at: " + r + appendRoundBracketed(joinWithComma(comments))); |
254 | setField(boardLocation := r); |
255 | board = clipBufferedImage(img, r); |
256 | |
257 | squareImages = bufferedImageMNGrid(board, 8, 8); |
258 | |
259 | if (isBoard != null) |
260 | isBoard.setImageAndZoomToDisplay( |
261 | mergeBufferedImagesVertically( |
262 | map(bufferedImageNVerticalSlices(8, board), i -> |
263 | mergeBufferedImagesHorizontally(bufferedImageNHorizontalSlices(8, i))))); |
264 | |
265 | recognized = new L; |
266 | for (BufferedImage img : concatLists(squareImages)) |
267 | recognized.add(pieceRecognizer.recognize(img)); |
268 | setField(recognitionScore := doubleAvg(pairsB(recognized)); |
269 | setField(fen := makeFEN()); |
270 | print(recognized); |
271 | recreate(); |
272 | } |
273 | |
274 | Rect wiggleRect(Rect r) { |
275 | if (r == null) null; |
276 | int range = 1; |
277 | ret intersectRects(imageRect(img), |
278 | rectFromPoints(varyInt(range, r.x), varyInt(range, r.y), |
279 | varyInt(range, r.x2()), varyInt(range, r.y2()))); |
280 | } |
281 | |
282 | abstract JComponent segmenterSection(); |
283 | abstract Iterator<ParameterizedSegmenter> makeSegmenters(); |
284 | } |
Began life as a copy of #1024870
download show line numbers debug dex old transpilations
Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #1024887 |
Snippet name: | ChessOCR_DynPipelinedRecognizer [dev.] |
Eternal ID of this version: | #1024887/1 |
Text MD5: | aa82c3540ed810f6c0fa674e501d06d7 |
Author: | stefan |
Category: | javax / chess ocr |
Type: | JavaX fragment (include) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2019-08-28 16:49:05 |
Source code size: | 11396 bytes / 284 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 212 / 207 |
Referenced in: | [show references] |