Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

307
LINES

< > BotCompany Repo | #1024870 // ChessOCR_DynChessBoardRecognizer

JavaX fragment (include) [tags: use-pretranspiled]

Transpiled version (19318L) is out of date.

1  
abstract sclass ChessOCR_DynChessBoardRecognizer extends DynModule {
2  
  replace Segmenter with IF1<BufferedImage, L<Rect>>.
3  
  
4  
  S segmenterPreset;
5  
  transient Iterator<? extends Segmenter> segmenters;
6  
  L<Rect> segments;
7  
  Rect boardLocation;
8  
  double recognitionScore;
9  
  S status;
10  
  S selectedSquare;
11  
  S piece;
12  
  LPair<S, Double> recognized;
13  
  S fen;
14  
15  
  switchable bool showSelectedSquare = true;
16  
  switchable bool showSegmenterSection = true;
17  
  switchable bool squareHeightAtBottom = true; // for YouTube videos
18  
  transient int recreatedSquareSize = 32;
19  
  transient ReliableSingleThread rst = dm_rst(module(), r recognize);
20  
  transient BufferedImage img, board, square, recreated;
21  
  transient LL<BufferedImage> squareImages;
22  
  transient ImageSurface isInput, isBoard, isSquare, isRecreated;
23  
  
24  
  transient LS pieces = ai_chessPieces();
25  
  transient Map<Bool, Map<S, BufferedImage>> recreateStock; // false = dark, true = light
26  
  transient ChessPieceRecognizer pieceRecognizer;
27  
  
28  
  visualize {
29  
    isBoard = jImageSurface();
30  
    imageSurfaceOnHover(isBoard, voidfunc(Pt p) {
31  
      if (board == null || p == null) ret with noToolTip(isBoard);
32  
      setToolTip(isBoard, toolTipForSquare(boardPointToSquare(p));
33  
    });
34  
    imageSurfaceOnLeftClick(isBoard, voidfunc(Pt p) {
35  
      selectSquare(boardPointToSquare(p))
36  
    });
37  
    
38  
    isRecreated = jImageSurface(recreated);
39  
    imageSurfaceOnHover(isRecreated, voidfunc(Pt p) {
40  
      if (recreated == null || p == null) ret with noToolTip(isRecreated);
41  
      setToolTip(isRecreated, toolTipForSquare(recreatedPointToSquare(p));
42  
    });
43  
    imageSurfaceOnLeftClick(isRecreated, voidfunc(Pt p) {
44  
      selectSquare(recreatedPointToSquare(p))
45  
    });
46  
    
47  
    isInput = jImageSurface(or_func(img, () -> whiteImage(100)));
48  
    isInput.onNewImage = rst;
49  
    isInput.zoomToWindow();
50  
    
51  
    JComponent recreatedSection = jCenteredSection("Recreated", jscroll_center(isRecreated)),
52  
      boardSection = jCenteredSection("Chess board in image",
53  
        centerAndSouthWithMargin(jscroll_center(isBoard),
54  
        dm_centeredCalculatedLabel(() -> "Recognition score: " + formatDouble(recognitionScore, 2))
55  
      ));
56  
      
57  
    JButton btnScreenshot = jbutton("Grab screenshot & recognize", rThread grabScreenshot);
58  
    JButton popDownButton = jPopDownButton_noText(
59  
      "Show possible chess boards found", rThread showSegments,
60  
      "Publish training image...", rThread publishImage,
61  
      //"---", null,
62  
      //"Screenshot (hiding OS)", rThread grabScreenshot
63  
    );
64  
    
65  
    ret withCalc(rst, jvsplit(
66  
      jCenteredSection("Input image", jscroll_center(isInput)),
67  
      northCenterAndSouthWithMargins(
68  
        !showSegmenterSection ? centeredLine(btnScreenshot, popDownButton)
69  
        : westCenterAndEastWithMargin(
70  
          btnScreenshot,
71  
          segmenterSection(),
72  
          hstackWithSpacing(
73  
            jbutton("Recognize", rst),
74  
            popDownButton
75  
          )),
76  
          
77  
        (!showSelectedSquare ? hgridWithSpacing(boardSection, recreatedSection) : hgridWithSpacing(
78  
          recreatedSection,
79  
          boardSection,
80  
          jCenteredLiveValueSection(dm_calculatedLiveValue(S, () -> "Selected square" + (empty(selectedSquare) ? "" : " (" + selectedSquare + ")")), centerAndSouthWithMargin(
81  
            jscroll_center(isSquare = jImageSurface(square)),
82  
            rightAlignedLine(
83  
              withLabel("Piece: ", dm_comboBox('piece, pieces)),
84  
              jbutton("Save & upload", rThread saveAndUploadPiece))
85  
        )))),
86  
        vstackWithSpacing(
87  
          dm_label('status),
88  
          rightAlignedLine(withLabel("FEN (chess position):", dm_calculatedBoldLabel(() -> or2(fen, "-"))),
89  
            jbutton("Copy", rThread copyFEN))
90  
        )
91  
      )));
92  
  }
93  
    
94  
  void recognize {
95  
    status("Recognizing...");
96  
    segmenters = makeSegmenters();
97  
    if (empty(segmenters)) ret with infoBox(status("No segmenter"));
98  
    //img = dm_shootScreenHidingOS();
99  
    if (isInput != null) img = isInput.getImage();
100  
    
101  
    if (pieceRecognizer == null)
102  
      pieceRecognizer = chessOCR_pieceRecognizer();
103  
    
104  
    ChessOCR_TwoStageRecognizer rec = new(pieceRecognizer, img, segmenters);
105  
    rec.verbose(true);
106  
107  
    clearImageSurfaces(isBoard, isRecreated);
108  
    setField(fen := "");
109  
          
110  
    bool done = false;
111  
    new Var<Rect> lastLocation;
112  
    while (!done && !rst.triggered()) {
113  
      done = !rec.step();
114  
      
115  
      Rect r = rec.bestBoardLocation();
116  
      if (r != null)
117  
        rec.tryRect(print("wiggling", wiggleRect(r)));
118  
      
119  
      r = rec.bestBoardLocation();
120  
      if (r != null && setVar_trueIfChanged(lastLocation, r))
121  
        newBestLocation(r, pieceRecognizer);
122  
    }
123  
    
124  
    if (done && !lastLocation.has())
125  
      status("No chess board found");
126  
  }
127  
  
128  
  S status(S status) { setField(+status); ret status; }
129  
  
130  
  O _getReloadData() { ret img; }
131  
  void _setReloadData(BufferedImage img) { this.img = img; }
132  
 
133  
  void setImage(BufferedImage img) {
134  
    this.img = img;
135  
    if (isInput != null) isInput.setImageAndZoomToDisplay(img);
136  
    rst.trigger();
137  
  }
138  
139  
  S boardPointToSquare(Pt p) {
140  
    double space = imageMergeSpacing();
141  
    double squareW = board.getWidth()/8.0+space;
142  
    double squareH = board.getHeight()/8.0+space;
143  
    int x = clamp(iratio_floor(p.x+space/2, squareW), 0, 7);
144  
    int y = clamp(iratio_floor(p.y+space/2, squareH), 0, 7);
145  
    ret strCharPlus('A', x) + (8-y);
146  
  }
147  
  
148  
  S recreatedPointToSquare(Pt p) {
149  
    int x = clamp(iratio_floor(p.x, recreatedSquareSize), 0, 7);
150  
    int y = clamp(iratio_floor(p.y, recreatedSquareSize), 0, 7);
151  
    ret strCharPlus('A', x) + (8-y);
152  
  }
153  
  
154  
  void saveAndUploadPiece enter {
155  
    if (square == null || empty(piece)) ret;
156  
    chessOCR_uploadPieceImage(selectedSquare, piece, square);
157  
    if (mapGet(mapGet(recreateStock, chess_isLightSquare(selectedSquare)), piece) == null) recreateStock = null;
158  
    rst.trigger();
159  
  }
160  
  
161  
  void recreate {
162  
    if (recognized == null) ret;
163  
    if (recreateStock == null)
164  
      recreateStock = mapToValues(ll(false, true), light ->
165  
        mapValues(pairsToMap(reversePairs(chessPieceImagesFromAGIBlue(light))), imageID -> resizeImage(loadImage2(imageID), recreatedSquareSize, recreatedSquareSize)));
166  
167  
    new L<BufferedImage> out;
168  
    int i = 0;
169  
    for (S piece : pairsA(recognized)) {
170  
      bool light = even(i/8+i%8);
171  
      out.add(or_func(mapGet(mapGet(recreateStock, light), piece), () -> whiteImage(recreatedSquareSize, recreatedSquareSize)));
172  
      ++i;
173  
    }
174  
      
175  
    new L<BufferedImage> rows;
176  
    for (L<BufferedImage> row : listToChunks(out, 8))
177  
      rows.add(mergeBufferedImagesHorizontally(row, spacing := 0));
178  
    recreated = mergeBufferedImagesVertically(rows, spacing := 0);
179  
    if (isRecreated != null) isRecreated.setImageAndZoomToDisplay(recreated);
180  
  }
181  
  
182  
  Pair<S, Double> recognitionForSquare(S square) {
183  
    int x, y = unpair chess_squareToPos(square);
184  
    ret _get(recognized, y*8+x);
185  
  }
186  
  
187  
  S toolTipForSquare(S square) {
188  
    Pair<S, Double> rec = recognitionForSquare(square);
189  
    ret square + " - " + (rec == null ? "?" : rec.a + " [" + iround(clamp(rec.b, 0, 100)) + "%]");
190  
  }
191  
  
192  
  void selectSquare(S _square) {
193  
    setField(selectedSquare := _square);
194  
    S rec = pairA(recognitionForSquare(selectedSquare));
195  
    if (rec != null) setField(piece := rec);
196  
    int x, y = unpair chess_squareToPos(selectedSquare);
197  
    square = _get(_get(squareImages, y), x);
198  
    
199  
    ChessPieceProfile1 profile = chessOCR_pieceProfileFromRawImage_1(square, withPreprocessedImages := true);
200  
    isSquare.setImage(mergeBufferedImagesHorizontally(
201  
      itemPlusList(square, map toBufferedImage(profile.preprocessedImages))));
202  
    //printVars_str("leftClick", +p, +selectedSquare, +x, +y, +square);
203  
  }
204  
  
205  
  void showSegments {
206  
    L<Rect> segments = this.segments;
207  
    showImage_centered(n2(segments, "segment") + " found",
208  
      mergeBufferedImagesVertically(map(segments, r -> clipBufferedImage(img, r))));
209  
  }
210  
  
211  
  void publishImage enter {
212  
    BufferedImage img = this.img;
213  
    S segmenterPreset = this.segmenterPreset;
214  
    
215  
    JCheckBox cbFound = jcheckbox(true), cbHasBoard = jcheckbox(true);
216  
    JTextField tfFEN = jtextfield(fen), tfComments = jtextfield(),
217  
      tfImageCredits = jtextfield();
218  
    
219  
    showFormTitled("Publish training image",
220  
      "Image", jImageSurface(scaleImageToWidth(img, 500)),
221  
      "Segmenter used", jlabel(segmenterPreset),
222  
      "Is there a chess board in the image?", cbHasBoard,
223  
      "FEN (empty if no board in image)", tfFEN,
224  
      "Was the chess board found?", cbFound, // TODO: make clearer what this means if there is no board in image
225  
      "Image credits (optional)", tfImageCredits,
226  
      "Random comments", tfComments,
227  
      "NOTE:", jMultiLineLabel("By clicking 'publish', you PUBLISH this image (fair use/public domain)."),
228  
      "", jThreadedButton("Publish", r {
229  
        temp tempInfoBox_noHide("Uploading training image...");
230  
        S imageURL = uploadToImageServer("Chess board recognition training image", img);
231  
        S desc = "Chess board recognition training image " + assertNempty("Parseable image URL", parseSnippetImageURL(imageURL));
232  
        new LinkedHashMap<S> map;
233  
        map.put("Input image", imageURL);
234  
        mapPutIfNemptyValue(map, "Image credits", gtt(tfImageCredits));
235  
        S fen = gtt(tfFEN);
236  
        map.put("Image has chess board", yesNo_short_firstUpper(isChecked(cbHasBoard) || nempty(fen)));
237  
        mapPutIfNemptyValue(map, "FEN", fen);
238  
        map.put("Segmenter used", segmenterPreset);
239  
        map.put("Chess board found", yesNo_short_firstUpper(isChecked(cbFound)));
240  
        if (isChecked(cbFound))
241  
          map.put("Board location", struct(boardLocation));
242  
        S sliceID = agiBlue_chessBoardRecognitionTrainingImagesSliceID();
243  
        new L<Map> toPost;
244  
        S name = agiBlue_createUnusedNumberedPage(sliceID, desc + " #");
245  
        for (S key, value : map)
246  
          toPost.add(litmap(q := name, +key, +value));
247  
        agiBot_postMulti(keyPairForProgram(), toPost, slice := sliceID);
248  
        infoBox("Training image uploaded!");
249  
        openURLInBrowser(agiBlue_linkForPhrase(name, slice := sliceID));
250  
        disposeWindow(heldButton());
251  
      })
252  
    );
253  
  }
254  
  
255  
  S makeFEN() {
256  
    ret chess_makeFEN(listToChunks(pairsA(recognized), 8));
257  
  }
258  
  
259  
  void newBestLocation(Rect r, ChessPieceRecognizer pieceRecognizer) {
260  
    setField(segments := singletonUnlessNull(r));
261  
    bool corrected = squareHeightAtBottom && abs(r.w-r.h) > 3;
262  
    if (corrected) r.h = r.w;
263  
    new LS comments;
264  
    if (corrected) comments.add("height-corrected");
265  
    status("Chess board found at: " + r + appendRoundBracketed(joinWithComma(comments)));
266  
    setField(boardLocation := r);
267  
    board = clipBufferedImage(img, r);
268  
    
269  
    squareImages = bufferedImageMNGrid(board, 8, 8);
270  
    
271  
    if (isBoard != null)
272  
      isBoard.setImageAndZoomToDisplay(
273  
        mergeBufferedImagesVertically(
274  
          map(bufferedImageNVerticalSlices(8, board), i ->
275  
            mergeBufferedImagesHorizontally(bufferedImageNHorizontalSlices(8, i)))));
276  
            
277  
    recognized = new L;
278  
    for (BufferedImage img : concatLists(squareImages))
279  
      recognized.add(pieceRecognizer.recognize(img));
280  
    setField(recognitionScore := doubleAvg(pairsB(recognized));
281  
    setField(fen := makeFEN());
282  
    print(recognized);
283  
    recreate();
284  
  }
285  
  
286  
  Rect wiggleRect(Rect r) {
287  
    if (r == null) null;
288  
    int range = 1;
289  
    ret intersectRects(imageRect(img),
290  
      rectFromPoints(varyInt(range, r.x), varyInt(range, r.y),
291  
        varyInt(range, r.x2()), varyInt(range, r.y2())));
292  
  }
293  
  
294  
  void copyFEN {
295  
    if (empty(fen)) ret;
296  
    copyToClipboard(fen);
297  
    infoBox("FEN copied to clipboard");
298  
  }
299  
  
300  
  void grabScreenshot {
301  
    isInput.setImageAndZoomToDisplay(dm_shootScreenHidingOS());
302  
    rst.trigger();
303  
  }
304  
305  
  abstract JComponent segmenterSection();
306  
  abstract Iterator<? extends Segmenter> makeSegmenters();
307  
}

Author comment

Began life as a copy of #1024674

download  show line numbers  debug dex  old transpilations   

Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #1024870
Snippet name: ChessOCR_DynChessBoardRecognizer
Eternal ID of this version: #1024870/26
Text MD5: d8c7564dd15e7fced1f32c38170b1d10
Author: stefan
Category: javax / chess ocr
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2019-08-29 14:29:06
Source code size: 12174 bytes / 307 lines
Pitched / IR pitched: No / No
Views / Downloads: 186 / 647
Version history: 25 change(s)
Referenced in: [show references]