Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

284
LINES

< > BotCompany Repo | #1024887 // ChessOCR_DynPipelinedRecognizer [dev.]

JavaX fragment (include)

1  
abstract sclass ChessOCR_DynPipelinedRecognizer extends DynModule {
2  
  S segmenterPreset;
3  
  transient Iterator<ParameterizedSegmenter> segmenters;
4  
  transient ChessOCR_Pipeline pipeline;
5  
  
6  
  Rect boardLocation;
7  
  double recognitionScore;
8  
  S status;
9  
  S selectedSquare;
10  
  S piece;
11  
  LPair<S, Double> recognized;
12  
  S fen;
13  
  ChessOCR_BoardsFound boardsFound;
14  
15  
  switchable bool squareHeightAtBottom = true; // for YouTube videos
16  
  transient int recreatedSquareSize = 32;
17  
  transient ReliableSingleThread rst = dm_rst(module(), r recognize);
18  
  transient BufferedImage img, board, square, recreated;
19  
  transient LL<BufferedImage> squareImages;
20  
  transient ImageSurface isInput, isBoard, isSquare, isRecreated;
21  
  
22  
  transient LS pieces = ai_chessPieces();
23  
  transient Map<Bool, Map<S, BufferedImage>> recreateStock; // false = dark, true = light
24  
  
25  
  visualize {
26  
    isBoard = jImageSurface();
27  
    imageSurfaceOnHover(isBoard, voidfunc(Pt p) {
28  
      if (board == null || p == null) ret with noToolTip(isBoard);
29  
      setToolTip(isBoard, toolTipForSquare(boardPointToSquare(p));
30  
    });
31  
    imageSurfaceOnLeftClick(isBoard, voidfunc(Pt p) {
32  
      selectSquare(boardPointToSquare(p))
33  
    });
34  
    
35  
    isRecreated = jImageSurface(recreated);
36  
    imageSurfaceOnHover(isRecreated, voidfunc(Pt p) {
37  
      if (recreated == null || p == null) ret with noToolTip(isRecreated);
38  
      setToolTip(isRecreated, toolTipForSquare(recreatedPointToSquare(p));
39  
    });
40  
    imageSurfaceOnLeftClick(isRecreated, voidfunc(Pt p) {
41  
      selectSquare(recreatedPointToSquare(p))
42  
    });
43  
    
44  
    isInput = jImageSurface(or_func(img, () -> whiteImage(100)));
45  
    isInput.onNewImage = rst;
46  
    
47  
    ret withCalc(rst, jvsplit(
48  
      jCenteredSection("Input image", jscroll_center(isInput)),
49  
      northCenterAndSouthWithMargins(
50  
        centerAndEastWithMargin(segmenterSection(),
51  
          hstackWithSpacing(
52  
            jbutton("Recognize", rst),
53  
            jPopDownButton_noText(
54  
              "Show possible chess boards found", rThread showSegments,
55  
              "Publish training image...", rThread publishImage,
56  
              "---", null,
57  
              "Screenshot (hiding OS)", rThread {
58  
                isInput.setImageAndZoomToDisplay(dm_shootScreenHidingOS());
59  
                rst.trigger();
60  
              })
61  
          )),
62  
        hgridWithSpacing(
63  
          jCenteredSection("Recreated", jscroll_center(isRecreated)),
64  
          jCenteredSection("Chess board in image",
65  
            centerAndSouthWithMargin(jscroll_center(isBoard),
66  
            dm_centeredCalculatedLabel(() -> "Recognition score: " + formatDouble(recognitionScore, 2))
67  
          )),
68  
          jCenteredLiveValueSection(dm_calculatedLiveValue(S, () -> "Selected square" + (empty(selectedSquare) ? "" : " (" + selectedSquare + ")")), centerAndSouthWithMargin(
69  
            jscroll_center(isSquare = jImageSurface(square)),
70  
            rightAlignedLine(
71  
              withLabel("Piece: ", dm_comboBox('piece, pieces)),
72  
              jbutton("Save & upload", rThread saveAndUploadPiece))
73  
        ))),
74  
        vstackWithSpacing(
75  
          dm_label('status),
76  
          withLabel("FEN:", dm_label('fen))
77  
        )
78  
      )));
79  
  }
80  
    
81  
  void recognize {
82  
    if (pipeline != null)
83  
      pipeline.cancel();
84  
    status("Recognizing...");
85  
    segmenters = makeSegmenters();
86  
    if (empty(segmenters)) ret with infoBox(status("No segmenter"));
87  
    //img = dm_shootScreenHidingOS();
88  
    if (isInput != null) img = isInput.getImage();
89  
    
90  
    ChessPieceRecognizer pieceRecognizer = chessOCR_pieceRecognizer();
91  
    
92  
    ChessOCR_TwoStageRecognizer rec = new(pieceRecognizer, img, segmenters);
93  
    rec.verbose(true);
94  
95  
    clearImageSurfaces(isBoard, isRecreated);
96  
    setField(fen := "");
97  
          
98  
    bool done = false;
99  
    new Var<Rect> lastLocation;
100  
    while (!done && !rst.triggered()) {
101  
      done = !rec.step();
102  
      
103  
      Rect r = rec.bestBoardLocation();
104  
      if (r != null)
105  
        rec.tryRect(print("wiggling", wiggleRect(r)));
106  
      
107  
      r = rec.bestBoardLocation();
108  
      if (r != null && setVar_trueIfChanged(lastLocation, r))
109  
        newBestLocation(r, pieceRecognizer);
110  
    }
111  
    
112  
    if (done && !lastLocation.has())
113  
      status("No chess board found");
114  
  }
115  
  
116  
  S status(S status) { setField(+status); ret status; }
117  
  
118  
  O _getReloadData() { ret img; }
119  
  void _setReloadData(BufferedImage img) { this.img = img; }
120  
 
121  
  void setImage(BufferedImage img) {
122  
    this.img = img;
123  
    if (isInput != null) isInput.setImageAndZoomToDisplay(img);
124  
    rst.trigger();
125  
  }
126  
127  
  S boardPointToSquare(Pt p) {
128  
    double space = imageMergeSpacing();
129  
    double squareW = board.getWidth()/8.0+space;
130  
    double squareH = board.getHeight()/8.0+space;
131  
    int x = clamp(iratio_floor(p.x+space/2, squareW), 0, 7);
132  
    int y = clamp(iratio_floor(p.y+space/2, squareH), 0, 7);
133  
    ret strCharPlus('A', x) + (8-y);
134  
  }
135  
  
136  
  S recreatedPointToSquare(Pt p) {
137  
    int x = clamp(iratio_floor(p.x, recreatedSquareSize), 0, 7);
138  
    int y = clamp(iratio_floor(p.y, recreatedSquareSize), 0, 7);
139  
    ret strCharPlus('A', x) + (8-y);
140  
  }
141  
  
142  
  void saveAndUploadPiece enter {
143  
    if (square == null || empty(piece)) ret;
144  
    chessOCR_uploadPieceImage(selectedSquare, piece, square);
145  
    if (mapGet(mapGet(recreateStock, chess_isLightSquare(selectedSquare)), piece) == null) recreateStock = null;
146  
    rst.trigger();
147  
  }
148  
  
149  
  void recreate {
150  
    if (recognized == null) ret;
151  
    if (recreateStock == null)
152  
      recreateStock = mapToValues(ll(false, true), light ->
153  
        mapValues(pairsToMap(reversePairs(chessPieceImagesFromAGIBlue(light))), imageID -> resizeImage(loadImage2(imageID), recreatedSquareSize, recreatedSquareSize)));
154  
155  
    new L<BufferedImage> out;
156  
    int i = 0;
157  
    for (S piece : pairsA(recognized)) {
158  
      bool light = even(i/8+i%8);
159  
      out.add(or_func(mapGet(mapGet(recreateStock, light), piece), () -> whiteImage(recreatedSquareSize, recreatedSquareSize)));
160  
      ++i;
161  
    }
162  
      
163  
    new L<BufferedImage> rows;
164  
    for (L<BufferedImage> row : listToChunks(out, 8))
165  
      rows.add(mergeBufferedImagesHorizontally(row, spacing := 0));
166  
    recreated = mergeBufferedImagesVertically(rows, spacing := 0);
167  
    if (isRecreated != null) isRecreated.setImageAndZoomToDisplay(recreated);
168  
  }
169  
  
170  
  Pair<S, Double> recognitionForSquare(S square) {
171  
    int x, y = unpair chess_squareToPos(square);
172  
    ret _get(recognized, y*8+x);
173  
  }
174  
  
175  
  S toolTipForSquare(S square) {
176  
    Pair<S, Double> rec = recognitionForSquare(square);
177  
    ret square + " - " + (rec == null ? "?" : rec.a + " [" + iround(clamp(rec.b, 0, 100)) + "%]");
178  
  }
179  
  
180  
  void selectSquare(S _square) {
181  
    setField(selectedSquare := _square);
182  
    S rec = pairA(recognitionForSquare(selectedSquare));
183  
    if (rec != null) setField(piece := rec);
184  
    int x, y = unpair chess_squareToPos(selectedSquare);
185  
    square = _get(_get(squareImages, y), x);
186  
    
187  
    ChessPieceProfile1 profile = chessOCR_pieceProfileFromRawImage_1(square, withPreprocessedImages := true);
188  
    isSquare.setImage(mergeBufferedImagesHorizontally(
189  
      itemPlusList(square, map toBufferedImage(profile.preprocessedImages))));
190  
    //printVars_str("leftClick", +p, +selectedSquare, +x, +y, +square);
191  
  }
192  
  
193  
  void showSegments {
194  
    L<Rect> segments = this.segments;
195  
    showImage_centered(n2(segments, "segment") + " found",
196  
      mergeBufferedImagesVertically(map(segments, r -> clipBufferedImage(img, r))));
197  
  }
198  
  
199  
  void publishImage enter {
200  
    BufferedImage img = this.img;
201  
    S segmenterPreset = this.segmenterPreset;
202  
    
203  
    JCheckBox cbFound = jcheckbox(true), cbHasBoard = jcheckbox(true);
204  
    JTextField tfFEN = jtextfield(fen), tfComments = jtextfield(),
205  
      tfImageCredits = jtextfield();
206  
    
207  
    showFormTitled("Publish training image",
208  
      "Image", jImageSurface(scaleImageToWidth(img, 500)),
209  
      "Segmenter used", jlabel(segmenterPreset),
210  
      "Is there a chess board in the image?", cbHasBoard,
211  
      "FEN (empty if no board in image)", tfFEN,
212  
      "Was the chess board found?", cbFound, // TODO: make clearer what this means if there is no board in image
213  
      "Image credits (optional)", tfImageCredits,
214  
      "Random comments", tfComments,
215  
      "NOTE:", jMultiLineLabel("By clicking 'publish', you PUBLISH this image (fair use/public domain)."),
216  
      "", jThreadedButton("Publish", r {
217  
        temp tempInfoBox_noHide("Uploading training image...");
218  
        S imageURL = uploadToImageServer("Chess board recognition training image", img);
219  
        S desc = "Chess board recognition training image " + assertNempty("Parseable image URL", parseSnippetImageURL(imageURL));
220  
        new LinkedHashMap<S> map;
221  
        map.put("Input image", imageURL);
222  
        mapPutIfNemptyValue(map, "Image credits", gtt(tfImageCredits));
223  
        S fen = gtt(tfFEN);
224  
        map.put("Image has chess board", yesNo_short_firstUpper(isChecked(cbHasBoard) || nempty(fen)));
225  
        mapPutIfNemptyValue(map, "FEN", fen);
226  
        map.put("Segmenter used", segmenterPreset);
227  
        map.put("Chess board found", yesNo_short_firstUpper(isChecked(cbFound)));
228  
        if (isChecked(cbFound))
229  
          map.put("Board location", struct(boardLocation));
230  
        S sliceID = agiBlue_chessBoardRecognitionTrainingImagesSliceID();
231  
        new L<Map> toPost;
232  
        S name = agiBlue_createUnusedNumberedPage(sliceID, desc + " #");
233  
        for (S key, value : map)
234  
          toPost.add(litmap(q := name, +key, +value));
235  
        agiBot_postMulti(keyPairForProgram(), toPost, slice := sliceID);
236  
        infoBox("Training image uploaded!");
237  
        openURLInBrowser(agiBlue_linkForPhrase(name, slice := sliceID));
238  
        disposeWindow(heldButton());
239  
      })
240  
    );
241  
  }
242  
  
243  
  S makeFEN() {
244  
    ret chess_makeFEN(listToChunks(pairsA(recognized), 8));
245  
  }
246  
  
247  
  void newBestLocation(Rect r, ChessPieceRecognizer pieceRecognizer) {
248  
    setField(segments := singletonUnlessNull(r));
249  
    bool corrected = squareHeightAtBottom && abs(r.w-r.h) > 3;
250  
    if (corrected) r.h = r.w;
251  
    new LS comments;
252  
    if (corrected) comments.add("height-corrected");
253  
    status("Chess board found at: " + r + appendRoundBracketed(joinWithComma(comments)));
254  
    setField(boardLocation := r);
255  
    board = clipBufferedImage(img, r);
256  
    
257  
    squareImages = bufferedImageMNGrid(board, 8, 8);
258  
    
259  
    if (isBoard != null)
260  
      isBoard.setImageAndZoomToDisplay(
261  
        mergeBufferedImagesVertically(
262  
          map(bufferedImageNVerticalSlices(8, board), i ->
263  
            mergeBufferedImagesHorizontally(bufferedImageNHorizontalSlices(8, i)))));
264  
            
265  
    recognized = new L;
266  
    for (BufferedImage img : concatLists(squareImages))
267  
      recognized.add(pieceRecognizer.recognize(img));
268  
    setField(recognitionScore := doubleAvg(pairsB(recognized));
269  
    setField(fen := makeFEN());
270  
    print(recognized);
271  
    recreate();
272  
  }
273  
  
274  
  Rect wiggleRect(Rect r) {
275  
    if (r == null) null;
276  
    int range = 1;
277  
    ret intersectRects(imageRect(img),
278  
      rectFromPoints(varyInt(range, r.x), varyInt(range, r.y),
279  
        varyInt(range, r.x2()), varyInt(range, r.y2())));
280  
  }
281  
  
282  
  abstract JComponent segmenterSection();
283  
  abstract Iterator<ParameterizedSegmenter> makeSegmenters();
284  
}

Author comment

Began life as a copy of #1024870

download  show line numbers  debug dex  old transpilations   

Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #1024887
Snippet name: ChessOCR_DynPipelinedRecognizer [dev.]
Eternal ID of this version: #1024887/1
Text MD5: aa82c3540ed810f6c0fa674e501d06d7
Author: stefan
Category: javax / chess ocr
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2019-08-28 16:49:05
Source code size: 11396 bytes / 284 lines
Pitched / IR pitched: No / No
Views / Downloads: 111 / 125
Referenced in: [show references]