Instant Full-Text Snippet Search v4 [using deep double word index, OK] [1029014]

!7

sclass SearchResult {
  S snippetID;
  S title;
  //int occurrences;
  
  sS _fieldOrder = "snippetID title";
}

cmodule FullTextSnippetSearch > DynTableWithInput2<SearchResult> {
  switchable int maxResults = 1000;
  switchable int maxTextLength = 100000;
  transient S status;
  
  JComponent mainComponent() {
    ret northAndCenterWithMargin(dm_rightAlignedLabel status(), super.mainComponent());
  }
  
  start {
    dontPersist();
    thread { warmUp(); }
  }
  
  void calc {
    setList(search(input, true));
  }
  
  void warmUp enter {
    dm_showPoorMansProfile(r {
      benchFor5Seconds(r { hijackPrint(r { search("this is cool", false) }) });
    });
  }
  
  L<SearchResult> search(S input, bool setStatus) {
    S query = upper(input);
    S index = dm_require("#1029010/SnippetsDeepDoubleWordIndex");
    long nanos = nanoTime();
    Iterable<S> preSnippetIDs = cast dm_call(index, 'snippetsContainingTextIC_preSearch, query);
    //print("Pre-search: " + (preSnippetIDs == null ? "-" : lstr(preSnippetIDs)));
    
    S mod = dm_require("#1028945/CacheAllSnippets");
    SS data = cast dm_rcall getData(mod);
    SS titles = cast dm_rcall getTitles(mod);
    new L<SearchResult> out;
    if (preSnippetIDs == null) preSnippetIDs = keys(data);
    int scanned = 0;
    if (nempty(query)) {
      fOr ping (S snippetID : preSnippetIDs) {
        S text = data.get(snippetID);
        continue if l(text) > maxTextLength;
        S title = titles.get(snippetID);
        /*int n = 
          countOccurrencesIC(title, query)
          + countOccurrences_optimizedForZero(text, query);
        bool found = n > 0;*/
        scanned += l(text);
        bool found = contains(text, query); 
        if (found) {
          new SearchResult r;
          r.snippetID = snippetID;
          r.title = title;
          //r.occurrences = n;
          out.add(r);
          if (l(out) >= maxResults) break;
        }
      }
    }
    nanos = max(nanoTime()-nanos, 0);
    print("Scanned chars: " + scanned);
    //sortInPlaceByCalculatedFieldDesc(out, r -> r.occurrences);
    if (setStatus)
      setField(status := empty(query) ? " " : "Found " + addPlusToCount(maxResults, l(out), nSnippets(out) + " in " + formatDouble(nanoSecondsToMilliseconds(nanos), 3) + " ms"));
    ret out;
  }
}

Travelled to 7 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv

Snippet ID:	#1029014
Snippet name:	Instant Full-Text Snippet Search v4 [using deep double word index, OK]
Eternal ID of this version:	#1029014/13
Text MD5:	6d7f9ab89aca39fc5e9ca2de68ca5c6e
Transpilation MD5:	263293352b671b5b67a317fb0bb9d7bd
Author:	stefan
Category:	javax
Type:	JavaX source code (Dynamic Module)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2020-07-17 00:21:55
Source code size:	2376 bytes / 76 lines
Pitched / IR pitched:	No / No
Views / Downloads:	157 / 1713
Version history:	12 change(s)
Referenced in:	[show references]

< > BotCompany Repo | #1029014 // Instant Full-Text Snippet Search v4 [using deep double word index, OK]

JavaX source code (Dynamic Module) [tags: use-pretranspiled] - run with: Stefan's OS

Author comment

1	!7
2
3	sclass SearchResult {
4	S snippetID;
5	S title;
6	//int occurrences;
7
8	sS _fieldOrder = "snippetID title";
9	}
10
11	cmodule FullTextSnippetSearch > DynTableWithInput2<SearchResult> {
12	switchable int maxResults = 1000;
13	switchable int maxTextLength = 100000;
14	transient S status;
15
16	JComponent mainComponent() {
17	ret northAndCenterWithMargin(dm_rightAlignedLabel status(), super.mainComponent());
18	}
19
20	start {
21	dontPersist();
22	thread { warmUp(); }
23	}
24
25	void calc {
26	setList(search(input, true));
27	}
28
29	void warmUp enter {
30	dm_showPoorMansProfile(r {
31	benchFor5Seconds(r { hijackPrint(r { search("this is cool", false) }) });
32	});
33	}
34
35	L<SearchResult> search(S input, bool setStatus) {
36	S query = upper(input);
37	S index = dm_require("#1029010/SnippetsDeepDoubleWordIndex");
38	long nanos = nanoTime();
39	Iterable<S> preSnippetIDs = cast dm_call(index, 'snippetsContainingTextIC_preSearch, query);
40	//print("Pre-search: " + (preSnippetIDs == null ? "-" : lstr(preSnippetIDs)));
41
42	S mod = dm_require("#1028945/CacheAllSnippets");
43	SS data = cast dm_rcall getData(mod);
44	SS titles = cast dm_rcall getTitles(mod);
45	new L<SearchResult> out;
46	if (preSnippetIDs == null) preSnippetIDs = keys(data);
47	int scanned = 0;
48	if (nempty(query)) {
49	fOr ping (S snippetID : preSnippetIDs) {
50	S text = data.get(snippetID);
51	continue if l(text) > maxTextLength;
52	S title = titles.get(snippetID);
53	/*int n =
54	countOccurrencesIC(title, query)
55	+ countOccurrences_optimizedForZero(text, query);
56	bool found = n > 0;*/
57	scanned += l(text);
58	bool found = contains(text, query);
59	if (found) {
60	new SearchResult r;
61	r.snippetID = snippetID;
62	r.title = title;
63	//r.occurrences = n;
64	out.add(r);
65	if (l(out) >= maxResults) break;
66	}
67	}
68	}
69	nanos = max(nanoTime()-nanos, 0);
70	print("Scanned chars: " + scanned);
71	//sortInPlaceByCalculatedFieldDesc(out, r -> r.occurrences);
72	if (setStatus)
73	setField(status := empty(query) ? " " : "Found " + addPlusToCount(maxResults, l(out), nSnippets(out) + " in " + formatDouble(nanoSecondsToMilliseconds(nanos), 3) + " ms"));
74	ret out;
75	}
76	}