Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

206
LINES

< > BotCompany Repo | #1029456 // BufferedDiskIntMemory64 [working backup before memory-mapped IO]

JavaX fragment (include) [tags: use-pretranspiled]

Uses 11335K of libraries. Click here for Pure Java version (4735L/31K).

1  
!include once #1027304 // Eclipse Collections
2  
3  
// read-only, so far. should be thread-safe
4  
final sclass BufferedDiskIntMemory64 implements IIntMemory64, AutoCloseable {
5  
  File file;
6  
  long size; // file size in ints
7  
  RandomAccessFile raf;
8  
  bool bigEndian = true;
9  
  bool writable = true;
10  
  bool debug, verboseEvictions;
11  
  long cacheSize = 128*1024*1024; // in bytes
12  
  
13  
  // pageSize is in ints
14  
  // page indices are ints interpreted as unsigned
15  
  int pageShift, pageSize, maxCachedPages;
16  
  new LongObjectHashMap<CacheEntry> cache;
17  
  CacheEntry newestCacheEntry, oldestCacheEntry;
18  
  
19  
  // stats
20  
  long pageLoads, evictions;
21  
  long pageLoadPrintInterval = 1000;
22  
  
23  
  static int defaultPageSize = 4096; // in bytes
24  
  
25  
  sclass CacheEntry {
26  
    long page;
27  
    bool dirty;
28  
    int[] data;
29  
    CacheEntry newer, older; // MRU list
30  
    
31  
    toString { ret "Page " + page; }
32  
  }
33  
  
34  
  *() {
35  
    setPageSize(4096);
36  
  }
37  
  
38  
  synchronized void setPageSize(int pageSizeInBytes) {
39  
    clearCache();
40  
    pageShift = 31-Int.numberOfLeadingZeros(pageSizeInBytes >> 2);
41  
    pageSize = 1 << pageShift;
42  
    updateMaxCachedPages();
43  
  }
44  
  
45  
  void setCacheSize(long bytes) { setMaxCachedBytes(bytes); }
46  
  synchronized void setMaxCachedBytes(long bytes) {
47  
    cacheSize = bytes;
48  
    updateMaxCachedPages();
49  
  }
50  
  
51  
  // internal
52  
  void updateMaxCachedPages {
53  
    maxCachedPages = (int) (cacheSize >> (pageShift+2));
54  
  }
55  
  
56  
  long maxCachedBytes() {
57  
    ret ((long) maxCachedPages) << (pageShift+2);
58  
  }
59  
  
60  
  long cachedBytes() {
61  
    ret ((long) cache.size()) << (pageShift+2);
62  
  }
63  
  
64  
  synchronized void clearCache {
65  
    flush();
66  
    cache = new LongObjectHashMap;
67  
    newestCacheEntry = oldestCacheEntry = null;
68  
  }
69  
  
70  
  
71  
  *(File *file) {
72  
    this(file, false);
73  
  }
74  
  
75  
  *(File *file, bool *writable) {
76  
    this();
77  
    size = fileSize(file)/4;
78  
    raf = newRandomAccessFile(file, writable ? "rw" : "r");
79  
  }
80  
  
81  
  public synchronized void close {
82  
    flush();
83  
    dispose raf;
84  
    cache = null;
85  
  }
86  
  
87  
  synchronized void flush {
88  
    for (CacheEntry e : cache.values())
89  
      flushPage(e);
90  
  }
91  
  
92  
  public synchronized int get(long idx) {
93  
    rangeCheck(idx, size);
94  
    ret loadCell(idx).data[(int) idx & (pageSize-1)];
95  
  }
96  
  
97  
  public synchronized void set(long idx, int val) {
98  
    checkWritable();
99  
    rangeCheck(idx, size);
100  
    CacheEntry e = loadCell(idx);
101  
    e.data[(int) idx & (pageSize-1)] = val;
102  
    e.dirty = true;
103  
  }
104  
  
105  
  CacheEntry loadCell(long idx) {
106  
    long page = idx >> pageShift;
107  
    CacheEntry e = cache.get(page);
108  
    if (e == null) {
109  
      if (debug) print("Accessing unloaded cell " + longToHex(idx));
110  
      e = loadPage(page);
111  
    } else
112  
      touchPage(e);
113  
    ret e;
114  
  }
115  
  
116  
  void touchPage(CacheEntry e) {
117  
    if (e == newestCacheEntry) ret;
118  
    if (debug) print("Touching page " + e.page + ". Older=" + e.older + ", newer=" + e.newer + ". Oldest=" + oldestCacheEntry + ", newest=" + newestCacheEntry);
119  
    
120  
    // Take out of list
121  
    removeFromLinkedList(e);
122  
    
123  
    // re-insert at top
124  
    e.newer = null; // nobody newer than us
125  
    e.older = newestCacheEntry; // point to previous newest
126  
    newestCacheEntry.newer = e; // point previous newest to us
127  
    newestCacheEntry = e; // make us the newest
128  
    
129  
    if (debug) print("Touched page " + e.page + ". Older=" + e.older + ", newer=" + e.newer + ". Oldest=" + oldestCacheEntry + ", newest=" + newestCacheEntry);
130  
  }
131  
  
132  
  void removeFromLinkedList(CacheEntry e) {
133  
    if (e.older != null)
134  
      e.older.newer = e.newer; // There is an older page, point it to who was on top of us
135  
    else
136  
      oldestCacheEntry = e.newer; // We were the oldest, point it to who was on top uf os
137  
    if (e.newer != null)
138  
      e.newer.older = e.older; // Point who was on top of us to who was behind us (or no one)
139  
    else
140  
      newestCacheEntry = e.older;
141  
  }
142  
  
143  
  bool cacheFull() { ret cache.size() >= maxCachedPages; }
144  
  
145  
  void evictAPage {
146  
    ++evictions;
147  
    CacheEntry e = oldestCacheEntry;
148  
    flushPage(e);
149  
    if (debug || verboseEvictions) print("Evicting page " + e.page);
150  
    cache.remove(e.page);
151  
    removeFromLinkedList(e);
152  
  }
153  
  
154  
  void flushPage(CacheEntry e) ctex {
155  
    if (!e.dirty) ret;
156  
    if (debug) print("Saving page " + e.page);
157  
    raf.seek(e.page << (pageShift+2));
158  
    byte[] buf = bigEndian ? intArrayToBytes(e.data) : intArrayToBytes_littleEndian(e.data);
159  
    raf.write(buf);
160  
    e.dirty = false;
161  
  }
162  
  
163  
  CacheEntry loadPage(long page) ctex {
164  
    if (((++pageLoads) % pageLoadPrintInterval) == 0)
165  
      print(fileName(file) + ": " + n2(pageLoads, "page load")
166  
        + ", cache size: " + toM(usedCacheSize()) + "/" + toM(maxCacheSize()) + "M");
167  
    if (cacheFull()) evictAPage();
168  
    if (debug) print("Loading page " + page);
169  
    raf.seek(page << (pageShift+2));
170  
    byte[] buf = new[pageSize*4];
171  
    raf.read(buf);
172  
    new CacheEntry e;
173  
    e.page = page;
174  
    e.data = bigEndian ? intArrayFromBytes(buf) : intArrayFromBytes_littleEndian(buf);
175  
    cache.put(page, e);
176  
    
177  
    // put e in front of MRU list
178  
    e.older = newestCacheEntry; // point to previous top
179  
    if (newestCacheEntry != null) // list is not empty, update "newer" pointer of current top
180  
      newestCacheEntry.newer = e;
181  
    else oldestCacheEntry = e; // otherwise, we are also the oldest entry
182  
    newestCacheEntry = e; // we are new top
183  
    
184  
    if (debug) print("Loaded page " + e.page + ". Older=" + e.older + ", newer=" + e.newer + ". Oldest=" + oldestCacheEntry + ", newest=" + newestCacheEntry);
185  
    ret e;
186  
  }
187  
  
188  
  void checkWritable {
189  
    if (!writable) fail("read-only");
190  
  }
191  
  
192  
  public long size() {
193  
    ret size;
194  
  }
195  
  
196  
  double cacheFullPercentage() {
197  
    ret percentRatio(cache.size(), maxCachedPages);
198  
  }
199  
  
200  
  long usedCacheSize() { ret cache.size()*(long) pageSize; }
201  
  long maxCacheSize() { ret cacheSize; }
202  
  
203  
  public synchronized void ensureSize(int size) {
204  
    this.size = max(this.size, size);
205  
  }
206  
}

Author comment

Began life as a copy of #1029446

download  show line numbers  debug dex  old transpilations   

Travelled to 7 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv

No comments. add comment

Snippet ID: #1029456
Snippet name: BufferedDiskIntMemory64 [working backup before memory-mapped IO]
Eternal ID of this version: #1029456/1
Text MD5: 9c3a5f5d6efc6617fd10685d8a038695
Transpilation MD5: 0b1bf039330f7fea232e19cb31a83cfc
Author: stefan
Category: javax
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2020-08-08 15:44:30
Source code size: 6080 bytes / 206 lines
Pitched / IR pitched: No / No
Views / Downloads: 247 / 324
Referenced in: [show references]