Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

90
LINES

< > BotCompany Repo | #1029532 // BufferedDiskByteMemory64 [handles files > 16 GB, dev.]

JavaX fragment (include) [tags: use-pretranspiled]

Uses 11335K of libraries. Click here for Pure Java version (5830L/37K).

1  
!include once #1027304 // Eclipse Collections
2  
3  
import java.nio.*;
4  
import java.nio.channels.*;
5  
6  
// read-only, so far. should be thread-safe
7  
final sclass BufferedDiskByteMemory64 implements IByteMemory64, AutoCloseable {
8  
  File file;
9  
  long size; // file size in ints
10  
  RandomAccessFile raf;
11  
  bool bigEndian = true;
12  
  bool writable = false;
13  
  bool debug;
14  
15  
  // byte buffers
16  
  int byteBufferShift = 30; // each buffer is 1 GB
17  
  int byteBufferSize = 1 << byteBufferShift;
18  
  MappedByteBuffer[] byteBuffers;
19  
  
20  
  *(File *file) {
21  
    this(file, false);
22  
  }
23  
  
24  
  *(File *file, bool *writable) {
25  
    load(file, writable);
26  
  }
27  
  
28  
  void load(File file, bool writable default false) ctex {
29  
    this.file = file;
30  
    this.writable = writable;
31  
    size = fileSize(file);
32  
    raf = newRandomAccessFile(file, writable ? "rw" : "r");
33  
    
34  
    byteBuffers = new MappedByteBuffer[toInt(rightShift_ceil(size, byteBufferShift))];
35  
    FileChannel channel = raf.getChannel();
36  
    for i over byteBuffers: {
37  
      long pos = (long) i << byteBufferShift;
38  
      int len = toInt(min(1 << byteBufferShift, size-pos));
39  
      //print("Mapping bytes " + longToHex(pos) + "-" + longToHex(pos+len));
40  
      byteBuffers[i] = channel.map(FileChannel.MapMode.READ_ONLY, pos, len);
41  
      byteBuffers[i].order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
42  
    }
43  
  }
44  
  
45  
  public void close {
46  
    dispose raf;
47  
  }
48  
  
49  
  public byte getByte(long idx) {
50  
    rangeCheck(idx, size);
51  
    ret byteBuffers[(int) (idx >> byteBufferShift)].get(((int) idx) & (byteBufferSize-1));
52  
  }
53  
  
54  
  public int getInt(long idx) {
55  
    int ofs = ((int) idx) & (byteBufferSize-1);
56  
    // check if we are crossing a buffer boundary
57  
    if (ofs > byteBufferSize-4)
58  
      ret getInt_manual(idx);
59  
    try {
60  
      ret byteBuffers[(int) (idx >> byteBufferShift)].getInt(ofs);
61  
    } on fail {
62  
      print(+idx);
63  
    }
64  
  }
65  
  
66  
  public int getInt_manual(long idx) {
67  
    assertTrue(+bigEndian);
68  
    ret ubyteToInt(getByte(idx)) << 24 |
69  
      ubyteToInt(getByte(idx+1)) << 16 |
70  
      ubyteToInt(getByte(idx+2)) << 8 |
71  
      ubyteToInt(getByte(idx+3));
72  
  }
73  
  
74  
  public void set(long idx, int val) {
75  
    checkWritable();
76  
    fail("todo");
77  
  }
78  
  
79  
  void checkWritable {
80  
    if (!writable) fail("read-only");
81  
  }
82  
  
83  
  public long size() {
84  
    ret size;
85  
  }
86  
  
87  
  public void ensureSize(int size) {
88  
    this.size = max(this.size, size);
89  
  }
90  
}

Author comment

Began life as a copy of #1029446

download  show line numbers  debug dex  old transpilations   

Travelled to 7 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv

No comments. add comment

Snippet ID: #1029532
Snippet name: BufferedDiskByteMemory64 [handles files > 16 GB, dev.]
Eternal ID of this version: #1029532/16
Text MD5: 080b366b6f0c90534987bc314ecdcba7
Transpilation MD5: 617e7c8f2b706196aa45f113b4aa1371
Author: stefan
Category: javax
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2020-08-18 15:13:18
Source code size: 2464 bytes / 90 lines
Pitched / IR pitched: No / No
Views / Downloads: 169 / 462
Version history: 15 change(s)
Referenced in: [show references]