Skip to content

Commit 14233f5

Browse files
devgianludevgianlu
authored
New cache system (#179)
* Removed H2 database + using journal file * Actually writing data to file + added test * Improved journal creation time * Added cache maintenance logic (clean up + remove corrupted entries) * Minor changes + logging * Rewritten test properly * Do not publish snapshot for PR
1 parent 0d0b6f2 commit 14233f5

9 files changed

Lines changed: 613 additions & 250 deletions

File tree

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ script:
99
- mvn clean test -Pdebug -B -U -Dgpg.skip -Dmaven.javadoc.skip=true
1010

1111
after_script:
12-
- test $TRAVIS_PULL_REQUEST = "false" && mvn deploy --settings .maven.xml -B -U -Prelease
12+
- test $TRAVIS_PULL_REQUEST = "false" && test $TRAVIS_BRANCH = "dev" && mvn deploy --settings .maven.xml -B -U -Prelease
1313

1414
before_deploy:
1515
- mvn help:evaluate -N -Dexpression=project.version|grep -v '\['

‎core/pom.xml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,13 +71,6 @@
7171
<version>1.1.2</version>
7272
</dependency>
7373

74-
<!-- H2 (cache) -->
75-
<dependency>
76-
<groupId>com.h2database</groupId>
77-
<artifactId>h2</artifactId>
78-
<version>1.4.200</version>
79-
</dependency>
80-
8174
<!-- HTTP -->
8275
<dependency>
8376
<groupId>com.squareup.okhttp3</groupId>

‎core/src/main/java/xyz/gianlu/librespot/cache/CacheJournal.java‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
package xyz.gianlu.librespot.cache;
2+
3+
import org.jetbrains.annotations.NotNull;
4+
import org.jetbrains.annotations.Nullable;
5+
import xyz.gianlu.librespot.common.Utils;
6+
7+
import java.io.*;
8+
import java.nio.charset.StandardCharsets;
9+
import java.util.*;
10+
11+
/**
12+
* A little journal implementation that stores information about the cache. The data is stored in this order:
13+
* - 40 bytes for the ID
14+
* - 2048 bytes for chunks
15+
* - 8 headers each of 1023 length + 1 byte for the ID
16+
* <p>
17+
* Headers are encoded to strings in order to take advantage of null terminators.
18+
*
19+
* @author Gianlu
20+
*/
21+
class CacheJournal implements Closeable {
22+
static final int MAX_CHUNKS_SIZE = 2048;
23+
static final int MAX_CHUNKS = MAX_CHUNKS_SIZE * 8;
24+
static final int MAX_HEADER_LENGTH = 1023;
25+
static final int MAX_ID_LENGTH = 40;
26+
private static final int MAX_HEADERS = 8;
27+
static final int JOURNAL_ENTRY_SIZE = MAX_ID_LENGTH + MAX_CHUNKS_SIZE + (1 + MAX_HEADER_LENGTH) * MAX_HEADERS;
28+
private static final byte[] ZERO_ARRAY = new byte[JOURNAL_ENTRY_SIZE];
29+
private final RandomAccessFile io;
30+
private final Map<String, Entry> entries = Collections.synchronizedMap(new HashMap<>(1024));
31+
32+
CacheJournal(@NotNull File parent) throws FileNotFoundException {
33+
File file = new File(parent, "journal.dat");
34+
io = new RandomAccessFile(file, "rwd");
35+
}
36+
37+
private static boolean checkId(@NotNull RandomAccessFile io, int first, @NotNull byte[] id) throws IOException {
38+
for (int i = 0; i < id.length; i++) {
39+
int read = i == 0 ? first : io.read();
40+
if (read == 0)
41+
return i != 0;
42+
43+
if (read != id[i])
44+
return false;
45+
}
46+
47+
return true;
48+
}
49+
50+
@NotNull
51+
private static String trimArrayToNullTerminator(byte[] bytes) {
52+
for (int i = 0; i < bytes.length; i++)
53+
if (bytes[i] == 0)
54+
return new String(bytes, 0, i);
55+
56+
return new String(bytes);
57+
}
58+
59+
boolean hasChunk(@NotNull String streamId, int index) throws IOException {
60+
if (index < 0 || index > MAX_CHUNKS) throw new IllegalArgumentException();
61+
62+
Entry entry = find(streamId);
63+
if (entry == null) throw new JournalException("Couldn't find entry on journal: " + streamId);
64+
65+
synchronized (io) {
66+
return entry.hasChunk(index);
67+
}
68+
}
69+
70+
void setChunk(@NotNull String streamId, int index, boolean val) throws IOException {
71+
if (index < 0 || index > MAX_CHUNKS) throw new IllegalArgumentException();
72+
73+
Entry entry = find(streamId);
74+
if (entry == null) throw new JournalException("Couldn't find entry on journal: " + streamId);
75+
76+
synchronized (io) {
77+
entry.setChunk(index, val);
78+
}
79+
}
80+
81+
@NotNull
82+
List<JournalHeader> getHeaders(@NotNull String streamId) throws IOException {
83+
Entry entry = find(streamId);
84+
if (entry == null) throw new JournalException("Couldn't find entry on journal: " + streamId);
85+
86+
synchronized (io) {
87+
return entry.getHeaders();
88+
}
89+
}
90+
91+
@Nullable
92+
JournalHeader getHeader(@NotNull String streamId, byte id) throws IOException {
93+
Entry entry = find(streamId);
94+
if (entry == null) throw new JournalException("Couldn't find entry on journal: " + streamId);
95+
96+
synchronized (io) {
97+
return entry.getHeader(id);
98+
}
99+
}
100+
101+
void setHeader(@NotNull String streamId, byte headerId, byte[] value) throws IOException {
102+
String strValue = Utils.bytesToHex(value);
103+
104+
if (strValue.length() > MAX_HEADER_LENGTH) throw new IllegalArgumentException();
105+
else if (headerId == 0) throw new IllegalArgumentException();
106+
107+
Entry entry = find(streamId);
108+
if (entry == null) throw new JournalException("Couldn't find entry on journal: " + streamId);
109+
110+
synchronized (io) {
111+
entry.setHeader(headerId, strValue);
112+
}
113+
}
114+
115+
void remove(@NotNull String streamId) throws IOException {
116+
Entry entry = find(streamId);
117+
if (entry == null) return;
118+
119+
synchronized (io) {
120+
entry.remove();
121+
}
122+
123+
entries.remove(streamId);
124+
}
125+
126+
@NotNull
127+
List<String> getEntries() throws IOException {
128+
List<String> list = new ArrayList<>(1024);
129+
130+
synchronized (io) {
131+
io.seek(0);
132+
133+
int i = 0;
134+
while (true) {
135+
io.seek(i * JOURNAL_ENTRY_SIZE);
136+
137+
int first = io.read();
138+
if (first == -1) // EOF
139+
break;
140+
141+
if (first == 0) { // Empty spot
142+
i++;
143+
continue;
144+
}
145+
146+
byte[] id = new byte[MAX_ID_LENGTH];
147+
id[0] = (byte) first;
148+
io.read(id, 1, MAX_ID_LENGTH - 1);
149+
150+
String idStr = trimArrayToNullTerminator(id);
151+
Entry entry = new Entry(idStr, i * JOURNAL_ENTRY_SIZE);
152+
entries.put(idStr, entry);
153+
list.add(idStr);
154+
155+
i++;
156+
}
157+
}
158+
159+
return list;
160+
}
161+
162+
@Nullable
163+
private Entry find(@NotNull String id) throws IOException {
164+
if (id.length() > MAX_ID_LENGTH) throw new IllegalArgumentException();
165+
166+
Entry entry = entries.get(id);
167+
if (entry != null) return entry;
168+
169+
byte[] idBytes = id.getBytes(StandardCharsets.US_ASCII);
170+
synchronized (io) {
171+
io.seek(0);
172+
173+
int i = 0;
174+
while (true) {
175+
io.seek(i * JOURNAL_ENTRY_SIZE);
176+
177+
int first = io.read();
178+
if (first == -1) // EOF
179+
return null;
180+
181+
if (first == 0) { // Empty spot
182+
i++;
183+
continue;
184+
}
185+
186+
if (checkId(io, first, idBytes)) {
187+
entry = new Entry(id, i * JOURNAL_ENTRY_SIZE);
188+
entries.put(id, entry);
189+
return entry;
190+
}
191+
192+
i++;
193+
}
194+
}
195+
}
196+
197+
void createIfNeeded(@NotNull String id) throws IOException {
198+
if (find(id) != null) return;
199+
200+
synchronized (io) {
201+
io.seek(0);
202+
203+
int i = 0;
204+
while (true) {
205+
io.seek(i * JOURNAL_ENTRY_SIZE);
206+
207+
int first = io.read();
208+
if (first == 0 || first == -1) { // First empty spot or EOF
209+
Entry entry = new Entry(id, i * JOURNAL_ENTRY_SIZE);
210+
entry.writeId();
211+
entries.put(id, entry);
212+
return;
213+
}
214+
215+
i++;
216+
}
217+
}
218+
}
219+
220+
@Override
221+
public void close() throws IOException {
222+
synchronized (io) {
223+
io.close();
224+
}
225+
}
226+
227+
private static class JournalException extends IOException {
228+
JournalException(String message) {
229+
super(message);
230+
}
231+
}
232+
233+
private class Entry {
234+
private final String id;
235+
private final int offset;
236+
237+
private Entry(@NotNull String id, int offset) {
238+
this.id = id;
239+
this.offset = offset;
240+
}
241+
242+
void writeId() throws IOException {
243+
io.seek(offset);
244+
io.write(id.getBytes(StandardCharsets.US_ASCII));
245+
io.write(ZERO_ARRAY, 0, JOURNAL_ENTRY_SIZE - id.length());
246+
}
247+
248+
void remove() throws IOException {
249+
io.seek(offset);
250+
io.write(0);
251+
}
252+
253+
private int findHeader(byte headerId) throws IOException {
254+
for (int i = 0; i < MAX_HEADERS; i++) {
255+
io.seek(offset + MAX_ID_LENGTH + MAX_CHUNKS_SIZE + i * (MAX_HEADER_LENGTH + 1));
256+
if (io.read() == headerId)
257+
return i;
258+
}
259+
260+
return -1;
261+
}
262+
263+
void setHeader(byte id, @NotNull String value) throws IOException {
264+
int index = findHeader(id);
265+
if (index == -1) {
266+
for (int i = 0; i < MAX_HEADERS; i++) {
267+
io.seek(offset + MAX_ID_LENGTH + MAX_CHUNKS_SIZE + i * (MAX_HEADER_LENGTH + 1));
268+
if (io.read() == 0) {
269+
index = i;
270+
break;
271+
}
272+
}
273+
274+
if (index == -1) throw new IllegalStateException();
275+
}
276+
277+
io.seek(offset + MAX_ID_LENGTH + MAX_CHUNKS_SIZE + index * (MAX_HEADER_LENGTH + 1));
278+
io.write(id);
279+
io.write(value.getBytes());
280+
}
281+
282+
@NotNull
283+
List<JournalHeader> getHeaders() throws IOException {
284+
List<JournalHeader> list = new ArrayList<>(MAX_HEADERS);
285+
for (int i = 0; i < MAX_HEADERS; i++) {
286+
io.seek(offset + MAX_ID_LENGTH + MAX_CHUNKS_SIZE + i * (MAX_HEADER_LENGTH + 1));
287+
int headerId;
288+
if ((headerId = io.read()) == 0)
289+
continue;
290+
291+
byte[] read = new byte[MAX_HEADER_LENGTH];
292+
io.read(read);
293+
294+
list.add(new JournalHeader((byte) headerId, trimArrayToNullTerminator(read)));
295+
}
296+
297+
return list;
298+
}
299+
300+
@Nullable
301+
JournalHeader getHeader(byte id) throws IOException {
302+
int index = findHeader(id);
303+
if (index == -1) return null;
304+
305+
io.seek(offset + MAX_ID_LENGTH + MAX_CHUNKS_SIZE + index * (MAX_HEADER_LENGTH + 1) + 1);
306+
byte[] read = new byte[MAX_HEADER_LENGTH];
307+
io.read(read);
308+
309+
return new JournalHeader(id, trimArrayToNullTerminator(read));
310+
}
311+
312+
void setChunk(int index, boolean val) throws IOException {
313+
int pos = offset + MAX_ID_LENGTH + (index / 8);
314+
io.seek(pos);
315+
int read = io.read();
316+
if (val) read |= (1 << (index % 8));
317+
else read &= ~(1 << (index % 8));
318+
io.seek(pos);
319+
io.write(read);
320+
}
321+
322+
boolean hasChunk(int index) throws IOException {
323+
io.seek(offset + MAX_ID_LENGTH + (index / 8));
324+
return ((io.read() >>> (index % 8)) & 0b00000001) == 1;
325+
}
326+
}
327+
}

0 commit comments

Comments
 (0)