deno.land / x / replicache@v10.0.0-beta.0 / dag / lazy-store.test.ts
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366import {expect} from '@esm-bundle/chai';import { assertHash, assertNotTempHash, isTempHash, makeNewTempHashFunction, newTempHash,} from '../hash';import type {Value} from '../kv/mod';import {makeTestChunkHasher} from './chunk';import {LazyStore} from './lazy-store';import {TestStore} from './test-store';
const DEFAULT_VALUE_SIZE = 100;function getSizeOfValueForTest(value: Value): number { if (value && typeof value === 'object' && !Array.isArray(value)) { const {size} = value as {size?: Value}; if (typeof size === 'number') { return size; } } return DEFAULT_VALUE_SIZE;}
const DEFAULT_CACHE_SIZE_LIMIT = 200;function createLazyStoreForTest( options: { cacheSizeLimit?: number; } = {},) { const {cacheSizeLimit = DEFAULT_CACHE_SIZE_LIMIT} = options; const sourceStoreChunkHasher = makeTestChunkHasher('source'); const sourceStore = new TestStore( undefined, sourceStoreChunkHasher, assertNotTempHash, ); return { sourceStore, sourceStoreChunkHasher, lazyStore: new LazyStore( sourceStore, cacheSizeLimit, makeNewTempHashFunction(), assertHash, getSizeOfValueForTest, ), };}
test('chunks with non-temp hashes are loaded from source store and cached if reachable from a LazyStore head', async () => { const {sourceStore, sourceStoreChunkHasher, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Hash = sourceStoreChunkHasher(testValue1); await lazyStore.withRead(async read => { expect(await read.getChunk(testValue1Hash)).to.be.undefined; }); await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy', testValue1Hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Hash))?.data).to.equal(testValue1); }); await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource'); await write.commit(); }); await sourceStore.withRead(async read => { expect(await read.getChunk(testValue1Hash)).to.be.undefined; }); await lazyStore.withRead(async read => { // value of testValue1Hash is cached expect((await read.getChunk(testValue1Hash))?.data).to.equal(testValue1); });});
test('chunks with non-temp hashes are loaded from source store but not cached if not reachable from a LazyStore head', async () => { const {sourceStore, sourceStoreChunkHasher, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Hash = sourceStoreChunkHasher(testValue1); await lazyStore.withRead(async read => { expect(await read.getChunk(testValue1Hash)).to.be.undefined; }); await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); }); await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Hash))?.data).to.equal(testValue1); }); await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource'); await write.commit(); }); await sourceStore.withRead(async read => { expect(await read.getChunk(testValue1Hash)).to.be.undefined; }); await lazyStore.withRead(async read => { // value of testValue1Hash is not cached expect(await read.getChunk(testValue1Hash)).to.be.undefined; });});
// Source store is never expected to contain temp-hashes, so no attempt// is made to load them from the source storetest('chunks with temp hashes are not loaded from source store', async () => { // For this tests we create a source store that can contain temp hashes const sourceStore = new TestStore( undefined, makeNewTempHashFunction(), assertHash, ); const lazyStore = new LazyStore( sourceStore, DEFAULT_CACHE_SIZE_LIMIT, makeNewTempHashFunction(), assertHash, getSizeOfValueForTest, ); const testValue1Chunk = await sourceStore.withWrite(async write => { const testValue1 = 'testValue1'; const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); return testValue1Chunk; }); expect(isTempHash(testValue1Chunk.hash)).to.be.true; await lazyStore.withRead(async read => { expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; });});
test('heads are *not* loaded from source store', async () => { const {sourceStore, sourceStoreChunkHasher, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Hash = sourceStoreChunkHasher(testValue1); await lazyStore.withRead(async read => { expect(await read.getChunk(testValue1Hash)).to.be.undefined; }); await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); }); await lazyStore.withRead(async read => { expect(await read.getHead('testHeadSource')).to.be.undefined; });});
test('setHead stores head in memory but does not write through to source store', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); await lazyStore.withRead(async read => { expect(await read.getHead('testHead1')).to.be.undefined; }); const tempHash1 = newTempHash(); await lazyStore.withWrite(async write => { await write.setHead('testHead1', tempHash1); await write.commit(); }); await lazyStore.withRead(async read => { expect(await read.getHead('testHead1')).to.equal(tempHash1); }); await sourceStore.withRead(async read => { expect(await read.getHead('testHead1')).to.be.undefined; });});
test('removeHead removes head from memory but does not write through to source store', async () => { const {sourceStore, sourceStoreChunkHasher, lazyStore} = createLazyStoreForTest(); await lazyStore.withRead(async read => { expect(await read.getHead('testHead1')).to.be.undefined; }); const tempHash1 = newTempHash(); await lazyStore.withWrite(async write => { await write.setHead('testHead1', tempHash1); await write.commit(); }); const testValue1 = 'testValue1'; const testValue1Hash = sourceStoreChunkHasher(testValue1); await sourceStore.withWrite(async write => { await write.setHead('testHead1', testValue1Hash); await write.commit(); }); await lazyStore.withRead(async read => { expect(await read.getHead('testHead1')).to.equal(tempHash1); }); await sourceStore.withRead(async read => { expect(await read.getHead('testHead1')).to.equal(testValue1Hash); }); await lazyStore.withWrite(async write => { await write.removeHead('testHead1'); await write.commit(); }); await lazyStore.withRead(async read => { expect(await read.getHead('testHead1')).to.be.undefined; }); await sourceStore.withRead(async read => { expect(await read.getHead('testHead1')).to.equal(testValue1Hash); });});
test('putChunk with non-temp hashes updates memory but does not write through to source store', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Chunk = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); return testValue1Chunk; }); await lazyStore.withWrite(async write => { await write.putChunk(testValue1Chunk); await write.setHead('testHeadLazy', testValue1Chunk.hash); await write.commit(); }); await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); }); await sourceStore.withRead(async read => { expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; });});
test('putChunk with temp hashes updates memory but does not write through to source store', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Chunk = await lazyStore.withWrite(async write => { return write.createChunk(testValue1, []); }); expect(isTempHash(testValue1Chunk.hash)).to.be.true; // Set a head to testValue1Chunk's hash so that if it was written through to source it wouldn't // be gc'd await sourceStore.withWrite(async write => { await write.setHead('testHeadSource', testValue1Chunk.hash); await write.commit(); }); await lazyStore.withWrite(async write => { await write.putChunk(testValue1Chunk); await write.setHead('testHeadLazy', testValue1Chunk.hash); await write.commit(); }); await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); }); await sourceStore.withRead(async read => { expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; });});
test('writes are visible within same write transaction but not other transactions when not committed', async () => { const {lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1'; const testValue1Chunk = await lazyStore.withWrite(async write => { return write.createChunk(testValue1, []); }); await lazyStore.withWrite(async write => { await write.putChunk(testValue1Chunk); await write.setHead('testHeadLazy', testValue1Chunk.hash); // visible within this write transaction expect((await write.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect(await write.getHead('testHeadLazy')).to.equal(testValue1Chunk.hash); // do not commit }); await lazyStore.withRead(async read => { // was never commited, so not visible in another transaction expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect(await read.getHead('testHeadLazy')).to.be.undefined; });});
test('cache evicts in lru fashion, basic test of just reads', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3'; const {testValue1Chunk, testValue2Chunk, testValue3Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); await write.commit(); return {testValue1Chunk, testValue2Chunk, testValue3Chunk}; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); // evicts testValue1Chunk expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk was evicted and is no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });});
test('source store values are reloaded if evicted from cache', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3'; const {testValue1Chunk, testValue2Chunk, testValue3Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); await write.commit(); return {testValue1Chunk, testValue2Chunk, testValue3Chunk}; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); // evicts testValue1Chunk expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
// do not gc chunks from base store, so they can be reloaded await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); });});
test('cache evicts in lru fashion, slightly more complex test with repeats of just reads', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3'; const {testValue1Chunk, testValue2Chunk, testValue3Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); await write.commit(); return {testValue1Chunk, testValue2Chunk, testValue3Chunk}; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); // evicts testValue2Chunk expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue2Chunk was evicted and is no longer available in base store expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });});
test('cache evicts in lru fashion, basic test of evict on write', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3'; const {testValue1Chunk, testValue2Chunk, testValue3Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); await write.commit(); return {testValue1Chunk, testValue2Chunk, testValue3Chunk}; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); });
await lazyStore.withWrite(async write => { // Evicts testValue1Chunk await write.putChunk(testValue3Chunk); await write.commit(); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk was evicted and is no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });});
test('cache will evict multiple chunks to make room for newly read chunk', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 300, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3', testValue4 = {name: 'testValue4', size: 200}; const {testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); const testValue4Chunk = write.createChunk(testValue4, []); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource4', testValue4Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.setHead('testHeadLazy4', testValue4Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); // evicts testValue1Chunk and testValue2Chunk as its size is 200 expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.removeHead('testHeadSource4'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk and testValue2Chunk were evicted and are no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });});
test('cache will evict multiple chunks to make room for newly written chunk', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 300, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3', testValue4 = {name: 'testValue4', size: 200}; const {testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); const testValue4Chunk = write.createChunk(testValue4, []); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource4', testValue4Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.setHead('testHeadLazy4', testValue4Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
await lazyStore.withWrite(async write => { // evicts testValue1Chunk and testValue2Chunk as its size is 200 await write.putChunk(testValue4Chunk); await write.commit(); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.removeHead('testHeadSource4'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk and testValue2Chunk were evicted and are no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });});
test('cache will evict all cached values to make room for new chunk', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 300, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3', testValue4 = {name: 'testValue4', size: 250}; const {testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); const testValue4Chunk = write.createChunk(testValue4, []); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource4', testValue4Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.setHead('testHeadLazy4', testValue4Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); // evicts testValue1Chunk, testValue2Chunk, and testValue3Chunk as its size is 250 expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.removeHead('testHeadSource4'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk, testValue2Chunk and testValue3Chunk were evicted and are no longer available in // base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });});
test('cache does not cache read chunks with size greater than cacheSizeLimit, and does not evict other chunks to try to make room', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 300, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3', testValue4 = {name: 'testValue4', size: 400}; const {testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); const testValue4Chunk = write.createChunk(testValue4, []); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource4', testValue4Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.setHead('testHeadLazy4', testValue4Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); // is not cached because its size exceeds cache size limit // other chunks are not evicted expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.removeHead('testHeadSource4'); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); expect((await read.getChunk(testValue4Chunk.hash))?.data).to.be.undefined; });});
test('cache does not cache put chunks with size greater than cacheSizeLimit, and does not evict other chunks to try to make room', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 200, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3', testValue4 = {name: 'testValue4', size: 400}, testValue5 = {name: 'testValue5', size: 400}; const { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, testValue5Chunk, } = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); const testValue4Chunk = write.createChunk(testValue4, []); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource4', testValue4Chunk.hash); const testValue5Chunk = write.createChunk(testValue5, []); await write.putChunk(testValue5Chunk); await write.setHead('testHeadSource5', testValue5Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, testValue5Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash); await write.setHead('testHeadLazy4', testValue4Chunk.hash); await write.setHead('testHeadLazy5', testValue5Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); });
await lazyStore.withWrite(async write => { // evicts testValue1Chunk await write.putChunk(testValue3Chunk); // testValue4Chunk and testValue5Chunk are not cached because each of // their sizes exceeds cache size limit. Other chunks are not evicted. await write.putChunk(testValue4Chunk); await write.putChunk(testValue5Chunk); await write.commit(); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.removeHead('testHeadSource4'); await write.removeHead('testHeadSource5'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk was evicted and is no longer available in base store expect((await read.getChunk(testValue1Chunk.hash))?.data).to.be.undefined; expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); expect((await read.getChunk(testValue4Chunk.hash))?.data).to.be.undefined; expect((await read.getChunk(testValue5Chunk.hash))?.data).to.be.undefined; });});
test('cache eviction updates ref counts and removes cache chunks when their ref count goes to zero', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 300, }); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = {name: 'testValue3', size: 200}, testValue4 = 'testValue4'; // 4 // / | // 3 2 // | // 1 const {testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); const testValue2Chunk = write.createChunk(testValue2, [ testValue1Chunk.hash, ]); await write.putChunk(testValue2Chunk); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); const testValue4Chunk = write.createChunk(testValue4, [ testValue2Chunk.hash, testValue3Chunk.hash, ]); await write.putChunk(testValue4Chunk); await write.setHead('testHeadSource', testValue4Chunk.hash); await write.commit(); return { testValue1Chunk, testValue2Chunk, testValue3Chunk, testValue4Chunk, }; });
await lazyStore.withWrite(async write => { await write.setHead('testHeadLazy', testValue4Chunk.hash); await write.commit(); });
await lazyStore.withRead(async read => { expect((await read.getChunk(testValue4Chunk.hash))?.data).to.equal( testValue4, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); // Current LRU order 4, 2, 1 // Update LRU order to 2, 4, 1 // 2, 1, 4 expect((await read.getChunk(testValue4Chunk.hash))?.data).to.equal( testValue4, ); // 2, 4, 1 expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); // To make room for 3 (of size 200), 2 chunks of size 100 need to be // removed from cache. 2 and 4 are least recently used. However, // first 2 is evicted, lowering 1's ref count to zero and is removed. // 4 is not removed since evicting 1 and processing gc from this eviction // made enough room expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource'); await write.commit(); });
await lazyStore.withRead(async read => { // testValue1Chunk and testValue2Chunk were evicted and are no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect(await read.getChunk(testValue2Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); expect((await read.getChunk(testValue4Chunk.hash))?.data).to.deep.equal( testValue4, ); });});
test('temp chunks are not evicted when cache size is exceeded', async () => { const {sourceStore, lazyStore} = createLazyStoreForTest(); const testValue1 = 'testValue1', testValue2 = 'testValue2', testValue3 = 'testValue3'; const {testValue1Chunk, testValue2Chunk, testValue3Chunk} = await sourceStore.withWrite(async write => { const testValue1Chunk = write.createChunk(testValue1, []); await write.putChunk(testValue1Chunk); await write.setHead('testHeadSource1', testValue1Chunk.hash); const testValue2Chunk = write.createChunk(testValue2, []); await write.putChunk(testValue2Chunk); await write.setHead('testHeadSource2', testValue2Chunk.hash); const testValue3Chunk = write.createChunk(testValue3, []); await write.putChunk(testValue3Chunk); await write.setHead('testHeadSource3', testValue3Chunk.hash); await write.commit(); return {testValue1Chunk, testValue2Chunk, testValue3Chunk}; }); const tempValue1 = 'tempValue1', tempValue2 = 'tempValue2'; const {tempValue1Chunk, tempValue2Chunk} = await lazyStore.withWrite( async write => { const tempValue1Chunk = write.createChunk(tempValue1, []); await write.putChunk(tempValue1Chunk); await write.setHead('tempHeadLazy1', tempValue1Chunk.hash); const tempValue2Chunk = write.createChunk(tempValue2, []); await write.putChunk(tempValue2Chunk); await write.setHead('tempHeadLazy2', tempValue2Chunk.hash);
await write.setHead('testHeadLazy1', testValue1Chunk.hash); await write.setHead('testHeadLazy2', testValue2Chunk.hash); await write.setHead('testHeadLazy3', testValue3Chunk.hash);
await write.commit(); return {tempValue1Chunk, tempValue2Chunk}; }, );
await lazyStore.withRead(async read => { expect((await read.getChunk(tempValue1Chunk.hash))?.data).to.equal( tempValue1, ); expect((await read.getChunk(tempValue2Chunk.hash))?.data).to.equal( tempValue2, ); expect((await read.getChunk(testValue1Chunk.hash))?.data).to.equal( testValue1, ); expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); // over cache size limit, should evict testValue1Chunk, but not any temp chunks expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });
// gc chunks from base store await sourceStore.withWrite(async write => { await write.removeHead('testHeadSource1'); await write.removeHead('testHeadSource2'); await write.removeHead('testHeadSource3'); await write.commit(); });
await lazyStore.withRead(async read => { // temp chunks were not evicted expect((await read.getChunk(tempValue1Chunk.hash))?.data).to.equal( tempValue1, ); expect((await read.getChunk(tempValue2Chunk.hash))?.data).to.equal( tempValue2, ); // testValue1Chunk was evicted and is no longer available in base store expect(await read.getChunk(testValue1Chunk.hash)).to.be.undefined; expect((await read.getChunk(testValue2Chunk.hash))?.data).to.equal( testValue2, ); expect((await read.getChunk(testValue3Chunk.hash))?.data).to.equal( testValue3, ); });});
test('[all temp chunks] chunk ref counts are updated on commit and are deleted when their ref count goes to zero', async () => { const {lazyStore} = createLazyStoreForTest();
// R // / | // A B // \ | // C // | // D
const {r, a, b, c, d} = await lazyStore.withWrite(async write => { const d = write.createChunk('d', []); const c = write.createChunk('c', [d.hash]); const b = write.createChunk('b', [c.hash]); const a = write.createChunk('a', [c.hash]); const r = write.createChunk('r', [a.hash, b.hash]); await write.putChunk(r); await write.putChunk(a); await write.putChunk(b); await write.putChunk(c); await write.putChunk(d); await write.setHead('test', r.hash); await write.commit(); return {r, a, b, c, d}; });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [r.hash, 1], [a.hash, 1], [b.hash, 1], [c.hash, 2], [d.hash, 1], ]), );
// E // | // D
const e = await lazyStore.withWrite(async write => { const e = write.createChunk('e', [d.hash]); await write.putChunk(e); await write.setHead('test', e.hash); await write.commit(); return e; });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [d.hash, 1], [e.hash, 1], ]), );
await lazyStore.withRead(async read => { expect(await read.getChunk(r.hash)).to.be.undefined; expect(await read.getChunk(a.hash)).to.be.undefined; expect(await read.getChunk(b.hash)).to.be.undefined; expect(await read.getChunk(c.hash)).to.be.undefined; expect(await read.getChunk(d.hash)).to.deep.equal(d); expect(await read.getChunk(e.hash)).to.deep.equal(e); });});
test('[all cached chunks] chunk ref counts are updated on commit and are deleted when their ref count goes to zero', async () => { // Make cache size large enough that eviction does not occur // during test const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 1000, });
// R // / | // A B // \ | // C // | // D
const {r, a, b, c, d} = await sourceStore.withWrite(async write => { const d = write.createChunk('d', []); const c = write.createChunk('c', [d.hash]); const b = write.createChunk('b', [c.hash]); const a = write.createChunk('a', [c.hash]); const r = write.createChunk('r', [a.hash, b.hash]); await write.putChunk(r); await write.putChunk(a); await write.putChunk(b); await write.putChunk(c); await write.putChunk(d); await write.setHead('testSource', r.hash); await write.commit(); return {r, a, b, c, d}; });
await lazyStore.withWrite(async write => { await write.putChunk(r); await write.putChunk(a); await write.putChunk(b); await write.putChunk(c); await write.putChunk(d); await write.setHead('testLazy', r.hash); await write.commit(); });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [r.hash, 1], [a.hash, 1], [b.hash, 1], [c.hash, 2], [d.hash, 1], ]), );
// E // | // D
const e = await sourceStore.withWrite(async write => { const e = write.createChunk('e', [d.hash]); await write.putChunk(e); await write.setHead('testSource', e.hash); await write.commit(); return e; });
await lazyStore.withWrite(async write => { await write.putChunk(e); await write.setHead('testLazy', e.hash); await write.commit(); return e; });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [d.hash, 1], [e.hash, 1], ]), );
await lazyStore.withRead(async read => { expect(await read.getChunk(r.hash)).to.be.undefined; expect(await read.getChunk(a.hash)).to.be.undefined; expect(await read.getChunk(b.hash)).to.be.undefined; expect(await read.getChunk(c.hash)).to.be.undefined; expect(await read.getChunk(d.hash)).to.deep.equal(d); expect(await read.getChunk(e.hash)).to.deep.equal(e); });});
test('[mix of temp and cached chunks] chunk ref counts are updated on commit and are deleted when their ref count goes to zero', async () => { // Make cache size large enough that eviction does not occur // during test const {sourceStore, lazyStore} = createLazyStoreForTest({ cacheSizeLimit: 1000, });
// R // / | // A B // \ | // C // | // D
const {r, a, b, c, d} = await sourceStore.withWrite(async write => { const d = write.createChunk('d', []); const c = write.createChunk('c', [d.hash]); const b = write.createChunk('b', [c.hash]); const a = write.createChunk('a', [c.hash]); const r = write.createChunk('r', [a.hash, b.hash]); await write.putChunk(r); await write.putChunk(a); await write.putChunk(b); await write.putChunk(c); await write.putChunk(d); await write.setHead('testSource', r.hash); await write.commit(); return {r, a, b, c, d}; });
await lazyStore.withWrite(async write => { await write.putChunk(r); await write.putChunk(a); await write.putChunk(b); await write.putChunk(c); await write.putChunk(d); await write.setHead('testLazy', r.hash); await write.commit(); });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [r.hash, 1], [a.hash, 1], [b.hash, 1], [c.hash, 2], [d.hash, 1], ]), );
// tempE // | // D
const tempE = await lazyStore.withWrite(async write => { const tempE = write.createChunk('e', [d.hash]); await write.putChunk(tempE); await write.setHead('testLazy', tempE.hash); await write.commit(); return tempE; });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [d.hash, 1], [tempE.hash, 1], ]), );
// E // | // D const e = await sourceStore.withWrite(async write => { const e = write.createChunk('e', [d.hash]); await write.putChunk(e); await write.setHead('testSource', e.hash); await write.commit(); return e; });
await lazyStore.withWrite(async write => { await write.putChunk(e); await write.setHead('testLazy', e.hash); await write.commit(); return e; });
expect(lazyStore.refCountsSnapshot).to.deep.equal( new Map([ [d.hash, 1], [e.hash, 1], ]), );
await lazyStore.withRead(async read => { expect(await read.getChunk(r.hash)).to.be.undefined; expect(await read.getChunk(a.hash)).to.be.undefined; expect(await read.getChunk(b.hash)).to.be.undefined; expect(await read.getChunk(c.hash)).to.be.undefined; expect(await read.getChunk(tempE.hash)).to.be.undefined; expect(await read.getChunk(d.hash)).to.deep.equal(d); expect(await read.getChunk(e.hash)).to.deep.equal(e); });});
Version Info