motiejus
/
jgit
1
Fork 0
Code Releases Activity

Merge changes from topic 'dfs-reftable' 

* changes:
  dfs: helper to open multiple reftables
  dfs: expose DfsReftable from DfsObjDatabase
  dfs: support reading reftables through DfsBlockCache
This commit is contained in:
Shawn Pearce 2017-08-30 20:42:23 -04:00 committed by Gerrit Code Review @ Eclipse.org
parent 0ca1fb0fe0 4f7ce2b46b
commit f4329b09d4
8 changed files with 458 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
View File

@ -53,7 +53,7 @@
import org.eclipse.jgit.internal.storage.pack.PackExt;
/** Block based file stored in {@link DfsBlockCache}. */
public abstract class BlockBasedFile {
abstract class BlockBasedFile {
/** Cache that owns this file and its data. */
final DfsBlockCache cache;
@ -129,6 +129,10 @@ else if (size < cache.getBlockSize())
return size;
}
DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
return cache.getOrLoad(this, pos, ctx, null);
}
DfsBlock readOneBlock(long pos, DfsReader ctx,
@Nullable ReadableChannel fileChannel) throws IOException {
if (invalid)

10
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java
View File

@ -46,6 +46,7 @@
package org.eclipse.jgit.internal.storage.dfs;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
@ -55,11 +56,8 @@
/** A cached slice of a {@link BlockBasedFile}. */
final class DfsBlock {
final DfsStreamKey stream;
final long start;
final long end;
private final byte[] block;
DfsBlock(DfsStreamKey p, long pos, byte[] buf) {
@ -73,6 +71,12 @@ int size() {
return block.length;
}
ByteBuffer zeroCopyByteBuffer(int n) {
ByteBuffer b = ByteBuffer.wrap(block);
b.position(n);
return b;
}
boolean contains(DfsStreamKey want, long pos) {
return stream.equals(want) && start <= pos && pos < end;
}

136
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
View File

@ -48,6 +48,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -61,7 +62,9 @@
/** Manages objects stored in {@link DfsPackFile} on a storage system. */
public abstract class DfsObjDatabase extends ObjectDatabase {
private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]) {
private static final PackList NO_PACKS = new PackList(
new DfsPackFile[0],
new DfsReftable[0]) {
@Override
boolean dirty() {
return true;
@ -191,6 +194,18 @@ public DfsPackFile[] getPacks() throws IOException {
return getPackList().packs;
}
/**
* Scan and list all available reftable files in the repository.
*
* @return list of available reftables. The returned array is shared with
* the implementation and must not be modified by the caller.
* @throws IOException
* the pack list cannot be initialized.
*/
public DfsReftable[] getReftables() throws IOException {
return getPackList().reftables;
}
/**
* Scan and list all available pack files in the repository.
*
@ -219,6 +234,16 @@ public DfsPackFile[] getCurrentPacks() {
return getCurrentPackList().packs;
}
/**
* List currently known reftable files in the repository, without scanning.
*
* @return list of available reftables. The returned array is shared with
* the implementation and must not be modified by the caller.
*/
public DfsReftable[] getCurrentReftables() {
return getCurrentPackList().reftables;
}
/**
* List currently known pack files in the repository, without scanning.
*
@ -428,7 +453,7 @@ void addPack(DfsPackFile newPack) throws IOException {
DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
packs[0] = newPack;
System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
n = new PackListImpl(packs);
n = new PackListImpl(packs, o.reftables);
} while (!packList.compareAndSet(o, n));
}
@ -454,59 +479,93 @@ PackList scanPacks(final PackList original) throws IOException {
private PackList scanPacksImpl(PackList old) throws IOException {
DfsBlockCache cache = DfsBlockCache.getInstance();
Map<DfsPackDescription, DfsPackFile> forReuse = reuseMap(old);
Map<DfsPackDescription, DfsPackFile> packs = packMap(old);
Map<DfsPackDescription, DfsReftable> reftables = reftableMap(old);
List<DfsPackDescription> scanned = listPacks();
Collections.sort(scanned);
List<DfsPackFile> list = new ArrayList<>(scanned.size());
List<DfsPackFile> newPacks = new ArrayList<>(scanned.size());
List<DfsReftable> newReftables = new ArrayList<>(scanned.size());
boolean foundNew = false;
for (DfsPackDescription dsc : scanned) {
DfsPackFile oldPack = forReuse.remove(dsc);
DfsPackFile oldPack = packs.remove(dsc);
if (oldPack != null) {
list.add(oldPack);
newPacks.add(oldPack);
} else if (dsc.hasFileExt(PackExt.PACK)) {
list.add(new DfsPackFile(cache, dsc));
newPacks.add(new DfsPackFile(cache, dsc));
foundNew = true;
}
DfsReftable oldReftable = reftables.remove(dsc);
if (oldReftable != null) {
newReftables.add(oldReftable);
} else if (dsc.hasFileExt(PackExt.REFTABLE)) {
newReftables.add(new DfsReftable(cache, dsc));
foundNew = true;
}
}
for (DfsPackFile p : forReuse.values())
p.close();
if (list.isEmpty())
return new PackListImpl(NO_PACKS.packs);
if (newPacks.isEmpty())
return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables);
if (!foundNew) {
old.clearDirty();
return old;
}
return new PackListImpl(list.toArray(new DfsPackFile[list.size()]));
Collections.sort(newReftables, reftableComparator());
return new PackListImpl(
newPacks.toArray(new DfsPackFile[0]),
newReftables.toArray(new DfsReftable[0]));
}
private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
private static Map<DfsPackDescription, DfsPackFile> packMap(PackList old) {
Map<DfsPackDescription, DfsPackFile> forReuse = new HashMap<>();
for (DfsPackFile p : old.packs) {
if (p.invalid()) {
// The pack instance is corrupted, and cannot be safely used
// again. Do not include it in our reuse map.
//
p.close();
continue;
}
DfsPackFile prior = forReuse.put(p.getPackDescription(), p);
if (prior != null) {
// This should never occur. It should be impossible for us
// to have two pack files with the same name, as all of them
// came out of the same directory. If it does, we promised to
// close any PackFiles we did not reuse, so close the second,
// readers are likely to be actively using the first.
//
forReuse.put(prior.getPackDescription(), prior);
p.close();
if (!p.invalid()) {
forReuse.put(p.desc, p);
}
}
return forReuse;
}
private static Map<DfsPackDescription, DfsReftable> reftableMap(PackList old) {
Map<DfsPackDescription, DfsReftable> forReuse = new HashMap<>();
for (DfsReftable p : old.reftables) {
if (!p.invalid()) {
forReuse.put(p.desc, p);
}
}
return forReuse;
}
/** @return comparator to sort {@link DfsReftable} by priority. */
protected Comparator<DfsReftable> reftableComparator() {
return (fa, fb) -> {
DfsPackDescription a = fa.getPackDescription();
DfsPackDescription b = fb.getPackDescription();
// GC, COMPACT reftables first by higher category.
int c = category(b) - category(a);
if (c != 0) {
return c;
}
// Lower maxUpdateIndex first.
c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex());
if (c != 0) {
return c;
}
// Older reftable first.
return Long.signum(a.getLastModified() - b.getLastModified());
};
}
static int category(DfsPackDescription d) {
PackSource s = d.getPackSource();
return s != null ? s.category : 0;
}
/** Clears the cached list of packs, forcing them to be scanned again. */
protected void clearCache() {
packList.set(NO_PACKS);
@ -514,12 +573,7 @@ protected void clearCache() {
@Override
public void close() {
// PackList packs = packList.get();
packList.set(NO_PACKS);
// TODO Close packs if they aren't cached.
// for (DfsPackFile p : packs.packs)
// p.close();
}
/** Snapshot of packs scanned in a single pass. */
@ -527,10 +581,14 @@ public static abstract class PackList {
/** All known packs, sorted. */
public final DfsPackFile[] packs;
/** All known reftables, sorted. */
public final DfsReftable[] reftables;
private long lastModified = -1;
PackList(DfsPackFile[] packs) {
PackList(DfsPackFile[] packs, DfsReftable[] reftables) {
this.packs = packs;
this.reftables = reftables;
}
/** @return last modified time of all packs, in milliseconds. */
@ -561,8 +619,8 @@ public long getLastModified() {
private static final class PackListImpl extends PackList {
private volatile boolean dirty;
PackListImpl(DfsPackFile[] packs) {
super(packs);
PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) {
super(packs, reftables);
}
@Override

58
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java
View File

@ -44,11 +44,13 @@
package org.eclipse.jgit.internal.storage.dfs;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import java.util.Arrays;
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.storage.pack.PackStatistics;
/**
@ -68,7 +70,11 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
private int[] blockSizeMap;
private long objectCount;
private long deltaCount;
private PackStatistics stats;
private long minUpdateIndex;
private long maxUpdateIndex;
private PackStatistics packStats;
private ReftableWriter.Stats refStats;
private int extensions;
private int indexVersion;
private long estimatedPackSize;
@ -170,6 +176,36 @@ public DfsPackDescription setLastModified(long timeMillis) {
return this;
}
/** @return minUpdateIndex for the reftable, if present. */
public long getMinUpdateIndex() {
return minUpdateIndex;
}
/**
* @param min
* minUpdateIndex for the reftable, or 0.
* @return {@code this}
*/
public DfsPackDescription setMinUpdateIndex(long min) {
minUpdateIndex = min;
return this;
}
/** @return maxUpdateIndex for the reftable, if present. */
public long getMaxUpdateIndex() {
return maxUpdateIndex;
}
/**
* @param max
* maxUpdateIndex for the reftable, or 0.
* @return {@code this}
*/
public DfsPackDescription setMaxUpdateIndex(long max) {
maxUpdateIndex = max;
return this;
}
/**
* @param ext
* the file extension.
@ -281,24 +317,38 @@ public DfsPackDescription setDeltaCount(long cnt) {
* is being committed to the repository.
*/
public PackStatistics getPackStats() {
return stats;
return packStats;
}
DfsPackDescription setPackStats(PackStatistics stats) {
this.stats = stats;
this.packStats = stats;
setFileSize(PACK, stats.getTotalBytes());
setObjectCount(stats.getTotalObjects());
setDeltaCount(stats.getTotalDeltas());
return this;
}
/** @return stats from the sibling reftable, if created. */
public ReftableWriter.Stats getReftableStats() {
return refStats;
}
void setReftableStats(ReftableWriter.Stats stats) {
this.refStats = stats;
setMinUpdateIndex(stats.minUpdateIndex());
setMaxUpdateIndex(stats.maxUpdateIndex());
setFileSize(REFTABLE, stats.totalBytes());
setBlockSize(REFTABLE, stats.refBlockSize());
}
/**
* Discard the pack statistics, if it was populated.
*
* @return {@code this}
*/
public DfsPackDescription clearPackStats() {
stats = null;
packStats = null;
refStats = null;
return this;
}

10
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
View File

@ -385,12 +385,6 @@ void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
idx(ctx).resolve(matches, id, matchLimit);
}
/** Release all memory used by this DfsPackFile instance. */
public void close() {
index = null;
reverseIndex = null;
}
/**
* Obtain the total number of objects available in this pack. This method
* relies on pack index, giving number of effectively available objects.
@ -739,10 +733,6 @@ private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
throw new EOFException();
}
DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
return cache.getOrLoad(this, pos, ctx, null);
}
ObjectLoader load(DfsReader ctx, long pos)
throws IOException {
try {

18
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java
View File

@ -655,7 +655,7 @@ public void copyPackAsIs(PackOutputStream out, CachedPack pack)
/**
* Copy bytes from the window to a caller supplied buffer.
*
* @param pack
* @param file
* the file the desired window is stored within.
* @param position
* position within the file to read from.
@ -674,24 +674,24 @@ public void copyPackAsIs(PackOutputStream out, CachedPack pack)
* this cursor does not match the provider or id and the proper
* window could not be acquired through the provider's cache.
*/
int copy(DfsPackFile pack, long position, byte[] dstbuf, int dstoff, int cnt)
throws IOException {
int copy(BlockBasedFile file, long position, byte[] dstbuf, int dstoff,
int cnt) throws IOException {
if (cnt == 0)
return 0;
long length = pack.length;
long length = file.length;
if (0 <= length && length <= position)
return 0;
int need = cnt;
do {
pin(pack, position);
pin(file, position);
int r = block.copy(position, dstbuf, dstoff, need);
position += r;
dstoff += r;
need -= r;
if (length < 0)
length = pack.length;
length = file.length;
} while (0 < need && position < length);
return cnt - need;
}
@ -756,14 +756,14 @@ private void prepareInflater() {
inf.reset();
}
void pin(DfsPackFile pack, long position) throws IOException {
if (block == null || !block.contains(pack.key, position)) {
void pin(BlockBasedFile file, long position) throws IOException {
if (block == null || !block.contains(file.key, position)) {
// If memory is low, we may need what is in our window field to
// be cleaned up by the GC during the get for the next window.
// So we always clear it, even though we are just going to set
// it again.
block = null;
block = pack.getOrLoadBlock(position, this);
block = file.getOrLoadBlock(position, this);
}
}

178
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java Normal file
View File

@ -0,0 +1,178 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.dfs;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.eclipse.jgit.internal.storage.io.BlockSource;
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
/** A reftable stored in {@link DfsBlockCache}. */
public class DfsReftable extends BlockBasedFile {
/**
* Construct a reader for an existing reftable.
*
* @param desc
* description of the reftable within the DFS.
*/
public DfsReftable(DfsPackDescription desc) {
this(DfsBlockCache.getInstance(), desc);
}
/**
* Construct a reader for an existing reftable.
*
* @param cache
* cache that will store the reftable data.
* @param desc
* description of the reftable within the DFS.
*/
public DfsReftable(DfsBlockCache cache, DfsPackDescription desc) {
super(cache, desc, REFTABLE);
int bs = desc.getBlockSize(REFTABLE);
if (bs > 0) {
setBlockSize(bs);
}
long sz = desc.getFileSize(REFTABLE);
length = sz > 0 ? sz : -1;
}
/** @return description that was originally used to configure this file. */
public DfsPackDescription getPackDescription() {
return desc;
}
/**
* Open reader on the reftable.
* <p>
* The returned reader is not thread safe.
*
* @param ctx
* reader to access the DFS storage.
* @return cursor to read the table; caller must close.
* @throws IOException
* table cannot be opened.
*/
public ReftableReader open(DfsReader ctx) throws IOException {
return new ReftableReader(new CacheSource(this, cache, ctx));
}
private static final class CacheSource extends BlockSource {
private final DfsReftable file;
private final DfsBlockCache cache;
private final DfsReader ctx;
private ReadableChannel ch;
private int readAhead;
CacheSource(DfsReftable file, DfsBlockCache cache, DfsReader ctx) {
this.file = file;
this.cache = cache;
this.ctx = ctx;
}
@Override
public ByteBuffer read(long pos, int cnt) throws IOException {
if (ch == null && readAhead > 0 && notInCache(pos)) {
open().setReadAheadBytes(readAhead);
}
DfsBlock block = cache.getOrLoad(file, pos, ctx, ch);
if (block.start == pos && block.size() >= cnt) {
return block.zeroCopyByteBuffer(cnt);
}
byte[] dst = new byte[cnt];
ByteBuffer buf = ByteBuffer.wrap(dst);
buf.position(ctx.copy(file, pos, dst, 0, cnt));
return buf;
}
private boolean notInCache(long pos) {
return cache.get(file.key, file.alignToBlock(pos)) == null;
}
@Override
public long size() throws IOException {
long n = file.length;
if (n < 0) {
n = open().size();
file.length = n;
}
return n;
}
@Override
public void adviseSequentialRead(long start, long end) {
int sz = ctx.getOptions().getStreamPackBufferSize();
if (sz > 0) {
readAhead = (int) Math.min(sz, end - start);
}
}
private ReadableChannel open() throws IOException {
if (ch == null) {
ch = ctx.db.openFile(file.desc, file.ext);
}
return ch;
}
@Override
public void close() {
if (ch != null) {
try {
ch.close();
} catch (IOException e) {
// Ignore read close failures.
} finally {
ch = null;
}
}
}
}
}

108
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.dfs;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.eclipse.jgit.internal.storage.reftable.Reftable;
/** Tracks multiple open {@link Reftable} instances. */
public class ReftableStack implements AutoCloseable {
/**
* Opens a stack of tables for reading.
*
* @param ctx
* context to read the tables with. This {@code ctx} will be
* retained by the stack and each of the table readers.
* @param tables
* the tables to open.
* @return stack reference to close the tables.
* @throws IOException
* a table could not be opened
*/
public static ReftableStack open(DfsReader ctx, List<DfsReftable> tables)
throws IOException {
ReftableStack stack = new ReftableStack(tables.size());
boolean close = true;
try {
for (DfsReftable t : tables) {
stack.tables.add(t.open(ctx));
}
close = false;
return stack;
} finally {
if (close) {
stack.close();
}
}
}
private final List<Reftable> tables;
private ReftableStack(int tableCnt) {
this.tables = new ArrayList<>(tableCnt);
}
/**
* @return unmodifiable list of tables, in the same order the files were
* passed to {@link #open(DfsReader, List)}.
*/
public List<Reftable> readers() {
return Collections.unmodifiableList(tables);
}
@Override
public void close() {
for (Reftable t : tables) {
try {
t.close();
} catch (IOException e) {
// Ignore close failures.
}
}
}
}