Merge changes I159e9154,I06c722b2
* changes: DfsObjectDatabase: Expose PackList and move markDirty there Invalidate DfsObjDatabase pack list when refs are updated
This commit is contained in:
commit
e790ec3fb1
|
@ -61,7 +61,17 @@
|
||||||
|
|
||||||
/** Manages objects stored in {@link DfsPackFile} on a storage system. */
|
/** Manages objects stored in {@link DfsPackFile} on a storage system. */
|
||||||
public abstract class DfsObjDatabase extends ObjectDatabase {
|
public abstract class DfsObjDatabase extends ObjectDatabase {
|
||||||
private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]);
|
private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]) {
|
||||||
|
@Override
|
||||||
|
boolean dirty() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void markDirty() {
|
||||||
|
// Always dirty.
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/** Sources for a pack file. */
|
/** Sources for a pack file. */
|
||||||
public static enum PackSource {
|
public static enum PackSource {
|
||||||
|
@ -173,7 +183,20 @@ public ObjectInserter newInserter() {
|
||||||
* the pack list cannot be initialized.
|
* the pack list cannot be initialized.
|
||||||
*/
|
*/
|
||||||
public DfsPackFile[] getPacks() throws IOException {
|
public DfsPackFile[] getPacks() throws IOException {
|
||||||
return scanPacks(NO_PACKS).packs;
|
return getPackList().packs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scan and list all available pack files in the repository.
|
||||||
|
*
|
||||||
|
* @return list of available packs, with some additional metadata. The
|
||||||
|
* returned array is shared with the implementation and must not be
|
||||||
|
* modified by the caller.
|
||||||
|
* @throws IOException
|
||||||
|
* the pack list cannot be initialized.
|
||||||
|
*/
|
||||||
|
public PackList getPackList() throws IOException {
|
||||||
|
return scanPacks(NO_PACKS);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return repository owning this object database. */
|
/** @return repository owning this object database. */
|
||||||
|
@ -188,7 +211,18 @@ protected DfsRepository getRepository() {
|
||||||
* implementation and must not be modified by the caller.
|
* implementation and must not be modified by the caller.
|
||||||
*/
|
*/
|
||||||
public DfsPackFile[] getCurrentPacks() {
|
public DfsPackFile[] getCurrentPacks() {
|
||||||
return packList.get().packs;
|
return getCurrentPackList().packs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List currently known pack files in the repository, without scanning.
|
||||||
|
*
|
||||||
|
* @return list of available packs, with some additional metadata. The
|
||||||
|
* returned array is shared with the implementation and must not be
|
||||||
|
* modified by the caller.
|
||||||
|
*/
|
||||||
|
public PackList getCurrentPackList() {
|
||||||
|
return packList.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -363,11 +397,11 @@ void addPack(DfsPackFile newPack) throws IOException {
|
||||||
DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
|
DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
|
||||||
packs[0] = newPack;
|
packs[0] = newPack;
|
||||||
System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
|
System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
|
||||||
n = new PackList(packs);
|
n = new PackListImpl(packs);
|
||||||
} while (!packList.compareAndSet(o, n));
|
} while (!packList.compareAndSet(o, n));
|
||||||
}
|
}
|
||||||
|
|
||||||
private PackList scanPacks(final PackList original) throws IOException {
|
PackList scanPacks(final PackList original) throws IOException {
|
||||||
PackList o, n;
|
PackList o, n;
|
||||||
synchronized (packList) {
|
synchronized (packList) {
|
||||||
do {
|
do {
|
||||||
|
@ -408,10 +442,10 @@ private PackList scanPacksImpl(PackList old) throws IOException {
|
||||||
for (DfsPackFile p : forReuse.values())
|
for (DfsPackFile p : forReuse.values())
|
||||||
p.close();
|
p.close();
|
||||||
if (list.isEmpty())
|
if (list.isEmpty())
|
||||||
return new PackList(NO_PACKS.packs);
|
return new PackListImpl(NO_PACKS.packs);
|
||||||
if (!foundNew)
|
if (!foundNew)
|
||||||
return old;
|
return old;
|
||||||
return new PackList(list.toArray(new DfsPackFile[list.size()]));
|
return new PackListImpl(list.toArray(new DfsPackFile[list.size()]));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
|
private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
|
||||||
|
@ -456,12 +490,42 @@ public void close() {
|
||||||
// p.close();
|
// p.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final class PackList {
|
/** Snapshot of packs scanned in a single pass. */
|
||||||
|
public static abstract class PackList {
|
||||||
/** All known packs, sorted. */
|
/** All known packs, sorted. */
|
||||||
final DfsPackFile[] packs;
|
public final DfsPackFile[] packs;
|
||||||
|
|
||||||
PackList(final DfsPackFile[] packs) {
|
PackList(DfsPackFile[] packs) {
|
||||||
this.packs = packs;
|
this.packs = packs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
abstract boolean dirty();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark pack list as dirty.
|
||||||
|
* <p>
|
||||||
|
* Used when the caller knows that new data might have been written to the
|
||||||
|
* repository that could invalidate open readers depending on this pack list,
|
||||||
|
* for example if refs are newly scanned.
|
||||||
|
*/
|
||||||
|
public abstract void markDirty();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class PackListImpl extends PackList {
|
||||||
|
private volatile boolean dirty;
|
||||||
|
|
||||||
|
PackListImpl(DfsPackFile[] packs) {
|
||||||
|
super(packs);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
boolean dirty() {
|
||||||
|
return dirty;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void markDirty() {
|
||||||
|
dirty = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,6 +53,7 @@
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.zip.DataFormatException;
|
||||||
|
@ -62,6 +63,7 @@
|
||||||
import org.eclipse.jgit.errors.MissingObjectException;
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
|
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
|
||||||
import org.eclipse.jgit.internal.JGitText;
|
import org.eclipse.jgit.internal.JGitText;
|
||||||
|
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList;
|
||||||
import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl;
|
import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl;
|
||||||
import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
|
import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
|
||||||
import org.eclipse.jgit.internal.storage.file.PackIndex;
|
import org.eclipse.jgit.internal.storage.file.PackIndex;
|
||||||
|
@ -91,6 +93,8 @@
|
||||||
* reader is not thread safe.
|
* reader is not thread safe.
|
||||||
*/
|
*/
|
||||||
public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
|
public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
|
||||||
|
private static final int MAX_RESOLVE_MATCHES = 256;
|
||||||
|
|
||||||
/** Temporary buffer large enough for at least one raw object id. */
|
/** Temporary buffer large enough for at least one raw object id. */
|
||||||
final byte[] tempId = new byte[OBJECT_ID_LENGTH];
|
final byte[] tempId = new byte[OBJECT_ID_LENGTH];
|
||||||
|
|
||||||
|
@ -163,22 +167,44 @@ public Collection<ObjectId> resolve(AbbreviatedObjectId id)
|
||||||
return Collections.singleton(id.toObjectId());
|
return Collections.singleton(id.toObjectId());
|
||||||
boolean noGarbage = avoidUnreachable;
|
boolean noGarbage = avoidUnreachable;
|
||||||
HashSet<ObjectId> matches = new HashSet<ObjectId>(4);
|
HashSet<ObjectId> matches = new HashSet<ObjectId>(4);
|
||||||
for (DfsPackFile pack : db.getPacks()) {
|
PackList packList = db.getPackList();
|
||||||
if (noGarbage && pack.isGarbage())
|
resolveImpl(packList, id, noGarbage, matches);
|
||||||
continue;
|
if (matches.size() < MAX_RESOLVE_MATCHES && packList.dirty()) {
|
||||||
pack.resolve(this, matches, id, 256);
|
resolveImpl(db.scanPacks(packList), id, noGarbage, matches);
|
||||||
if (256 <= matches.size())
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
return matches;
|
return matches;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void resolveImpl(PackList packList, AbbreviatedObjectId id,
|
||||||
|
boolean noGarbage, HashSet<ObjectId> matches) throws IOException {
|
||||||
|
for (DfsPackFile pack : packList.packs) {
|
||||||
|
if (noGarbage && pack.isGarbage()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
pack.resolve(this, matches, id, MAX_RESOLVE_MATCHES);
|
||||||
|
if (matches.size() >= MAX_RESOLVE_MATCHES) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean has(AnyObjectId objectId) throws IOException {
|
public boolean has(AnyObjectId objectId) throws IOException {
|
||||||
if (last != null && last.hasObject(this, objectId))
|
if (last != null && last.hasObject(this, objectId))
|
||||||
return true;
|
return true;
|
||||||
boolean noGarbage = avoidUnreachable;
|
boolean noGarbage = avoidUnreachable;
|
||||||
for (DfsPackFile pack : db.getPacks()) {
|
PackList packList = db.getPackList();
|
||||||
|
if (hasImpl(packList, objectId, noGarbage)) {
|
||||||
|
return true;
|
||||||
|
} else if (packList.dirty()) {
|
||||||
|
return hasImpl(db.scanPacks(packList), objectId, noGarbage);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasImpl(PackList packList, AnyObjectId objectId,
|
||||||
|
boolean noGarbage) throws IOException {
|
||||||
|
for (DfsPackFile pack : packList.packs) {
|
||||||
if (pack == last || (noGarbage && pack.isGarbage()))
|
if (pack == last || (noGarbage && pack.isGarbage()))
|
||||||
continue;
|
continue;
|
||||||
if (pack.hasObject(this, objectId)) {
|
if (pack.hasObject(this, objectId)) {
|
||||||
|
@ -193,19 +219,22 @@ public boolean has(AnyObjectId objectId) throws IOException {
|
||||||
public ObjectLoader open(AnyObjectId objectId, int typeHint)
|
public ObjectLoader open(AnyObjectId objectId, int typeHint)
|
||||||
throws MissingObjectException, IncorrectObjectTypeException,
|
throws MissingObjectException, IncorrectObjectTypeException,
|
||||||
IOException {
|
IOException {
|
||||||
|
ObjectLoader ldr;
|
||||||
if (last != null) {
|
if (last != null) {
|
||||||
ObjectLoader ldr = last.get(this, objectId);
|
ldr = last.get(this, objectId);
|
||||||
if (ldr != null)
|
if (ldr != null)
|
||||||
return ldr;
|
return ldr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PackList packList = db.getPackList();
|
||||||
boolean noGarbage = avoidUnreachable;
|
boolean noGarbage = avoidUnreachable;
|
||||||
for (DfsPackFile pack : db.getPacks()) {
|
ldr = openImpl(packList, objectId, noGarbage);
|
||||||
if (pack == last || (noGarbage && pack.isGarbage()))
|
if (ldr != null) {
|
||||||
continue;
|
return ldr;
|
||||||
ObjectLoader ldr = pack.get(this, objectId);
|
}
|
||||||
|
if (packList.dirty()) {
|
||||||
|
ldr = openImpl(db.scanPacks(packList), objectId, noGarbage);
|
||||||
if (ldr != null) {
|
if (ldr != null) {
|
||||||
last = pack;
|
|
||||||
return ldr;
|
return ldr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -216,6 +245,21 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint)
|
||||||
throw new MissingObjectException(objectId.copy(), typeHint);
|
throw new MissingObjectException(objectId.copy(), typeHint);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ObjectLoader openImpl(PackList packList, AnyObjectId objectId,
|
||||||
|
boolean noGarbage) throws IOException {
|
||||||
|
for (DfsPackFile pack : packList.packs) {
|
||||||
|
if (pack == last || (noGarbage && pack.isGarbage())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ObjectLoader ldr = pack.get(this, objectId);
|
||||||
|
if (ldr != null) {
|
||||||
|
last = pack;
|
||||||
|
return ldr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Set<ObjectId> getShallowCommits() {
|
public Set<ObjectId> getShallowCommits() {
|
||||||
return Collections.emptySet();
|
return Collections.emptySet();
|
||||||
|
@ -253,39 +297,58 @@ private static class FoundObject<T extends ObjectId> {
|
||||||
|
|
||||||
private <T extends ObjectId> Iterable<FoundObject<T>> findAll(
|
private <T extends ObjectId> Iterable<FoundObject<T>> findAll(
|
||||||
Iterable<T> objectIds) throws IOException {
|
Iterable<T> objectIds) throws IOException {
|
||||||
ArrayList<FoundObject<T>> r = new ArrayList<FoundObject<T>>();
|
Collection<T> pending = new LinkedList<>();
|
||||||
DfsPackFile[] packList = db.getPacks();
|
for (T id : objectIds) {
|
||||||
if (packList.length == 0) {
|
pending.add(id);
|
||||||
for (T t : objectIds)
|
}
|
||||||
|
|
||||||
|
PackList packList = db.getPackList();
|
||||||
|
List<FoundObject<T>> r = new ArrayList<>();
|
||||||
|
findAllImpl(packList, pending, r);
|
||||||
|
if (!pending.isEmpty() && packList.dirty()) {
|
||||||
|
findAllImpl(db.scanPacks(packList), pending, r);
|
||||||
|
}
|
||||||
|
for (T t : pending) {
|
||||||
r.add(new FoundObject<T>(t));
|
r.add(new FoundObject<T>(t));
|
||||||
|
}
|
||||||
|
Collections.sort(r, FOUND_OBJECT_SORT);
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private <T extends ObjectId> void findAllImpl(PackList packList,
|
||||||
|
Collection<T> pending, List<FoundObject<T>> r) {
|
||||||
|
DfsPackFile[] packs = packList.packs;
|
||||||
|
if (packs.length == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
int lastIdx = 0;
|
int lastIdx = 0;
|
||||||
DfsPackFile lastPack = packList[lastIdx];
|
DfsPackFile lastPack = packs[lastIdx];
|
||||||
boolean noGarbage = avoidUnreachable;
|
boolean noGarbage = avoidUnreachable;
|
||||||
|
|
||||||
OBJECT_SCAN: for (T t : objectIds) {
|
OBJECT_SCAN: for (Iterator<T> it = pending.iterator(); it.hasNext();) {
|
||||||
|
T t = it.next();
|
||||||
try {
|
try {
|
||||||
long p = lastPack.findOffset(this, t);
|
long p = lastPack.findOffset(this, t);
|
||||||
if (0 < p) {
|
if (0 < p) {
|
||||||
r.add(new FoundObject<T>(t, lastIdx, lastPack, p));
|
r.add(new FoundObject<T>(t, lastIdx, lastPack, p));
|
||||||
|
it.remove();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
// Fall though and try to examine other packs.
|
// Fall though and try to examine other packs.
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < packList.length; i++) {
|
for (int i = 0; i < packs.length; i++) {
|
||||||
if (i == lastIdx)
|
if (i == lastIdx)
|
||||||
continue;
|
continue;
|
||||||
DfsPackFile pack = packList[i];
|
DfsPackFile pack = packs[i];
|
||||||
if (noGarbage && pack.isGarbage())
|
if (noGarbage && pack.isGarbage())
|
||||||
continue;
|
continue;
|
||||||
try {
|
try {
|
||||||
long p = pack.findOffset(this, t);
|
long p = pack.findOffset(this, t);
|
||||||
if (0 < p) {
|
if (0 < p) {
|
||||||
r.add(new FoundObject<T>(t, i, pack, p));
|
r.add(new FoundObject<T>(t, i, pack, p));
|
||||||
|
it.remove();
|
||||||
lastIdx = i;
|
lastIdx = i;
|
||||||
lastPack = pack;
|
lastPack = pack;
|
||||||
continue OBJECT_SCAN;
|
continue OBJECT_SCAN;
|
||||||
|
@ -294,13 +357,9 @@ private <T extends ObjectId> Iterable<FoundObject<T>> findAll(
|
||||||
// Examine other packs.
|
// Examine other packs.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
r.add(new FoundObject<T>(t));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Collections.sort(r, FOUND_OBJECT_SORT);
|
|
||||||
last = lastPack;
|
last = lastPack;
|
||||||
return r;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -418,24 +477,43 @@ public long getObjectSize(AnyObjectId objectId, int typeHint)
|
||||||
IOException {
|
IOException {
|
||||||
if (last != null) {
|
if (last != null) {
|
||||||
long sz = last.getObjectSize(this, objectId);
|
long sz = last.getObjectSize(this, objectId);
|
||||||
if (0 <= sz)
|
if (0 <= sz) {
|
||||||
return sz;
|
return sz;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (DfsPackFile pack : db.getPacks()) {
|
PackList packList = db.getPackList();
|
||||||
if (pack == last)
|
long sz = getObjectSizeImpl(packList, objectId);
|
||||||
|
if (0 <= sz) {
|
||||||
|
return sz;
|
||||||
|
}
|
||||||
|
if (packList.dirty()) {
|
||||||
|
sz = getObjectSizeImpl(packList, objectId);
|
||||||
|
if (0 <= sz) {
|
||||||
|
return sz;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeHint == OBJ_ANY) {
|
||||||
|
throw new MissingObjectException(objectId.copy(),
|
||||||
|
JGitText.get().unknownObjectType2);
|
||||||
|
}
|
||||||
|
throw new MissingObjectException(objectId.copy(), typeHint);
|
||||||
|
}
|
||||||
|
|
||||||
|
private long getObjectSizeImpl(PackList packList, AnyObjectId objectId)
|
||||||
|
throws IOException {
|
||||||
|
for (DfsPackFile pack : packList.packs) {
|
||||||
|
if (pack == last) {
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
long sz = pack.getObjectSize(this, objectId);
|
long sz = pack.getObjectSize(this, objectId);
|
||||||
if (0 <= sz) {
|
if (0 <= sz) {
|
||||||
last = pack;
|
last = pack;
|
||||||
return sz;
|
return sz;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return -1;
|
||||||
if (typeHint == OBJ_ANY)
|
|
||||||
throw new MissingObjectException(objectId.copy(),
|
|
||||||
JGitText.get().unknownObjectType2);
|
|
||||||
throw new MissingObjectException(objectId.copy(), typeHint);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) {
|
public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) {
|
||||||
|
@ -451,6 +529,8 @@ public int compare(DfsObjectToPack a, DfsObjectToPack b) {
|
||||||
public void selectObjectRepresentation(PackWriter packer,
|
public void selectObjectRepresentation(PackWriter packer,
|
||||||
ProgressMonitor monitor, Iterable<ObjectToPack> objects)
|
ProgressMonitor monitor, Iterable<ObjectToPack> objects)
|
||||||
throws IOException, MissingObjectException {
|
throws IOException, MissingObjectException {
|
||||||
|
// Don't check dirty bit on PackList; assume ObjectToPacks all came from the
|
||||||
|
// current list.
|
||||||
for (DfsPackFile pack : db.getPacks()) {
|
for (DfsPackFile pack : db.getPacks()) {
|
||||||
List<DfsObjectToPack> tmp = findAllFromPack(pack, objects);
|
List<DfsObjectToPack> tmp = findAllFromPack(pack, objects);
|
||||||
if (tmp.isEmpty())
|
if (tmp.isEmpty())
|
||||||
|
|
|
@ -310,6 +310,7 @@ protected RefCache scanAllRefs() throws IOException {
|
||||||
}
|
}
|
||||||
ids.sort();
|
ids.sort();
|
||||||
sym.sort();
|
sym.sort();
|
||||||
|
objdb.getCurrentPackList().markDirty();
|
||||||
return new RefCache(ids.toRefList(), sym.toRefList());
|
return new RefCache(ids.toRefList(), sym.toRefList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue