From a84a2d22abef94eb632e3da711622638e354e516 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 25 Apr 2018 11:59:21 -0600 Subject: [PATCH] Split out packfile handling from ObjectDirectory The ObjectDirectory class manages the interactions for the entire object database, this includes loose objects, packfiles, alternates, and shallow commits. To help reduce the complexity of this class, abstract some of the packfile specific details into a class which understands just this, leaving the ObjectDirectory to focus more on the interactions between the different mechanisms. Change-Id: I5cc87b964434b0afa860b3fe23867a77b3c3a4f2 Signed-off-by: Martin Fick Signed-off-by: Matthias Sohn --- .../storage/file/ObjectDirectory.java | 392 +------------- .../internal/storage/file/PackDirectory.java | 498 ++++++++++++++++++ 2 files changed, 513 insertions(+), 377 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index d32182864..b4a336f7b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -24,20 +24,14 @@ import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.PackInvalidException; -import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -45,7 +39,6 @@ import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; @@ -79,9 +72,6 @@ public class ObjectDirectory extends FileObjectDatabase { private static final Logger LOG = LoggerFactory .getLogger(ObjectDirectory.class); - private static final PackList NO_PACKS = new PackList( - FileSnapshot.DIRTY, new PackFile[0]); - /** Maximum number of candidates offered as resolutions of abbreviation. */ private static final int RESOLVE_ABBREV_LIMIT = 256; @@ -93,7 +83,7 @@ public class ObjectDirectory extends FileObjectDatabase { private final File infoDirectory; - private final File packDirectory; + private final PackDirectory packed; private final File preservedDirectory; @@ -111,8 +101,6 @@ public class ObjectDirectory extends FileObjectDatabase { private Set shallowCommitsIds; - final AtomicReference packList; - /** * Initialize a reference to an on-disk object directory. * @@ -136,10 +124,10 @@ public ObjectDirectory(final Config cfg, final File dir, config = cfg; objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ - packDirectory = new File(objects, "pack"); //$NON-NLS-1$ + File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); - packList = new AtomicReference<>(NO_PACKS); + packed = new PackDirectory(config, packDirectory); unpackedObjectCache = new UnpackedObjectCache(); this.fs = fs; this.shallowFile = shallowFile; @@ -167,7 +155,7 @@ public final File getDirectory() { * @return the location of the pack directory. */ public final File getPackDirectory() { - return packDirectory; + return packed.getDirectory(); } /** @@ -190,7 +178,7 @@ public boolean exists() { public void create() throws IOException { FileUtils.mkdirs(objects); FileUtils.mkdir(infoDirectory); - FileUtils.mkdir(packDirectory); + packed.create(); } /** {@inheritDoc} */ @@ -214,11 +202,7 @@ public PackInserter newPackInserter() { public void close() { unpackedObjectCache.clear(); - final PackList packs = packList.get(); - if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (PackFile p : packs.packs) - p.close(); - } + packed.close(); // Fully close all loaded alternates and clear the alternate list. AlternateHandle[] alt = alternates.get(); @@ -231,11 +215,7 @@ public void close() { /** {@inheritDoc} */ @Override public Collection getPacks() { - PackList list = packList.get(); - if (list == NO_PACKS) - list = scanPacks(list); - PackFile[] packs = list.packs; - return Collections.unmodifiableCollection(Arrays.asList(packs)); + return packed.getPacks(); } /** @@ -264,7 +244,7 @@ public PackFile openPack(File pack) } PackFile res = new PackFile(pack, extensions); - insertPack(res); + packed.insert(res); return res; } @@ -315,25 +295,7 @@ private boolean hasLooseInSelfOrAlternate(AnyObjectId objectId, } boolean hasPackedObject(AnyObjectId objectId) { - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - if (p.hasObject(objectId)) - return true; - } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. - LOG.warn(MessageFormat.format( - JGitText.get().unableToReadPackfile, - p.getPackFile().getAbsolutePath()), e); - removePack(p); - } - } - } while (searchPacksAgain(pList)); - return false; + return packed.has(objectId); } @Override @@ -345,23 +307,8 @@ void resolve(Set matches, AbbreviatedObjectId id) private void resolve(Set matches, AbbreviatedObjectId id, Set skips) throws IOException { - // Go through the packs once. If we didn't find any resolutions - // scan for new packs and check once more. - int oldSize = matches.size(); - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - p.resolve(matches, id, RESOLVE_ABBREV_LIMIT); - p.resetTransientErrorCount(); - } catch (IOException e) { - handlePackError(e, p); - } - if (matches.size() > RESOLVE_ABBREV_LIMIT) - return; - } - } while (matches.size() == oldSize && searchPacksAgain(pList)); + if (!packed.resolve(matches, id, RESOLVE_ABBREV_LIMIT)) + return; String fanOut = id.name().substring(0, 2); String[] entries = new File(getDirectory(), fanOut).list(); @@ -446,28 +393,7 @@ private ObjectLoader openLooseFromSelfOrAlternate(WindowCursor curs, } ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - ObjectLoader ldr = p.get(curs, objectId); - p.resetTransientErrorCount(); - if (ldr != null) - return ldr; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return null; + return packed.open(curs, objectId); } @Override @@ -504,7 +430,7 @@ long getObjectSize(WindowCursor curs, AnyObjectId id) private long getPackedSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set skips) { - long len = getPackedObjectSize(curs, id); + long len = packed.getSize(curs, id); if (0 <= len) { return len; } @@ -538,31 +464,6 @@ private long getLooseSizeFromSelfOrAlternate(WindowCursor curs, return -1; } - private long getPackedObjectSize(WindowCursor curs, AnyObjectId id) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - long len = p.getObjectSize(curs, id); - p.resetTransientErrorCount(); - if (0 <= len) - return len; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return -1; - } - private long getLooseObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { File f = fileFor(id); @@ -586,25 +487,7 @@ void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs, Set skips) throws IOException { - PackList pList = packList.get(); - SEARCH: for (;;) { - for (PackFile p : pList.packs) { - try { - LocalObjectRepresentation rep = p.representation(curs, otp); - p.resetTransientErrorCount(); - if (rep != null) - packer.select(otp, rep); - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - // - pList = scanPacks(pList); - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } + packed.selectRepresentation(packer, otp, curs); skips = addMe(skips); for (AlternateHandle h : myAlternates()) { @@ -614,54 +497,6 @@ private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, } } - private void handlePackError(IOException e, PackFile p) { - String warnTmpl = null; - int transientErrorCount = 0; - String errTmpl = JGitText.get().exceptionWhileReadingPack; - if ((e instanceof CorruptObjectException) - || (e instanceof PackInvalidException)) { - warnTmpl = JGitText.get().corruptPack; - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - // Assume the pack is corrupted, and remove it from the list. - removePack(p); - } else if (e instanceof FileNotFoundException) { - if (p.getPackFile().exists()) { - errTmpl = JGitText.get().packInaccessible; - transientErrorCount = p.incrementTransientErrorCount(); - } else { - warnTmpl = JGitText.get().packWasDeleted; - removePack(p); - } - } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { - warnTmpl = JGitText.get().packHandleIsStale; - removePack(p); - } else { - transientErrorCount = p.incrementTransientErrorCount(); - } - if (warnTmpl != null) { - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - } else { - if (doLogExponentialBackoff(transientErrorCount)) { - // Don't remove the pack from the list, as the error may be - // transient. - LOG.error(MessageFormat.format(errTmpl, - p.getPackFile().getAbsolutePath(), - Integer.valueOf(transientErrorCount)), e); - } - } - } - - /** - * @param n - * count of consecutive failures - * @return @{code true} if i is a power of 2 - */ - private boolean doLogExponentialBackoff(int n) { - return (n & (n - 1)) == 0; - } - @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { @@ -727,21 +562,6 @@ private InsertLooseObjectResult tryMove(File tmp, File dst, return InsertLooseObjectResult.INSERTED; } - boolean searchPacksAgain(PackList old) { - // Whether to trust the pack folder's modification time. If set - // to false we will always scan the .git/objects/pack folder to - // check for new pack files. If set to true (default) we use the - // lastmodified attribute of the folder and assume that no new - // pack files can be in this folder if his modification time has - // not changed. - boolean trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - - return ((!trustFolderStat) || old.snapshot.isModified(packDirectory)) - && old != scanPacks(old); - } - @Override Config getConfig() { return config; @@ -780,182 +600,13 @@ Set getShallowCommits() throws IOException { return shallowCommitsIds; } - private void insertPack(PackFile pf) { - PackList o, n; - do { - o = packList.get(); - - // If the pack in question is already present in the list - // (picked up by a concurrent thread that did a scan?) we - // do not want to insert it a second time. - // - final PackFile[] oldList = o.packs; - final String name = pf.getPackFile().getName(); - for (PackFile p : oldList) { - if (name.equals(p.getPackFile().getName())) - return; - } - - final PackFile[] newList = new PackFile[1 + oldList.length]; - newList[0] = pf; - System.arraycopy(oldList, 0, newList, 1, oldList.length); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - } - - private void removePack(PackFile deadPack) { - PackList o, n; - do { - o = packList.get(); - - final PackFile[] oldList = o.packs; - final int j = indexOf(oldList, deadPack); - if (j < 0) - break; - - final PackFile[] newList = new PackFile[oldList.length - 1]; - System.arraycopy(oldList, 0, newList, 0, j); - System.arraycopy(oldList, j + 1, newList, j, newList.length - j); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - deadPack.close(); - } - - private static int indexOf(PackFile[] list, PackFile pack) { - for (int i = 0; i < list.length; i++) { - if (list[i] == pack) - return i; - } - return -1; - } - - private PackList scanPacks(PackList original) { - synchronized (packList) { - PackList o, n; - do { - o = packList.get(); - if (o != original) { - // Another thread did the scan for us, while we - // were blocked on the monitor above. - // - return o; - } - n = scanPacksImpl(o); - if (n == o) - return n; - } while (!packList.compareAndSet(o, n)); - return n; - } - } - - private PackList scanPacksImpl(PackList old) { - final Map forReuse = reuseMap(old); - final FileSnapshot snapshot = FileSnapshot.save(packDirectory); - final Set names = listPackDirectory(); - final List list = new ArrayList<>(names.size() >> 2); - boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) //$NON-NLS-1$ - continue; - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) - extensions |= ext.getBit(); - } - - if ((extensions & PACK.getBit()) == 0) { - // Sometimes C Git's HTTP fetch transport leaves a - // .idx file behind and does not download the .pack. - // We have to skip over such useless indexes. - // - continue; - } - - final String packName = base + PACK.getExtension(); - final File packFile = new File(packDirectory, packName); - final PackFile oldPack = forReuse.get(packName); - if (oldPack != null - && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); - list.add(oldPack); - continue; - } - - list.add(new PackFile(packFile, extensions)); - foundNew = true; - } - - // If we did not discover any new files, the modification time was not - // changed, and we did not remove any files, then the set of files is - // the same as the set we were given. Instead of building a new object - // return the same collection. - // - if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { - old.snapshot.setClean(snapshot); - return old; - } - - for (PackFile p : forReuse.values()) { - p.close(); - } - - if (list.isEmpty()) - return new PackList(snapshot, NO_PACKS.packs); - - final PackFile[] r = list.toArray(new PackFile[0]); - Arrays.sort(r, PackFile.SORT); - return new PackList(snapshot, r); - } - - private static Map reuseMap(PackList old) { - final Map forReuse = new HashMap<>(); - for (PackFile p : old.packs) { - if (p.invalid()) { - // The pack instance is corrupted, and cannot be safely used - // again. Do not include it in our reuse map. - // - p.close(); - continue; - } - - final PackFile prior = forReuse.put(p.getPackFile().getName(), p); - if (prior != null) { - // This should never occur. It should be impossible for us - // to have two pack files with the same name, as all of them - // came out of the same directory. If it does, we promised to - // close any PackFiles we did not reuse, so close the second, - // readers are likely to be actively using the first. - // - forReuse.put(prior.getPackFile().getName(), prior); - p.close(); - } - } - return forReuse; - } - - private Set listPackDirectory() { - final String[] nameList = packDirectory.list(); - if (nameList == null) - return Collections.emptySet(); - final Set nameSet = new HashSet<>(nameList.length << 1); - for (String name : nameList) { - if (name.startsWith("pack-")) //$NON-NLS-1$ - nameSet.add(name); - } - return nameSet; - } - void closeAllPackHandles(File packFile) { // if the packfile already exists (because we are rewriting a // packfile for the same set of objects maybe with different // PackConfig) then make sure we get rid of all handles on the file. // Windows will not allow for rename otherwise. if (packFile.exists()) { - for (PackFile p : getPacks()) { + for (PackFile p : packed.getPacks()) { if (packFile.getPath().equals(p.getPackFile().getPath())) { p.close(); break; @@ -1037,19 +688,6 @@ public File fileFor(AnyObjectId objectId) { return new File(new File(getDirectory(), d), f); } - static final class PackList { - /** State just before reading the pack directory. */ - final FileSnapshot snapshot; - - /** All known packs, sorted by {@link PackFile#SORT}. */ - final PackFile[] packs; - - PackList(FileSnapshot monitor, PackFile[] packs) { - this.snapshot = monitor; - this.packs = packs; - } - } - static class AlternateHandle { static class Id { String alternateId; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java new file mode 100644 index 000000000..f89b23eeb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -0,0 +1,498 @@ +/* + * Copyright (C) 2009, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackInvalidException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Traditional file system packed objects directory handler. + *

+ * This is the {@code PackFile}s object representation for a Git object database, + * where objects are stored in compressed containers + * known as {@link org.eclipse.jgit.internal.storage.file.PackFile}s. + */ +class PackDirectory { + private final static Logger LOG = LoggerFactory + .getLogger(PackDirectory.class); + + private static final PackList NO_PACKS = new PackList( + FileSnapshot.DIRTY, new PackFile[0]); + + private final Config config; + + private final File directory; + + private final AtomicReference packList; + + /** + * Initialize a reference to an on-disk 'pack' directory. + * + * @param config + * configuration this directory consults for write settings. + * @param directory + * the location of the {@code pack} directory. + */ + PackDirectory(Config config, File directory) { + this.config = config; + this.directory = directory; + packList = new AtomicReference<>(NO_PACKS); + } + + /** + * Getter for the field {@code directory}. + * + * @return the location of the {@code pack} directory. + */ + File getDirectory() { + return directory; + } + + void create() throws IOException { + FileUtils.mkdir(directory); + } + + void close() { + PackList packs = packList.get(); + if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { + for (PackFile p : packs.packs) + p.close(); + } + } + + Collection getPacks() { + PackList list = packList.get(); + if (list == NO_PACKS) + list = scanPacks(list); + PackFile[] packs = list.packs; + return Collections.unmodifiableCollection(Arrays.asList(packs)); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return "PackDirectory[" + getDirectory() + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } + + /** + * Does the requested object exist in this PackDirectory? + * + * @param objectId + * identity of the object to test for existence of. + * @return true if the specified object is stored in this PackDirectory. + */ + boolean has(AnyObjectId objectId) { + PackList pList; + do { + pList = packList.get(); + for (PackFile p : pList.packs) { + try { + if (p.hasObject(objectId)) + return true; + } catch (IOException e) { + // The hasObject call should have only touched the index, + // so any failure here indicates the index is unreadable + // by this process, and the pack is likewise not readable. + LOG.warn(MessageFormat.format( + JGitText.get().unableToReadPackfile, + p.getPackFile().getAbsolutePath()), e); + remove(p); + } + } + } while (searchPacksAgain(pList)); + return false; + } + + /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + * @return {@code true} if the matches were exhausted before reaching + * {@code maxLimit}. + */ + boolean resolve(Set matches, AbbreviatedObjectId id, + int matchLimit) { + // Go through the packs once. If we didn't find any resolutions + // scan for new packs and check once more. + int oldSize = matches.size(); + PackList pList; + do { + pList = packList.get(); + for (PackFile p : pList.packs) { + try { + p.resolve(matches, id, matchLimit); + p.resetTransientErrorCount(); + } catch (IOException e) { + handlePackError(e, p); + } + if (matches.size() > matchLimit) + return false; + } + } while (matches.size() == oldSize && searchPacksAgain(pList)); + return true; + } + + ObjectLoader open(WindowCursor curs, AnyObjectId objectId) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (PackFile p : pList.packs) { + try { + ObjectLoader ldr = p.get(curs, objectId); + p.resetTransientErrorCount(); + if (ldr != null) + return ldr; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) + continue SEARCH; + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return null; + } + + long getSize(WindowCursor curs, AnyObjectId id) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (PackFile p : pList.packs) { + try { + long len = p.getObjectSize(curs, id); + p.resetTransientErrorCount(); + if (0 <= len) + return len; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) + continue SEARCH; + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return -1; + } + + void selectRepresentation(PackWriter packer, ObjectToPack otp, + WindowCursor curs) { + PackList pList = packList.get(); + SEARCH: for (;;) { + for (PackFile p : pList.packs) { + try { + LocalObjectRepresentation rep = p.representation(curs, otp); + p.resetTransientErrorCount(); + if (rep != null) + packer.select(otp, rep); + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + // + pList = scanPacks(pList); + continue SEARCH; + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } + + private void handlePackError(IOException e, PackFile p) { + String warnTmpl = null; + int transientErrorCount = 0; + String errTmpl = JGitText.get().exceptionWhileReadingPack; + if ((e instanceof CorruptObjectException) + || (e instanceof PackInvalidException)) { + warnTmpl = JGitText.get().corruptPack; + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + // Assume the pack is corrupted, and remove it from the list. + remove(p); + } else if (e instanceof FileNotFoundException) { + if (p.getPackFile().exists()) { + errTmpl = JGitText.get().packInaccessible; + transientErrorCount = p.incrementTransientErrorCount(); + } else { + warnTmpl = JGitText.get().packWasDeleted; + remove(p); + } + } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { + warnTmpl = JGitText.get().packHandleIsStale; + remove(p); + } else { + transientErrorCount = p.incrementTransientErrorCount(); + } + if (warnTmpl != null) { + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + } else { + if (doLogExponentialBackoff(transientErrorCount)) { + // Don't remove the pack from the list, as the error may be + // transient. + LOG.error(MessageFormat.format(errTmpl, + p.getPackFile().getAbsolutePath(), + Integer.valueOf(transientErrorCount)), e); + } + } + } + + /** + * @param n + * count of consecutive failures + * @return @{code true} if i is a power of 2 + */ + private boolean doLogExponentialBackoff(int n) { + return (n & (n - 1)) == 0; + } + + boolean searchPacksAgain(PackList old) { + // Whether to trust the pack folder's modification time. If set + // to false we will always scan the .git/objects/pack folder to + // check for new pack files. If set to true (default) we use the + // lastmodified attribute of the folder and assume that no new + // pack files can be in this folder if his modification time has + // not changed. + boolean trustFolderStat = config.getBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + + return ((!trustFolderStat) || old.snapshot.isModified(directory)) + && old != scanPacks(old); + } + + void insert(PackFile pf) { + PackList o, n; + do { + o = packList.get(); + + // If the pack in question is already present in the list + // (picked up by a concurrent thread that did a scan?) we + // do not want to insert it a second time. + // + final PackFile[] oldList = o.packs; + final String name = pf.getPackFile().getName(); + for (PackFile p : oldList) { + if (name.equals(p.getPackFile().getName())) + return; + } + + final PackFile[] newList = new PackFile[1 + oldList.length]; + newList[0] = pf; + System.arraycopy(oldList, 0, newList, 1, oldList.length); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + } + + private void remove(PackFile deadPack) { + PackList o, n; + do { + o = packList.get(); + + final PackFile[] oldList = o.packs; + final int j = indexOf(oldList, deadPack); + if (j < 0) + break; + + final PackFile[] newList = new PackFile[oldList.length - 1]; + System.arraycopy(oldList, 0, newList, 0, j); + System.arraycopy(oldList, j + 1, newList, j, newList.length - j); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + deadPack.close(); + } + + private static int indexOf(PackFile[] list, PackFile pack) { + for (int i = 0; i < list.length; i++) { + if (list[i] == pack) + return i; + } + return -1; + } + + private PackList scanPacks(PackList original) { + synchronized (packList) { + PackList o, n; + do { + o = packList.get(); + if (o != original) { + // Another thread did the scan for us, while we + // were blocked on the monitor above. + // + return o; + } + n = scanPacksImpl(o); + if (n == o) + return n; + } while (!packList.compareAndSet(o, n)); + return n; + } + } + + private PackList scanPacksImpl(PackList old) { + final Map forReuse = reuseMap(old); + final FileSnapshot snapshot = FileSnapshot.save(directory); + final Set names = listPackDirectory(); + final List list = new ArrayList<>(names.size() >> 2); + boolean foundNew = false; + for (String indexName : names) { + // Must match "pack-[0-9a-f]{40}.idx" to be an index. + // + if (indexName.length() != 49 || !indexName.endsWith(".idx")) //$NON-NLS-1$ + continue; + + final String base = indexName.substring(0, indexName.length() - 3); + int extensions = 0; + for (PackExt ext : PackExt.values()) { + if (names.contains(base + ext.getExtension())) + extensions |= ext.getBit(); + } + + if ((extensions & PACK.getBit()) == 0) { + // Sometimes C Git's HTTP fetch transport leaves a + // .idx file behind and does not download the .pack. + // We have to skip over such useless indexes. + // + continue; + } + + final String packName = base + PACK.getExtension(); + final File packFile = new File(directory, packName); + final PackFile oldPack = forReuse.get(packName); + if (oldPack != null + && !oldPack.getFileSnapshot().isModified(packFile)) { + forReuse.remove(packName); + list.add(oldPack); + continue; + } + + list.add(new PackFile(packFile, extensions)); + foundNew = true; + } + + // If we did not discover any new files, the modification time was not + // changed, and we did not remove any files, then the set of files is + // the same as the set we were given. Instead of building a new object + // return the same collection. + // + if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { + old.snapshot.setClean(snapshot); + return old; + } + + for (PackFile p : forReuse.values()) { + p.close(); + } + + if (list.isEmpty()) + return new PackList(snapshot, NO_PACKS.packs); + + final PackFile[] r = list.toArray(new PackFile[0]); + Arrays.sort(r, PackFile.SORT); + return new PackList(snapshot, r); + } + + private static Map reuseMap(PackList old) { + final Map forReuse = new HashMap<>(); + for (PackFile p : old.packs) { + if (p.invalid()) { + // The pack instance is corrupted, and cannot be safely used + // again. Do not include it in our reuse map. + // + p.close(); + continue; + } + + final PackFile prior = forReuse.put(p.getPackFile().getName(), p); + if (prior != null) { + // This should never occur. It should be impossible for us + // to have two pack files with the same name, as all of them + // came out of the same directory. If it does, we promised to + // close any PackFiles we did not reuse, so close the second, + // readers are likely to be actively using the first. + // + forReuse.put(prior.getPackFile().getName(), prior); + p.close(); + } + } + return forReuse; + } + + private Set listPackDirectory() { + final String[] nameList = directory.list(); + if (nameList == null) + return Collections.emptySet(); + final Set nameSet = new HashSet<>(nameList.length << 1); + for (String name : nameList) { + if (name.startsWith("pack-")) //$NON-NLS-1$ + nameSet.add(name); + } + return nameSet; + } + + static final class PackList { + /** State just before reading the pack directory. */ + final FileSnapshot snapshot; + + /** All known packs, sorted by {@link PackFile#SORT}. */ + final PackFile[] packs; + + PackList(FileSnapshot monitor, PackFile[] packs) { + this.snapshot = monitor; + this.packs = packs; + } + } +}