From 7f59cfe14354a19a38ccc5920c2ce5e49de9ae0d Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Sat, 12 Aug 2017 08:25:16 -0700 Subject: [PATCH 1/2] Support symbolic references in ReceiveCommand Allow creating symbolic references with link, and deleting them or switching to ObjectId with unlink. How this happens is up to the individual RefDatabase. The default implementation detaches RefUpdate if a symbolic reference is involved, supporting these command instances on RefDirectory. Unfortunately the packed-refs file does not support storing symrefs, so atomic transactions involving more than one symref command are failed early. Updating InMemoryRepository is deferred until reftable lands, as I plan to switch InMemoryRepository to use reftable for its internal storage representation. Change-Id: Ibcae068b17a2fc6d958f767f402a570ad88d9151 Signed-off-by: Minh Thai Signed-off-by: Terry Parker --- .../storage/reftree/RefTreeDatabaseTest.java | 3 +- .../eclipse/jgit/internal/JGitText.properties | 3 + .../org/eclipse/jgit/internal/JGitText.java | 3 + .../storage/file/PackedBatchRefUpdate.java | 23 +- .../internal/storage/reftree/Command.java | 17 +- .../jgit/transport/ReceiveCommand.java | 241 ++++++++++++++++-- 6 files changed, 268 insertions(+), 22 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java index 9aef94369..1684afa4e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java @@ -657,7 +657,8 @@ public boolean apply(ObjectReader reader, RefTree tree) Ref old = tree.exactRef(reader, name); Command n; try (RevWalk rw = new RevWalk(repo)) { - n = new Command(old, Command.toRef(rw, id, name, true)); + n = new Command(old, + Command.toRef(rw, id, null, name, true)); } return tree.apply(Collections.singleton(n)); } diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 43dd9482d..f586aee93 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -22,6 +22,7 @@ atLeastOnePatternIsRequired=At least one pattern is required. atLeastTwoFiltersNeeded=At least two filters needed. atomicPushNotSupported=Atomic push not supported. atomicRefUpdatesNotSupported=Atomic ref updates not supported +atomicSymRefNotSupported=Atomic symref not supported authenticationNotSupported=authentication not supported badBase64InputCharacterAt=Bad Base64 input character at {0} : {1} (decimal) badEntryDelimiter=Bad entry delimiter @@ -41,6 +42,7 @@ blameNotCommittedYet=Not Committed Yet blobNotFound=Blob not found: {0} blobNotFoundForPath=Blob not found: {0} for path: {1} blockSizeNotPowerOf2=blockSize must be a power of 2 +bothRefTargetsMustNotBeNull=both old and new ref targets must not be null. branchNameInvalid=Branch name {0} is not allowed buildingBitmaps=Building bitmaps cachedPacksPreventsIndexCreation=Using cached packs prevents index creation @@ -434,6 +436,7 @@ month=month months=months monthsAgo={0} months ago multipleMergeBasesFor=Multiple merge bases for:\n {0}\n {1} found:\n {2}\n {3} +nameMustNotBeNullOrEmpty=Ref name must not be null or empty. need2Arguments=Need 2 arguments needPackOut=need packOut needsAtLeastOneEntry=Needs at least one entry diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 6b3631601..c41fc51bd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -81,6 +81,7 @@ public static JGitText get() { /***/ public String atLeastTwoFiltersNeeded; /***/ public String atomicPushNotSupported; /***/ public String atomicRefUpdatesNotSupported; + /***/ public String atomicSymRefNotSupported; /***/ public String authenticationNotSupported; /***/ public String badBase64InputCharacterAt; /***/ public String badEntryDelimiter; @@ -100,6 +101,7 @@ public static JGitText get() { /***/ public String blobNotFound; /***/ public String blobNotFoundForPath; /***/ public String blockSizeNotPowerOf2; + /***/ public String bothRefTargetsMustNotBeNull; /***/ public String branchNameInvalid; /***/ public String buildingBitmaps; /***/ public String cachedPacksPreventsIndexCreation; @@ -493,6 +495,7 @@ public static JGitText get() { /***/ public String months; /***/ public String monthsAgo; /***/ public String multipleMergeBasesFor; + /***/ public String nameMustNotBeNullOrEmpty; /***/ public String need2Arguments; /***/ public String needPackOut; /***/ public String needsAtLeastOneEntry; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java index b328eb83e..ad2500059 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java @@ -47,6 +47,7 @@ import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; import java.io.IOException; import java.text.MessageFormat; @@ -142,6 +143,12 @@ public void execute(RevWalk walk, ProgressMonitor monitor, super.execute(walk, monitor, options); return; } + if (containsSymrefs(pending)) { + // packed-refs file cannot store symrefs + reject(pending.get(0), REJECTED_OTHER_REASON, + JGitText.get().atomicSymRefNotSupported, pending); + return; + } // Required implementation details copied from super.execute. if (!blockUntilTimestamps(MAX_WAIT)) { @@ -209,6 +216,15 @@ public void execute(RevWalk walk, ProgressMonitor monitor, writeReflog(pending); } + private static boolean containsSymrefs(List commands) { + for (ReceiveCommand cmd : commands) { + if (cmd.getOldSymref() != null || cmd.getNewSymref() != null) { + return true; + } + } + return false; + } + private boolean checkConflictingNames(List commands) throws IOException { Set takenNames = new HashSet<>(); @@ -510,7 +526,12 @@ private static void lockFailure(ReceiveCommand cmd, private static void reject(ReceiveCommand cmd, ReceiveCommand.Result result, List commands) { - cmd.setResult(result); + reject(cmd, result, null, commands); + } + + private static void reject(ReceiveCommand cmd, ReceiveCommand.Result result, + String why, List commands) { + cmd.setResult(result, why); for (ReceiveCommand c2 : commands) { if (c2.getResult() == ReceiveCommand.Result.OK) { // Undo OK status so ReceiveCommand#abort aborts it. Assumes this method diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java index dd08375f2..92cfe3d89 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java @@ -61,6 +61,7 @@ import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.SymbolicRef; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevWalk; @@ -153,14 +154,20 @@ public Command(@Nullable Ref oldRef, @Nullable Ref newRef) { */ public Command(RevWalk rw, ReceiveCommand cmd) throws MissingObjectException, IOException { - this.oldRef = toRef(rw, cmd.getOldId(), cmd.getRefName(), false); - this.newRef = toRef(rw, cmd.getNewId(), cmd.getRefName(), true); + this.oldRef = toRef(rw, cmd.getOldId(), cmd.getOldSymref(), + cmd.getRefName(), false); + this.newRef = toRef(rw, cmd.getNewId(), cmd.getNewSymref(), + cmd.getRefName(), true); this.cmd = cmd; } - static Ref toRef(RevWalk rw, ObjectId id, String name, - boolean mustExist) throws MissingObjectException, IOException { - if (ObjectId.zeroId().equals(id)) { + static Ref toRef(RevWalk rw, ObjectId id, @Nullable String target, + String name, boolean mustExist) + throws MissingObjectException, IOException { + if (target != null) { + return new SymbolicRef(name, + new ObjectIdRef.Unpeeled(NETWORK, target, id)); + } else if (ObjectId.zeroId().equals(id)) { return null; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceiveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceiveCommand.java index e9681b34c..374df6a67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceiveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceiveCommand.java @@ -52,6 +52,7 @@ import java.util.Collection; import java.util.List; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AnyObjectId; @@ -196,8 +197,8 @@ public static void abort(Iterable commands) { * * @param cmd * command. - * @return whether the command failed due to transaction aborted, as in {@link - * #abort(Iterable)}. + * @return whether the command failed due to transaction aborted, as in + * {@link #abort(Iterable)}. * @since 4.9 */ public static boolean isTransactionAborted(ReceiveCommand cmd) { @@ -205,14 +206,71 @@ public static boolean isTransactionAborted(ReceiveCommand cmd) { && cmd.getMessage().equals(JGitText.get().transactionAborted); } + /** + * Create a command to switch a reference from object to symbolic. + * + * @param oldId + * expected oldId. May be {@code zeroId} to create. + * @param newTarget + * new target; must begin with {@code "refs/"}. + * @param name + * name of the reference to make symbolic. + * @return command instance. + * @since 4.10 + */ + public static ReceiveCommand link(@NonNull ObjectId oldId, + @NonNull String newTarget, @NonNull String name) { + return new ReceiveCommand(oldId, newTarget, name); + } + + /** + * Create a command to switch a symbolic reference's target. + * + * @param oldTarget + * expected old target. May be null to create. + * @param newTarget + * new target; must begin with {@code "refs/"}. + * @param name + * name of the reference to make symbolic. + * @return command instance. + * @since 4.10 + */ + public static ReceiveCommand link(@Nullable String oldTarget, + @NonNull String newTarget, @NonNull String name) { + return new ReceiveCommand(oldTarget, newTarget, name); + } + + /** + * Create a command to switch a reference from symbolic to object. + * + * @param oldTarget + * expected old target. + * @param newId + * new object identifier. May be {@code zeroId()} to delete. + * @param name + * name of the reference to convert from symbolic. + * @return command instance. + * @since 4.10 + */ + public static ReceiveCommand unlink(@NonNull String oldTarget, + @NonNull ObjectId newId, @NonNull String name) { + return new ReceiveCommand(oldTarget, newId, name); + } + private final ObjectId oldId; + private final String oldSymref; + private final ObjectId newId; + private final String newSymref; + private final String name; private Type type; + private boolean typeIsCorrect; + private Ref ref; private Result status = Result.NOT_ATTEMPTED; @@ -227,8 +285,6 @@ public static boolean isTransactionAborted(ReceiveCommand cmd) { private Boolean forceRefLog; - private boolean typeIsCorrect; - /** * Create a new command for {@link BaseReceivePack}. * @@ -244,13 +300,21 @@ public static boolean isTransactionAborted(ReceiveCommand cmd) { public ReceiveCommand(final ObjectId oldId, final ObjectId newId, final String name) { if (oldId == null) { - throw new IllegalArgumentException(JGitText.get().oldIdMustNotBeNull); + throw new IllegalArgumentException( + JGitText.get().oldIdMustNotBeNull); } if (newId == null) { - throw new IllegalArgumentException(JGitText.get().newIdMustNotBeNull); + throw new IllegalArgumentException( + JGitText.get().newIdMustNotBeNull); + } + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + JGitText.get().nameMustNotBeNullOrEmpty); } this.oldId = oldId; + this.oldSymref = null; this.newId = newId; + this.newSymref = null; this.name = name; type = Type.UPDATE; @@ -275,19 +339,28 @@ public ReceiveCommand(final ObjectId oldId, final ObjectId newId, * name of the ref being affected. * @param type * type of the command. Must be {@link Type#CREATE} if {@code - * oldId} is zero, or {@link Type#DELETE} if {@code newId} is zero. + * oldId} is zero, or {@link Type#DELETE} if {@code newId} is + * zero. * @since 2.0 */ public ReceiveCommand(final ObjectId oldId, final ObjectId newId, final String name, final Type type) { if (oldId == null) { - throw new IllegalArgumentException(JGitText.get().oldIdMustNotBeNull); + throw new IllegalArgumentException( + JGitText.get().oldIdMustNotBeNull); } if (newId == null) { - throw new IllegalArgumentException(JGitText.get().newIdMustNotBeNull); + throw new IllegalArgumentException( + JGitText.get().newIdMustNotBeNull); + } + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + JGitText.get().nameMustNotBeNullOrEmpty); } this.oldId = oldId; + this.oldSymref = null; this.newId = newId; + this.newSymref = null; this.name = name; switch (type) { case CREATE: @@ -311,21 +384,144 @@ public ReceiveCommand(final ObjectId oldId, final ObjectId newId, } break; default: - throw new IllegalStateException(JGitText.get().enumValueNotSupported0); + throw new IllegalStateException( + JGitText.get().enumValueNotSupported0); } this.type = type; } + /** + * Create a command to switch a reference from object to symbolic. + * + * @param oldId + * the old object id; must not be null. Use + * {@link ObjectId#zeroId()} to indicate a ref creation. + * @param newSymref + * new target, must begin with {@code "refs/"}. Use {@code null} + * to indicate a ref deletion. + * @param name + * name of the reference to make symbolic. + * @since 4.10 + */ + private ReceiveCommand(ObjectId oldId, String newSymref, String name) { + if (oldId == null) { + throw new IllegalArgumentException( + JGitText.get().oldIdMustNotBeNull); + } + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + JGitText.get().nameMustNotBeNullOrEmpty); + } + this.oldId = oldId; + this.oldSymref = null; + this.newId = ObjectId.zeroId(); + this.newSymref = newSymref; + this.name = name; + if (AnyObjectId.equals(ObjectId.zeroId(), oldId)) { + type = Type.CREATE; + } else if (newSymref != null) { + type = Type.UPDATE; + } else { + type = Type.DELETE; + } + typeIsCorrect = true; + } + + /** + * Create a command to switch a reference from symbolic to object. + * + * @param oldSymref + * expected old target. Use {@code null} to indicate a ref + * creation. + * @param newId + * the new object id; must not be null. Use + * {@link ObjectId#zeroId()} to indicate a ref deletion. + * @param name + * name of the reference to convert from symbolic. + * @since 4.10 + */ + private ReceiveCommand(String oldSymref, ObjectId newId, String name) { + if (newId == null) { + throw new IllegalArgumentException( + JGitText.get().newIdMustNotBeNull); + } + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + JGitText.get().nameMustNotBeNullOrEmpty); + } + this.oldId = ObjectId.zeroId(); + this.oldSymref = oldSymref; + this.newId = newId; + this.newSymref = null; + this.name = name; + if (oldSymref == null) { + type = Type.CREATE; + } else if (!AnyObjectId.equals(ObjectId.zeroId(), newId)) { + type = Type.UPDATE; + } else { + type = Type.DELETE; + } + typeIsCorrect = true; + } + + /** + * Create a command to switch a symbolic reference's target. + * + * @param oldTarget + * expected old target. Use {@code null} to indicate a ref + * creation. + * @param newTarget + * new target. Use {@code null} to indicate a ref deletion. + * @param name + * name of the reference to make symbolic. + * @since 4.10 + */ + private ReceiveCommand(@Nullable String oldTarget, String newTarget, String name) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + JGitText.get().nameMustNotBeNullOrEmpty); + } + this.oldId = ObjectId.zeroId(); + this.oldSymref = oldTarget; + this.newId = ObjectId.zeroId(); + this.newSymref = newTarget; + this.name = name; + if (oldTarget == null) { + if (newTarget == null) { + throw new IllegalArgumentException( + JGitText.get().bothRefTargetsMustNotBeNull); + } + type = Type.CREATE; + } else if (newTarget != null) { + type = Type.UPDATE; + } else { + type = Type.DELETE; + } + typeIsCorrect = true; + } + /** @return the old value the client thinks the ref has. */ public ObjectId getOldId() { return oldId; } + /** @return expected old target for a symbolic reference. */ + @Nullable + public String getOldSymref() { + return oldSymref; + } + /** @return the requested new value for this ref. */ public ObjectId getNewId() { return newId; } + /** @return requested new target for a symbolic reference. */ + @Nullable + public String getNewSymref() { + return newSymref; + } + /** @return the name of the ref being updated. */ public String getRefName() { return name; @@ -452,8 +648,8 @@ public boolean isRefLogIncludingResult() { /** * Check whether the reflog should be written regardless of repo defaults. * - * @return whether force writing is enabled; null if {@code - * #setForceRefLog(boolean)} was never called. + * @return whether force writing is enabled; {@code null} if + * {@code #setForceRefLog(boolean)} was never called. * @since 4.9 */ @Nullable @@ -525,7 +721,18 @@ public void updateType(RevWalk walk) throws IOException { */ public void execute(final BaseReceivePack rp) { try { - final RefUpdate ru = rp.getRepository().updateRef(getRefName()); + String expTarget = getOldSymref(); + boolean detach = getNewSymref() != null + || (type == Type.DELETE && expTarget != null); + RefUpdate ru = rp.getRepository().updateRef(getRefName(), detach); + if (expTarget != null) { + if (!ru.getRef().isSymbolic() || !ru.getRef().getTarget() + .getName().equals(expTarget)) { + setResult(Result.LOCK_FAILURE); + return; + } + } + ru.setRefLogIdent(rp.getRefLogIdent()); ru.setRefLogMessage(refLogMessage, refLogIncludeResult); switch (getType()) { @@ -546,9 +753,13 @@ public void execute(final BaseReceivePack rp) { case UPDATE_NONFASTFORWARD: ru.setForceUpdate(rp.isAllowNonFastForwards()); ru.setExpectedOldObjectId(getOldId()); - ru.setNewObjectId(getNewId()); ru.setRefLogMessage("push", true); //$NON-NLS-1$ - setResult(ru.update(rp.getRevWalk())); + if (getNewSymref() != null) { + setResult(ru.link(getNewSymref())); + } else { + ru.setNewObjectId(getNewId()); + setResult(ru.update(rp.getRevWalk())); + } break; } } catch (IOException err) { From 1d31257a5d9d599d7a4e6b0333baa15c94cf1b0c Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Sat, 12 Aug 2017 12:19:30 -0700 Subject: [PATCH 2/2] dfs: reftable backed DfsRefDatabase DfsReftableDatabase is a new alternative for DfsRefDatabase that handles more operations for the implementor by delegating through reftables. All reftable files are stored in sibling DfsObjDatabase using PackExt.REFTABLE and PackSource.INSERT. Its assumed the DfsObjDatabase periodically runs compactions and GCs using DfsPackCompactor and DfsGarbageCollector. Those passes are essential to collapsing the stack of reftables. Change-Id: Ia03196ff6fd9ae2d0623c3747cfa84357c6d0c79 Signed-off-by: Minh Thai Signed-off-by: Terry Parker --- .../internal/storage/dfs/DfsObjDatabase.java | 26 + .../internal/storage/dfs/DfsRefDatabase.java | 4 +- .../storage/dfs/DfsReftableDatabase.java | 361 ++++++++++++++ .../storage/dfs/ReftableBatchRefUpdate.java | 460 ++++++++++++++++++ .../internal/storage/dfs/ReftableStack.java | 19 +- 5 files changed, 864 insertions(+), 6 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 943982201..6e9d7e07e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -52,6 +52,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -457,6 +458,31 @@ void addPack(DfsPackFile newPack) throws IOException { } while (!packList.compareAndSet(o, n)); } + void addReftable(DfsPackDescription add, Set remove) + throws IOException { + PackList o, n; + do { + o = packList.get(); + if (o == NO_PACKS) { + o = scanPacks(o); + for (DfsReftable t : o.reftables) { + if (t.getPackDescription().equals(add)) { + return; + } + } + } + + List tables = new ArrayList<>(1 + o.reftables.length); + for (DfsReftable t : o.reftables) { + if (!remove.contains(t.getPackDescription())) { + tables.add(t); + } + } + tables.add(new DfsReftable(add)); + n = new PackListImpl(o.packs, tables.toArray(new DfsReftable[0])); + } while (!packList.compareAndSet(o, n)); + } + PackList scanPacks(final PackList original) throws IOException { PackList o, n; synchronized (packList) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefDatabase.java index b41c18b6c..d11286ac0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefDatabase.java @@ -180,7 +180,7 @@ public Ref peel(Ref ref) throws IOException { return recreate(ref, newLeaf); } - private Ref doPeel(final Ref leaf) throws MissingObjectException, + Ref doPeel(Ref leaf) throws MissingObjectException, IOException { try (RevWalk rw = new RevWalk(repository)) { RevObject obj = rw.parseAny(leaf.getObjectId()); @@ -199,7 +199,7 @@ private Ref doPeel(final Ref leaf) throws MissingObjectException, } } - private static Ref recreate(Ref old, Ref leaf) { + static Ref recreate(Ref old, Ref leaf) { if (old.isSymbolic()) { Ref dst = recreate(old.getTarget(), leaf); return new SymbolicRef(old.getName(), dst); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java new file mode 100644 index 000000000..09fb2d688 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -0,0 +1,361 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.reftable.MergedReftable; +import org.eclipse.jgit.internal.storage.reftable.RefCursor; +import org.eclipse.jgit.internal.storage.reftable.Reftable; +import org.eclipse.jgit.lib.BatchRefUpdate; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.RefList; +import org.eclipse.jgit.util.RefMap; + +/** + * A {@link DfsRefDatabase} that uses reftable for storage. + *

+ * A {@code DfsRefDatabase} instance is thread-safe. + *

+ * Implementors may wish to use {@link DfsPackDescription#getMaxUpdateIndex()} + * as the primary key identifier for a {@link PackExt#REFTABLE} only pack + * description, ensuring that when there are competing transactions one wins, + * and one will fail. + */ +public class DfsReftableDatabase extends DfsRefDatabase { + private final ReentrantLock lock = new ReentrantLock(true); + + private DfsReader ctx; + + private ReftableStack tableStack; + + private MergedReftable mergedTables; + + /** + * Initialize the reference database for a repository. + * + * @param repo + * the repository this database instance manages references for. + */ + protected DfsReftableDatabase(DfsRepository repo) { + super(repo); + } + + @Override + public boolean performsAtomicTransactions() { + return true; + } + + @Override + public BatchRefUpdate newBatchUpdate() { + DfsObjDatabase odb = getRepository().getObjectDatabase(); + return new ReftableBatchRefUpdate(this, odb); + } + + /** @return the lock protecting this instance's state. */ + protected ReentrantLock getLock() { + return lock; + } + + /** + * @return {@code true} if commit of a new small reftable should try to + * replace a prior small reftable by performing a compaction, + * instead of extending the stack depth. + */ + protected boolean compactDuringCommit() { + return true; + } + + /** + * Obtain a handle to the merged reader. + * + * @return (possibly cached) handle to the merged reader. + * @throws IOException + * if tables cannot be opened. + */ + protected Reftable reader() throws IOException { + lock.lock(); + try { + if (mergedTables == null) { + mergedTables = new MergedReftable(stack().readers()); + } + return mergedTables; + } finally { + lock.unlock(); + } + } + + /** + * Obtain a handle to the stack of reftables. + * + * @return (possibly cached) handle to the stack. + * @throws IOException + * if tables cannot be opened. + */ + protected ReftableStack stack() throws IOException { + lock.lock(); + try { + if (tableStack == null) { + DfsObjDatabase odb = getRepository().getObjectDatabase(); + if (ctx == null) { + ctx = odb.newReader(); + } + tableStack = ReftableStack.open(ctx, + Arrays.asList(odb.getReftables())); + } + return tableStack; + } finally { + lock.unlock(); + } + } + + @Override + public boolean isNameConflicting(String refName) throws IOException { + lock.lock(); + try { + Reftable table = reader(); + + // Cannot be nested within an existing reference. + int lastSlash = refName.lastIndexOf('/'); + while (0 < lastSlash) { + if (table.hasRef(refName.substring(0, lastSlash))) { + return true; + } + lastSlash = refName.lastIndexOf('/', lastSlash - 1); + } + + // Cannot be the container of an existing reference. + return table.hasRef(refName + '/'); + } finally { + lock.unlock(); + } + } + + @Override + public Ref exactRef(String name) throws IOException { + lock.lock(); + try { + Reftable table = reader(); + Ref ref = table.exactRef(name); + if (ref != null && ref.isSymbolic()) { + return table.resolve(ref); + } + return ref; + } finally { + lock.unlock(); + } + } + + @Override + public Ref getRef(String needle) throws IOException { + for (String prefix : SEARCH_PATH) { + Ref ref = exactRef(prefix + needle); + if (ref != null) { + return ref; + } + } + return null; + } + + @Override + public Map getRefs(String prefix) throws IOException { + RefList.Builder all = new RefList.Builder<>(); + lock.lock(); + try { + Reftable table = reader(); + try (RefCursor rc = ALL.equals(prefix) ? table.allRefs() + : table.seekRef(prefix)) { + while (rc.next()) { + Ref ref = table.resolve(rc.getRef()); + if (ref != null) { + all.add(ref); + } + } + } + } finally { + lock.unlock(); + } + + RefList none = RefList.emptyList(); + return new RefMap(prefix, all.toRefList(), none, none); + } + + @Override + public Ref peel(Ref ref) throws IOException { + Ref oldLeaf = ref.getLeaf(); + if (oldLeaf.isPeeled() || oldLeaf.getObjectId() == null) { + return ref; + } + return recreate(ref, doPeel(oldLeaf)); + } + + @Override + boolean exists() throws IOException { + DfsObjDatabase odb = getRepository().getObjectDatabase(); + return odb.getReftables().length > 0; + } + + @Override + void clearCache() { + lock.lock(); + try { + if (tableStack != null) { + tableStack.close(); + tableStack = null; + } + if (ctx != null) { + ctx.close(); + ctx = null; + } + mergedTables = null; + } finally { + lock.unlock(); + } + } + + @Override + protected boolean compareAndPut(Ref oldRef, @Nullable Ref newRef) + throws IOException { + ReceiveCommand cmd = toCommand(oldRef, newRef); + try (RevWalk rw = new RevWalk(getRepository())) { + newBatchUpdate().setAllowNonFastForwards(true).addCommand(cmd) + .execute(rw, NullProgressMonitor.INSTANCE); + } + switch (cmd.getResult()) { + case OK: + return true; + case REJECTED_OTHER_REASON: + throw new IOException(cmd.getMessage()); + case LOCK_FAILURE: + default: + return false; + } + } + + private static ReceiveCommand toCommand(Ref oldRef, Ref newRef) { + ObjectId oldId = toId(oldRef); + ObjectId newId = toId(newRef); + String name = toName(oldRef, newRef); + + if (oldRef != null && oldRef.isSymbolic()) { + if (newRef != null) { + if (newRef.isSymbolic()) { + return ReceiveCommand.link(oldRef.getTarget().getName(), + newRef.getTarget().getName(), name); + } else { + return ReceiveCommand.unlink(oldRef.getTarget().getName(), + newId, name); + } + } else { + return ReceiveCommand.unlink(oldRef.getTarget().getName(), + ObjectId.zeroId(), name); + } + } + + if (newRef != null && newRef.isSymbolic()) { + if (oldRef != null) { + if (oldRef.isSymbolic()) { + return ReceiveCommand.link(oldRef.getTarget().getName(), + newRef.getTarget().getName(), name); + } else { + return ReceiveCommand.link(oldId, + newRef.getTarget().getName(), name); + } + } else { + return ReceiveCommand.link(ObjectId.zeroId(), + newRef.getTarget().getName(), name); + } + } + + return new ReceiveCommand(oldId, newId, name); + } + + private static ObjectId toId(Ref ref) { + if (ref != null) { + ObjectId id = ref.getObjectId(); + if (id != null) { + return id; + } + } + return ObjectId.zeroId(); + } + + private static String toName(Ref oldRef, Ref newRef) { + return oldRef != null ? oldRef.getName() : newRef.getName(); + } + + @Override + protected boolean compareAndRemove(Ref oldRef) throws IOException { + return compareAndPut(oldRef, null); + } + + @Override + protected RefCache scanAllRefs() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + void stored(Ref ref) { + // Unnecessary; ReftableBatchRefUpdate calls clearCache(). + } + + @Override + void removed(String refName) { + // Unnecessary; ReftableBatchRefUpdate calls clearCache(). + } + + @Override + protected void cachePeeledState(Ref oldLeaf, Ref newLeaf) { + // Do not cache peeled state in reftable. + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java new file mode 100644 index 000000000..c2a4603bf --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; +import static org.eclipse.jgit.lib.Ref.Storage.NEW; +import static org.eclipse.jgit.lib.Ref.Storage.PACKED; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_MISSING_OBJECT; +import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; +import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantLock; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; +import org.eclipse.jgit.internal.storage.io.BlockSource; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.reftable.RefCursor; +import org.eclipse.jgit.internal.storage.reftable.Reftable; +import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; +import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; +import org.eclipse.jgit.internal.storage.reftable.ReftableReader; +import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.BatchRefUpdate; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.ReflogEntry; +import org.eclipse.jgit.lib.SymbolicRef; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.ReceiveCommand; + +/** {@link BatchRefUpdate} for {@link DfsReftableDatabase}. */ +public class ReftableBatchRefUpdate extends BatchRefUpdate { + private static final int AVG_BYTES = 36; + + private final DfsReftableDatabase refdb; + + private final DfsObjDatabase odb; + + private final ReentrantLock lock; + + private final ReftableConfig reftableConfig; + + /** + * Initialize batch update. + * + * @param refdb + * database the update will modify. + * @param odb + * object database to store the reftable. + */ + protected ReftableBatchRefUpdate(DfsReftableDatabase refdb, + DfsObjDatabase odb) { + super(refdb); + this.refdb = refdb; + this.odb = odb; + lock = refdb.getLock(); + reftableConfig = new ReftableConfig(refdb.getRepository().getConfig()); + } + + @Override + public void execute(RevWalk rw, ProgressMonitor pm, List options) { + List pending = getPending(); + if (pending.isEmpty()) { + return; + } + if (options != null) { + setPushOptions(options); + } + try { + if (!checkObjectExistence(rw, pending)) { + return; + } + if (!checkNonFastForwards(rw, pending)) { + return; + } + + lock.lock(); + try { + Reftable table = refdb.reader(); + if (!checkExpected(table, pending)) { + return; + } + if (!checkConflicting(pending)) { + return; + } + if (!blockUntilTimestamps(MAX_WAIT)) { + return; + } + applyUpdates(rw, pending); + for (ReceiveCommand cmd : pending) { + cmd.setResult(OK); + } + } finally { + lock.unlock(); + } + } catch (IOException e) { + pending.get(0).setResult(LOCK_FAILURE, "io error"); //$NON-NLS-1$ + ReceiveCommand.abort(pending); + } + } + + private List getPending() { + return ReceiveCommand.filter(getCommands(), NOT_ATTEMPTED); + } + + private boolean checkObjectExistence(RevWalk rw, + List pending) throws IOException { + for (ReceiveCommand cmd : pending) { + try { + if (!cmd.getNewId().equals(ObjectId.zeroId())) { + rw.parseAny(cmd.getNewId()); + } + } catch (MissingObjectException e) { + // ReceiveCommand#setResult(Result) converts REJECTED to + // REJECTED_NONFASTFORWARD, even though that result is also + // used for a missing object. Eagerly handle this case so we + // can set the right result. + cmd.setResult(REJECTED_MISSING_OBJECT); + ReceiveCommand.abort(pending); + return false; + } + } + return true; + } + + private boolean checkNonFastForwards(RevWalk rw, + List pending) throws IOException { + if (isAllowNonFastForwards()) { + return true; + } + for (ReceiveCommand cmd : pending) { + cmd.updateType(rw); + if (cmd.getType() == UPDATE_NONFASTFORWARD) { + cmd.setResult(REJECTED_NONFASTFORWARD); + ReceiveCommand.abort(pending); + return false; + } + } + return true; + } + + private boolean checkConflicting(List pending) + throws IOException { + Set names = new HashSet<>(); + for (ReceiveCommand cmd : pending) { + names.add(cmd.getRefName()); + } + + boolean ok = true; + for (ReceiveCommand cmd : pending) { + String name = cmd.getRefName(); + if (refdb.isNameConflicting(name)) { + cmd.setResult(LOCK_FAILURE); + ok = false; + } else { + int s = name.lastIndexOf('/'); + while (0 < s) { + if (names.contains(name.substring(0, s))) { + cmd.setResult(LOCK_FAILURE); + ok = false; + break; + } + s = name.lastIndexOf('/', s - 1); + } + } + } + if (!ok && isAtomic()) { + ReceiveCommand.abort(pending); + return false; + } + return ok; + } + + private boolean checkExpected(Reftable table, List pending) + throws IOException { + for (ReceiveCommand cmd : pending) { + Ref ref; + try (RefCursor rc = table.seekRef(cmd.getRefName())) { + ref = rc.next() ? rc.getRef() : null; + } + if (!matchOld(cmd, ref)) { + cmd.setResult(LOCK_FAILURE); + if (isAtomic()) { + ReceiveCommand.abort(pending); + return false; + } + } + } + return true; + } + + private static boolean matchOld(ReceiveCommand cmd, @Nullable Ref ref) { + if (ref == null) { + return AnyObjectId.equals(ObjectId.zeroId(), cmd.getOldId()) + && cmd.getOldSymref() == null; + } else if (ref.isSymbolic()) { + return ref.getTarget().getName().equals(cmd.getOldSymref()); + } + ObjectId id = ref.getObjectId(); + if (id == null) { + id = ObjectId.zeroId(); + } + return cmd.getOldId().equals(id); + } + + private void applyUpdates(RevWalk rw, List pending) + throws IOException { + List newRefs = toNewRefs(rw, pending); + long updateIndex = nextUpdateIndex(); + Set prune = Collections.emptySet(); + DfsPackDescription pack = odb.newPack(PackSource.INSERT); + try (DfsOutputStream out = odb.writeFile(pack, REFTABLE)) { + ReftableConfig cfg = DfsPackCompactor + .configureReftable(reftableConfig, out); + + ReftableWriter.Stats stats; + if (refdb.compactDuringCommit() + && newRefs.size() * AVG_BYTES <= cfg.getRefBlockSize() + && canCompactTopOfStack(cfg)) { + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + write(tmp, cfg, updateIndex, newRefs, pending); + stats = compactTopOfStack(out, cfg, tmp.toByteArray()); + prune = toPruneTopOfStack(); + } else { + stats = write(out, cfg, updateIndex, newRefs, pending); + } + pack.addFileExt(REFTABLE); + pack.setReftableStats(stats); + } + + odb.commitPack(Collections.singleton(pack), prune); + odb.addReftable(pack, prune); + refdb.clearCache(); + } + + private ReftableWriter.Stats write(OutputStream os, ReftableConfig cfg, + long updateIndex, List newRefs, List pending) + throws IOException { + ReftableWriter writer = new ReftableWriter(cfg) + .setMinUpdateIndex(updateIndex).setMaxUpdateIndex(updateIndex) + .begin(os).sortAndWriteRefs(newRefs); + if (!isRefLogDisabled()) { + writeLog(writer, updateIndex, pending); + } + writer.finish(); + return writer.getStats(); + } + + private void writeLog(ReftableWriter writer, long updateIndex, + List pending) throws IOException { + Map cmds = new HashMap<>(); + List byName = new ArrayList<>(pending.size()); + for (ReceiveCommand cmd : pending) { + cmds.put(cmd.getRefName(), cmd); + byName.add(cmd.getRefName()); + } + Collections.sort(byName); + + PersonIdent ident = getRefLogIdent(); + if (ident == null) { + ident = new PersonIdent(refdb.getRepository()); + } + for (String name : byName) { + ReceiveCommand cmd = cmds.get(name); + if (isRefLogDisabled(cmd)) { + continue; + } + String msg = getRefLogMessage(cmd); + if (isRefLogIncludingResult(cmd)) { + String strResult = toResultString(cmd); + if (strResult != null) { + msg = msg.isEmpty() ? strResult : msg + ": " + strResult; //$NON-NLS-1$ + } + } + writer.writeLog(name, updateIndex, ident, cmd.getOldId(), + cmd.getNewId(), msg); + } + } + + private String toResultString(ReceiveCommand cmd) { + switch (cmd.getType()) { + case CREATE: + return ReflogEntry.PREFIX_CREATED; + case UPDATE: + // Match the behavior of a single RefUpdate. In that case, setting + // the force bit completely bypasses the potentially expensive + // isMergedInto check, by design, so the reflog message may be + // inaccurate. + // + // Similarly, this class bypasses the isMergedInto checks when the + // force bit is set, meaning we can't actually distinguish between + // UPDATE and UPDATE_NONFASTFORWARD when isAllowNonFastForwards() + // returns true. + return isAllowNonFastForwards() ? ReflogEntry.PREFIX_FORCED_UPDATE + : ReflogEntry.PREFIX_FAST_FORWARD; + case UPDATE_NONFASTFORWARD: + return ReflogEntry.PREFIX_FORCED_UPDATE; + default: + return null; + } + } + + private static List toNewRefs(RevWalk rw, List pending) + throws IOException { + List refs = new ArrayList<>(pending.size()); + for (ReceiveCommand cmd : pending) { + String name = cmd.getRefName(); + ObjectId newId = cmd.getNewId(); + String newSymref = cmd.getNewSymref(); + if (AnyObjectId.equals(ObjectId.zeroId(), newId) + && newSymref == null) { + refs.add(new ObjectIdRef.Unpeeled(NEW, name, null)); + continue; + } else if (newSymref != null) { + refs.add(new SymbolicRef(name, + new ObjectIdRef.Unpeeled(NEW, newSymref, null))); + continue; + } + + RevObject obj = rw.parseAny(newId); + RevObject peel = null; + if (obj instanceof RevTag) { + peel = rw.peel(obj); + } + if (peel != null) { + refs.add(new ObjectIdRef.PeeledTag(PACKED, name, newId, + peel.copy())); + } else { + refs.add(new ObjectIdRef.PeeledNonTag(PACKED, name, newId)); + } + } + return refs; + } + + private long nextUpdateIndex() throws IOException { + long updateIndex = 0; + for (Reftable r : refdb.stack().readers()) { + if (r instanceof ReftableReader) { + updateIndex = Math.max(updateIndex, + ((ReftableReader) r).maxUpdateIndex()); + } + } + return updateIndex + 1; + } + + private boolean canCompactTopOfStack(ReftableConfig cfg) + throws IOException { + ReftableStack stack = refdb.stack(); + List readers = stack.readers(); + if (readers.isEmpty()) { + return false; + } + + int lastIdx = readers.size() - 1; + DfsReftable last = stack.files().get(lastIdx); + DfsPackDescription desc = last.getPackDescription(); + if (desc.getPackSource() != PackSource.INSERT + || !packOnlyContainsReftable(desc)) { + return false; + } + + Reftable table = readers.get(lastIdx); + int bs = cfg.getRefBlockSize(); + return table instanceof ReftableReader + && ((ReftableReader) table).size() <= 3 * bs; + } + + private ReftableWriter.Stats compactTopOfStack(OutputStream out, + ReftableConfig cfg, byte[] newTable) throws IOException { + List stack = refdb.stack().readers(); + Reftable last = stack.get(stack.size() - 1); + + List tables = new ArrayList<>(2); + tables.add(last); + tables.add(new ReftableReader(BlockSource.from(newTable))); + + ReftableCompactor compactor = new ReftableCompactor(); + compactor.setConfig(cfg); + compactor.addAll(tables); + compactor.compact(out); + return compactor.getStats(); + } + + private Set toPruneTopOfStack() throws IOException { + List stack = refdb.stack().files(); + DfsReftable last = stack.get(stack.size() - 1); + return Collections.singleton(last.getPackDescription()); + } + + private boolean packOnlyContainsReftable(DfsPackDescription desc) { + for (PackExt ext : PackExt.values()) { + if (ext != REFTABLE && desc.hasFileExt(ext)) { + return false; + } + } + return true; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java index 8d1cc989d..365688457 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java @@ -58,18 +58,19 @@ public class ReftableStack implements AutoCloseable { * @param ctx * context to read the tables with. This {@code ctx} will be * retained by the stack and each of the table readers. - * @param tables + * @param files * the tables to open. * @return stack reference to close the tables. * @throws IOException * a table could not be opened */ - public static ReftableStack open(DfsReader ctx, List tables) + public static ReftableStack open(DfsReader ctx, List files) throws IOException { - ReftableStack stack = new ReftableStack(tables.size()); + ReftableStack stack = new ReftableStack(files.size()); boolean close = true; try { - for (DfsReftable t : tables) { + for (DfsReftable t : files) { + stack.files.add(t); stack.tables.add(t.open(ctx)); } close = false; @@ -81,12 +82,22 @@ public static ReftableStack open(DfsReader ctx, List tables) } } + private final List files; private final List tables; private ReftableStack(int tableCnt) { + this.files = new ArrayList<>(tableCnt); this.tables = new ArrayList<>(tableCnt); } + /** + * @return unmodifiable list of DfsRefatble files, in the same order the + * files were passed to {@link #open(DfsReader, List)}. + */ + public List files() { + return Collections.unmodifiableList(files); + } + /** * @return unmodifiable list of tables, in the same order the files were * passed to {@link #open(DfsReader, List)}.