Merge branch 'stable-4.8'

* stable-4.8:
  SubmoduleUpdateCommand#setCallback should return 'this'
  CloneCommand#setCallback should return 'this'
  Prepare 4.7.2-SNAPSHOT builds
  JGit v4.7.1.201706071930-r
  ArchiveCommand: Create prefix entry with commit time
  Run auto GC in the background
  Update Orbit to the Oxygen version R20170516192513

Change-Id: Ibf90b4899d097474e7836e6baab8829e66fca524
Signed-off-by: David Pursehouse <david.pursehouse@gmail.com>
This commit is contained in:
David Pursehouse 2017-06-10 14:14:12 +09:00
commit a7949c1e35
20 changed files with 355 additions and 19 deletions

View File

@ -62,6 +62,7 @@
import org.eclipse.jgit.dircache.DirCache;
import org.eclipse.jgit.dircache.DirCacheEntry;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
@ -121,6 +122,12 @@ public void setUp() throws Exception {
mockSystemReader = new MockSystemReader();
mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp,
"usergitconfig"), FS.DETECTED);
// We have to set autoDetach to false for tests, because tests expect to be able
// to clean up by recursively removing the repository, and background GC might be
// in the middle of writing or deleting files, which would disrupt this.
mockSystemReader.userGitConfig.setBoolean(ConfigConstants.CONFIG_GC_SECTION,
null, ConfigConstants.CONFIG_KEY_AUTODETACH, false);
mockSystemReader.userGitConfig.save();
ceilTestDirectories(getCeilings());
SystemReader.setInstance(mockSystemReader);

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?pde?>
<!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
<target name="jgit-4.5" sequenceNumber="1494529263">
<target name="jgit-4.5" sequenceNumber="1496008880">
<locations>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.jetty.client" version="9.4.5.v20170502"/>
@ -98,7 +98,7 @@
<unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
<unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
<unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/R20170516192513/repository"/>
</location>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.osgi" version="0.0.0"/>

View File

@ -2,7 +2,7 @@ target "jgit-4.5" with source configurePhase
include "projects/jetty-9.4.5.tpd"
include "orbit/R20160221192158-Mars.tpd"
include "orbit/S20170306214312-Oxygen.tpd"
include "orbit/R20170516192513-Oxygen.tpd"
location "http://download.eclipse.org/releases/mars/" {
org.eclipse.osgi lazy

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?pde?>
<!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
<target name="jgit-4.6" sequenceNumber="1494529259">
<target name="jgit-4.6" sequenceNumber="1496008884">
<locations>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.jetty.client" version="9.4.5.v20170502"/>
@ -60,7 +60,7 @@
<unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
<unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
<unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/R20170516192513/repository"/>
</location>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.osgi" version="0.0.0"/>

View File

@ -1,7 +1,7 @@
target "jgit-4.6" with source configurePhase
include "projects/jetty-9.4.5.tpd"
include "orbit/S20170306214312-Oxygen.tpd"
include "orbit/R20170516192513-Oxygen.tpd"
location "http://download.eclipse.org/releases/neon/" {
org.eclipse.osgi lazy

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?pde?>
<!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
<target name="jgit-4.7" sequenceNumber="1491140544">
<target name="jgit-4.7" sequenceNumber="1496008862">
<locations>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.jetty.client" version="9.4.5.v20170502"/>
@ -60,7 +60,7 @@
<unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
<unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
<unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
<repository location="http://download.eclipse.org/tools/orbit/downloads/drops/R20170516192513/repository"/>
</location>
<location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
<unit id="org.eclipse.osgi" version="0.0.0"/>

View File

@ -1,7 +1,7 @@
target "jgit-4.7" with source configurePhase
include "projects/jetty-9.4.5.tpd"
include "orbit/S20170306214312-Oxygen.tpd"
include "orbit/R20170516192513-Oxygen.tpd"
location "http://download.eclipse.org/releases/oxygen/" {
org.eclipse.osgi lazy

View File

@ -1,7 +1,7 @@
target "S20170306214312-Oxygen" with source configurePhase
target "R20170516192513-Oxygen" with source configurePhase
// see http://download.eclipse.org/tools/orbit/downloads/
location "http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository" {
location "http://download.eclipse.org/tools/orbit/downloads/drops/R20170516192513/repository" {
org.apache.ant [1.9.6.v201510161327,1.9.6.v201510161327]
org.apache.ant.source [1.9.6.v201510161327,1.9.6.v201510161327]
org.apache.commons.compress [1.6.0.v201310281400,1.6.0.v201310281400]

View File

@ -348,7 +348,7 @@ public void testArchivePrefixOption() throws Exception {
commitBazAndFooSlashBar();
byte[] result = CLIGitCommand.executeRaw(
"git archive --prefix=x/ --format=zip master", db).outBytes();
String[] expect = { "x/baz", "x/foo/", "x/foo/bar" };
String[] expect = { "x/", "x/baz", "x/foo/", "x/foo/bar" };
String[] actual = listZipEntries(result);
Arrays.sort(expect);
@ -361,7 +361,7 @@ public void testTarPrefixOption() throws Exception {
commitBazAndFooSlashBar();
byte[] result = CLIGitCommand.executeRaw(
"git archive --prefix=x/ --format=tar master", db).outBytes();
String[] expect = { "x/baz", "x/foo/", "x/foo/bar" };
String[] expect = { "x/", "x/baz", "x/foo/", "x/foo/bar" };
String[] actual = listTarEntries(result);
Arrays.sort(expect);
@ -380,7 +380,7 @@ public void testPrefixDoesNotNormalizeDoubleSlash() throws Exception {
commitFoo();
byte[] result = CLIGitCommand.executeRaw(
"git archive --prefix=x// --format=zip master", db).outBytes();
String[] expect = { "x//foo" };
String[] expect = { "x/", "x//foo" };
assertArrayEquals(expect, listZipEntries(result));
}
@ -389,7 +389,7 @@ public void testPrefixDoesNotNormalizeDoubleSlashInTar() throws Exception {
commitFoo();
byte[] result = CLIGitCommand.executeRaw(
"git archive --prefix=x// --format=tar master", db).outBytes();
String[] expect = { "x//foo" };
String[] expect = { "x/", "x//foo" };
assertArrayEquals(expect, listTarEntries(result));
}

View File

@ -74,6 +74,7 @@ public void testListConfig() throws Exception {
String[] output = execute("git config --list");
List<String> expect = new ArrayList<>();
expect.add("gc.autoDetach=false");
expect.add("core.filemode=" + !isWindows);
expect.add("core.logallrefupdates=true");
if (isMac)

View File

@ -1,5 +1,21 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<component id="org.eclipse.jgit" version="2">
<resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants">
<filter id="1141899266">
<message_arguments>
<message_argument value="4.7"/>
<message_argument value="4.8"/>
<message_argument value="CONFIG_KEY_AUTODETACH"/>
</message_arguments>
</filter>
<filter id="1141899266">
<message_arguments>
<message_argument value="4.7"/>
<message_argument value="4.8"/>
<message_argument value="CONFIG_KEY_LOGEXPIRY"/>
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/merge/MergeStrategy.java" type="org.eclipse.jgit.merge.MergeStrategy">
<filter comment="OSGi semantic versioning allows breaking implementors of an API in a minor version" id="336695337">
<message_arguments>

View File

@ -299,6 +299,8 @@ flagNotFromThis={0} not from this.
flagsAlreadyCreated={0} flags already created.
funnyRefname=funny refname
gcFailed=Garbage collection failed.
gcLogExists=A previous GC run reported an error: ''{0}''. Automatic gc will fail until ''{1}'' is removed.
gcTooManyUnpruned=Too many loose, unpruneable objects after garbage collection. Consider adjusting gc.auto or gc.pruneExpire.
gitmodulesNotFound=.gitmodules not found in tree.
headRequiredToStash=HEAD required to stash local changes
hoursAgo={0} hours ago

View File

@ -403,6 +403,12 @@ private <T extends Closeable> OutputStream writeArchive(Format<T> fmt) {
if (!paths.isEmpty())
walk.setFilter(PathFilterGroup.createFromStrings(paths));
// Put base directory into archive
if (pfx.endsWith("/")) { //$NON-NLS-1$
fmt.putEntry(outa, tree, pfx.replaceAll("[/]+$", "/"), //$NON-NLS-1$ //$NON-NLS-2$
FileMode.TREE, null);
}
while (walk.next()) {
String name = pfx + walk.getPathString();
FileMode mode = walk.getFileMode(0);

View File

@ -611,10 +611,12 @@ public CloneCommand setNoCheckout(boolean noCheckout) {
*
* @param callback
* the callback
* @return {@code this}
* @since 4.8
*/
public void setCallback(Callback callback) {
public CloneCommand setCallback(Callback callback) {
this.callback = callback;
return this;
}
private static void validateDirs(File directory, File gitDir, boolean bare)

View File

@ -240,9 +240,11 @@ public SubmoduleUpdateCommand setStrategy(MergeStrategy strategy) {
*
* @param callback
* the callback
* @return {@code this}
* @since 4.8
*/
public void setCallback(CloneCommand.Callback callback) {
public SubmoduleUpdateCommand setCallback(CloneCommand.Callback callback) {
this.callback = callback;
return this;
}
}

View File

@ -358,6 +358,8 @@ public static JGitText get() {
/***/ public String flagsAlreadyCreated;
/***/ public String funnyRefname;
/***/ public String gcFailed;
/***/ public String gcLogExists;
/***/ public String gcTooManyUnpruned;
/***/ public String gitmodulesNotFound;
/***/ public String headRequiredToStash;
/***/ public String hoursAgo;

View File

@ -637,12 +637,18 @@ static void loadRulesFromFile(AttributesNode r, File attrs)
}
private boolean shouldAutoDetach() {
return getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION,
ConfigConstants.CONFIG_KEY_AUTODETACH, true);
}
@Override
public void autoGC(ProgressMonitor monitor) {
GC gc = new GC(this);
gc.setPackConfig(new PackConfig(this));
gc.setProgressMonitor(monitor);
gc.setAuto(true);
gc.setBackground(shouldAutoDetach());
try {
gc.gc();
} catch (ParseException | IOException e) {

View File

@ -50,6 +50,8 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
@ -73,11 +75,17 @@
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.dircache.DirCacheIterator;
import org.eclipse.jgit.errors.CancelledException;
import org.eclipse.jgit.errors.CorruptObjectException;
@ -143,6 +151,8 @@ public class GC {
private static final int DEFAULT_AUTOLIMIT = 6700;
private static ExecutorService executor = Executors.newFixedThreadPool(1);
private final FileRepository repo;
private ProgressMonitor pm;
@ -177,6 +187,11 @@ public class GC {
*/
private boolean automatic;
/**
* Whether to run gc in a background thread
*/
private boolean background;
/**
* Creates a new garbage collector with default values. An expirationTime of
* two weeks and <code>null</code> as progress monitor will be used.
@ -202,13 +217,73 @@ public GC(FileRepository repo) {
* first check whether any housekeeping is required; if not, it exits
* without performing any work.
*
* If {@link #setBackground(boolean)} was set to {@code true}
* {@code collectGarbage} will start the gc in the background, and then
* return immediately. In this case, errors will not be reported except in
* gc.log.
*
* @return the collection of {@link PackFile}'s which are newly created
* @throws IOException
* @throws ParseException
* If the configuration parameter "gc.pruneexpire" couldn't be
* parsed
*/
// TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>>
public Collection<PackFile> gc() throws IOException, ParseException {
final GcLog gcLog = background ? new GcLog(repo) : null;
if (gcLog != null && !gcLog.lock(background)) {
// there is already a background gc running
return Collections.emptyList();
}
Callable<Collection<PackFile>> gcTask = () -> {
try {
Collection<PackFile> newPacks = doGc();
if (automatic && tooManyLooseObjects() && gcLog != null) {
String message = JGitText.get().gcTooManyUnpruned;
gcLog.write(message);
gcLog.commit();
}
return newPacks;
} catch (IOException | ParseException e) {
if (background) {
if (gcLog == null) {
// Lacking a log, there's no way to report this.
return Collections.emptyList();
}
try {
gcLog.write(e.getMessage());
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
gcLog.write(sw.toString());
gcLog.commit();
} catch (IOException e2) {
e2.addSuppressed(e);
LOG.error(e2.getMessage(), e2);
}
} else {
throw new JGitInternalException(e.getMessage(), e);
}
} finally {
if (gcLog != null) {
gcLog.unlock();
}
}
return Collections.emptyList();
};
Future<Collection<PackFile>> result = executor.submit(gcTask);
if (background) {
// TODO(ms): in 5.0 change signature and return the Future
return Collections.emptyList();
}
try {
return result.get();
} catch (InterruptedException | ExecutionException e) {
throw new IOException(e);
}
}
private Collection<PackFile> doGc() throws IOException, ParseException {
if (automatic && !needGc()) {
return Collections.emptyList();
}
@ -1365,6 +1440,14 @@ public void setAuto(boolean auto) {
this.automatic = auto;
}
/**
* @param background
* whether to run the gc in a background thread.
*/
void setBackground(boolean background) {
this.background = background;
}
private boolean needGc() {
if (tooManyPacks()) {
addRepackAllOption();
@ -1403,8 +1486,7 @@ boolean tooManyPacks() {
* @return {@code true} if number of loose objects > gc.auto (default 6700)
*/
boolean tooManyLooseObjects() {
int auto = repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
int auto = getLooseObjectLimit();
if (auto <= 0) {
return false;
}
@ -1436,4 +1518,9 @@ public boolean accept(Path file) throws IOException {
}
return false;
}
private int getLooseObjectLimit() {
return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
}
}

View File

@ -0,0 +1,191 @@
/*
* Copyright (C) 2017 Two Sigma Open Source
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.file;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.util.GitDateParser;
import org.eclipse.jgit.util.SystemReader;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.attribute.FileTime;
import java.text.MessageFormat;
import java.text.ParseException;
import java.time.Instant;
/**
* This class manages the gc.log file for a {@link FileRepository}.
*/
class GcLog {
private final FileRepository repo;
private final File logFile;
private final LockFile lock;
private Instant gcLogExpire;
private static final String LOG_EXPIRY_DEFAULT = "1.day.ago"; //$NON-NLS-1$
private boolean nonEmpty = false;
/**
* Construct a GcLog object for a {@link FileRepository}
*
* @param repo
* the repository
*/
GcLog(FileRepository repo) {
this.repo = repo;
logFile = new File(repo.getDirectory(), "gc.log"); //$NON-NLS-1$
lock = new LockFile(logFile);
}
private Instant getLogExpiry() throws ParseException {
if (gcLogExpire == null) {
String logExpiryStr = repo.getConfig().getString(
ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_LOGEXPIRY);
if (logExpiryStr == null) {
logExpiryStr = LOG_EXPIRY_DEFAULT;
}
gcLogExpire = GitDateParser.parse(logExpiryStr, null,
SystemReader.getInstance().getLocale()).toInstant();
}
return gcLogExpire;
}
private boolean autoGcBlockedByOldLockFile(boolean background) {
try {
FileTime lastModified = Files.getLastModifiedTime(logFile.toPath());
if (lastModified.toInstant().compareTo(getLogExpiry()) > 0) {
// There is an existing log file, which is too recent to ignore
if (!background) {
try (BufferedReader reader = Files
.newBufferedReader(logFile.toPath())) {
char[] buf = new char[1000];
int len = reader.read(buf, 0, 1000);
String oldError = new String(buf, 0, len);
throw new JGitInternalException(MessageFormat.format(
JGitText.get().gcLogExists, oldError, logFile));
}
}
return true;
}
} catch (NoSuchFileException e) {
// No existing log file, OK.
} catch (IOException | ParseException e) {
throw new JGitInternalException(e.getMessage(), e);
}
return false;
}
/**
* Lock the GC log file for updates
*
* @param background
* If true, and if gc.log already exists, unlock and return false
* @return {@code true} if we hold the lock
*/
boolean lock(boolean background) {
try {
if (!lock.lock()) {
return false;
}
} catch (IOException e) {
throw new JGitInternalException(e.getMessage(), e);
}
if (autoGcBlockedByOldLockFile(background)) {
lock.unlock();
return false;
}
return true;
}
/**
* Unlock (roll back) the GC log lock
*/
void unlock() {
lock.unlock();
}
/**
* Commit changes to the gc log, if there have been any writes. Otherwise,
* just unlock and delete the existing file (if any)
*
* @return true if committing (or unlocking/deleting) succeeds.
*/
boolean commit() {
if (nonEmpty) {
return lock.commit();
} else {
logFile.delete();
lock.unlock();
return true;
}
}
/**
* Write to the pending gc log. Content will be committed upon a call to
* commit()
*
* @param content
* The content to write
* @throws IOException
*/
void write(String content) throws IOException {
if (content.length() > 0) {
nonEmpty = true;
}
lock.write(content.getBytes(UTF_8));
}
}

View File

@ -290,6 +290,20 @@ public class ConfigConstants {
*/
public static final String CONFIG_KEY_PRUNEPACKEXPIRE = "prunepackexpire";
/**
* The "logexpiry" key
*
* @since 4.7
*/
public static final String CONFIG_KEY_LOGEXPIRY = "logExpiry";
/**
* The "autodetach" key
*
* @since 4.7
*/
public static final String CONFIG_KEY_AUTODETACH = "autoDetach";
/**
* The "aggressiveDepth" key
* @since 3.6