GC: Remove empty references folders
After packaging references, the folders containing these references are not deleted. In a busy repository, this causes operations to slow down as traversing the references tree becomes longer. Delete empty reference folders after the loose references have been packed. To avoid deleting a folder that was just created by another concurrent operation, only delete folders that were not modified in the last 30 seconds. Signed-off-by: Hector Oswaldo Caballero <hector.caballero@ericsson.com> Change-Id: Ie79447d6121271cf5e25171be377ea396c7028e0 Signed-off-by: Luca Milanesio <luca.milanesio@gmail.com> Signed-off-by: David Pursehouse <david.pursehouse@gmail.com>
This commit is contained in:
parent
5a8ad44208
commit
4eea4ea508
|
@ -0,0 +1,118 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2018 Ericsson
|
||||||
|
* and other copyright owners as documented in the project's IP log.
|
||||||
|
*
|
||||||
|
* This program and the accompanying materials are made available
|
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which
|
||||||
|
* accompanies this distribution, is reproduced below, and is
|
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
*
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or
|
||||||
|
* without modification, are permitted provided that the following
|
||||||
|
* conditions are met:
|
||||||
|
*
|
||||||
|
* - Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* - Redistributions in binary form must reproduce the above
|
||||||
|
* copyright notice, this list of conditions and the following
|
||||||
|
* disclaimer in the documentation and/or other materials provided
|
||||||
|
* with the distribution.
|
||||||
|
*
|
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the
|
||||||
|
* names of its contributors may be used to endorse or promote
|
||||||
|
* products derived from this software without specific prior
|
||||||
|
* written permission.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.file;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.nio.file.attribute.FileTime;
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class GcDeleteEmptyRefsFoldersTest extends GcTestCase {
|
||||||
|
private static final String REF_FOLDER_01 = "01";
|
||||||
|
private static final String REF_FOLDER_02 = "02";
|
||||||
|
|
||||||
|
private Path refsDir;
|
||||||
|
private Path heads;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
refsDir = Paths.get(repo.getDirectory().getAbsolutePath())
|
||||||
|
.resolve("refs");
|
||||||
|
heads = refsDir.resolve("heads");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void emptyRefFoldersAreDeleted() throws Exception {
|
||||||
|
FileTime fileTime = FileTime.from(Instant.now().minusSeconds(31));
|
||||||
|
Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
|
||||||
|
Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
|
||||||
|
Files.setLastModifiedTime(refDir01, fileTime);
|
||||||
|
Files.setLastModifiedTime(refDir02, fileTime);
|
||||||
|
assertTrue(refDir01.toFile().exists());
|
||||||
|
assertTrue(refDir02.toFile().exists());
|
||||||
|
gc.gc();
|
||||||
|
|
||||||
|
assertFalse(refDir01.toFile().exists());
|
||||||
|
assertFalse(refDir02.toFile().exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void emptyRefFoldersAreKeptIfTheyAreTooRecent()
|
||||||
|
throws Exception {
|
||||||
|
Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
|
||||||
|
Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
|
||||||
|
assertTrue(refDir01.toFile().exists());
|
||||||
|
assertTrue(refDir02.toFile().exists());
|
||||||
|
gc.gc();
|
||||||
|
|
||||||
|
assertTrue(refDir01.toFile().exists());
|
||||||
|
assertTrue(refDir02.toFile().exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void nonEmptyRefsFoldersAreKept() throws Exception {
|
||||||
|
Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
|
||||||
|
Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
|
||||||
|
Path ref01 = Files.createFile(refDir01.resolve("ref01"));
|
||||||
|
Path ref02 = Files.createFile(refDir01.resolve("ref02"));
|
||||||
|
assertTrue(refDir01.toFile().exists());
|
||||||
|
assertTrue(refDir02.toFile().exists());
|
||||||
|
assertTrue(ref01.toFile().exists());
|
||||||
|
assertTrue(ref02.toFile().exists());
|
||||||
|
gc.gc();
|
||||||
|
assertTrue(refDir01.toFile().exists());
|
||||||
|
assertTrue(refDir02.toFile().exists());
|
||||||
|
assertTrue(ref01.toFile().exists());
|
||||||
|
assertTrue(ref02.toFile().exists());
|
||||||
|
}
|
||||||
|
}
|
|
@ -61,6 +61,8 @@
|
||||||
import java.nio.file.StandardCopyOption;
|
import java.nio.file.StandardCopyOption;
|
||||||
import java.text.MessageFormat;
|
import java.text.MessageFormat;
|
||||||
import java.text.ParseException;
|
import java.text.ParseException;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.temporal.ChronoUnit;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
@ -865,6 +867,7 @@ else if (RefTreeNames.isRefTree(refdb, ref.getName()))
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
prunePacked();
|
prunePacked();
|
||||||
|
deleteEmptyRefsFolders();
|
||||||
deleteOrphans();
|
deleteOrphans();
|
||||||
|
|
||||||
lastPackedRefs = refsBefore;
|
lastPackedRefs = refsBefore;
|
||||||
|
@ -880,6 +883,49 @@ private static boolean isTag(Ref ref) {
|
||||||
return ref.getName().startsWith(Constants.R_TAGS);
|
return ref.getName().startsWith(Constants.R_TAGS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void deleteEmptyRefsFolders() throws IOException {
|
||||||
|
Path refs = repo.getDirectory().toPath().resolve("refs"); //$NON-NLS-1$
|
||||||
|
try (Stream<Path> entries = Files.list(refs)) {
|
||||||
|
Iterator<Path> iterator = entries.iterator();
|
||||||
|
while (iterator.hasNext()) {
|
||||||
|
try (Stream<Path> s = Files.list(iterator.next())) {
|
||||||
|
s.forEach(this::deleteDir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteDir(Path dir) {
|
||||||
|
try (Stream<Path> dirs = Files.walk(dir)) {
|
||||||
|
dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
|
||||||
|
.forEach(this::delete);
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isDirectory(Path p) {
|
||||||
|
return p.toFile().isDirectory();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean delete(Path d) {
|
||||||
|
try {
|
||||||
|
// Avoid deleting a folder that was just created so that concurrent
|
||||||
|
// operations trying to create a reference are not impacted
|
||||||
|
Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
|
||||||
|
Instant lastModified = Files.getLastModifiedTime(d).toInstant();
|
||||||
|
if (lastModified.isBefore(threshold)) {
|
||||||
|
// If the folder is not empty, the delete operation will fail
|
||||||
|
// silently. This is a cheaper alternative to filtering the
|
||||||
|
// stream in the calling method.
|
||||||
|
return d.toFile().delete();
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Deletes orphans
|
* Deletes orphans
|
||||||
* <p>
|
* <p>
|
||||||
|
|
Loading…
Reference in New Issue