Clean up orphan files in GC
An orphan file is either a bitmap or an idx file in pack folder, and its corresponding pack file is missing. Change-Id: I3c4cb1f7aa99dd7b398bdb8d513f528d7761edff Signed-off-by: Hongkai Liu <hongkai.liu@ericsson.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
This commit is contained in:
parent
efbecb9764
commit
8fd500e20c
|
@ -0,0 +1,119 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2017 Ericsson
|
||||||
|
* and other copyright owners as documented in the project's IP log.
|
||||||
|
*
|
||||||
|
* This program and the accompanying materials are made available
|
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which
|
||||||
|
* accompanies this distribution, is reproduced below, and is
|
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
*
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or
|
||||||
|
* without modification, are permitted provided that the following
|
||||||
|
* conditions are met:
|
||||||
|
*
|
||||||
|
* - Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* - Redistributions in binary form must reproduce the above
|
||||||
|
* copyright notice, this list of conditions and the following
|
||||||
|
* disclaimer in the documentation and/or other materials provided
|
||||||
|
* with the distribution.
|
||||||
|
*
|
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the
|
||||||
|
* names of its contributors may be used to endorse or promote
|
||||||
|
* products derived from this software without specific prior
|
||||||
|
* written permission.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.file;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class GcOrphanFilesTest extends GcTestCase {
|
||||||
|
private final static String PACK = "pack";
|
||||||
|
|
||||||
|
private final static String BITMAP_File_1 = PACK + "-1.bitmap";
|
||||||
|
|
||||||
|
private final static String IDX_File_2 = PACK + "-2.idx";
|
||||||
|
|
||||||
|
private final static String IDX_File_malformed = PACK + "-1234idx";
|
||||||
|
|
||||||
|
private final static String PACK_File_2 = PACK + "-2.pack";
|
||||||
|
|
||||||
|
private final static String PACK_File_3 = PACK + "-3.pack";
|
||||||
|
|
||||||
|
private File packDir;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
packDir = new File(repo.getObjectsDirectory(), PACK);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void bitmapAndIdxDeletedButPackNot() throws Exception {
|
||||||
|
createFileInPackFolder(BITMAP_File_1);
|
||||||
|
createFileInPackFolder(IDX_File_2);
|
||||||
|
createFileInPackFolder(PACK_File_3);
|
||||||
|
gc.gc();
|
||||||
|
assertFalse(new File(packDir, BITMAP_File_1).exists());
|
||||||
|
assertFalse(new File(packDir, IDX_File_2).exists());
|
||||||
|
assertTrue(new File(packDir, PACK_File_3).exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void bitmapDeletedButIdxAndPackNot() throws Exception {
|
||||||
|
createFileInPackFolder(BITMAP_File_1);
|
||||||
|
createFileInPackFolder(IDX_File_2);
|
||||||
|
createFileInPackFolder(PACK_File_2);
|
||||||
|
createFileInPackFolder(PACK_File_3);
|
||||||
|
gc.gc();
|
||||||
|
assertFalse(new File(packDir, BITMAP_File_1).exists());
|
||||||
|
assertTrue(new File(packDir, IDX_File_2).exists());
|
||||||
|
assertTrue(new File(packDir, PACK_File_2).exists());
|
||||||
|
assertTrue(new File(packDir, PACK_File_3).exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void malformedIdxNotDeleted() throws Exception {
|
||||||
|
createFileInPackFolder(IDX_File_malformed);
|
||||||
|
gc.gc();
|
||||||
|
assertTrue(new File(packDir, IDX_File_malformed).exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createFileInPackFolder(String fileName) throws IOException {
|
||||||
|
if (!packDir.exists() || !packDir.isDirectory()) {
|
||||||
|
assertTrue(packDir.mkdirs());
|
||||||
|
}
|
||||||
|
assertTrue(new File(packDir, fileName).createNewFile());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void noSuchPackFolder() throws Exception {
|
||||||
|
assertTrue(packDir.delete());
|
||||||
|
gc.gc();
|
||||||
|
}
|
||||||
|
}
|
|
@ -55,6 +55,7 @@
|
||||||
import java.nio.file.DirectoryStream;
|
import java.nio.file.DirectoryStream;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
import java.nio.file.StandardCopyOption;
|
import java.nio.file.StandardCopyOption;
|
||||||
import java.text.MessageFormat;
|
import java.text.MessageFormat;
|
||||||
import java.text.ParseException;
|
import java.text.ParseException;
|
||||||
|
@ -73,6 +74,8 @@
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
import org.eclipse.jgit.annotations.NonNull;
|
import org.eclipse.jgit.annotations.NonNull;
|
||||||
import org.eclipse.jgit.dircache.DirCacheIterator;
|
import org.eclipse.jgit.dircache.DirCacheIterator;
|
||||||
|
@ -126,6 +129,13 @@ public class GC {
|
||||||
private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
|
private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
|
||||||
.compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
|
.compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
|
||||||
|
|
||||||
|
private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$
|
||||||
|
|
||||||
|
private static final String BITMAP_EXT = "." //$NON-NLS-1$
|
||||||
|
+ PackExt.BITMAP_INDEX.getExtension();
|
||||||
|
|
||||||
|
private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$
|
||||||
|
|
||||||
private static final int DEFAULT_AUTOPACKLIMIT = 50;
|
private static final int DEFAULT_AUTOPACKLIMIT = 50;
|
||||||
|
|
||||||
private static final int DEFAULT_AUTOLIMIT = 6700;
|
private static final int DEFAULT_AUTOLIMIT = 6700;
|
||||||
|
@ -654,12 +664,55 @@ else if (RefTreeNames.isRefTree(refdb, ref.getName()))
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
prunePacked();
|
prunePacked();
|
||||||
|
deleteOrphans();
|
||||||
|
|
||||||
lastPackedRefs = refsBefore;
|
lastPackedRefs = refsBefore;
|
||||||
lastRepackTime = time;
|
lastRepackTime = time;
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes orphans
|
||||||
|
* <p>
|
||||||
|
* A file is considered an orphan if it is either a "bitmap" or an index
|
||||||
|
* file, and its corresponding pack file is missing in the list.
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
private void deleteOrphans() {
|
||||||
|
Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
|
||||||
|
"pack"); //$NON-NLS-1$
|
||||||
|
List<String> fileNames = null;
|
||||||
|
try (Stream<Path> files = Files.list(packDir)) {
|
||||||
|
fileNames = files.map(path -> path.getFileName().toString())
|
||||||
|
.filter(name -> {
|
||||||
|
return (name.endsWith(PACK_EXT)
|
||||||
|
|| name.endsWith(BITMAP_EXT)
|
||||||
|
|| name.endsWith(INDEX_EXT));
|
||||||
|
}).sorted(Collections.reverseOrder())
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
} catch (IOException e1) {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
if (fileNames == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String base = null;
|
||||||
|
for (String n : fileNames) {
|
||||||
|
if (n.endsWith(PACK_EXT)) {
|
||||||
|
base = n.substring(0, n.lastIndexOf('.'));
|
||||||
|
} else {
|
||||||
|
if (base == null || !n.startsWith(base)) {
|
||||||
|
try {
|
||||||
|
Files.delete(new File(packDir.toFile(), n).toPath());
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param ref
|
* @param ref
|
||||||
* the ref which log should be inspected
|
* the ref which log should be inspected
|
||||||
|
|
Loading…
Reference in New Issue