From 78009782cdabdba42841f2a71fbd5867f2ae683f Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Mon, 3 Dec 2012 16:08:04 -0800 Subject: [PATCH] archive: Add tar support Unlike ZIP files, tar files do not treat symlinks as ordinary files with a different mode, so tar support involves a little more code than would be ideal. Change-Id: Ica2568f4a0e443bf4b955ef0c029bc8eec62d369 --- .../tst/org/eclipse/jgit/pgm/ArchiveTest.java | 167 +++++++++++++++++- org.eclipse.jgit.pgm/META-INF/MANIFEST.MF | 1 + .../org/eclipse/jgit/pgm/CLIText.properties | 2 +- .../src/org/eclipse/jgit/pgm/Archive.java | 38 +++- 4 files changed, 199 insertions(+), 9 deletions(-) diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ArchiveTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ArchiveTest.java index cad379774..bcf272852 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ArchiveTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ArchiveTest.java @@ -53,11 +53,15 @@ import java.io.InputStreamReader; import java.io.IOException; import java.io.OutputStream; - +import java.lang.Object; import java.lang.String; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -91,6 +95,13 @@ public void testEmptyArchive() throws Exception { assertArrayEquals(new String[0], listZipEntries(result)); } + @Test + public void testEmptyTar() throws Exception { + final byte[] result = CLIGitCommand.rawExecute( // + "git archive --format=tar " + emptyTree, db); + assertArrayEquals(new String[0], listTarEntries(result)); + } + @Test public void testArchiveWithFiles() throws Exception { writeTrashFile("a", "a file with content!"); @@ -132,6 +143,32 @@ public void testArchiveWithSubdir() throws Exception { assertArrayEquals(expect, actual); } + @Test + public void testTarWithSubdir() throws Exception { + writeTrashFile("a", "a file with content!"); + writeTrashFile("b.c", "before subdir in git sort order"); + writeTrashFile("b0c", "after subdir in git sort order"); + writeTrashFile("c", ""); + git.add().addFilepattern("a").call(); + git.add().addFilepattern("b.c").call(); + git.add().addFilepattern("b0c").call(); + git.add().addFilepattern("c").call(); + git.commit().setMessage("populate toplevel").call(); + writeTrashFile("b/b", "file in subdirectory"); + writeTrashFile("b/a", "another file in subdirectory"); + git.add().addFilepattern("b").call(); + git.commit().setMessage("add subdir").call(); + + final byte[] result = CLIGitCommand.rawExecute( // + "git archive --format=tar master", db); + String[] expect = { "a", "b.c", "b0c", "b/a", "b/b", "c" }; + String[] actual = listTarEntries(result); + + Arrays.sort(expect); + Arrays.sort(actual); + assertArrayEquals(expect, actual); + } + @Test public void testArchivePreservesMode() throws Exception { writeTrashFile("plain", "a file with content"); @@ -158,6 +195,32 @@ public void testArchivePreservesMode() throws Exception { assertContainsEntryWithMode("zip-with-modes.zip", "l", "symlink"); } + @Test + public void testTarPreservesMode() throws Exception { + writeTrashFile("plain", "a file with content"); + writeTrashFile("executable", "an executable file"); + writeTrashFile("symlink", "plain"); + git.add().addFilepattern("plain").call(); + git.add().addFilepattern("executable").call(); + git.add().addFilepattern("symlink").call(); + + DirCache cache = db.lockDirCache(); + cache.getEntry("executable").setFileMode(FileMode.EXECUTABLE_FILE); + cache.getEntry("symlink").setFileMode(FileMode.SYMLINK); + cache.write(); + cache.commit(); + cache.unlock(); + + git.commit().setMessage("three files with different modes").call(); + + final byte[] archive = CLIGitCommand.rawExecute( // + "git archive --format=tar master", db); + writeRaw("with-modes.tar", archive); + assertTarContainsEntry("with-modes.tar", "-rw-r--r--", "plain"); + assertTarContainsEntry("with-modes.tar", "-rwxr-xr-x", "executable"); + assertTarContainsEntry("with-modes.tar", "l", "symlink -> plain"); + } + @Test public void testArchivePreservesContent() throws Exception { final String payload = "“The quick brown fox jumps over the lazy dog!”"; @@ -171,23 +234,45 @@ public void testArchivePreservesContent() throws Exception { zipEntryContent(result, "xyzzy")); } - private void assertContainsEntryWithMode(String zipFilename, String mode, String name) // - throws Exception { + @Test + public void testTarPreservesContent() throws Exception { + final String payload = "“The quick brown fox jumps over the lazy dog!”"; + writeTrashFile("xyzzy", payload); + git.add().addFilepattern("xyzzy").call(); + git.commit().setMessage("add file with content").call(); + + final byte[] result = CLIGitCommand.rawExecute( // + "git archive --format=tar HEAD", db); + assertArrayEquals(new String[] { payload }, // + tarEntryContent(result, "xyzzy")); + } + + private Process spawnAssumingCommandPresent(String... cmdline) { final File cwd = db.getWorkTree(); - final ProcessBuilder procBuilder = new ProcessBuilder("zipinfo", zipFilename) // + final ProcessBuilder procBuilder = new ProcessBuilder(cmdline) // .directory(cwd) // .redirectErrorStream(true); Process proc = null; try { proc = procBuilder.start(); } catch (IOException e) { - // On machines without a "zipinfo" command, let the test pass. + // On machines without `cmdline[0]`, let the test pass. assumeNoException(e); } - proc.getOutputStream().close(); - final BufferedReader reader = new BufferedReader( // + return proc; + } + + private BufferedReader readFromProcess(Process proc) throws Exception { + return new BufferedReader( // new InputStreamReader(proc.getInputStream(), "UTF-8")); + } + + private void grepForEntry(String name, String mode, String... cmdline) // + throws Exception { + final Process proc = spawnAssumingCommandPresent(cmdline); + proc.getOutputStream().close(); + final BufferedReader reader = readFromProcess(proc); try { String line; while ((line = reader.readLine()) != null) @@ -201,6 +286,16 @@ private void assertContainsEntryWithMode(String zipFilename, String mode, String } } + private void assertContainsEntryWithMode(String zipFilename, String mode, String name) // + throws Exception { + grepForEntry(name, mode, "zipinfo", zipFilename); + } + + private void assertTarContainsEntry(String tarfile, String mode, String name) // + throws Exception { + grepForEntry(name, mode, "tar", "tvf", tarfile); + } + private void writeRaw(String filename, byte[] data) // throws IOException { final File path = new File(db.getWorkTree(), filename); @@ -224,6 +319,43 @@ private static String[] listZipEntries(byte[] zipData) throws IOException { return l.toArray(new String[l.size()]); } + private static Future writeAsync(final OutputStream stream, final byte[] data) { + final ExecutorService executor = Executors.newSingleThreadExecutor(); + + return executor.submit(new Callable() { // + public Object call() throws IOException { + try { + stream.write(data); + return null; + } finally { + stream.close(); + } + } + }); + } + + private String[] listTarEntries(byte[] tarData) throws Exception { + final List l = new ArrayList(); + final Process proc = spawnAssumingCommandPresent("tar", "tf", "-"); + final BufferedReader reader = readFromProcess(proc); + final OutputStream out = proc.getOutputStream(); + + // Dump tarball to tar stdin in background + final Future writing = writeAsync(out, tarData); + + try { + String line; + while ((line = reader.readLine()) != null) + l.add(line); + + return l.toArray(new String[l.size()]); + } finally { + writing.get(); + reader.close(); + proc.destroy(); + } + } + private static String[] zipEntryContent(byte[] zipData, String path) // throws IOException { final ZipInputStream in = new ZipInputStream( // @@ -246,4 +378,25 @@ private static String[] zipEntryContent(byte[] zipData, String path) // // not found return null; } + + private String[] tarEntryContent(byte[] tarData, String path) // + throws Exception { + final List l = new ArrayList(); + final Process proc = spawnAssumingCommandPresent("tar", "Oxf", "-", path); + final BufferedReader reader = readFromProcess(proc); + final OutputStream out = proc.getOutputStream(); + final Future writing = writeAsync(out, tarData); + + try { + String line; + while ((line = reader.readLine()) != null) + l.add(line); + + return l.toArray(new String[l.size()]); + } finally { + writing.get(); + reader.close(); + proc.destroy(); + } + } } diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index 247f93cb2..b2a988c3a 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -7,6 +7,7 @@ Bundle-Vendor: %provider_name Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: J2SE-1.5 Import-Package: org.apache.commons.compress.archivers;version="[1.3,2.0)", + org.apache.commons.compress.archivers.tar;version="[1.3,2.0)", org.apache.commons.compress.archivers.zip;version="[1.3,2.0)", org.eclipse.jgit.api;version="[2.2.0,2.3.0)", org.eclipse.jgit.api.errors;version="[2.2.0,2.3.0)", diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/CLIText.properties index fe70e7132..5586a2820 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/CLIText.properties @@ -197,7 +197,7 @@ usage_addFileContentsToTheIndex=Add file contents to the index usage_alterTheDetailShown=alter the detail shown usage_approveDestructionOfRepository=approve destruction of repository usage_archive=zip up files from the named tree -usage_archiveFormat=archive format. Currently supported formats: 'zip' +usage_archiveFormat=archive format. Currently supported formats: 'tar', 'zip' usage_blameLongRevision=show long revision usage_blameRange=annotate only the given range usage_blameRawTimestamp=show raw timestamp diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Archive.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Archive.java index ad3263823..4a5bf1c55 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Archive.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Archive.java @@ -53,6 +53,9 @@ import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarConstants; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.eclipse.jgit.lib.FileMode; @@ -111,7 +114,8 @@ static private void warnArchiveEntryModeIgnored(String name) { } public enum Format { - ZIP + ZIP, + TAR }; private static interface Archiver { @@ -149,6 +153,38 @@ else if (mode == FileMode.EXECUTABLE_FILE || out.closeArchiveEntry(); } }); + fmts.put(Format.TAR, new Archiver() { + @Override + public ArchiveOutputStream createArchiveOutputStream(OutputStream s) { + return new TarArchiveOutputStream(s); + } + + @Override + public void putEntry(String path, FileMode mode, // + ObjectLoader loader, ArchiveOutputStream out) // + throws IOException { + if (mode == FileMode.SYMLINK) { + final TarArchiveEntry entry = new TarArchiveEntry( // + path, TarConstants.LF_SYMLINK); + entry.setLinkName(new String( // + loader.getCachedBytes(100), "UTF-8")); + out.putArchiveEntry(entry); + out.closeArchiveEntry(); + return; + } + + final TarArchiveEntry entry = new TarArchiveEntry(path); + if (mode == FileMode.REGULAR_FILE || + mode == FileMode.EXECUTABLE_FILE) + entry.setMode(mode.getBits()); + else + warnArchiveEntryModeIgnored(path); + entry.setSize(loader.getSize()); + out.putArchiveEntry(entry); + loader.copyTo(out); + out.closeArchiveEntry(); + } + }); formats = fmts; } }