Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

zip files supported as resources #210

Merged
merged 3 commits into from
May 9, 2019
Merged

zip files supported as resources #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c17882f
zip files support
sergiorussia Apr 29, 2019
4973da5
revert non-related changes, small fixes
sergiorussia May 5, 2019
7cf4ce0
unit tests
sergiorussia May 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions core/src/main/java/com/threerings/getdown/cache/GarbageCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
package com.threerings.getdown.cache;

import java.io.File;

import com.threerings.getdown.data.Resource;
import com.threerings.getdown.util.FileUtil;

/**
Expand Down Expand Up @@ -55,9 +57,9 @@ public static void collectNative (File cacheDir, final long retentionPeriodMilli
if (subdirs != null) {
for (File dir : subdirs) {
if (dir.isDirectory()) {
// Get all the native jars in the directory (there should only be one)
// Get all the native jars or zips in the directory (there should only be one)
for (File file : dir.listFiles()) {
if (!file.getName().endsWith(".jar")) {
if (!Resource.isJar(file) && !Resource.isZip(file)) {
continue;
}
File cachedFile = getCachedFile(file);
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/com/threerings/getdown/data/PathBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.jar.JarFile;
import java.util.zip.ZipFile;

import com.threerings.getdown.cache.GarbageCollector;
import com.threerings.getdown.cache.ResourceCache;
Expand Down Expand Up @@ -112,7 +112,7 @@ public static ClassPath buildLibsPath (Application app,

if (!unpackedIndicator.exists()) {
try {
FileUtil.unpackJar(new JarFile(cachedFile), cachedParent, false);
FileUtil.unpackJar(new ZipFile(cachedFile), cachedParent, false);
unpackedIndicator.createNewFile();
} catch (IOException ioe) {
log.warning("Failed to unpack native jar",
Expand Down
71 changes: 40 additions & 31 deletions core/src/main/java/com/threerings/getdown/data/Resource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,23 @@

package com.threerings.getdown.data;

import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.MessageDigest;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import com.threerings.getdown.util.FileUtil;
import com.threerings.getdown.util.ProgressObserver;
import com.threerings.getdown.util.StringUtil;

import static com.threerings.getdown.Log.log;

/**
Expand Down Expand Up @@ -60,29 +62,29 @@ public static String computeDigest (int version, File target, MessageDigest md,
byte[] buffer = new byte[DIGEST_BUFFER_SIZE];
int read;

boolean isJar = isJar(target.getPath());
boolean isPacked200Jar = isPacked200Jar(target.getPath());
boolean isZip = isJar(target) || isZip(target); // jar is a zip too
boolean isPacked200Jar = isPacked200Jar(target);

// if this is a jar, we need to compute the digest in a "timestamp and file order" agnostic
// manner to properly correlate jardiff patched jars with their unpatched originals
if (isJar || isPacked200Jar){
if (isPacked200Jar || isZip){
File tmpJarFile = null;
JarFile jar = null;
ZipFile zip = null;
try {
// if this is a compressed jar file, uncompress it to compute the jar file digest
// if this is a compressed zip file, uncompress it to compute the zip file digest
if (isPacked200Jar){
tmpJarFile = new File(target.getPath() + ".tmp");
FileUtil.unpackPacked200Jar(target, tmpJarFile);
jar = new JarFile(tmpJarFile);
tmpJarFile.deleteOnExit();
zip = FileUtil.unpackPacked200Jar(target, tmpJarFile);
} else{
jar = new JarFile(target);
zip = new ZipFile(target);
}

List<JarEntry> entries = Collections.list(jar.entries());
List<? extends ZipEntry> entries = Collections.list(zip.entries());
Collections.sort(entries, ENTRY_COMP);

int eidx = 0;
for (JarEntry entry : entries) {
for (ZipEntry entry : entries) {
// old versions of the digest code skipped metadata
if (version < 2) {
if (entry.getName().startsWith("META-INF")) {
Expand All @@ -91,7 +93,7 @@ public static String computeDigest (int version, File target, MessageDigest md,
}
}

try (InputStream in = jar.getInputStream(entry)) {
try (InputStream in = zip.getInputStream(entry)) {
while ((read = in.read(buffer)) != -1) {
md.update(buffer, 0, read);
}
Expand All @@ -101,11 +103,11 @@ public static String computeDigest (int version, File target, MessageDigest md,
}

} finally {
if (jar != null) {
if (zip != null) {
try {
jar.close();
zip.close();
} catch (IOException ioe) {
log.warning("Error closing jar", "path", target, "jar", jar, "error", ioe);
log.warning("Error closing zip", "path", target, "zip", zip, "error", ioe);
}
}
if (tmpJarFile != null) {
Expand Down Expand Up @@ -135,14 +137,13 @@ public Resource (String path, URL remote, File local, EnumSet<Attr> attrs)
_remote = remote;
_local = local;
_localNew = new File(local.toString() + "_new");
String lpath = _local.getPath();
_marker = new File(lpath + "v");
_marker = new File(_local.getPath() + "v");

_attrs = attrs;
_isJar = isJar(lpath);
_isPacked200Jar = isPacked200Jar(lpath);
_isZip = isJar(local) || isZip(local);
_isPacked200Jar = isPacked200Jar(local);
boolean unpack = attrs.contains(Attr.UNPACK);
if (unpack && _isJar) {
if (unpack && _isZip) {
_unpacked = _local.getParentFile();
} else if(unpack && _isPacked200Jar) {
String dotJar = ".jar", lname = _local.getName();
Expand Down Expand Up @@ -298,11 +299,11 @@ public void install (boolean validate) throws IOException {
public void unpack () throws IOException
{
// sanity check
if (!_isJar && !_isPacked200Jar) {
if (!_isZip && !_isPacked200Jar) {
throw new IOException("Requested to unpack non-jar file '" + _local + "'.");
}
if (_isJar) {
try (JarFile jar = new JarFile(_local)) {
if (_isZip) {
try (ZipFile jar = new ZipFile(_local)) {
FileUtil.unpackJar(jar, _unpacked, _attrs.contains(Attr.CLEAN));
}
} else {
Expand Down Expand Up @@ -366,26 +367,34 @@ protected static void updateProgress (ProgressObserver obs, long pos, long total
}
}

protected static boolean isJar (String path)
public static boolean isJar (File file)
{
String path = file.getName();
return path.endsWith(".jar") || path.endsWith(".jar_new");
}

protected static boolean isPacked200Jar (String path)
public static boolean isPacked200Jar (File file)
{
String path = file.getName();
return path.endsWith(".jar.pack") || path.endsWith(".jar.pack_new") ||
path.endsWith(".jar.pack.gz")|| path.endsWith(".jar.pack.gz_new");
}

public static boolean isZip (File file)
{
String path = file.getName();
return path.endsWith(".zip") || path.endsWith(".zip_new");
}

protected String _path;
protected URL _remote;
protected File _local, _localNew, _marker, _unpacked;
protected EnumSet<Attr> _attrs;
protected boolean _isJar, _isPacked200Jar;
protected boolean _isZip, _isPacked200Jar;

/** Used to sort the entries in a jar file. */
protected static final Comparator<JarEntry> ENTRY_COMP = new Comparator<JarEntry>() {
@Override public int compare (JarEntry e1, JarEntry e2) {
protected static final Comparator<ZipEntry> ENTRY_COMP = new Comparator<ZipEntry>() {
@Override public int compare (ZipEntry e1, ZipEntry e2) {
return e1.getName().compareTo(e2.getName());
}
};
Expand Down
28 changes: 12 additions & 16 deletions core/src/main/java/com/threerings/getdown/tools/Differ.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,12 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;

import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;

import java.security.MessageDigest;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

import com.threerings.getdown.data.Application;
import com.threerings.getdown.data.Digest;
Expand Down Expand Up @@ -99,7 +96,7 @@ protected void createPatch (File patch, ArrayList<Resource> orsrcs,
MessageDigest md = Digest.getMessageDigest(version);
try (FileOutputStream fos = new FileOutputStream(patch);
BufferedOutputStream buffered = new BufferedOutputStream(fos);
JarOutputStream jout = new JarOutputStream(buffered)) {
ZipOutputStream jout = new ZipOutputStream(buffered)) {

// for each file in the new application, it either already exists
// in the old application, or it is new
Expand Down Expand Up @@ -172,13 +169,13 @@ protected File rebuildJar (File target)
throws IOException
{
File temp = File.createTempFile("differ", "jar");
try (JarFile jar = new JarFile(target);
FileOutputStream tempFos = new FileOutputStream(temp);
BufferedOutputStream tempBos = new BufferedOutputStream(tempFos);
JarOutputStream jout = new JarOutputStream(tempBos)) {
try (ZipFile jar = new ZipFile(target);
FileOutputStream tempFos = new FileOutputStream(temp);
BufferedOutputStream tempBos = new BufferedOutputStream(tempFos);
ZipOutputStream jout = new ZipOutputStream(tempBos)) {
byte[] buffer = new byte[4096];
for (Enumeration< JarEntry > iter = jar.entries(); iter.hasMoreElements();) {
JarEntry entry = iter.nextElement();
for (Enumeration<? extends ZipEntry> iter = jar.entries(); iter.hasMoreElements();) {
ZipEntry entry = iter.nextElement();
entry.setCompressedSize(-1);
jout.putNextEntry(entry);
try (InputStream in = jar.getInputStream(entry)) {
Expand All @@ -193,7 +190,7 @@ protected File rebuildJar (File target)
return temp;
}

protected void jarDiff (File ofile, File nfile, JarOutputStream jout)
protected void jarDiff (File ofile, File nfile, ZipOutputStream jout)
throws IOException
{
JarDiff.createPatch(ofile.getPath(), nfile.getPath(), jout, false);
Expand Down Expand Up @@ -222,8 +219,7 @@ public static void main (String[] args)
}
}

protected static void pipe (File file, JarOutputStream jout)
throws IOException
protected static void pipe (File file, ZipOutputStream jout) throws IOException
{
try (FileInputStream fin = new FileInputStream(file)) {
StreamUtil.copy(fin, jout);
Expand Down
Loading