From f2a780d088bacb5f73efc9ddc812fc37fa4e2dc8 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 12 Feb 2015 12:22:35 +0100 Subject: [PATCH] Upgrading to java 7 and using posix nio API --- build.sbt | 3 +- .../com/typesafe/sbt/packager/FileUtil.scala | 62 ++++++++-- .../sbt/packager/universal/ZipHelper.scala | 91 ++++++++------- .../sbt/packager/DeleteDirectoryVisitor.scala | 20 ++++ .../typesafe/sbt/packager/FileUtilSpec.scala | 28 +++++ .../packager/universal/ZipHelperSpec.scala | 109 ++++++++++++++++++ 6 files changed, 261 insertions(+), 52 deletions(-) create mode 100644 src/test/scala/com/typesafe/sbt/packager/DeleteDirectoryVisitor.scala create mode 100644 src/test/scala/com/typesafe/sbt/packager/FileUtilSpec.scala create mode 100644 src/test/scala/com/typesafe/sbt/packager/universal/ZipHelperSpec.scala diff --git a/build.sbt b/build.sbt index 2ef79ca2b..65dcf86c7 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,8 @@ scalacOptions in Compile ++= Seq("-deprecation", "-target:jvm-1.6") libraryDependencies ++= Seq( "org.apache.commons" % "commons-compress" % "1.4.1", - "org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar")) + "org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar")), + "org.scalatest" %% "scalatest" % "2.2.4" % "test" ) site.settings diff --git a/src/main/scala/com/typesafe/sbt/packager/FileUtil.scala b/src/main/scala/com/typesafe/sbt/packager/FileUtil.scala index d942a39b9..f33035d12 100644 --- a/src/main/scala/com/typesafe/sbt/packager/FileUtil.scala +++ b/src/main/scala/com/typesafe/sbt/packager/FileUtil.scala @@ -1,18 +1,60 @@ package com.typesafe.sbt package packager -import java.io.File -import sbt.Process +import java.io.{ File, IOException } +import java.nio.file.{ Paths, Files } +import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions } +/** + * Setting the file permissions + */ object chmod { + + /** + * Using java 7 nio API to set the permissions. + * + * @param file + * @param perms in octal format + */ def apply(file: File, perms: String): Unit = - Process(Seq("chmod", perms, file.getAbsolutePath)).! match { - case 0 => () - case n => sys.error("Error running chmod " + perms + " " + file) - } - def safe(file: File, perms: String): Unit = - try apply(file, perms) - catch { - case e: RuntimeException => () + try { + Files.setPosixFilePermissions(file.toPath, permissions(perms)) + } catch { + case e: IOException => sys.error("Error setting permissions " + perms + " on " + file.getAbsolutePath + ": " + e.getMessage) } + +} + +/** + * Converts a octal unix permission representation into + * a java `PosiFilePermissions` compatible string. + */ +object permissions { + + /** + * @param perms in octal format + * @return java 7 posix file permissions + */ + def apply(perms: String): java.util.Set[PosixFilePermission] = PosixFilePermissions fromString convert(perms) + + def convert(perms: String): String = { + require(perms.length == 4, s"Permissions must have 4 digits, got [$perms]") + // ignore setuid/setguid/sticky bit + val user = Character getNumericValue (perms charAt 1) + val group = Character getNumericValue (perms charAt 2) + val other = Character getNumericValue (perms charAt 3) + + asString(user) + asString(group) + asString(other) + } + + private def asString(perm: Int): String = perm match { + case 0 => "---" + case 1 => "--x" + case 2 => "-w-" + case 3 => "-wx" + case 4 => "r--" + case 5 => "r-x" + case 6 => "rw-" + case 7 => "rwx" + } } \ No newline at end of file diff --git a/src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala b/src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala index 687dc1408..4ce5ba3a7 100644 --- a/src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala +++ b/src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala @@ -10,9 +10,22 @@ import org.apache.commons.compress.compressors.{ } import java.util.zip.Deflater import org.apache.commons.compress.utils.IOUtils +import java.nio.file.{ Paths, Files, FileSystems, FileSystem, StandardCopyOption } +import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions } +import java.net.URI +import scala.collection.JavaConverters._ +/** + * + * + * + * @see http://stackoverflow.com/questions/17888365/file-permissions-are-not-being-preserved-while-after-zip + * @see http://stackoverflow.com/questions/3450250/is-it-possible-to-create-a-script-to-save-and-restore-permissions + * @see http://stackoverflow.com/questions/1050560/maintain-file-permissions-when-extracting-from-a-zip-file-using-jdk-5-api + * @see http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/zipfilesystemprovider.html + */ object ZipHelper { - case class FileMapping(file: File, name: String, unixMode: Option[Int] = None) + case class FileMapping(file: File, name: String) /** * Creates a zip file attempting to give files the appropriate unix permissions using Java 6 APIs. @@ -41,36 +54,22 @@ object ZipHelper { } /** - * Creates a zip file attempting to give files the appropriate unix permissions using Java 6 APIs. + * Creates a zip file attempting to give files the appropriate unix permissions using Java 7 APIs. + * * Note: This is known to have some odd issues on MacOSX whereby executable permissions * are not actually discovered, even though the Info-Zip headers exist and work on * many variants of linux. Yay Apple. + * * @param sources The files to include in the zip file. * @param outputZip The location of the output file. */ def zip(sources: Traversable[(File, String)], outputZip: File): Unit = { - val mappings = - for { - (file, name) <- sources.toSeq - // TODO - Figure out if this is good enough.... - perm = if (file.isDirectory || file.canExecute) 0755 else 0644 - } yield FileMapping(file, name, Some(perm)) + val mappings = sources.toSeq.map { + case (file, name) => FileMapping(file, name) + } archive(mappings, outputZip) } - /** - * Creates a zip file using the given set of filters - * @param sources The files to include in the zip file. A File, Location, Permission pairing. - * @param outputZip The location of the output file. - */ - def zipWithPerms(sources: Traversable[(File, String, Int)], outputZip: File): Unit = { - val mappings = - for { - (file, name, perm) <- sources - } yield FileMapping(file, name, Some(perm)) - archive(mappings.toSeq, outputZip) - } - /** * Replaces windows backslash file separator with a forward slash, this ensures the zip file entry is correct for * any system it is extracted on. @@ -84,31 +83,41 @@ object ZipHelper { path.replace(sep, '/') } + /** + * + */ private def archive(sources: Seq[FileMapping], outputFile: File): Unit = { - if (outputFile.isDirectory) sys.error("Specified output file " + outputFile + " is a directory.") - else { - val outputDir = outputFile.getParentFile - IO createDirectory outputDir - withZipOutput(outputFile) { output => - for (FileMapping(file, name, mode) <- sources; if !file.isDirectory) { - val entry = new ZipArchiveEntry(file, normalizePath(name)) - // Now check to see if we have permissions for this sucker. - mode foreach (entry.setUnixMode) - output putArchiveEntry entry - // TODO - Write file into output? - IOUtils.copy(new java.io.FileInputStream(file), output) - output.closeArchiveEntry() - } + require(!outputFile.isDirectory, "Specified output file " + outputFile + " is a directory.") + + // make sure everything is available + val outputDir = outputFile.getParentFile + IO createDirectory outputDir + + // zipping the sources into the output zip + withZipFilesystem(outputFile) { system => + sources foreach { + case FileMapping(dir, name) if dir.isDirectory => Files createDirectories (system getPath name) + case FileMapping(file, name) => Files copy (file.toPath, system getPath name, StandardCopyOption.COPY_ATTRIBUTES) } } + } - private def withZipOutput(file: File)(f: ZipArchiveOutputStream => Unit): Unit = { - val zipOut = new ZipArchiveOutputStream(file) - zipOut setLevel Deflater.BEST_COMPRESSION - try { f(zipOut) } - finally { - zipOut.close() + /** + * Opens a zip filesystem and creates the file if neccessary + * + * @param zipFile + * @param f: FileSystem => Unit, logic working in the filesystem + */ + def withZipFilesystem(zipFile: File)(f: FileSystem => Unit) { + val env = Map("create" -> "true").asJava + val uri = URI.create("jar:file:" + zipFile.getAbsolutePath) + + val system = FileSystems.newFileSystem(uri, env) + try { + f(system) + } finally { + system.close() } } } diff --git a/src/test/scala/com/typesafe/sbt/packager/DeleteDirectoryVisitor.scala b/src/test/scala/com/typesafe/sbt/packager/DeleteDirectoryVisitor.scala new file mode 100644 index 000000000..e0f608fa2 --- /dev/null +++ b/src/test/scala/com/typesafe/sbt/packager/DeleteDirectoryVisitor.scala @@ -0,0 +1,20 @@ +package com.typesafe.sbt.packager + +import java.nio.file._ +import java.nio.file.attribute.BasicFileAttributes +import java.io.IOException + +class DeleteDirectoryVisitor extends SimpleFileVisitor[Path] { + + override def visitFile(file: Path, attrs: BasicFileAttributes) = { + Files delete file + FileVisitResult.CONTINUE + } + + override def postVisitDirectory(dir: Path, exc: IOException) = { + Files delete dir + FileVisitResult.CONTINUE + } + +} + diff --git a/src/test/scala/com/typesafe/sbt/packager/FileUtilSpec.scala b/src/test/scala/com/typesafe/sbt/packager/FileUtilSpec.scala new file mode 100644 index 000000000..c9b2cc033 --- /dev/null +++ b/src/test/scala/com/typesafe/sbt/packager/FileUtilSpec.scala @@ -0,0 +1,28 @@ +package com.typesafe.sbt.packager + +import org.scalatest._ +import java.nio.file.attribute.PosixFilePermission._ + +class FileUtilSpec extends FlatSpec with Matchers { + + "permissions" should "convert octal to symbolic correctly" in { + permissions convert "0000" should be("---------") + permissions convert "0600" should be("rw-------") + permissions convert "0755" should be("rwxr-xr-x") + permissions convert "0777" should be("rwxrwxrwx") + } + + it should "generate valid java PosixFilePermission" in { + permissions("0000") should be(empty) + + val perm1 = permissions("0600") + perm1 should not be (empty) + perm1 should contain only (OWNER_READ, OWNER_WRITE) + + val perm2 = permissions("0755") + perm2 should not be (empty) + perm2 should contain only (OWNER_READ, OWNER_WRITE, OWNER_EXECUTE, GROUP_READ, GROUP_EXECUTE, OTHERS_READ, OTHERS_EXECUTE) + + } + +} \ No newline at end of file diff --git a/src/test/scala/com/typesafe/sbt/packager/universal/ZipHelperSpec.scala b/src/test/scala/com/typesafe/sbt/packager/universal/ZipHelperSpec.scala new file mode 100644 index 000000000..4212431ba --- /dev/null +++ b/src/test/scala/com/typesafe/sbt/packager/universal/ZipHelperSpec.scala @@ -0,0 +1,109 @@ +package com.typesafe.sbt.packager.universal + +import com.typesafe.sbt.packager.DeleteDirectoryVisitor +import com.typesafe.sbt.packager.permissions +import org.scalatest._ +import java.nio.file.{ Path, Paths, Files } +import java.nio.file.attribute.PosixFilePermission._ + +class ZipHelperSpec extends FlatSpec with Matchers with BeforeAndAfterAll { + + var tmp: Path = _ + + override def beforeAll { + tmp = Files createTempDirectory "_sbt-native-packager" + } + + override def afterAll { + Files.walkFileTree(tmp, new DeleteDirectoryVisitor) + } + + "The zip helper" should "create a zip with a single file" in { + // setup + val out = tmp resolve "single.zip" + val file = tmp resolve "single.txt" + Files createFile file + + ZipHelper.zip(List(file.toFile -> "single.txt"), out.toFile) + Files exists out should be(true) + + ZipHelper.withZipFilesystem(out.toFile) { system => + val zippedFile = system getPath "single.txt" + Files exists zippedFile should be(true) + } + } + + it should "create a zip with nested directories" in { + // setup + val out = tmp resolve "single.zip" + val dir = tmp resolve "dir" + val nested = dir resolve "nested" + Files createDirectories nested + + ZipHelper.zip(List(nested.toFile -> "dir/nested"), out.toFile) + + ZipHelper.withZipFilesystem(out.toFile) { system => + val zDir = system getPath "dir" + Files exists zDir should be(true) + Files isDirectory zDir should be(true) + + val zNested = zDir resolve "nested" + Files exists zNested should be(true) + Files isDirectory zNested should be(true) + } + } + + it should "create a zip with nested directories containing file" in { + // setup + val out = tmp resolve "single.zip" + val dir = tmp resolve "dir" + val file = dir resolve "file.txt" + Files createDirectories dir + Files createFile file + + ZipHelper.zip(List(file.toFile -> "dir/file.txt"), out.toFile) + + ZipHelper.withZipFilesystem(out.toFile) { system => + val zDir = system getPath "dir" + Files exists zDir should be(true) + Files isDirectory zDir should be(true) + + val zFile = zDir resolve "file.txt" + Files exists zFile should be(true) + Files isDirectory zFile should be(false) + } + } + + /* + * This is currently not possible. + */ + it should "preserve the executable bit" ignore { + // setup + val out = tmp resolve "exec.zip" + val exec = tmp resolve "exec" + Files createFile exec + Files.setPosixFilePermissions(exec, permissions("0755")) + + val perms = Files getPosixFilePermissions exec + perms should contain only (OWNER_READ, OWNER_WRITE, OWNER_EXECUTE, GROUP_READ, GROUP_EXECUTE, OTHERS_READ, OTHERS_EXECUTE) + + ZipHelper.zip(List(exec.toFile -> "exec"), out.toFile) + Files exists out should be(true) + + Files.copy(out, Paths get "/home/muki/test.zip") + + val unzipped = tmp resolve "unzipped-exec" + ZipHelper.withZipFilesystem(out.toFile) { system => + val zippedFile = system getPath "exec" + Files exists zippedFile should be(true) + + Files.copy(zippedFile, unzipped) + } + + // checking permissions + val unzippedPerms = Files getPosixFilePermissions unzipped + unzippedPerms should contain only (OWNER_READ, OWNER_WRITE, OWNER_EXECUTE, GROUP_READ, GROUP_EXECUTE, OTHERS_READ, OTHERS_EXECUTE) + + } + +} \ No newline at end of file