Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrading to java 7 and using posix nio API #487

Merged
merged 4 commits into from
Feb 15, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ language: scala
os:
- linux
script:
- sbt ++$TRAVIS_SCALA_VERSION "test"
- sbt ++$TRAVIS_SCALA_VERSION "scripted rpm/* debian/* universal/*"
scala:
- 2.10.3
jdk:
- openjdk6
- openjdk7
- oraclejdk8
notifications:
Expand Down
3 changes: 2 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ scalacOptions in Compile ++= Seq("-deprecation", "-target:jvm-1.6")

libraryDependencies ++= Seq(
"org.apache.commons" % "commons-compress" % "1.4.1",
"org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar"))
"org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb", "jar", "jar")),
"org.scalatest" %% "scalatest" % "2.2.4" % "test"
)

site.settings
Expand Down
63 changes: 53 additions & 10 deletions src/main/scala/com/typesafe/sbt/packager/FileUtil.scala
Original file line number Diff line number Diff line change
@@ -1,18 +1,61 @@
package com.typesafe.sbt
package packager

import java.io.File
import sbt.Process
import java.io.{ File, IOException }
import java.nio.file.{ Paths, Files }
import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions }

/**
* Setting the file permissions
*/
object chmod {

/**
* Using java 7 nio API to set the permissions.
*
* @param file
* @param perms in octal format
*/
def apply(file: File, perms: String): Unit =
Process(Seq("chmod", perms, file.getAbsolutePath)).! match {
case 0 => ()
case n => sys.error("Error running chmod " + perms + " " + file)
}
def safe(file: File, perms: String): Unit =
try apply(file, perms)
catch {
case e: RuntimeException => ()
try {
Files.setPosixFilePermissions(file.toPath, permissions(perms))
} catch {
case e: IOException => sys.error("Error setting permissions " + perms + " on " + file.getAbsolutePath + ": " + e.getMessage)
}

}

/**
* Converts a octal unix permission representation into
* a java `PosiFilePermissions` compatible string.
*/
object permissions {

/**
* @param perms in octal format
* @return java 7 posix file permissions
*/
def apply(perms: String): java.util.Set[PosixFilePermission] = PosixFilePermissions fromString convert(perms)

def convert(perms: String): String = {
require(perms.length == 4 || perms.length == 3, s"Permissions must have 3 or 4 digits, got [$perms]")
// ignore setuid/setguid/sticky bit
val i = if (perms.length == 3) 0 else 1
val user = Character getNumericValue (perms charAt i)
val group = Character getNumericValue (perms charAt i + 1)
val other = Character getNumericValue (perms charAt i + 2)

asString(user) + asString(group) + asString(other)
}

private def asString(perm: Int): String = perm match {
case 0 => "---"
case 1 => "--x"
case 2 => "-w-"
case 3 => "-wx"
case 4 => "r--"
case 5 => "r-x"
case 6 => "rw-"
case 7 => "rwx"
}
}
78 changes: 63 additions & 15 deletions src/main/scala/com/typesafe/sbt/packager/universal/Archives.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,44 @@ import sbt._
/** Helper methods to package up files into compressed archives. */
object Archives {

/** Makes a zip file in the given target directory using the given name. */
def makeZip(target: File, name: String, mappings: Seq[(File, String)]): File = {
/**
* Makes a zip file in the given target directory using the given name.
*
* @param target folder to build package in
* @param name of output (without extension)
* @param mappings included in the output
* @param top level directory
* @return zip file
*/
def makeZip(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
val zip = target / (name + ".zip")
// TODO - If mappings already start with the given name, don't add it?
val m2 = mappings map { case (f, p) => f -> (name + "/" + p) }

// add top level directory if defined
val m2 = top map { dir =>
mappings map { case (f, p) => f -> (dir + "/" + p) }
} getOrElse (mappings)

ZipHelper.zip(m2, zip)
zip
}

/** Makes a zip file in the given target directory using the given name. */
def makeNativeZip(target: File, name: String, mappings: Seq[(File, String)]): File = {
/**
* Makes a zip file in the given target directory using the given name.
*
* @param target folder to build package in
* @param name of output (without extension)
* @param mappings included in the output
* @param top level directory
* @return zip file
*/
def makeNativeZip(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
val zip = target / (name + ".zip")
// TODO - If mappings already start with the given name, don't add it?
val m2 = mappings map { case (f, p) => f -> (name + "/" + p) }

// add top level directory if defined
val m2 = top map { dir =>
mappings map { case (f, p) => f -> (dir + "/" + p) }
} getOrElse (mappings)

ZipHelper.zipNative(m2, zip)
zip
}
Expand All @@ -29,8 +53,14 @@ object Archives {
* Makes a dmg file in the given target directory using the given name.
*
* Note: Only works on OSX
*
* @param target folder to build package in
* @param name of output (without extension)
* @param mappings included in the output
* @param top level directory : NOT USED
* @return dmg file
*/
def makeDmg(target: File, name: String, mappings: Seq[(File, String)]): File = {
def makeDmg(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
val t = target / "dmg"
val dmg = target / (name + ".dmg")
if (!t.isDirectory) IO.createDirectory(t)
Expand Down Expand Up @@ -113,25 +143,43 @@ object Archives {
val makeTgz = makeTarball(gzip, ".tgz") _
val makeTar = makeTarball(identity, ".tar") _

/** Helper method used to construct tar-related compression functions. */
def makeTarball(compressor: File => File, ext: String)(target: File, name: String, mappings: Seq[(File, String)]): File = {
/**
* Helper method used to construct tar-related compression functions.
* @param target folder to build package in
* @param name of output (without extension)
* @param mappings included in the output
* @param top level directory
* @return tar file
*
*/
def makeTarball(compressor: File => File, ext: String)(target: File, name: String, mappings: Seq[(File, String)], top: Option[String]): File = {
val relname = name
val tarball = target / (name + ext)
IO.withTemporaryDirectory { f =>
val rdir = f / relname
val m2 = mappings map { case (f, p) => f -> (rdir / name / p) }
val m2 = top map { dir =>
mappings map { case (f, p) => f -> (rdir / dir / p) }
} getOrElse {
mappings map { case (f, p) => f -> (rdir / p) }
}

IO.copy(m2)
// TODO - Is this enough?
for (f <- (m2 map { case (_, f) => f }); if f.getAbsolutePath contains "/bin/") {
println("Making " + f.getAbsolutePath + " executable")
f.setExecutable(true, false)
}

IO.createDirectory(tarball.getParentFile)
val distdir = IO.listFiles(rdir).headOption.getOrElse {
sys.error("Unable to find tarball in directory: " + rdir.getAbsolutePath + ".\n This could be an issue with the temporary filesystem used to create tarballs.")

// all directories that should be zipped
val distdirs = top map (_ :: Nil) getOrElse {
IO.listFiles(rdir).map(_.getName).toList // no top level dir, use all available
}

val tmptar = f / (relname + ".tar")
Process(Seq("tar", "-pcvf", tmptar.getAbsolutePath, distdir.getName), Some(rdir)).! match {

Process(Seq("tar", "-pcvf", tmptar.getAbsolutePath) ++ distdirs, Some(rdir)).! match {
case 0 => ()
case n => sys.error("Error tarballing " + tarball + ". Exit code: " + n)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ trait UniversalKeys {
val stage = TaskKey[File]("stage", "Create a local directory with all the files laid out as they would be in the final distribution.")
val dist = TaskKey[File]("dist", "Creates the distribution packages.")
val stagingDirectory = SettingKey[File]("stagingDirectory", "Directory where we stage distributions/releases.")
val topLevelDirectory = SettingKey[Option[String]]("topLevelDirectory", "Top level dir in compressed output file.")
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ object UniversalPlugin extends AutoPlugin {
name in UniversalDocs <<= name in Universal,
name in UniversalSrc <<= name in Universal,
packageName in Universal <<= packageName,
topLevelDirectory := Some((packageName in Universal).value),
executableScriptName in Universal <<= executableScriptName
) ++
makePackageSettingsForConfig(Universal) ++
Expand Down Expand Up @@ -95,12 +96,12 @@ object UniversalPlugin extends AutoPlugin {
dist
}

private type Packager = (File, String, Seq[(File, String)]) => File
private type Packager = (File, String, Seq[(File, String)], Option[String]) => File
/** Creates packaging settings for a given package key, configuration + archive type. */
private[this] def makePackageSettings(packageKey: TaskKey[File], config: Configuration)(packager: Packager): Seq[Setting[_]] =
inConfig(config)(Seq(
mappings in packageKey <<= mappings map checkMappings,
packageKey <<= (target, packageName, mappings in packageKey) map packager
packageKey <<= (target, packageName, mappings in packageKey, topLevelDirectory) map packager
))

/** check that all mapped files actually exist */
Expand Down
98 changes: 77 additions & 21 deletions src/main/scala/com/typesafe/sbt/packager/universal/ZipHelper.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,20 @@ import org.apache.commons.compress.compressors.{
}
import java.util.zip.Deflater
import org.apache.commons.compress.utils.IOUtils
import java.nio.file.{ Paths, Files, FileSystems, FileSystem, StandardCopyOption }
import java.nio.file.attribute.{ PosixFilePermission, PosixFilePermissions }
import java.net.URI
import scala.collection.JavaConverters._

/**
*
*
*
* @see http://stackoverflow.com/questions/17888365/file-permissions-are-not-being-preserved-while-after-zip
* @see http://stackoverflow.com/questions/3450250/is-it-possible-to-create-a-script-to-save-and-restore-permissions
* @see http://stackoverflow.com/questions/1050560/maintain-file-permissions-when-extracting-from-a-zip-file-using-jdk-5-api
* @see http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/zipfilesystemprovider.html
*/
object ZipHelper {
case class FileMapping(file: File, name: String, unixMode: Option[Int] = None)

Expand Down Expand Up @@ -41,10 +54,12 @@ object ZipHelper {
}

/**
* Creates a zip file attempting to give files the appropriate unix permissions using Java 6 APIs.
* Creates a zip file with the apache commons compressor library.
*
* Note: This is known to have some odd issues on MacOSX whereby executable permissions
* are not actually discovered, even though the Info-Zip headers exist and work on
* many variants of linux. Yay Apple.
*
* @param sources The files to include in the zip file.
* @param outputZip The location of the output file.
*/
Expand All @@ -59,29 +74,32 @@ object ZipHelper {
}

/**
* Creates a zip file using the given set of filters
* @param sources The files to include in the zip file. A File, Location, Permission pairing.
* Creates a zip file attempting to give files the appropriate unix permissions using Java 7 APIs.
*
* @param sources The files to include in the zip file.
* @param outputZip The location of the output file.
*/
def zipWithPerms(sources: Traversable[(File, String, Int)], outputZip: File): Unit = {
val mappings =
for {
(file, name, perm) <- sources
} yield FileMapping(file, name, Some(perm))
archive(mappings.toSeq, outputZip)
}
def zipNIO(sources: Traversable[(File, String)], outputZip: File): Unit = {
require(!outputZip.isDirectory, "Specified output file " + outputZip + " is a directory.")
val mappings = sources.toSeq.map {
case (file, name) => FileMapping(file, name)
}

/**
* Replaces windows backslash file separator with a forward slash, this ensures the zip file entry is correct for
* any system it is extracted on.
* @param path The path of the file in the zip file
*/
private def normalizePath(path: String) = {
val sep = java.io.File.separatorChar
if (sep == '/')
path
else
path.replace(sep, '/')
// make sure everything is available
val outputDir = outputZip.getParentFile
IO createDirectory outputDir

// zipping the sources into the output zip
withZipFilesystem(outputZip) { system =>
mappings foreach {
case FileMapping(dir, name, _) if dir.isDirectory => Files createDirectories (system getPath name)
case FileMapping(file, name, _) =>
val dest = system getPath name
// create parent directories if available
Option(dest.getParent) foreach (Files createDirectories _)
Files copy (file.toPath, dest, StandardCopyOption.COPY_ATTRIBUTES)
}
}
}

private def archive(sources: Seq[FileMapping], outputFile: File): Unit = {
Expand All @@ -103,6 +121,9 @@ object ZipHelper {
}
}

/**
* using apache commons compress
*/
private def withZipOutput(file: File)(f: ZipArchiveOutputStream => Unit): Unit = {
val zipOut = new ZipArchiveOutputStream(file)
zipOut setLevel Deflater.BEST_COMPRESSION
Expand All @@ -111,4 +132,39 @@ object ZipHelper {
zipOut.close()
}
}

/**
* Replaces windows backslash file separator with a forward slash, this ensures the zip file entry is correct for
* any system it is extracted on.
* @param path The path of the file in the zip file
*/
private def normalizePath(path: String) = {
val sep = java.io.File.separatorChar
if (sep == '/')
path
else
path.replace(sep, '/')
}

/**
* Opens a zip filesystem and creates the file if necessary.
*
* Note: This will override an existing zipFile if existent!
*
* @param zipFile
* @param f: FileSystem => Unit, logic working in the filesystem
*/
def withZipFilesystem(zipFile: File, overwrite: Boolean = true)(f: FileSystem => Unit) {
if (overwrite) Files deleteIfExists zipFile.toPath
val env = Map("create" -> "true").asJava
val uri = URI.create("jar:file:" + zipFile.getAbsolutePath)

val system = FileSystems.newFileSystem(uri, env)
try {
f(system)
} finally {
system.close()
}
}

}
7 changes: 7 additions & 0 deletions src/sbt-test/universal/test-zips-no-top-level-dir/build.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
enablePlugins(JavaAppPackaging)

name := "simple-test"

version := "0.1.0"

topLevelDirectory := None
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Test configuration to include in zips.
Loading