update-webkit-dependency

#!/usr/bin/env perl

# Copyright (C) 2005, 2006, 2007 Apple Inc.  All rights reserved.
# Copyright (C) 2011 Carl Lobo.  All rights reserved.
# Copyright (C) 2016 Jeremy Zerfas.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 3.  Neither the name of Apple Inc. ("Apple") nor the names of
#     its contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Updates a dependency for the development environment.

use strict;
use warnings;

use Archive::Zip qw( :ERROR_CODES );
use File::Copy;
use File::Find;
use File::Spec;
use File::Temp ();
use FindBin;
use HTTP::Date qw(str2time time2str);
use HTTP::Request;
use LWP::Simple;
use LWP::UserAgent;
use POSIX;
use lib $FindBin::Bin;
use webkitdirs;

if ($#ARGV != 1) {
    die <<EOF;
Usage:
        update-webkit-dependancy <URL with the dependancy zip file> <*prefix dir inside zip without filename>

        * If filename is requirements.zip and the contents of the zip file are "requirements/x" then prefix = "."
        * If filename is xyz.zip and the contents of the zip file are xyz/abc/x" then prefix = "abc"
        * x is lib or include or bin.
EOF
}

sub getLibraryName($);

# Time in seconds that the new zip file must be newer than the old for us to
# consider them to be different. If the difference in modification time is less
# than this threshold, we assume that the files are the same. We need this
# because the zip file is served from a set of mirrors with slightly different
# Last-Modified times.
my $newnessThreshold = 30;

my $libsURL = shift;
my $prefixInZip = shift;
my $sourceDir = sourceDir();
my $file = getLibraryName($libsURL);
my $zipFile = "$file.zip";
my $webkitLibrariesDir = $ENV{'WEBKIT_LIBRARIES'} || File::Spec->catdir($sourceDir, "WebKitLibraries", "win");
my $tmpRelativeDir = File::Temp::tempdir("webkitlibsXXXXXXX", TMPDIR => 1, CLEANUP => 1);
my $tmpAbsDir = File::Spec->rel2abs($tmpRelativeDir);
my $ua = LWP::UserAgent->new();
$ua->env_proxy;

print "Checking for newer version of $zipFile...\n";

# Ideally we could just use a previous modification time and/or ETag to do a single conditional request for the file,
# however there are some problems with this approach. Some servers may not support conditional requests (Dropbox and
# GitHub don't currently support conditional requests using If-Modified-Since headers plus they also don't emit
# Last-Modified headers either). If mirrors are being used, some servers may have different ETags for the same file.
# Some servers may not implement conditional requests properly. Etc...
#
# Instead we will assume that we need to download a new version of the file unless we can be reasonably certain that we
# already previously got an up to date copy of the file. We can do this by making a quick, small request to check for
# the presence of Last-Modified and/or ETag headers (or alternatively a separate *.headers file that contains a
# Last-Modified header) and see if they match values that we may have gotten previously.

# It would be nice to be able to just use a HEAD request for this initial check but developer.apple.com returns 401
# errors for those requests. Instead we can just set a size limit for a GET request (which we can also set the range for
# to request just the first byte) so that we don't have to retrieve the entire file.
$ua->max_size(0);
my $response = $ua->get($libsURL, Accept => '*/*', Range => "bytes=0-0");
$ua->max_size(undef);

if (! $response->is_success) {
    print STDERR "Could not access $libsURL:\n";
    print STDERR $response->message, "\n";
    print STDERR $response->headers_as_string, "\n";
    print STDERR "Please ensure that $libsURL is reachable";
    if ($libsURL =~ /^https/i) {
        print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
        print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
    }
    print STDERR ".\n";

    if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
        print STDERR "Falling back to existing version of $file.\n";
        exit 0;
    } else {
        print STDERR "No existing version of $file to fall back to.\n";
        exit 1;
    }
}

my $contentType = $response->header('Content-Type');
# Try to determine the file size based on the Content-Range or Content-Length headers. Normally we will need to use the
# Content-Range header since we are just requesting a range but if the server doesn't support ranges then we will try
# using the Content-Length header instead. Also note that neither header is affected by using max_size() to limit the
# response size and the Content-Length header also shouldn't be affected from the use of HTTP compression since LWP
# doesn't use it by default.
my $contentLength = $response->header('Content-Length');
# If the Content-Range header is defined and has a number after the slash, then that's the file size we want.
if (defined $response->header('Content-Range') && $response->header('Content-Range') =~ /^.+\/(\d+)/) {
    $contentLength=$1;
}
my $lastModified = $response->header('Last-Modified');
my $etag = $response->header('ETag');

print "Located a file";
print " of type $contentType" if defined $contentType;
print " of size $contentLength" if defined $contentLength;
print ".\n";

# Get any old headers that are available.
my $oldLastModified;
my $oldETag;
if (open OLDHEADERSFILE, "<", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
    local $/ = undef;
    my $oldHeaders = <OLDHEADERSFILE>;
    close OLDHEADERSFILE;

    ($oldLastModified) = $oldHeaders =~ /^Last-Modified: (.+)$/mi;
    ($oldETag) = $oldHeaders =~ /^ETag: (.+)$/mi;
}

# If we have matching old and new ETags, then that is a very good indication that we already had gotten an up to date
# copy of the file previously. On the other hand if we have ETags that don't match, that doesn't necessarily indicate
# that the files are different since different servers may compute different ETags for the same file. If they don't
# match we will continue to see if we can skip downloading the file via other tests.
if (defined $etag && defined $oldETag && $etag eq $oldETag) {
    print "Current $file is up to date.\n";
    exit 0;
}

# If on the initial request we didn't get a Last-Modified header, then we will try getting the Last-Modified header from
# a separate *.headers file that may have been put on the server.
if (! defined $lastModified) {
    my $headerURL = $libsURL;
    $headerURL =~ s/\.zip$/\.headers/;

    $response = $ua->get($headerURL);

    if (! $response->is_success) {
        print STDERR "Could not access $headerURL:\n";
        print STDERR $response->message, "\n";
        print STDERR $response->headers_as_string, "\n";
        print STDERR "Please ensure that $headerURL is reachable";
        if ($headerURL =~ /^https/i) {
            print STDERR " and that Perl can use LWP::Simple to connect to HTTPS URLs.\n";
            print STDERR "You may have to run \"\$ cpan LWP::Protocol::https\"";
        }
        print STDERR ".\n";
    }

    ($lastModified) = $response->content =~ /^Last-Modified: (.+)$/mi;
}

my $lastModifiedTime = str2time($lastModified);
my $oldLastModifiedTime = str2time($oldLastModified);
if (defined $lastModifiedTime && defined $oldLastModifiedTime
    && abs($lastModifiedTime-$oldLastModifiedTime) < $newnessThreshold) {
    print "Current $file is up to date.\n";
    exit 0;
}

print "Downloading $zipFile...\n\n";
print "$libsURL\n";
my $request = HTTP::Request->new(GET => $libsURL);
$request->header(Accept => "*/*");
$response = $ua->request($request, File::Spec->catfile($tmpAbsDir, $zipFile));
die "Couldn't download $zipFile!" if is_error($response->code);

my $zip = Archive::Zip->new(File::Spec->catfile($tmpAbsDir, $zipFile));

# Make sure the zip file contains a directory with the same name as the zip file (minus the extension) and a prefix
# directory if applicable.
my $prefixDirectoryPathInZipFile = "$file/".(($prefixInZip eq ".") ? "" : "$prefixInZip/");
if (! $zip->memberNamed($prefixDirectoryPathInZipFile)) {
    print STDERR "Couldn't find $prefixDirectoryPathInZipFile directory in zip file.\n";

    if (-f File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
        print STDERR "Falling back to existing version of $file.\n";
        exit 0;
    } else {
        print STDERR "No existing version of $file to fall back to.\n";
        exit 1;
    }
}

my $result = $zip->extractTree("", $tmpAbsDir);
die "Couldn't unzip $zipFile." if $result != AZ_OK;

print "\nInstalling $file...\n";

sub wanted
{
    my $relativeName = File::Spec->abs2rel($File::Find::name, File::Spec->catdir($tmpAbsDir, $file, $prefixInZip));
    my $destination = File::Spec->catfile($webkitLibrariesDir, $relativeName);

    if (-d $_) {
        mkdir $destination;
        return;
    }

    copy($_, $destination);
}

File::Find::find(\&wanted, File::Spec->catfile($tmpAbsDir, $file));

if (open HEADERSFILE, ">", File::Spec->catfile($webkitLibrariesDir, "$file.headers")) {
    print HEADERSFILE "Last-Modified: $lastModified\n" if defined $lastModified;
    print HEADERSFILE "ETag: $etag\n" if defined $etag;
    close HEADERSFILE;
} else {
    print STDERR "Couldn't write $file.headers to $webkitLibrariesDir.\n"
}

print "The $file has been sucessfully installed in\n $webkitLibrariesDir\n";
exit;

sub getLibraryName($)
{
    my $url = shift;
    $url =~ m#/([^/]+)\.zip$#;
    return $1;
}