From f105badf536b948f801fb53c2bb7d47b6d1fbcf9 Mon Sep 17 00:00:00 2001 From: Andrii Nikitin Date: Thu, 16 Nov 2023 15:06:15 +0100 Subject: [PATCH] Adjust switch Digest::Zsync to separate package This also drops the need for Inline-C and gcc on backstage zsync hash calculation --- lib/Digest/Zsync.pm | 86 ------------------- lib/MirrorCache/Task/FolderHashesCreate.pm | 2 +- t/lib/Dockerfile.environ.mariadb | 2 +- t/lib/Dockerfile.environ.mariadb.experimental | 2 +- t/lib/Dockerfile.environ.postgresql | 2 +- 5 files changed, 4 insertions(+), 90 deletions(-) delete mode 100644 lib/Digest/Zsync.pm diff --git a/lib/Digest/Zsync.pm b/lib/Digest/Zsync.pm deleted file mode 100644 index 846f25c9..00000000 --- a/lib/Digest/Zsync.pm +++ /dev/null @@ -1,86 +0,0 @@ -package Digest::Zsync; - -use Mojo::Base -base, -signatures; -use Digest::MD4; -use Data::Dumper; -use bytes; - -has 'block_size' => 2048; -has seq_matches => 1; -has rsum_len => 2; -has checksum_len => 3; -has 'hashes'; - -sub digest ($self) { - my $arr = $self->hashes; - my @arr = @$arr; - join '', @arr; -} - -sub init ($self, $size) { - $self->block_size(4096) if $size > 100000000; # default behavior of zsyncmake - $self->block_size(2*4096) if $size > 1024*1024*1024; - $self->block_size(4*4096) if $size > 1024*1024*1024*16; - - $self->seq_matches(2) if $size >= $self->block_size; - my $rsum_len = int(0.99 + ((log($size // 1) + log($self->block_size)) / log(2) - 8.6) / $self->seq_matches / 8); - $rsum_len = 4 if $rsum_len > 4; - $rsum_len = 2 if $rsum_len < 2; - $self->rsum_len($rsum_len); - my $checksum_len = int(0.99 + - (20 + (log($size // 1) + log(1 + $size / $self->block_size)) / log(2)) - / $self->seq_matches / 8); - - my $checksum_len2 = int((7.9 + (20 + log(1 + $size / $self->block_size) / log(2))) / 8); - $checksum_len = $checksum_len2 if $checksum_len < $checksum_len2; - $self->checksum_len($checksum_len); - - my @hashes; - $self->hashes(\@hashes); - - return $self; -} - -sub add($self, $data) { - my $zhashes = $self->hashes; - my $block_size = $self->block_size; - use bytes; - while (length($data)) { - (my $block, $data) = unpack("a${block_size}a*", $data); - my $diff = $self->block_size - length($block); - $block .= (chr(0) x $diff) if $diff; - push @$zhashes, zsync_rsum06($block, $block_size, $self->rsum_len); - push @$zhashes, substr(Digest::MD4::md4($block),0,$self->checksum_len); - } - no bytes; - return $self; -} - -sub lengths($self) { - return $self->seq_matches . ',' . $self->rsum_len . ',' . $self->checksum_len; -} - -use Inline C => <<'EOC'; -struct rsum { - unsigned short a; - unsigned short b; -} __attribute__((packed)); - -SV* zsync_rsum06(char* data, size_t len, size_t x) { - register unsigned short a = 0; - register unsigned short b = 0; - while (len) { - unsigned char c = *data++; - a += c; - b += len * c; - len--; - } - struct rsum r = { htons(a), htons(b) }; - char* buffer = (char*)&r; - - return newSVpv(buffer + 4-x, x); -} -EOC - - -1; diff --git a/lib/MirrorCache/Task/FolderHashesCreate.pm b/lib/MirrorCache/Task/FolderHashesCreate.pm index 3ff1230b..b5221ec0 100644 --- a/lib/MirrorCache/Task/FolderHashesCreate.pm +++ b/lib/MirrorCache/Task/FolderHashesCreate.pm @@ -126,7 +126,7 @@ sub calcMetalink { my ($zsync_lengths, $zsync_block_size, $zsync_hashes); if ($zsync) { $zsync_lengths = $zsync->lengths; - $zsync_block_size = $zsync->block_size; + $zsync_block_size = $zsync->{block_size}; $zsync_hashes = $zsync->digest; } $schema->resultset('Hash')->store($file_id, $mtime, $size, $dmd5->hexdigest, $d1->hexdigest, $d256->hexdigest, $d512->hexdigest, $block_size, $pieceshex, $zsync_lengths, $zsync_block_size, $zsync_hashes); diff --git a/t/lib/Dockerfile.environ.mariadb b/t/lib/Dockerfile.environ.mariadb index 365350cd..24bb0baf 100644 --- a/t/lib/Dockerfile.environ.mariadb +++ b/t/lib/Dockerfile.environ.mariadb @@ -15,7 +15,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-P perl-Config-IniFiles # optional dependencies used in testing -RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc perl-DateTime-Format-MySQL libxml2-tools +RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools # this hack is needed because old nginx versions cannot run as non-root RUN bbe -e 's,/var/log/nginx/error.log,/tmp/log_nginx_error.log,' /usr/sbin/nginx > /usr/sbin/nginx.hacked diff --git a/t/lib/Dockerfile.environ.mariadb.experimental b/t/lib/Dockerfile.environ.mariadb.experimental index 40183e08..862c46b6 100644 --- a/t/lib/Dockerfile.environ.mariadb.experimental +++ b/t/lib/Dockerfile.environ.mariadb.experimental @@ -17,7 +17,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-P perl-Config-IniFiles # optional dependencies used in testing -RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc perl-DateTime-Format-MySQL libxml2-tools +RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools RUN zypper -vvv -n install MariaDB-server-compat MariaDB-client-compat diff --git a/t/lib/Dockerfile.environ.postgresql b/t/lib/Dockerfile.environ.postgresql index 00381acb..3e1735f6 100644 --- a/t/lib/Dockerfile.environ.postgresql +++ b/t/lib/Dockerfile.environ.postgresql @@ -13,7 +13,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-P apache2 perl-Digest-MD4 tidy nginx bbe perl-DateTime-HiRes perl-Config-IniFiles # optional dependencies used in testing -RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc libxml2-tools +RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync libxml2-tools # this hack is needed because old nginx versions cannot run as non-root RUN bbe -e 's,/var/log/nginx/error.log,/tmp/log_nginx_error.log,' /usr/sbin/nginx > /usr/sbin/nginx.hacked