From 6419fa43474c95eca59d1cea5fbdbfc8b6715eb3 Mon Sep 17 00:00:00 2001 From: St ef Date: Thu, 10 Jun 2021 21:54:39 +0200 Subject: [PATCH 1/4] "Solves https://github.com/composer/mirror/issues/10" --- mirror.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mirror.php b/mirror.php index 6b4a2cc..f5fd58b 100755 --- a/mirror.php +++ b/mirror.php @@ -147,7 +147,7 @@ public function syncV2() $this->delete($action['package']); } } - + sleep(1); $result = $this->downloadV2Files($requests); if (!$result) { return false; @@ -270,15 +270,16 @@ private function downloadV2Files(array $requests) // got an outdated file, possibly fetched from a mirror which was not yet up to date, so retry after 2sec if ($is404 || $mtime < $userData['minimumFilemtime']) { - if ($userData['retries'] > 2) { - // 404s after 3 retries should be deemed to have really been deleted, so we stop retrying + sleep($userData['retries']); + if ($userData['retries'] > 10) { + // 404s after 10 retries should be deemed to have really been deleted, so we stop retrying if ($is404) { return false; } throw new \Exception('Too many retries, could not update '.$userData['path'].' as the origin server returns an older file ('.$mtime.', expected '.$userData['minimumFilemtime'].')'); } $hasRetries = true; - $this->output('R'); + $this->output('R['.$userData['retries'].']'); $this->statsdIncrement('mirror.retry_provider_v2'); $userData['retries']++; $headers = file_exists($this->target.$userData['path'].'.gz') ? ['If-Modified-Since' => gmdate('D, d M Y H:i:s T', filemtime($this->target.$userData['path'].'.gz'))] : []; From 1066162847044e966981fe56b4156e34a7e0fd64 Mon Sep 17 00:00:00 2001 From: St ef Date: Thu, 10 Jun 2021 22:03:54 +0200 Subject: [PATCH 2/4] Workaround for https://github.com/composer/mirror/issues/10 --- mirror.php | 1 - 1 file changed, 1 deletion(-) diff --git a/mirror.php b/mirror.php index f5fd58b..44686fa 100755 --- a/mirror.php +++ b/mirror.php @@ -147,7 +147,6 @@ public function syncV2() $this->delete($action['package']); } } - sleep(1); $result = $this->downloadV2Files($requests); if (!$result) { return false; From b55e52f7e9cf5799077e0500483a17d7bbe99803 Mon Sep 17 00:00:00 2001 From: St ef Date: Sat, 12 Jun 2021 13:57:36 +0200 Subject: [PATCH 3/4] adds verbosity --- mirror.php | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/mirror.php b/mirror.php index 44686fa..a068845 100755 --- a/mirror.php +++ b/mirror.php @@ -28,6 +28,18 @@ throw new \ErrorException($errstr, $errno, E_ERROR, $errfile, $errline); }); +Class DateTimeExt extends DateTime { + + public function __construct(string $timestamp) { + parent::__construct(); + $this->createFromFormat('u', $timestamp); + } + + public function __toString() { + return $this->format('Y-m-d H:i'); + } +} + class Mirror { private $target; private $context; @@ -78,12 +90,13 @@ public function syncRootOnV2() $hash = hash('sha256', $rootData); if ($hash === $this->getHash('/packages.json')) { + $this->output('No work - /packages.json hash unchanged' . PHP_EOL); return; } $gzipped = gzencode($rootData, 8); $this->write('/packages.json', $rootData, $gzipped, strtotime($rootResp->getHeaders()['last-modified'][0])); - $this->output('X'); + $this->output(' X(P) '); $this->statsdIncrement('mirror.sync_root'); } @@ -95,6 +108,7 @@ public function getV2Timestamp(): int $resp = $this->client->request('GET', $this->apiUrl.'/metadata/changes.json', ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); $content = json_decode($resp->getContent(false), true); if ($resp->getStatusCode() === 400 && null !== $content) { + $this->output('API Timestamp is '. new DateTimeExt($content['timestamp']).PHP_EOL); return $content['timestamp']; } throw new \Exception('Failed to fetch timestamp from API, got invalid response '.$resp->getStatusCode().': '.$resp->getContent()); @@ -117,7 +131,7 @@ public function syncV2() throw new \UnexpectedValueException('Cannot save last timestamp to last_metadata_timestamp in '.getcwd().'. Make sure the file is writable.'); } $lastTime = trim(file_get_contents($this->getTimestampStorePath())); - + $this->out('Last update was on '.new DateTimeExt($lastTime)); $changesResp = $this->client->request('GET', $this->apiUrl.'/metadata/changes.json?since='.$lastTime, ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); if ($changesResp->getHeaders()['content-encoding'][0] !== 'gzip') { throw new \Exception('Expected gzip encoded responses, something is off'); @@ -125,7 +139,7 @@ public function syncV2() $changes = json_decode($changesResp->getContent(), true); if ([] === $changes['actions']) { - $this->output('No work' . PHP_EOL); + $this->output('No changes since '. new DateTimeExt($lastTime) . PHP_EOL); $this->writeLastTimestamp($changes['timestamp']); return true; } @@ -161,7 +175,7 @@ public function syncV2() public function resync(int $timestamp) { - $this->output('Resync requested'.PHP_EOL); + $this->output('Resync requested from '. new DateTimeExt($timestamp) .PHP_EOL); $listingResp = $this->client->request('GET', $this->apiUrl.'/packages/list.json?'.md5(uniqid()), ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); if ($listingResp->getHeaders()['content-encoding'][0] !== 'gzip') { @@ -221,7 +235,7 @@ public function resync(int $timestamp) $appendRequest('/p2/'.$pkg.'.json'); $appendRequest('/p2/'.$pkg.'~dev.json'); } - + $result = $this->downloadV2Files($requests); if (!$result) { return false; @@ -426,7 +440,7 @@ public function sync() $hash = hash('sha256', $rootData); if ($hash === $this->getHash('/packages.json')) { - $this->output('No work' . PHP_EOL); + $this->output('No work - /packages.json hash unchanged' . PHP_EOL); return true; } @@ -530,7 +544,7 @@ public function sync() $gzipped = gzencode($rootData, 8); $this->write('/packages.json', $rootData, $gzipped, strtotime($rootResp->getHeaders()['last-modified'][0])); - $this->output('X'); + $this->output(' X(P) '); $this->statsdIncrement('mirror.sync_root'); $this->output(PHP_EOL); @@ -551,6 +565,7 @@ public function gc() $rootFile = $this->target.'/packages.json.gz'; if (!file_exists($rootFile)) { + $this->output($rootFile.' doesn\'t exists' . PHP_EOL); return; } $rootJson = json_decode(gzdecode(file_get_contents($rootFile)), true); From cd1f58d01f91f308784793604cca36952c5c9070 Mon Sep 17 00:00:00 2001 From: St ef Date: Tue, 15 Jun 2021 07:52:14 +0200 Subject: [PATCH 4/4] typo error on call to output() --- mirror.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror.php b/mirror.php index a068845..57c6b46 100755 --- a/mirror.php +++ b/mirror.php @@ -131,7 +131,7 @@ public function syncV2() throw new \UnexpectedValueException('Cannot save last timestamp to last_metadata_timestamp in '.getcwd().'. Make sure the file is writable.'); } $lastTime = trim(file_get_contents($this->getTimestampStorePath())); - $this->out('Last update was on '.new DateTimeExt($lastTime)); + $this->output('Last update was on '.new DateTimeExt($lastTime).PHP_EOL); $changesResp = $this->client->request('GET', $this->apiUrl.'/metadata/changes.json?since='.$lastTime, ['headers' => ['Host' => parse_url($this->apiUrl, PHP_URL_HOST)]]); if ($changesResp->getHeaders()['content-encoding'][0] !== 'gzip') { throw new \Exception('Expected gzip encoded responses, something is off');