Skip to content

Commit

Permalink
Merge pull request #19 from xaviered/xaviered/optimize_memory_usage
Browse files Browse the repository at this point in the history 
Use the least amount of memory as possible.
  • Loading branch information
@xaviered
xaviered authored Sep 9, 2019
2 parents bb5052d + 3e7a37c commit e1160af
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 24 deletions.
39 changes: 24 additions & 15 deletions src/org/apache/hadoop/WebHDFS.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,10 @@ public function open($path, $offset = '', $length = '', $bufferSize = '',$localF
}

$result = $this->curl->getWithRedirect($url);
if ($this->curl->validateLastRequest()) {
if ($this->curl->validateLastRequest(true)) {
return $result;
}
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult(true));
}

public function mkdirs($path, $permission = '')
Expand Down Expand Up @@ -171,31 +171,41 @@ public function truncate($path, $newLength = 0)
public function getFileStatus($path)
{
$url = $this->_buildUrl($path, array('op' => 'GETFILESTATUS'));
$r = $this->curl->get($url);
$this->curl->cleanLastRequest();

return $this->curl->get($url);
return $r;
}

public function listStatus($path)
{
return $this->_listStatus($path, true);
}

private function _listStatus($path, $cleanLastRequest = false)
{
$url = $this->_buildUrl($path, array('op' => 'LISTSTATUS'));
if ($result = $this->curl->get($url)) {
if ($cleanLastRequest) {
$this->curl->cleanLastRequest();
}
$result = json_decode($result);
if (!is_null($result)) {
return $result;
} else {
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult($cleanLastRequest));
}
} else {
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult($cleanLastRequest));
}

return false;
}

public function listFiles($path, $recursive = false, $includeFileMetaData = false, $maxAmountOfFiles = false)
{
$result = array();
$listStatusResult = $this->listStatus($path);
$listStatusResult = $this->_listStatus($path);
$r = $this->curl->getLastRequestContentResult(true);

if (isset($listStatusResult->FileStatuses->FileStatus)) {
foreach ($listStatusResult->FileStatuses->FileStatus AS $fileEntity) {
switch ($fileEntity->type) {
Expand All @@ -220,7 +230,7 @@ public function listFiles($path, $recursive = false, $includeFileMetaData = fals
}
}
} else {
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($r);
}

return $result;
Expand All @@ -229,7 +239,9 @@ public function listFiles($path, $recursive = false, $includeFileMetaData = fals
public function listDirectories($path, $recursive = false, $includeFileMetaData = false)
{
$result = array();
$listStatusResult = $this->listStatus($path);
$listStatusResult = $this->_listStatus($path);
$r = $this->curl->getLastRequestContentResult(true);

if (isset($listStatusResult->FileStatuses->FileStatus)) {
foreach ($listStatusResult->FileStatuses->FileStatus AS $fileEntity) {
switch ($fileEntity->type) {
Expand All @@ -249,7 +261,7 @@ public function listDirectories($path, $recursive = false, $includeFileMetaData
}
}
} else {
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($r);
}

return $result;
Expand All @@ -259,14 +271,13 @@ public function listDirectories($path, $recursive = false, $includeFileMetaData

public function getContentSummary($path)
{
$result = false;
$url = $this->_buildUrl($path, array('op' => 'GETCONTENTSUMMARY'));
$rawResult = $this->curl->get($url);
$resultDecoded = json_decode($rawResult);
if (isset($resultDecoded->ContentSummary)) {
$result = $resultDecoded->ContentSummary;
} else {
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult());
throw $this->getResponseErrorException($this->curl->getLastRequestContentResult(true));
}

return $result;
Expand Down Expand Up @@ -435,5 +446,3 @@ private function getResponseErrorException($responseData)
return new WebHDFS_Exception($exceptionMessage, $exceptionCode);
}
}

?>
2 changes: 0 additions & 2 deletions src/org/apache/hadoop/WebHDFS/Exception.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,3 @@ class WebHDFS_Exception extends \Exception

const LOCAL_FILE_OPEN_ERR = 4;
}

?>
36 changes: 29 additions & 7 deletions src/org/apache/hadoop/tools/Curl.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,15 @@ public function setOption($key,$value) {
$this->options[$key] = $value;
}

public function cleanLastRequest()
{
unset($this->lastRequestInfoResult);
unset($this->lastRequestContentResult);
gc_collect_cycles();

return $this;
}

public function getWithRedirect($url)
{
return $this->get($url, array(CURLOPT_FOLLOWLOCATION => true));
Expand Down Expand Up @@ -136,22 +145,35 @@ public function delete($url)
return $this->_exec($options);
}

public function getLastRequestContentResult()
public function getLastRequestContentResult($cleanLastRequest = false)
{
return $this->lastRequestContentResult;
$r = $this->lastRequestContentResult;
if ($cleanLastRequest) {
$this->cleanLastRequest();
}

return $r;
}

public function getLastRequestInfoResult()
public function getLastRequestInfoResult($cleanLastRequest = false)
{
return $this->lastRequestInfoResult;
$r = $this->lastRequestInfoResult;
if ($cleanLastRequest) {
$this->cleanLastRequest();
}

return $r;
}

public function validateLastRequest()
public function validateLastRequest($cleanLastRequestIfValid = false)
{
$http_code = $this->getLastRequestInfoResult()['http_code'];
if ($http_code >= 400 && $http_code <= 500) {
return false;
}
if ($cleanLastRequestIfValid) {
$this->cleanLastRequest();
}

return true;
}
Expand Down Expand Up @@ -180,6 +202,8 @@ private function _exec($options, $returnInfo = false)
fflush($fp);
}
curl_setopt_array($ch, $options);
// force clean memory before getting more data
$this->cleanLastRequest();
$result = curl_exec($ch);
$this->lastRequestContentResult = $result;
$this->lastRequestInfoResult = curl_getinfo($ch);
Expand All @@ -195,5 +219,3 @@ private function _exec($options, $returnInfo = false)
}

}

?>

0 comments on commit e1160af

Please sign in to comment.