dumpyara: Initial release

Change-Id: I869977019333cf23937e6d8bff1efd555d262fde

Author: SebaUbuntu

.gitignore

@@ -0,0 +1,92 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+roject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# editors
+.idea/
+.vscode/
+
+# Don't ignore library folder
+!/dumpyara/lib/
+
+# Ignore output folder
+working/

README.md

@@ -0,0 +1,42 @@
+# Dumpyara
+
+## Installation
+
+```
+pip3 install dumpyara
+```
+The module is supported on Python 3.8 and above.
+
+## How to use
+
+```
+$ python3 -m dumpyara -h
+Dumpyara
+Version 1.0.0
+
+usage: python3 -m dumpyara [-h] [-o OUTPUT] [-v] file
+
+positional arguments:
+  file                  path to a device OTA
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -o OUTPUT, --output OUTPUT
+                        custom output folder
+  -v, --verbose         enable debugging logging
+```
+
+## Supported formats
+
+### Archives
+- All the ones supported by shutil's extract_archive
+
+### Images
+- Raw images
+- Brotli compressed images
+- sdat images
+- Sparsed images
+
+## Credits
+- lpunpack: unix3dgforce
+- sdat2img: xpirt

dumpyara/__init__.py

@@ -0,0 +1,13 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+import os
+from pathlib import Path
+
+__version__ = "1.0.0"
+
+module_path = Path(__file__).parent
+current_path = Path(os.getcwd())

dumpyara/__main__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from dumpyara.main import main
+
+if __name__ == '__main__':
+	main()

dumpyara/dumpyara.py

@@ -0,0 +1,85 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from dumpyara.lib.liblogging import LOGD, LOGI
+from dumpyara.lib.liblpunpack import LpUnpack
+from dumpyara.utils.partitions import can_be_partition, extract_partition
+import fnmatch
+from os import walk
+from pathlib import Path
+from shutil import unpack_archive
+from tempfile import TemporaryDirectory
+
+class Dumpyara:
+	"""
+	A class representing an Android dump
+	"""
+	def __init__(self, file: Path, output_path: Path) -> None:
+		"""Initialize dumpyara class."""
+		self.file = file
+		self.output_path = output_path
+
+		# Create working folder dir
+		self.output_path.mkdir(exist_ok=True, parents=True)
+
+		# Create a temporary directory where we will extract the images
+		self.tempdir = TemporaryDirectory()
+		self.tempdir_path = Path(self.tempdir.name)
+
+		# Output folder
+		self.path = self.output_path / self.file.stem
+		self.path.mkdir()
+
+		self.fileslist = []
+
+		LOGI("Extracting package...")
+		unpack_archive(self.file, self.tempdir_path)
+		self.update_tempdir_files_list()
+		LOGD(f"All files in package: {', '.join(self.fileslist)}")
+
+		# Extract super first if it exists
+		# It contains all the partitions that we are interested in
+		super_match = fnmatch.filter(self.fileslist, "*super.img*")
+		if super_match:
+			LOGI("Super partition detected, first extracting it")
+			LpUnpack(SUPER_IMAGE=self.tempdir_path / super_match[0], OUTPUT_DIR=self.tempdir_path).unpack()
+			self.update_tempdir_files_list()
+
+		# Process all files
+		for file in [self.tempdir_path / file for file in self.fileslist]:
+			# Might have been deleted by previous step
+			if not file.is_file():
+				continue
+
+			if can_be_partition(file):
+				extract_partition(file, self.path)
+			else:
+				LOGI(f"Skipping {file}")
+
+		# We don't need artifacts anymore
+		self.tempdir.cleanup()
+
+		# Create all_files.txt
+		LOGI("Creating all_files.txt")
+		with open(self.path / "all_files.txt", "w") as f:
+			f.write("\n".join(self.get_recursive_files_list(self.path, relative=True)))
+
+	@staticmethod
+	def get_recursive_files_list(path: Path, relative=False):
+		files_list = []
+
+		for currentpath, _, files in walk(path):
+			for file in files:
+				file_path = Path(currentpath) / file
+				if relative:
+					file_path = file_path.relative_to(path)
+				files_list.append(str(file_path))
+
+		return files_list
+
+	def update_tempdir_files_list(self):
+		self.fileslist.clear()
+		self.fileslist.extend(self.get_recursive_files_list(self.tempdir_path))

dumpyara/lib/liblogging/__init__.py

@@ -0,0 +1,13 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from logging import debug, info, warning, error, fatal
+
+LOGD = debug
+LOGI = info
+LOGW = warning
+LOGE = error
+LOGF = fatal

dumpyara/lib/liblpunpack/__init__.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#====================================================
+#          FILE: sdat2img.py
+#       AUTHORS: xpirt - luxi78 - howellzhu
+#          DATE: 2018-10-27 10:33:21 CEST
+#====================================================
+
+from __future__ import print_function
+import sys, os, errno
+
+def main(TRANSFER_LIST_FILE, NEW_DATA_FILE, OUTPUT_IMAGE_FILE):
+    __version__ = '1.2'
+
+    if sys.hexversion < 0x02070000:
+        print >> sys.stderr, "Python 2.7 or newer is required."
+        try:
+            input = raw_input
+        except NameError: pass
+        input('Press ENTER to exit...')
+        sys.exit(1)
+    else:
+        print('sdat2img binary - version: {}\n'.format(__version__))
+
+    def rangeset(src):
+        src_set = src.split(',')
+        num_set =  [int(item) for item in src_set]
+        if len(num_set) != num_set[0]+1:
+            print('Error on parsing following data to rangeset:\n{}'.format(src), file=sys.stderr)
+            sys.exit(1)
+
+        return tuple ([ (num_set[i], num_set[i+1]) for i in range(1, len(num_set), 2) ])
+
+    def parse_transfer_list_file(path):
+        trans_list = open(TRANSFER_LIST_FILE, 'r')
+
+        # First line in transfer list is the version number
+        version = int(trans_list.readline())
+
+        # Second line in transfer list is the total number of blocks we expect to write
+        new_blocks = int(trans_list.readline())
+
+        if version >= 2:
+            # Third line is how many stash entries are needed simultaneously
+            trans_list.readline()
+            # Fourth line is the maximum number of blocks that will be stashed simultaneously
+            trans_list.readline()
+
+        # Subsequent lines are all individual transfer commands
+        commands = []
+        for line in trans_list:
+            line = line.split(' ')
+            cmd = line[0]
+            if cmd in ['erase', 'new', 'zero']:
+                commands.append([cmd, rangeset(line[1])])
+            else:
+                # Skip lines starting with numbers, they are not commands anyway
+                if not cmd[0].isdigit():
+                    print('Command "{}" is not valid.'.format(cmd), file=sys.stderr)
+                    trans_list.close()
+                    sys.exit(1)
+
+        trans_list.close()
+        return version, new_blocks, commands
+
+    BLOCK_SIZE = 4096
+    
+    version, new_blocks, commands = parse_transfer_list_file(TRANSFER_LIST_FILE)
+
+    if version == 1:
+        print('Android Lollipop 5.0 detected!\n')
+    elif version == 2:
+        print('Android Lollipop 5.1 detected!\n')
+    elif version == 3:
+        print('Android Marshmallow 6.x detected!\n')
+    elif version == 4:
+        print('Android Nougat 7.x / Oreo 8.x detected!\n')
+    else:
+        print('Unknown Android version!\n')
+
+    # Don't clobber existing files to avoid accidental data loss
+    try:
+        output_img = open(OUTPUT_IMAGE_FILE, 'wb')
+    except IOError as e:
+        if e.errno == errno.EEXIST:
+            print('Error: the output file "{}" already exists'.format(e.filename), file=sys.stderr)
+            print('Remove it, rename it, or choose a different file name.', file=sys.stderr)
+            sys.exit(e.errno)
+        else:
+            raise
+
+    new_data_file = open(NEW_DATA_FILE, 'rb')
+    all_block_sets = [i for command in commands for i in command[1]]
+    max_file_size = max(pair[1] for pair in all_block_sets)*BLOCK_SIZE
+
+    for command in commands:
+        if command[0] == 'new':
+            for block in command[1]:
+                begin = block[0]
+                end = block[1]
+                block_count = end - begin
+                print('Copying {} blocks into position {}...'.format(block_count, begin))
+
+                # Position output file
+                output_img.seek(begin*BLOCK_SIZE)
+                
+                # Copy one block at a time
+                while(block_count > 0):
+                    output_img.write(new_data_file.read(BLOCK_SIZE))
+                    block_count -= 1
+        else:
+            print('Skipping command {}...'.format(command[0]))
+
+    # Make file larger if necessary
+    if(output_img.tell() < max_file_size):
+        output_img.truncate(max_file_size)
+
+    output_img.close()
+    new_data_file.close()
+    print('Done! Output image: {}'.format(os.path.realpath(output_img.name)))
+
+if __name__ == '__main__':
+    try:
+        TRANSFER_LIST_FILE = str(sys.argv[1])
+        NEW_DATA_FILE = str(sys.argv[2])
+    except IndexError:
+        print('\nUsage: sdat2img.py <transfer_list> <system_new_file> [system_img]\n')
+        print('    <transfer_list>: transfer list file')
+        print('    <system_new_file>: system new dat file')
+        print('    [system_img]: output system image\n\n')
+        print('Visit xda thread for more information.\n')
+        try:
+            input = raw_input
+        except NameError: pass
+        input('Press ENTER to exit...')
+        sys.exit()
+
+    try:
+        OUTPUT_IMAGE_FILE = str(sys.argv[3])
+    except IndexError:
+        OUTPUT_IMAGE_FILE = 'system.img'
+
+    main(TRANSFER_LIST_FILE, NEW_DATA_FILE, OUTPUT_IMAGE_FILE)

dumpyara/lib/libsdat2img/__init__.py

@@ -0,0 +1,10 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from subprocess import check_output
+
+def sevenz(command: str):
+	return check_output(f"7z {command}", shell=True)

dumpyara/lib/libsevenz/__init__.py

@@ -0,0 +1,35 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from argparse import ArgumentParser
+from dumpyara import __version__ as version, current_path
+from dumpyara.dumpyara import Dumpyara
+from dumpyara.utils.logging import setup_logging
+from pathlib import Path
+
+def main():
+	print(f"Dumpyara\n"
+		  f"Version {version}\n")
+
+	parser = ArgumentParser(prog='python3 -m dumpyara')
+
+	# Main arguments
+	parser.add_argument("file", type=Path,
+						help="path to a device OTA")
+	parser.add_argument("-o", "--output", type=Path, default=current_path / "working",
+						help="custom output folder")
+
+	# Optional arguments
+	parser.add_argument("-v", "--verbose", action='store_true',
+						help="enable debugging logging")
+
+	args = parser.parse_args()
+
+	setup_logging(args.verbose)
+
+	dumpyara = Dumpyara(args.file, args.output)
+
+	print(f"\nDone! You can find the dump in {str(dumpyara.path)}")

dumpyara/main.py

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from pathlib import Path
+from typing import Union
+
+class BuildProp:
+	"""
+	A class representing a build prop.
+
+	This class basically mimics Android props system, with both getprop and setprop commands
+	"""
+	def __init__(self, file: Path):
+		"""
+		Create a dictionary containing all the key-value from a build prop.
+		"""
+		self.file = file.read_text()
+		self.props = {}
+
+		for prop in self.file.splitlines():
+			if prop.startswith("#"):
+				continue
+			try:
+				prop_name, prop_value = prop.split("=", 1)
+			except ValueError:
+				continue
+			else:
+				self.set_prop(prop_name, prop_value)
+	
+	def get_prop(self, prop: str) -> Union[str, None]:
+		"""
+		From a prop name, return the prop value.
+
+		Returns a string if it exists, else None
+		"""
+		try:
+			return self.props[prop]
+		except KeyError:
+			return None
+
+	def set_prop(self, prop_name: str, prop_value: str):
+		self.props[prop_name] = prop_value

dumpyara/utils/build_prop.py

@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from logging import basicConfig, INFO, DEBUG
+
+def setup_logging(verbose=False):
+	if verbose:
+		basicConfig(format='[%(filename)s:%(lineno)s %(levelname)s] %(funcName)s: %(message)s',
+					level=DEBUG)
+	else:
+		basicConfig(format='[%(levelname)s] %(message)s', level=INFO)

dumpyara/utils/logging.py

@@ -0,0 +1,104 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+from dumpyara.lib.liblogging import LOGE, LOGI
+from dumpyara.lib.libsevenz import sevenz
+from dumpyara.utils.raw_image import get_raw_image
+from pathlib import Path
+from shutil import copyfile
+from subprocess import CalledProcessError
+
+# partition: is_raw
+PARTITIONS = {
+	"boot": True,
+	"dtbo": True,
+	"cust": False,
+	"factory": False,
+	"india": True,
+	"my_preload": True,
+	"my_odm": True,
+	"my_stock": True,
+	"my_operator": True,
+	"my_country": True,
+	"my_product": True,
+	"my_company": True,
+	"my_engineering": True,
+	"my_heytap": True,
+	"odm": False,
+	"oem": False,
+	"oppo_product": False,
+	"opproduct": True,
+	"preload_common": False,
+	"product": False,
+	"recovery": True,
+	"reserve": True,
+	"system": False,
+	"system_ext": False,
+	"system_other": False,
+	"systemex": False,
+	"vendor": False,
+	"xrom": False,
+	"modem": True,
+	"tz": True,
+}
+
+# alternative name: generic name
+ALTERNATIVE_PARTITION_NAMES = {
+	"boot-verified": "boot",
+	"dtbo-verified": "dtbo",
+	"NON-HLOS": "modem",
+}
+
+def get_partition_name(file: Path):
+	for partition in list(PARTITIONS) + list(ALTERNATIVE_PARTITION_NAMES):
+		possible_names = [
+			partition,
+			f"{partition}.bin",
+			f"{partition}.ext4",
+			f"{partition}.image",
+			f"{partition}.img",
+			f"{partition}.mbn",
+			f"{partition}.new.dat",
+			f"{partition}.new.dat.br",
+			f"{partition}.raw",
+			f"{partition}.raw.img",
+		]
+		if file.name in possible_names:
+			return partition
+
+	return None
+
+def can_be_partition(file: Path):
+	"""Check if the file can be a partition."""
+	return get_partition_name(file) is not None
+
+def extract_partition(file: Path, output_path: Path):
+	"""
+	Extract files from partition image.
+	
+	If the partition is raw, the file will simply be copied to the output folder,
+	else extracted using 7z (unsparsed if needed).
+	"""
+	partition_name = get_partition_name(file)
+	if not partition_name:
+		LOGI(f"Skipping {file.stem}")
+		return
+
+	new_partition_name = ALTERNATIVE_PARTITION_NAMES.get(partition_name, partition_name)
+
+	if PARTITIONS[new_partition_name]:
+		copyfile(file, output_path / f"{new_partition_name}.img", follow_symlinks=True)
+	else:
+		# Make sure we have a raw image
+		raw_image = get_raw_image(partition_name, file.parent)
+
+		# TODO: EROFS
+		try:
+			sevenz(f'x {raw_image} -y -o"{output_path / new_partition_name}"/')
+		except CalledProcessError as e:
+			LOGE(f"Error extracting {file.stem}")
+			LOGE(f"{e.output}")
+			raise e

dumpyara/utils/partitions.py

@@ -0,0 +1,48 @@
+#
+# Copyright (C) 2022 Dumpyara Project
+#
+# SPDX-License-Identifier: GPL-3.0
+#
+
+import brotli
+from dumpyara.lib.liblogging import LOGI
+from dumpyara.lib.liblpunpack import SparseImage
+from dumpyara.lib.libsdat2img import main as sdat2img
+from pathlib import Path
+from shutil import move
+
+def get_raw_image(partition: str, path: Path):
+	"""
+	Convert a partition image to a raw image.
+
+	This function handles brotli compression, sdat and sparse images.
+	"""
+	brotli_image = path / f"{partition}.new.dat.br"
+	dat_image = path / f"{partition}.new.dat"
+	transfer_list = path / f"{partition}.transfer.list"
+	raw_image = path / f"{partition}.img"
+
+	if brotli_image.is_file():
+		LOGI(f"Decompressing {brotli_image}")
+		with brotli_image.open("rb") as f:
+			with dat_image.open("wb") as g:
+				g.write(brotli.decompress(f.read()))
+		brotli_image.unlink()
+
+	if dat_image.is_file() and transfer_list.is_file():
+		LOGI(f"Converting {dat_image} to {raw_image}")
+		sdat2img(transfer_list, dat_image, raw_image)
+		dat_image.unlink()
+		transfer_list.unlink()
+
+	if raw_image.is_file():
+		with raw_image.open("rb") as f:
+			sparse_image = SparseImage(f)
+			if sparse_image.check():
+				LOGI(f"{raw_image.stem} is a sparse image, extracting...")
+				unsparse_file = sparse_image.unsparse()
+				move(unsparse_file, raw_image)
+
+		return raw_image
+
+	return None

dumpyara/utils/raw_image.py

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "dumpyara"
+version = "1.0.0"
+description = "Dumpyara"
+license = "GPL-3.0"
+readme = "README.md"
+repository = "https://github.com/SebaUbuntu/TWRP-device-tree-generator"
+authors = ["Sebastiano Barezzi <barezzisebastiano@gmail.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+Brotli = "^1.0.9"
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.1"
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"