Skip to content

Commit

Permalink
Merge pull request #916 from altendky/bladebit
Browse files Browse the repository at this point in the history
  • Loading branch information
@altendky
altendky authored Sep 12, 2021
2 parents 0acc490 + 1a49f8d commit 1e89f3a
Show file tree
Hide file tree
Showing 9 changed files with 1,082 additions and 3 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
([#898](https://github.com/ericaltendorf/plotman/pull/898))
- Output same entries to plotman.log from 'plotman interactive' and ' plotman plot/archive' "daemons".
([#878](https://github.com/ericaltendorf/plotman/pull/878))
- [BladeBit](https://github.com/harold-b/bladebit) support.
Requires BladeBit v1.1.0 for proper log monitoring.
([#916](https://github.com/ericaltendorf/plotman/pull/916))

## [0.5.1] - 2021-07-15
### Fixed
Expand Down
77 changes: 77 additions & 0 deletions src/plotman/_tests/plotters/test_bladebit.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import importlib.resources

import pendulum

import plotman.job
import plotman.plotters.bladebit
import plotman._tests.resources


def test_byte_by_byte_full_load() -> None:
read_bytes = importlib.resources.read_binary(
package=plotman._tests.resources,
resource="bladebit.plot.log",
)

parser = plotman.plotters.bladebit.Plotter()

for byte in (bytes([byte]) for byte in read_bytes):
parser.update(chunk=byte)

assert parser.info == plotman.plotters.bladebit.SpecificInfo(
phase=plotman.job.Phase(major=5, minor=1),
started_at=pendulum.datetime(2021, 8, 29, 22, 22, 0, tz=None),
plot_id="1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3",
threads=88,
plot_size=32,
dst_dir="/mnt/tmp/01/manual-transfer/",
phase1_duration_raw=313.98,
phase2_duration_raw=44.60,
phase3_duration_raw=203.26,
phase4_duration_raw=1.11,
total_time_raw=582.91,
filename="plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot",
plot_name="plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3",
)


def test_log_phases() -> None:
# TODO: CAMPid 0978413087474699698142013249869897439887
read_bytes = importlib.resources.read_binary(
package=plotman._tests.resources,
resource="bladebit.marked",
)

parser = plotman.plotters.bladebit.Plotter()

wrong = []

for marked_line in read_bytes.splitlines(keepends=True):
phase_bytes, _, line_bytes = marked_line.partition(b",")
major, _, minor = phase_bytes.decode("utf-8").partition(":")
phase = plotman.job.Phase(major=int(major), minor=int(minor))

parser.update(chunk=line_bytes)

if parser.info.phase != phase: # pragma: nocov
wrong.append([parser.info.phase, phase, line_bytes.decode("utf-8")])

assert wrong == []


def test_marked_log_matches() -> None:
# TODO: CAMPid 909831931987460871349879878609830987138931700871340870
marked_bytes = importlib.resources.read_binary(
package=plotman._tests.resources,
resource="bladebit.marked",
)
log_bytes = importlib.resources.read_binary(
package=plotman._tests.resources,
resource="bladebit.plot.log",
)

for marked_line, log_line in zip(
marked_bytes.splitlines(keepends=True), log_bytes.splitlines(keepends=True)
):
_, _, marked_just_line = marked_line.partition(b",")
assert marked_just_line == log_line
146 changes: 146 additions & 0 deletions src/plotman/_tests/plotters/test_init.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import plotman.errors
import plotman.job
import plotman.plotters
import plotman.plotters.bladebit
import plotman.plotters.chianetwork
import plotman.plotters.madmax
import plotman._tests.resources
Expand Down Expand Up @@ -92,6 +93,27 @@ class CommandLineExample:
cwd: str = ""


default_bladebit_arguments = dict(
sorted(
{
"threads": None,
"count": 1,
"farmer_key": None,
"pool_key": None,
"pool_contract": None,
"warm_start": False,
"plot_id": None,
"memo": None,
"show_memo": False,
"verbose": False,
"no_numa": False,
"no_cpu_affinity": False,
"out_dir": pathlib.PosixPath("."),
}.items()
)
)


default_chia_network_arguments = dict(
sorted(
{
Expand Down Expand Up @@ -139,6 +161,129 @@ class CommandLineExample:
)


bladebit_command_line_examples: typing.List[CommandLineExample] = [
CommandLineExample(
line=["bladebit"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={**default_bladebit_arguments},
),
),
CommandLineExample(
line=["bladebit", "-h"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=True,
parameters={**default_bladebit_arguments},
),
),
CommandLineExample(
line=["bladebit", "--help"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=True,
parameters={**default_bladebit_arguments},
),
),
CommandLineExample(
line=["bladebit", "--invalid-option"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=click.NoSuchOption("--invalid-option"),
help=False,
parameters={},
),
),
CommandLineExample(
line=["bladebit", "--pool-contract", "xch123abc", "--farmer-key", "abc123"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_bladebit_arguments,
"pool_contract": "xch123abc",
"farmer_key": "abc123",
},
),
),
CommandLineExample(
line=["here/there/bladebit"],
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={**default_bladebit_arguments},
),
),
CommandLineExample(
line=[
"bladebit",
"final/dir",
],
cwd="/cwd",
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_bladebit_arguments,
"out_dir": pathlib.Path("/", "cwd", "final", "dir"),
},
),
),
CommandLineExample(
line=plotman.plotters.bladebit.create_command_line(
options=plotman.plotters.bladebit.Options(),
tmpdir="",
tmp2dir=None,
dstdir="/farm/dst/dir",
farmer_public_key=None,
pool_public_key=None,
pool_contract_address=None,
),
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_bladebit_arguments,
"verbose": True,
"out_dir": pathlib.Path("/farm/dst/dir"),
},
),
),
CommandLineExample(
line=plotman.plotters.bladebit.create_command_line(
options=plotman.plotters.bladebit.Options(),
tmpdir="/farm/tmp/dir",
tmp2dir="/farm/tmp2/dir",
dstdir="/farm/dst/dir",
farmer_public_key="farmerpublickey",
pool_public_key="poolpublickey",
pool_contract_address="poolcontractaddress",
),
plotter=plotman.plotters.bladebit.Plotter,
parsed=plotman.job.ParsedChiaPlotsCreateCommand(
error=None,
help=False,
parameters={
**default_bladebit_arguments,
"farmer_key": "farmerpublickey",
"pool_key": "poolpublickey",
"pool_contract": "poolcontractaddress",
"verbose": True,
"out_dir": pathlib.Path("/farm/dst/dir"),
},
),
),
]


chianetwork_command_line_examples: typing.List[CommandLineExample] = [
CommandLineExample(
line=["python", "chia", "plots", "create"],
Expand Down Expand Up @@ -487,6 +632,7 @@ class CommandLineExample:


command_line_examples: typing.List[CommandLineExample] = [
*bladebit_command_line_examples,
*chianetwork_command_line_examples,
*madmax_command_line_examples,
]
Expand Down
129 changes: 129 additions & 0 deletions src/plotman/_tests/resources/bladebit.marked
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
0:0,Creating 1 plots:
0:0, Output path : /mnt/tmp/01/manual-transfer/
0:0, Thread count : 88
0:0, Warm start enabled : false
0:0, Farmer public key : b0a374845f4f4d6eab62fc4c5e17965d82ad7eee105818e5bd0cfcb46275a16acc4cd30955779bec841a716473416b21
0:0, Pool contract address : xch1u8ll2ztwhseej45d6u2zp9j4mlnzhwseccr0axqws9fl2tyj5u0svdy04y
0:0,
0:0,System Memory: 348/503 GiB.
0:0,Memory required: 416 GiB.
0:0,Warning: Not enough memory available. Buffer allocation may fail.
0:1,Allocating buffers.
0:2,Generating plot 1 / 1: 1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3
0:2,
1:0,Running Phase 1
1:0,Generating F1...
1:1,Finished F1 generation in 6.93 seconds.
1:1,Sorting F1...
1:1,Finished F1 sort in 18.23 seconds.
1:2,Forward propagating to table 2...
1:2, Pairing L/R groups...
1:2, Finished pairing L/R groups in 13.2870 seconds. Created 4294962218 pairs.
1:2, Average of 236.1403 pairs per group.
1:2, Computing Fx...
1:2, Finished computing Fx in 9.0360 seconds.
1:2, Sorting entries...
1:2, Finished sorting in 33.83 seconds.
1:2,Finished forward propagating table 2 in 56.61 seconds.
1:3,Forward propagating to table 3...
1:3, Pairing L/R groups...
1:3, Finished pairing L/R groups in 10.9170 seconds. Created 4294967296 pairs.
1:3, Average of 236.1406 pairs per group.
1:3, Computing Fx...
1:3, Finished computing Fx in 8.6420 seconds.
1:3, Sorting entries...
1:3, Finished sorting in 33.37 seconds.
1:3,Finished forward propagating table 3 in 53.39 seconds.
1:4,Forward propagating to table 4...
1:4, Pairing L/R groups...
1:4, Finished pairing L/R groups in 10.9450 seconds. Created 4294947733 pairs.
1:4, Average of 236.1396 pairs per group.
1:4, Computing Fx...
1:4, Finished computing Fx in 9.3490 seconds.
1:4, Sorting entries...
1:4, Finished sorting in 32.60 seconds.
1:4,Finished forward propagating table 4 in 53.35 seconds.
1:5,Forward propagating to table 5...
1:5, Pairing L/R groups...
1:5, Finished pairing L/R groups in 10.8420 seconds. Created 4294889963 pairs.
1:5, Average of 236.1364 pairs per group.
1:5, Computing Fx...
1:5, Finished computing Fx in 9.5180 seconds.
1:5, Sorting entries...
1:5, Finished sorting in 32.60 seconds.
1:5,Finished forward propagating table 5 in 53.42 seconds.
1:6,Forward propagating to table 6...
1:6, Pairing L/R groups...
1:6, Finished pairing L/R groups in 10.9870 seconds. Created 4294907255 pairs.
1:6, Average of 236.1373 pairs per group.
1:6, Computing Fx...
1:6, Finished computing Fx in 8.5110 seconds.
1:6, Sorting entries...
1:6, Finished sorting in 31.58 seconds.
1:6,Finished forward propagating table 6 in 51.54 seconds.
1:7,Forward propagating to table 7...
1:7, Pairing L/R groups...
1:7, Finished pairing L/R groups in 11.0050 seconds. Created 4294773122 pairs.
1:7, Average of 236.1300 pairs per group.
1:7, Computing Fx...
1:7, Finished computing Fx in 9.0510 seconds.
1:7,Finished forward propagating table 7 in 20.51 seconds.
2:0,Finished Phase 1 in 313.98 seconds.
2:0,Running Phase 2
2:1, Prunning table 6...
2:1, Finished prunning table 6 in 0.59 seconds.
2:2, Prunning table 5...
2:2, Finished prunning table 5 in 11.53 seconds.
2:3, Prunning table 4...
2:3, Finished prunning table 4 in 10.86 seconds.
2:4, Prunning table 3...
2:4, Finished prunning table 3 in 10.57 seconds.
2:5, Prunning table 2...
2:5, Finished prunning table 2 in 10.60 seconds.
3:0,Finished Phase 2 in 44.60 seconds.
3:0,Running Phase 3
3:1, Compressing tables 1 and 2...
3:1, Finished compressing tables 1 and 2 in 31.20 seconds
3:1, Table 1 now has 3429423491 / 4294962218 entries ( 79.85% ).
3:2, Compressing tables 2 and 3...
3:2, Finished compressing tables 2 and 3 in 35.05 seconds
3:2, Table 2 now has 3439923954 / 4294967296 entries ( 80.09% ).
3:3, Compressing tables 3 and 4...
3:3, Finished compressing tables 3 and 4 in 32.41 seconds
3:3, Table 3 now has 3466101892 / 4294947733 entries ( 80.70% ).
3:4, Compressing tables 4 and 5...
3:4, Finished compressing tables 4 and 5 in 33.40 seconds
3:4, Table 4 now has 3532981230 / 4294889963 entries ( 82.26% ).
3:5, Compressing tables 5 and 6...
3:5, Finished compressing tables 5 and 6 in 34.78 seconds
3:5, Table 5 now has 3713621551 / 4294907255 entries ( 86.47% ).
3:6, Compressing tables 6 and 7...
3:6, Finished compressing tables 6 and 7 in 36.41 seconds
3:6, Table 6 now has 4294773122 / 4294773122 entries ( 100.00% ).
4:0,Finished Phase 3 in 203.26 seconds.
4:0,Running Phase 4
4:1, Writing P7.
4:1, Finished writing P7 in 0.71 seconds.
4:2, Writing C1 table.
4:2, Finished writing C1 table in 0.00 seconds.
4:3, Writing C2 table.
4:3, Finished writing C2 table in 0.00 seconds.
4:4, Writing C3 table.
4:4, Finished writing C3 table in 0.40 seconds.
5:0,Finished Phase 4 in 1.11 seconds.
5:1,Writing final plot tables to disk
5:1,
5:1,Plot /mnt/tmp/01/manual-transfer/plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot finished writing to disk:
5:1, Table 1 pointer : 4096 ( 0x0000000000001000 )
5:1, Table 2 pointer : 14839635968 ( 0x000000037482e000 )
5:1, Table 3 pointer : 28822732800 ( 0x00000006b5f80000 )
5:1, Table 4 pointer : 42912239616 ( 0x00000009fdc4d000 )
5:1, Table 5 pointer : 57273606144 ( 0x0000000d55c5e000 )
5:1, Table 6 pointer : 72369262592 ( 0x00000010d98b5000 )
5:1, Table 7 pointer : 89827270656 ( 0x00000014ea1f6000 )
5:1, C1 table pointer : 107543220224 ( 0x000000190a135000 )
5:1, C2 table pointer : 107544940544 ( 0x000000190a2d9000 )
5:1, C3 table pointer : 107544944640 ( 0x000000190a2da000 )
5:1,
5:1,Finished writing tables to disk in 19.96 seconds.
5:1,Finished plotting in 582.91 seconds (9.72 minutes).
Loading

0 comments on commit 1e89f3a

Please sign in to comment.