diff --git a/Dockerfile b/Dockerfile index ed8d61eab..177fd5362 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ ##################################### ARG LOTUS_TEST_IMAGE=curio/lotus-all-in-one:latest -FROM ${LOTUS_TEST_IMAGE} as lotus-test -FROM golang:1.22.3-bullseye AS curio-builder -MAINTAINER Curio Development Team +FROM ${LOTUS_TEST_IMAGE} AS lotus-test +FROM golang:1.22.8-bullseye AS curio-builder +LABEL Maintainer = "Curio Development Team" RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev @@ -43,6 +43,7 @@ RUN make clean deps ARG RUSTFLAGS="" ARG GOFLAGS="" +ARG CURIO_TAGS="" RUN make build diff --git a/Makefile b/Makefile index 7e683b4a1..9e7a61c60 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ deps: $(BUILD_DEPS) ## ldflags -s -w strips binary -CURIO_TAGS := cunative +CURIO_TAGS ?= cunative curio: $(BUILD_DEPS) rm -f curio @@ -264,7 +264,7 @@ build_lotus?=0 curio_docker_user?=curio curio_base_image=$(curio_docker_user)/curio-all-in-one:latest-debug ffi_from_source?=0 -lotus_version?=v1.30.0-rc2 +lotus_version?=v1.32.0-rc1 ifeq ($(build_lotus),1) # v1: building lotus image with provided lotus version @@ -301,7 +301,7 @@ curio_docker_build_cmd=docker build --build-arg CURIO_TEST_IMAGE=$(curio_base_im docker/curio-all-in-one: $(curio_docker_build_cmd) -f Dockerfile --target curio-all-in-one \ - -t $(curio_base_image) --build-arg GOFLAGS=-tags=debug . + -t $(curio_base_image) --build-arg CURIO_TAGS="cunative debug" . .PHONY: docker/curio-all-in-one docker/lotus: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 1a1c45e95..3776b1b04 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.8' name: curio-devnet x-logging: @@ -27,6 +26,7 @@ services: - LOTUS_FEVM_ENABLEETHRPC=true - LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/1234/http - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/9090 + - LOTUS_CHAININDEXER_ENABLEINDEXER=true restart: unless-stopped logging: *default-logging volumes: diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 346dc9732..bf38c16d7 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -187,6 +187,7 @@ type StorageDealSummary struct { Miner string `json:"miner"` IsLegacy bool `json:"is_legacy"` Indexed *bool `db:"indexed" json:"indexed"` + IsDDO bool `db:"is_ddo" json:"is_ddo"` } func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDealSummary, error) { @@ -207,30 +208,83 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal if !isLegacy { var summaries []StorageDealSummary err = a.deps.DB.Select(ctx, &summaries, `SELECT - md.uuid, - md.sp_id, - md.created_at, - md.signed_proposal_cid, - md.offline, - md.verified, - md.start_epoch, - md.end_epoch, - md.client_peer_id, - md.chain_deal_id, - md.publish_cid, - md.piece_cid, - md.piece_size, - md.fast_retrieval, - md.announce_to_ipni, - md.url, - md.url_headers, - md.error, - mpd.sector_num, - mpm.indexed - FROM market_mk12_deals md - LEFT JOIN market_piece_deal mpd ON mpd.id = md.uuid AND mpd.sp_id = md.sp_id - LEFT JOIN market_piece_metadata mpm ON mpm.piece_cid = md.piece_cid - WHERE md.uuid = $1`, id.String()) + deal.uuid, + deal.sp_id, + deal.created_at, + deal.signed_proposal_cid, + deal.offline, + deal.verified, + deal.start_epoch, + deal.end_epoch, + deal.client_peer_id, + deal.chain_deal_id, + deal.publish_cid, + deal.piece_cid, + deal.piece_size, + deal.fast_retrieval, + deal.announce_to_ipni, + deal.url, + deal.url_headers, + deal.error, + mpd.sector_num, + mpm.indexed, + deal.is_ddo -- New column indicating whether the deal is from market_direct_deals + FROM ( + -- Query from market_mk12_deals (default, original table) + SELECT + md.uuid, + md.sp_id, + md.created_at, + md.signed_proposal_cid, + md.offline, + md.verified, + md.start_epoch, + md.end_epoch, + md.client_peer_id, + md.chain_deal_id, + md.publish_cid, + md.piece_cid, + md.piece_size, + md.fast_retrieval, + md.announce_to_ipni, + md.url, + md.url_headers, + md.error, + FALSE AS is_ddo -- Not from market_direct_deals + FROM market_mk12_deals md + WHERE md.uuid = $1 + + UNION ALL + + -- Query from market_direct_deals (new table) + SELECT + mdd.uuid, + mdd.sp_id, + mdd.created_at, + '' AS signed_proposal_cid, + mdd.offline, + mdd.verified, + mdd.start_epoch, + mdd.end_epoch, + '' AS client_peer_id, + 0 AS chain_deal_id, + '' AS publish_cid, + mdd.piece_cid, + mdd.piece_size, + mdd.fast_retrieval, + mdd.announce_to_ipni, + '' AS url, + '{}' AS url_headers, + '' AS error, + TRUE AS is_ddo -- From market_direct_deals + FROM market_direct_deals mdd + WHERE mdd.uuid = $1 + ) AS deal + LEFT JOIN market_piece_deal mpd + ON mpd.id = deal.uuid AND mpd.sp_id = deal.sp_id + LEFT JOIN market_piece_metadata mpm + ON mpm.piece_cid = deal.piece_cid; + `, id.String()) if err != nil { return &StorageDealSummary{}, xerrors.Errorf("select deal summary: %w", err) diff --git a/web/api/webrpc/pipeline_porep.go b/web/api/webrpc/pipeline_porep.go index 85e13a9ff..064bb0ff9 100644 --- a/web/api/webrpc/pipeline_porep.go +++ b/web/api/webrpc/pipeline_porep.go @@ -26,16 +26,18 @@ type PipelineTask struct { AfterSDR bool `db:"after_sdr"` StartedSDR bool `db:"started_sdr"` - TaskTreeD *int64 `db:"task_id_tree_d"` - AfterTreeD bool `db:"after_tree_d"` - StartedTreeD bool `db:"started_tree_d"` + TaskTreeD *int64 `db:"task_id_tree_d"` + AfterTreeD bool `db:"after_tree_d"` + StartedTreeD bool `db:"started_tree_d"` + TreeD *string `db:"tree_d_cid"` TaskTreeC *int64 `db:"task_id_tree_c"` AfterTreeC bool `db:"after_tree_c"` StartedTreeRC bool `db:"started_tree_rc"` - TaskTreeR *int64 `db:"task_id_tree_r"` - AfterTreeR bool `db:"after_tree_r"` + TaskTreeR *int64 `db:"task_id_tree_r"` + AfterTreeR bool `db:"after_tree_r"` + TreeR *string `db:"tree_r_cid"` TaskSynthetic *int64 `db:"task_id_synth"` AfterSynthetic bool `db:"after_synth"` @@ -47,8 +49,9 @@ type PipelineTask struct { AfterPrecommitMsg bool `db:"after_precommit_msg"` StartedPrecommitMsg bool `db:"started_precommit_msg"` - AfterPrecommitMsgSuccess bool `db:"after_precommit_msg_success"` - SeedEpoch *int64 `db:"seed_epoch"` + AfterPrecommitMsgSuccess bool `db:"after_precommit_msg_success"` + PreCommitMsgCid *string `db:"precommit_msg_cid"` + SeedEpoch *int64 `db:"seed_epoch"` TaskPoRep *int64 `db:"task_id_porep"` PoRepProof []byte `db:"porep_proof"` @@ -69,7 +72,8 @@ type PipelineTask struct { AfterCommitMsg bool `db:"after_commit_msg"` StartedCommitMsg bool `db:"started_commit_msg"` - AfterCommitMsgSuccess bool `db:"after_commit_msg_success"` + AfterCommitMsgSuccess bool `db:"after_commit_msg_success"` + CommitMsgCid *string `db:"commit_msg_cid"` Failed bool `db:"failed"` FailedReason string `db:"failed_reason"` diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index 4c834b3ea..a1834ee3f 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -2,16 +2,19 @@ package webrpc import ( "context" + "fmt" "strconv" "strings" "time" + "github.com/docker/go-units" "github.com/samber/lo" "github.com/snadrus/must" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" @@ -19,9 +22,28 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +const verifiedPowerGainMul = 9 + type SectorInfo struct { - SectorNumber int64 - SpID uint64 + SectorNumber int64 + SpID uint64 + Miner string + PreCommitMsg string + CommitMsg string + ActivationEpoch abi.ChainEpoch + ExpirationEpoch *int64 + DealWeight string + Deadline *int64 + Partition *int64 + UnsealedCid string + SealedCid string + UpdatedUnsealedCid string + UpdatedSealedCid string + IsSnap bool + UpdateMsg string + UnsealedState bool + HasUnsealed bool + PipelinePoRep *sectorListEntry PipelineSnap *sectorSnapListEntry @@ -49,14 +71,15 @@ type SnapPipelineTask struct { UpdateUnsealedCID *string `db:"update_unsealed_cid"` UpdateSealedCID *string `db:"update_sealed_cid"` - TaskEncode *int64 `db:"task_id_encode"` - AfterEncode bool `db:"after_encode"` - TaskProve *int64 `db:"task_id_prove"` - AfterProve bool `db:"after_prove"` - TaskSubmit *int64 `db:"task_id_submit"` - AfterSubmit bool `db:"after_submit"` - AfterProveMsgSuccess bool `db:"after_prove_msg_success"` - ProveMsgTsk []byte `db:"prove_msg_tsk"` + TaskEncode *int64 `db:"task_id_encode"` + AfterEncode bool `db:"after_encode"` + TaskProve *int64 `db:"task_id_prove"` + AfterProve bool `db:"after_prove"` + TaskSubmit *int64 `db:"task_id_submit"` + AfterSubmit bool `db:"after_submit"` + AfterProveMsgSuccess bool `db:"after_prove_msg_success"` + ProveMsgTsk []byte `db:"prove_msg_tsk"` + UpdateMsgCid *string `db:"prove_msg_cid"` TaskMoveStorage *int64 `db:"task_id_move_storage"` AfterMoveStorage bool `db:"after_move_storage"` @@ -94,9 +117,10 @@ type SectorPieceMeta struct { PieceCid string `db:"piece_cid"` PieceSize int64 `db:"piece_size"` - DataUrl string `db:"data_url"` - DataRawSize int64 `db:"data_raw_size"` - DeleteOnFinalize bool `db:"data_delete_on_finalize"` + DealID *string `db:"deal_id"` + DataUrl *string `db:"data_url"` + DataRawSize int64 `db:"data_raw_size"` + DeleteOnFinalize *bool `db:"data_delete_on_finalize"` F05PublishCid *string `db:"f05_publish_cid"` F05DealID *int64 `db:"f05_deal_id"` @@ -117,7 +141,10 @@ type SectorPieceMeta struct { PieceParkTaskID *int64 `db:"-"` PieceParkCleanupTaskID *int64 `db:"-"` - IsSnapPiece bool `db:"-"` + IsSnapPiece bool `db:"is_snap"` + + MK12Deal *bool `db:"boost_deal"` + LegacyDeal *bool `db:"legacy_deal"` } type FileLocations struct { @@ -135,6 +162,26 @@ type LocationTable struct { Locations []FileLocations } +type SectorMeta struct { + OrigUnsealedCid string `db:"orig_unsealed_cid"` + OrigSealedCid string `db:"orig_sealed_cid"` + + UpdatedUnsealedCid string `db:"cur_unsealed_cid"` + UpdatedSealedCid string `db:"cur_sealed_cid"` + + PreCommitCid string `db:"msg_cid_precommit"` + CommitCid string `db:"msg_cid_commit"` + UpdateCid *string `db:"msg_cid_update"` + + IsCC *bool `db:"is_cc"` + ExpirationEpoch *int64 `db:"expiration_epoch"` + + Deadline *int64 `db:"deadline"` + Partition *int64 `db:"partition"` + + UnsealedState *bool `db:"target_unseal_state"` +} + func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*SectorInfo, error) { maddr, err := address.NewFromString(sp) @@ -147,6 +194,14 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto return nil, xerrors.Errorf("invalid sp") } + fmt.Println("SPID", spid) + + si := &SectorInfo{ + SpID: spid, + Miner: maddr.String(), + SectorNumber: intid, + } + var tasks []PipelineTask // Fetch PoRep pipeline data @@ -154,23 +209,27 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto sp_id, sector_number, create_time, task_id_sdr, after_sdr, - task_id_tree_d, after_tree_d, + task_id_tree_d, after_tree_d, tree_d_cid, task_id_tree_c, after_tree_c, - task_id_tree_r, after_tree_r, + task_id_tree_r, after_tree_r, tree_r_cid, task_id_synth, after_synth, task_id_precommit_msg, after_precommit_msg, - after_precommit_msg_success, seed_epoch, + after_precommit_msg_success, precommit_msg_cid, seed_epoch, task_id_porep, porep_proof, after_porep, task_id_finalize, after_finalize, task_id_move_storage, after_move_storage, task_id_commit_msg, after_commit_msg, - after_commit_msg_success, + after_commit_msg_success, commit_msg_cid, failed, failed_reason FROM sectors_sdr_pipeline WHERE sp_id = $1 AND sector_number = $2`, spid, intid) if err != nil { return nil, xerrors.Errorf("failed to fetch pipeline task info: %w", err) } + if len(tasks) == 0 { + fmt.Println("NO PIPELINE") + } + // Fetch SnapDeals pipeline data var snapTasks []SnapPipelineTask @@ -185,7 +244,7 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto task_id_prove, after_prove, task_id_submit, after_submit, after_prove_msg_success, prove_msg_tsk, - task_id_move_storage, after_move_storage, + task_id_move_storage, after_move_storage, prove_msg_cid, failed, failed_at, failed_reason, failed_reason_msg, submit_after FROM sectors_snap_pipeline WHERE sp_id = $1 AND sector_number = $2`, spid, intid) @@ -206,7 +265,35 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto var sle *sectorListEntry if len(tasks) > 0 { + fmt.Println("FOUND THE PIPELINE") task := tasks[0] + if task.PreCommitMsgCid != nil { + si.PreCommitMsg = *task.PreCommitMsgCid + } else { + si.PreCommitMsg = "" + } + + if task.CommitMsgCid != nil { + si.CommitMsg = *task.CommitMsgCid + } else { + si.CommitMsg = "" + } + + if task.TreeD != nil { + si.UnsealedCid = *task.TreeD + si.UpdatedUnsealedCid = *task.TreeD + } else { + si.UnsealedCid = "" + si.UpdatedUnsealedCid = "" + } + + if task.TreeR != nil { + si.SealedCid = *task.TreeR + si.UpdatedSealedCid = *task.TreeR + } else { + si.SealedCid = "" + si.UpdatedSealedCid = "" + } sle = §orListEntry{ PipelineTask: tasks[0], AfterSeed: task.SeedEpoch != nil && *task.SeedEpoch <= int64(epoch), @@ -223,6 +310,22 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto var sleSnap *sectorSnapListEntry if len(snapTasks) > 0 { task := snapTasks[0] + if task.UpdateUnsealedCID != nil { + si.UpdatedUnsealedCid = *task.UpdateUnsealedCID + } else { + si.UpdatedUnsealedCid = "" + } + if task.UpdateUnsealedCID != nil { + si.UpdatedSealedCid = *task.UpdateUnsealedCID + } else { + si.UpdatedSealedCid = "" + } + if task.UpdateMsgCid != nil { + si.UpdateMsg = *task.UpdateMsgCid + } else { + si.UpdateMsg = "" + } + si.IsSnap = true sleSnap = §orSnapListEntry{ SnapPipelineTask: task, } @@ -246,6 +349,9 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto for i, loc := range sectorLocations { loc := loc + if loc.FileType == storiface.FTUnsealed { + si.HasUnsealed = true + } urlList := strings.Split(loc.Urls, paths.URLSeparator) @@ -307,39 +413,117 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto } - var pieces []SectorPieceMeta + var sectorMetas []SectorMeta - // Fetch PoRep pieces - err = a.deps.DB.Select(ctx, &pieces, `SELECT piece_index, piece_cid, piece_size, - data_url, data_raw_size, data_delete_on_finalize, - f05_publish_cid, f05_deal_id, direct_piece_activation_manifest FROM sectors_sdr_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid) + // Fetch SectorMeta from DB + err = a.deps.DB.Select(ctx, §orMetas, `SELECT orig_sealed_cid, + orig_unsealed_cid, cur_sealed_cid, cur_unsealed_cid, + msg_cid_precommit, msg_cid_commit, msg_cid_update, + expiration_epoch, deadline, partition, target_unseal_state, + is_cc FROM sectors_meta + WHERE sp_id = $1 AND sector_num = $2`, spid, intid) if err != nil { - return nil, xerrors.Errorf("failed to fetch sector pieces: %w", err) + return nil, xerrors.Errorf("failed to fetch sector metadata: %w", err) + } + + if len(sectorMetas) > 0 { + sectormeta := sectorMetas[0] + si.UnsealedCid = sectormeta.OrigUnsealedCid + si.SealedCid = sectormeta.OrigSealedCid + si.UpdatedUnsealedCid = sectormeta.UpdatedUnsealedCid + si.UpdatedSealedCid = sectormeta.UpdatedSealedCid + si.PreCommitMsg = sectormeta.PreCommitCid + si.CommitMsg = sectormeta.CommitCid + if sectormeta.UpdateCid != nil { + si.UpdateMsg = *sectormeta.UpdateCid + } + if sectormeta.IsCC != nil { + si.IsSnap = !*sectormeta.IsCC + } else { + si.IsSnap = false + } + + if sectormeta.ExpirationEpoch != nil { + si.ExpirationEpoch = sectormeta.ExpirationEpoch + } + if sectormeta.Deadline != nil { + d := *sectormeta.Deadline + si.Deadline = &d + } + if sectormeta.Partition != nil { + p := *sectormeta.Partition + si.Partition = &p + } + if sectormeta.UnsealedState != nil { + si.UnsealedState = *sectormeta.UnsealedState + } } - // Fetch SnapDeals pieces - var snapPieces []SectorPieceMeta + var pieces []SectorPieceMeta - err = a.deps.DB.Select(ctx, &snapPieces, `SELECT piece_index, piece_cid, piece_size, - data_url, data_raw_size, data_delete_on_finalize, - NULL as f05_publish_cid, NULL as f05_deal_id, direct_piece_activation_manifest FROM sectors_snap_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid) + err = a.deps.DB.Select(ctx, &pieces, `SELECT piece_index, combined.piece_cid, combined.piece_size, + data_url, data_raw_size, data_delete_on_finalize, + f05_deal_id, direct_piece_activation_manifest, + mpd.id AS deal_id, -- Extracted id from market_piece_deal + mpd.boost_deal, -- Retrieved boost_deal from market_piece_deal + mpd.legacy_deal, -- Retrieved legacy_deal from market_piece_deal + is_snap -- New column indicating whether the piece is a snap deal + FROM ( + -- Meta table entries (permanent, prioritized) + SELECT meta.piece_num AS piece_index, meta.piece_cid, meta.piece_size, + NULL AS data_url, meta.raw_data_size AS data_raw_size, + NOT meta.requested_keep_data AS data_delete_on_finalize, + meta.f05_deal_id, meta.ddo_pam AS direct_piece_activation_manifest, + meta.sp_id, + NOT sm.is_cc AS is_snap -- is_snap based on is_cc from sectors_meta + FROM sectors_meta_pieces meta + JOIN sectors_meta sm ON meta.sp_id = sm.sp_id AND meta.sector_num = sm.sector_num + WHERE meta.sp_id = $1 AND meta.sector_num = $2 + + UNION ALL + + -- SDR pipeline entries (temporary, non-snap pieces) + SELECT sdr.piece_index, sdr.piece_cid, sdr.piece_size, + sdr.data_url, sdr.data_raw_size, sdr.data_delete_on_finalize, + sdr.f05_deal_id, sdr.direct_piece_activation_manifest, + sdr.sp_id, + FALSE AS is_snap -- SDR pipeline pieces are never snap deals + FROM sectors_sdr_initial_pieces sdr + WHERE sdr.sp_id = $1 AND sdr.sector_number = $2 + AND NOT EXISTS ( + SELECT 1 + FROM sectors_meta_pieces meta + WHERE meta.sp_id = sdr.sp_id AND meta.piece_cid = sdr.piece_cid + ) + + UNION ALL + + -- Snap pipeline entries (temporary, always snap deals) + SELECT snap.piece_index, snap.piece_cid, snap.piece_size, + snap.data_url, snap.data_raw_size, snap.data_delete_on_finalize, + NULL AS f05_deal_id, snap.direct_piece_activation_manifest, + snap.sp_id, + TRUE AS is_snap -- Snap pipeline pieces are always snap deals + FROM sectors_snap_initial_pieces snap + WHERE snap.sp_id = $1 AND snap.sector_number = $2 + AND NOT EXISTS ( + SELECT 1 + FROM sectors_meta_pieces meta + WHERE meta.sp_id = snap.sp_id AND meta.piece_cid = snap.piece_cid + ) + ) AS combined + LEFT JOIN market_piece_deal mpd + ON combined.sp_id = mpd.sp_id AND combined.piece_cid = mpd.piece_cid; + `, spid, intid) if err != nil { - return nil, xerrors.Errorf("failed to fetch snap sector pieces: %w", err) - } - - // Mark SnapDeals pieces - for i := range snapPieces { - snapPieces[i].IsSnapPiece = true + return nil, xerrors.Errorf("failed to fetch sector pieces: %w", err) } - // Combine both slices - pieces = append(pieces, snapPieces...) - for i := range pieces { pieces[i].StrPieceSize = types.SizeStr(types.NewInt(uint64(pieces[i].PieceSize))) pieces[i].StrDataRawSize = types.SizeStr(types.NewInt(uint64(pieces[i].DataRawSize))) - id, isPiecePark := strings.CutPrefix(pieces[i].DataUrl, "pieceref:") + id, isPiecePark := strings.CutPrefix(derefOrZero(pieces[i].DataUrl), "pieceref:") if !isPiecePark { continue } @@ -509,20 +693,59 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto } } - return &SectorInfo{ - SectorNumber: intid, - SpID: spid, - PipelinePoRep: sle, - PipelineSnap: sleSnap, + onChainInfo, err := a.deps.Chain.StateSectorGetInfo(ctx, maddr, abi.SectorNumber(intid), types.EmptyTSK) + if err != nil { + return nil, xerrors.Errorf("failed to get on chain info for the sector: %w", err) + } + + if onChainInfo != nil { + dw, vp := .0, .0 + dealWeight := "CC" + { + rdw := big.Add(onChainInfo.DealWeight, onChainInfo.VerifiedDealWeight) + dw = float64(big.Div(rdw, big.NewInt(int64(onChainInfo.Expiration-onChainInfo.PowerBaseEpoch))).Uint64()) + vp = float64(big.Div(big.Mul(onChainInfo.VerifiedDealWeight, big.NewInt(verifiedPowerGainMul)), big.NewInt(int64(onChainInfo.Expiration-onChainInfo.PowerBaseEpoch))).Uint64()) + if vp > 0 { + dw = vp + } + if dw > 0 { + dealWeight = units.BytesSize(dw) + } + } + + if si.Deadline == nil || si.Partition == nil { + part, err := a.deps.Chain.StateSectorPartition(ctx, maddr, abi.SectorNumber(intid), types.EmptyTSK) + if err != nil { + return nil, xerrors.Errorf("failed to get partition info for the sector: %w", err) + } + + d := int64(part.Deadline) + si.Deadline = &d + + p := int64(part.Partition) + si.Partition = &p + } + + si.ActivationEpoch = onChainInfo.Activation + if si.ExpirationEpoch == nil || *si.ExpirationEpoch != int64(onChainInfo.Expiration) { + expr := int64(onChainInfo.Expiration) + si.ExpirationEpoch = &expr + } + si.DealWeight = dealWeight + } + + si.PipelinePoRep = sle + si.PipelineSnap = sleSnap + + si.Pieces = pieces + si.Locations = locs + si.Tasks = htasks + si.TaskHistory = th - Pieces: pieces, - Locations: locs, - Tasks: htasks, - TaskHistory: th, + si.Resumable = hasAnyStuckTask + si.Restart = hasAnyStuckTask && (sle == nil || !sle.AfterSynthetic) - Resumable: hasAnyStuckTask, - Restart: hasAnyStuckTask && (sle == nil || !sle.AfterSynthetic), - }, nil + return si, nil } func (a *WebRPC) SectorResume(ctx context.Context, spid, id int64) error { @@ -606,3 +829,10 @@ func (a *WebRPC) SectorRestart(ctx context.Context, spid, id int64) error { } return nil } + +func derefOrZero[T any](a *T) T { + if a == nil { + return *new(T) + } + return *a +} diff --git a/web/static/pages/mk12-deal/deal.mjs b/web/static/pages/mk12-deal/deal.mjs index 3f9e458ab..35bee2068 100644 --- a/web/static/pages/mk12-deal/deal.mjs +++ b/web/static/pages/mk12-deal/deal.mjs @@ -33,6 +33,7 @@ class DealDetails extends LitElement { {property: 'Offline', value: entry.offline}, {property: 'Verified', value: entry.verified}, {property: 'Is Legacy', value: entry.is_legacy}, + {property: 'Is DDO', value: entry.is_ddo}, {property: 'Start Epoch', value: html``}, {property: 'End Epoch', value: html``}, {property: 'Client Peer ID', value: entry.client_peer_id}, diff --git a/web/static/pages/pipeline_porep/pipeline-porep-sectors.mjs b/web/static/pages/pipeline_porep/pipeline-porep-sectors.mjs index f7c2786a0..3a878ce02 100644 --- a/web/static/pages/pipeline_porep/pipeline-porep-sectors.mjs +++ b/web/static/pages/pipeline_porep/pipeline-porep-sectors.mjs @@ -2,25 +2,7 @@ import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/al import RPCCall from '/lib/jsonrpc.mjs'; import { formatDateTwo } from '/lib/dateutil.mjs'; -class PipelinePorepSectors extends LitElement { - static properties = { - data: { type: Array }, - }; - - constructor() { - super(); - this.data = []; - this.loadData(); - } - - async loadData() { - this.data = await RPCCall('PipelinePorepSectors'); - // Refresh every 3 seconds - setTimeout(() => this.loadData(), 3000); - this.requestUpdate(); - } - - static styles = css` +export const pipelineStyles = css` .porep-pipeline-table, .porep-state { color: #d0d0d0; @@ -87,6 +69,26 @@ class PipelinePorepSectors extends LitElement { } `; +class PipelinePorepSectors extends LitElement { + static properties = { + data: { type: Array }, + }; + + constructor() { + super(); + this.data = []; + this.loadData(); + } + + async loadData() { + this.data = await RPCCall('PipelinePorepSectors'); + // Refresh every 3 seconds + setTimeout(() => this.loadData(), 3000); + this.requestUpdate(); + } + + static styles = [pipelineStyles]; + render() { // Count how many are "waiting for precommit": // (PreCommitReadyAt != null && !AfterPrecommitMsg && !TaskPrecommitMsg) @@ -169,7 +171,7 @@ class PipelinePorepSectors extends LitElement { - ${this.renderSectorPipeline(sector)} + ${renderSectorPipeline(sector)} @@ -200,221 +202,224 @@ class PipelinePorepSectors extends LitElement {
${timeStr}
`; } +} - renderSectorPipeline(sector) { - return html` +customElements.define('pipeline-porep-sectors', PipelinePorepSectors); + +export function renderSectorPipeline(sector) { + return html` - ${this.renderSectorState( - 'SDR', - 1, - sector, - sector.TaskSDR, - sector.AfterSDR, - sector.StartedSDR - )} - ${this.renderSectorState( - 'TreeC', - 1, - sector, - sector.TaskTreeC, - sector.AfterTreeC, - sector.StartedTreeRC - )} - ${this.renderSectorState( - 'Synthetic', - 2, - sector, - sector.TaskSynthetic, - sector.AfterSynthetic, - sector.StartedSynthetic - )} - ${this.renderSectorState( - 'PComm Msg', - 2, - sector, - sector.TaskPrecommitMsg, - sector.AfterPrecommitMsg, - sector.StartedPrecommitMsg - )} - ${this.renderSectorStateNoTask( - 'PComm Wait', - 2, - sector.AfterPrecommitMsg, - sector.AfterPrecommitMsgSuccess - )} + ${renderSectorState( + 'SDR', + 1, + sector, + sector.TaskSDR, + sector.AfterSDR, + sector.StartedSDR + )} + ${renderSectorState( + 'TreeC', + 1, + sector, + sector.TaskTreeC, + sector.AfterTreeC, + sector.StartedTreeRC + )} + ${renderSectorState( + 'Synthetic', + 2, + sector, + sector.TaskSynthetic, + sector.AfterSynthetic, + sector.StartedSynthetic + )} + ${renderSectorState( + 'PComm Msg', + 2, + sector, + sector.TaskPrecommitMsg, + sector.AfterPrecommitMsg, + sector.StartedPrecommitMsg + )} + ${renderSectorStateNoTask( + 'PComm Wait', + 2, + sector.AfterPrecommitMsg, + sector.AfterPrecommitMsgSuccess + )} - ${this.renderSectorState( - 'PoRep', - 2, - sector, - sector.TaskPoRep, - sector.AfterPoRep, - sector.StartedPoRep - )} - ${this.renderSectorState( - 'Clear Cache', - 1, - sector, - sector.TaskFinalize, - sector.AfterFinalize, - sector.StartedFinalize - )} - ${this.renderSectorState( - 'Move Storage', - 1, - sector, - sector.TaskMoveStorage, - sector.AfterMoveStorage, - sector.StartedMoveStorage - )} + ${renderSectorState( + 'PoRep', + 2, + sector, + sector.TaskPoRep, + sector.AfterPoRep, + sector.StartedPoRep + )} + ${renderSectorState( + 'Clear Cache', + 1, + sector, + sector.TaskFinalize, + sector.AfterFinalize, + sector.StartedFinalize + )} + ${renderSectorState( + 'Move Storage', + 1, + sector, + sector.TaskMoveStorage, + sector.AfterMoveStorage, + sector.StartedMoveStorage + )} - ${this.renderSectorState( - 'TreeD', - 1, - sector, - sector.TaskTreeD, - sector.AfterTreeD, - sector.StartedTreeD - )} - ${this.renderSectorState( - 'TreeR', - 1, - sector, - sector.TaskTreeR, - sector.AfterTreeR, - sector.StartedTreeRC - )} + ${renderSectorState( + 'TreeD', + 1, + sector, + sector.TaskTreeD, + sector.AfterTreeD, + sector.StartedTreeD + )} + ${renderSectorState( + 'TreeR', + 1, + sector, + sector.TaskTreeR, + sector.AfterTreeR, + sector.StartedTreeRC + )} - ${this.renderSectorState( - 'Commit Msg', - 1, - sector, - sector.TaskCommitMsg, - sector.AfterCommitMsg, - sector.StartedCommitMsg - )} - ${this.renderSectorStateNoTask( - 'Commit Wait', - 1, - sector.AfterCommitMsg, - sector.AfterCommitMsgSuccess - )} + ${renderSectorState( + 'Commit Msg', + 1, + sector, + sector.TaskCommitMsg, + sector.AfterCommitMsg, + sector.StartedCommitMsg + )} + ${renderSectorStateNoTask( + 'Commit Wait', + 1, + sector.AfterCommitMsg, + sector.AfterCommitMsgSuccess + )}
Wait Seed
${sector.AfterSeed ? 'done' : sector.SeedEpoch}
On Chain
${sector.ChainSector - ? 'yes' - : sector.ChainAlloc - ? 'allocated' - : 'no'} + ? 'yes' + : sector.ChainAlloc + ? 'allocated' + : 'no'}
State
${sector.Failed - ? 'Failed' - : sector.ChainActive - ? 'Sealed' - : 'Sealing'} + ? 'Failed' + : sector.ChainActive + ? 'Sealed' + : 'Sealing'}
Active
${sector.ChainActive - ? 'yes' - : sector.ChainUnproven - ? 'unproven' - : sector.ChainFaulty - ? 'faulty' - : 'no'} + ? 'yes' + : sector.ChainUnproven + ? 'unproven' + : sector.ChainFaulty + ? 'faulty' + : 'no'}
`; - } +} - /** - * Renders a stage cell with a task ID (if present) or a "Done / --" state. - * Also applies special "waiting" color if: - * - "PComm Msg" and sector is waiting for precommit - * - "Commit Msg" and sector is waiting for commit - */ - renderSectorState(name, rowspan, sector, task, after, started) { - // 1) "waiting for precommit" - if ( - name === 'PComm Msg' && - sector.PreCommitReadyAt && - !sector.AfterPrecommitMsg && - !sector.TaskPrecommitMsg - ) { - return html` +/** + * Renders a stage cell with a task ID (if present) or a "Done / --" state. + * Also applies special "waiting" color if: + * - "PComm Msg" and sector is waiting for precommit + * - "Commit Msg" and sector is waiting for commit + */ +export function renderSectorState(name, rowspan, sector, task, after, started) { + // 1) "waiting for precommit" + if ( + name === 'PComm Msg' && + sector.PreCommitReadyAt && + !sector.AfterPrecommitMsg && + !sector.TaskPrecommitMsg + ) { + return html`
${name}
Waiting
`; - } - // 2) "waiting for commit" - if ( - name === 'Commit Msg' && - sector.CommitReadyAt && - !sector.AfterCommitMsg && - !sector.TaskCommitMsg - ) { - return html` + } + // 2) "waiting for commit" + if ( + name === 'Commit Msg' && + sector.CommitReadyAt && + !sector.AfterCommitMsg && + !sector.TaskCommitMsg + ) { + return html`
${name}
Waiting
`; - } + } - // Normal logic for tasks with an ID - if (task) { - const missing = - sector.MissingTasks && sector.MissingTasks.includes(task); - return html` + // Normal logic for tasks with an ID + if (task) { + const missing = + sector.MissingTasks && sector.MissingTasks.includes(task); + return html`
${name}
@@ -424,34 +429,31 @@ class PipelinePorepSectors extends LitElement { ${missing ? html`
FAILED
` : ''} `; - } + } - // No task ID => either done or not started - return html` + // No task ID => either done or not started + return html`
${name}
${after ? 'done' : '--'}
`; - } +} - /** - * Renders a stage cell for tasks that don't have an associated Task ID - * but do have after/active states to display. - */ - renderSectorStateNoTask(name, rowspan, active, after) { - return html` +/** + * Renders a stage cell for tasks that don't have an associated Task ID + * but do have after/active states to display. + */ +export function renderSectorStateNoTask(name, rowspan, active, after) { + return html`
${name}
${after ? 'done' : '--'}
`; - } } - -customElements.define('pipeline-porep-sectors', PipelinePorepSectors); diff --git a/web/static/pages/sector/sector-info.mjs b/web/static/pages/sector/sector-info.mjs index 7d3ab33e7..167e52f6d 100644 --- a/web/static/pages/sector/sector-info.mjs +++ b/web/static/pages/sector/sector-info.mjs @@ -1,54 +1,7 @@ import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; import RPCCall from '/lib/jsonrpc.mjs'; -import '/pages/pipeline_porep/pipeline-porep-sectors.mjs'; - -customElements.define('sector-snap-state', class SectorSnapState extends LitElement { - static properties = { - data: { type: Object } - }; - - render() { - if (!this.data) { - return html`
No SnapDeals data available.
`; - } - - return html` - ${this.data.Failed ? html` -

- ${this.data.FailedReason}: ${this.data.FailedReasonMsg} -

- `: ''} - - - - - - - - - - - - - - - - - - - - - - - - - - - -
StageStatusTask ID
Encode${this.data.AfterEncode ? 'Completed' : 'Pending'}${this.data.TaskEncode || 'N/A'}
Prove${this.data.AfterProve ? 'Completed' : 'Pending'}${this.data.TaskProve || 'N/A'}
Submit${this.data.AfterSubmit ? 'Completed' : 'Pending'}${this.data.TaskSubmit || 'N/A'}
Move Storage${this.data.AfterMoveStorage ? 'Completed' : 'Pending'}${this.data.TaskMoveStorage || 'N/A'}
- `; - } -}); +import { renderSectorPipeline, pipelineStyles } from '/pages/pipeline_porep/pipeline-porep-sectors.mjs'; +import { renderSectorSnapPipeline, snapPipelineStyles} from '/snap/upgrade-sectors.mjs'; customElements.define('sector-info',class SectorInfo extends LitElement { constructor() { @@ -74,6 +27,8 @@ customElements.define('sector-info',class SectorInfo extends LitElement { window.location.reload(); } + static styles = [pipelineStyles, snapPipelineStyles]; + render() { if (!this.data) { return html`
Loading...
`; @@ -99,17 +54,56 @@ customElements.define('sector-info',class SectorInfo extends LitElement { ${this.data.Resumable ? html`` : ''}
+
+

Sector Info

+ + + + + + + + + + + + + + + + + + + + +
Miner ID${this.data.Miner}
Sector Number${this.data.SectorNumber}
PreCommit Message${this.data.PreCommitMsg}
Commit Message${this.data.CommitMsg}
Activation Epoch${this.data.ActivationEpoch}
Expiration Epoch${this.data.ExpirationEpoch}
Deal Weight${this.data.DealWeight}
Deadline${this.data.Deadline}
Partition${this.data.Partition}
Unsealed CID${this.data.UnsealedCid}
Sealed CID${this.data.SealedCid}
Updated Unsealed CID${this.data.UpdatedUnsealedCid}
Updated Sealed CID${this.data.UpdatedSealedCid}
Is Snap${this.data.IsSnap}
Update Message${this.data.UpdateMsg}
Unsealed State + ${this.data.UnsealedState == null + ? 'Either' + : this.data.UnsealedState + ? 'Keep Unsealed' + : 'Remove Unsealed'} +
+
${this.data.PipelinePoRep ? html`

PoRep Pipeline

- - ` : ''} + ${renderSectorPipeline(this.data.PipelinePoRep)} + ` : html` +

No data available for the PoRep pipeline.

+ `}
${this.data.PipelineSnap ? html` -

SnapDeals Pipeline

- - ` : ''} +

SnapDeals Pipeline

+ ${renderSectorSnapPipeline(this.data.PipelineSnap)} + ` : html` +

No data available for the SnapDeals pipeline.

+ `}

Pieces

@@ -118,6 +112,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement { Piece Index Piece CID Piece Size + Deal ID Data URL Data Raw Size Delete On Finalize @@ -136,9 +131,10 @@ customElements.define('sector-info',class SectorInfo extends LitElement { ${piece.PieceIndex} ${piece.PieceCid} ${piece.PieceSize} + ${piece.DealID} ${piece.DataUrl} ${piece.DataRawSize} - ${piece.DeleteOnFinalize} + ${piece.DeleteOnFinalize === null ? 'Either' : piece.DeleteOnFinalize} ${piece.IsSnapPiece ? 'SnapDeals' : 'PoRep'} ${piece.F05PublishCid} ${piece.F05DealID} diff --git a/web/static/snap/upgrade-sectors.mjs b/web/static/snap/upgrade-sectors.mjs index 493a48805..a158ce027 100644 --- a/web/static/snap/upgrade-sectors.mjs +++ b/web/static/snap/upgrade-sectors.mjs @@ -2,26 +2,7 @@ import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/al import RPCCall from '/lib/jsonrpc.mjs'; import { formatDateTwo } from '/lib/dateutil.mjs'; - -class UpgradeSectors extends LitElement { - static properties = { - data: { type: Array }, - }; - - constructor() { - super(); - this.data = []; - this.loadData(); - } - - async loadData() { - this.data = await RPCCall('UpgradeSectors'); - // Refresh every 3 seconds - setTimeout(() => this.loadData(), 3000); - this.requestUpdate(); - } - - static styles = css` +export const snapPipelineStyles = css` .porep-pipeline-table, .porep-state { color: #d0d0d0; @@ -74,6 +55,26 @@ class UpgradeSectors extends LitElement { } `; +class UpgradeSectors extends LitElement { + static properties = { + data: { type: Array }, + }; + + constructor() { + super(); + this.data = []; + this.loadData(); + } + + static styles = [snapPipelineStyles]; + + async loadData() { + this.data = await RPCCall('UpgradeSectors'); + // Refresh every 3 seconds + setTimeout(() => this.loadData(), 3000); + this.requestUpdate(); + } + render() { // Count how many are "waiting for submit": // (UpdateReadyAt != null && !AfterSubmit && !TaskIDSubmit) @@ -136,7 +137,7 @@ class UpgradeSectors extends LitElement { - ${this.renderSectorPipeline(sector)} + ${renderSectorSnapPipeline(sector)} @@ -158,45 +159,48 @@ class UpgradeSectors extends LitElement {
${timeStr}
`; } +} - renderSectorPipeline(sector) { - return html` +customElements.define('upgrade-sectors', UpgradeSectors); + +export function renderSectorSnapPipeline(sector) { + return html` - ${this.renderSectorState( - 'Encode', - sector, - sector.TaskIDEncode, - sector.AfterEncode, - sector.StartedEncode - )} - ${this.renderSectorState( - 'Prove', - sector, - sector.TaskIDProve, - sector.AfterProve, - sector.StartedProve - )} - ${this.renderSectorState( - 'Submit', - sector, - sector.TaskIDSubmit, - sector.AfterSubmit, - sector.StartedSubmit - )} - ${this.renderSectorState( - 'Move Storage', - sector, - sector.TaskIDMoveStorage, - sector.AfterMoveStorage, - sector.StartedMoveStorage - )} - ${this.renderSectorStateNoTask( - 'Prove Msg Landed', - sector.AfterSubmit, - sector.AfterProveSuccess - )} + ${renderSectorSnapState( + 'Encode', + sector, + sector.TaskIDEncode, + sector.AfterEncode, + sector.StartedEncode + )} + ${renderSectorSnapState( + 'Prove', + sector, + sector.TaskIDProve, + sector.AfterProve, + sector.StartedProve + )} + ${renderSectorSnapState( + 'Submit', + sector, + sector.TaskIDSubmit, + sector.AfterSubmit, + sector.StartedSubmit + )} + ${renderSectorSnapState( + 'Move Storage', + sector, + sector.TaskIDMoveStorage, + sector.AfterMoveStorage, + sector.StartedMoveStorage + )} + ${renderSectorSnapStateNoTask( + 'Prove Msg Landed', + sector.AfterSubmit, + sector.AfterProveSuccess + )}
State
@@ -206,41 +210,41 @@ class UpgradeSectors extends LitElement {
`; - } +} - /** - * Renders a stage cell for the pipeline. - * If this is the "Submit" stage and the sector is waiting for submit, - * we apply the orange "pipeline-waiting-submit" style and show "Waiting". - */ - renderSectorState(name, sector, taskID, after, started) { - // Special case: waiting for submit - if ( - name === 'Submit' && - sector.UpdateReadyAt && - !sector.AfterSubmit && - !sector.TaskIDSubmit - ) { - return html` +/** + * Renders a stage cell for the pipeline. + * If this is the "Submit" stage and the sector is waiting for submit, + * we apply the orange "pipeline-waiting-submit" style and show "Waiting". + */ +export function renderSectorSnapState(name, sector, taskID, after, started) { + // Special case: waiting for submit + if ( + name === 'Submit' && + sector.UpdateReadyAt && + !sector.AfterSubmit && + !sector.TaskIDSubmit + ) { + return html`
${name}
Waiting
`; - } + } - // Otherwise, do normal logic - if (taskID) { - const missing = - sector.MissingTasks && sector.MissingTasks.includes(taskID); + // Otherwise, do normal logic + if (taskID) { + const missing = + sector.MissingTasks && sector.MissingTasks.includes(taskID); - return html` + return html`
${name}
@@ -249,26 +253,23 @@ class UpgradeSectors extends LitElement { ${missing ? html`
FAILED
` : ''} `; - } else { - return html` + } else { + return html`
${name}
${after ? 'Done' : '--'}
`; - } } +} - renderSectorStateNoTask(name, active, after) { - return html` +export function renderSectorSnapStateNoTask(name, active, after) { + return html` + ? 'pipeline-success' + : ''}">
${name}
${after ? 'Done' : '--'}
`; - } } - -customElements.define('upgrade-sectors', UpgradeSectors);