From 628a3db6d0999b4ce72a04052a7655625249192e Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 8 Jan 2025 17:36:58 +0530 Subject: [PATCH] update deal section --- web/api/webrpc/market.go | 102 ++++++++++++++++++------ web/api/webrpc/sector.go | 80 ++++++++++++++----- web/static/pages/mk12-deal/deal.mjs | 1 + web/static/pages/sector/sector-info.mjs | 2 + 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 346dc9732..bf38c16d7 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -187,6 +187,7 @@ type StorageDealSummary struct { Miner string `json:"miner"` IsLegacy bool `json:"is_legacy"` Indexed *bool `db:"indexed" json:"indexed"` + IsDDO bool `db:"is_ddo" json:"is_ddo"` } func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDealSummary, error) { @@ -207,30 +208,83 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal if !isLegacy { var summaries []StorageDealSummary err = a.deps.DB.Select(ctx, &summaries, `SELECT - md.uuid, - md.sp_id, - md.created_at, - md.signed_proposal_cid, - md.offline, - md.verified, - md.start_epoch, - md.end_epoch, - md.client_peer_id, - md.chain_deal_id, - md.publish_cid, - md.piece_cid, - md.piece_size, - md.fast_retrieval, - md.announce_to_ipni, - md.url, - md.url_headers, - md.error, - mpd.sector_num, - mpm.indexed - FROM market_mk12_deals md - LEFT JOIN market_piece_deal mpd ON mpd.id = md.uuid AND mpd.sp_id = md.sp_id - LEFT JOIN market_piece_metadata mpm ON mpm.piece_cid = md.piece_cid - WHERE md.uuid = $1`, id.String()) + deal.uuid, + deal.sp_id, + deal.created_at, + deal.signed_proposal_cid, + deal.offline, + deal.verified, + deal.start_epoch, + deal.end_epoch, + deal.client_peer_id, + deal.chain_deal_id, + deal.publish_cid, + deal.piece_cid, + deal.piece_size, + deal.fast_retrieval, + deal.announce_to_ipni, + deal.url, + deal.url_headers, + deal.error, + mpd.sector_num, + mpm.indexed, + deal.is_ddo -- New column indicating whether the deal is from market_direct_deals + FROM ( + -- Query from market_mk12_deals (default, original table) + SELECT + md.uuid, + md.sp_id, + md.created_at, + md.signed_proposal_cid, + md.offline, + md.verified, + md.start_epoch, + md.end_epoch, + md.client_peer_id, + md.chain_deal_id, + md.publish_cid, + md.piece_cid, + md.piece_size, + md.fast_retrieval, + md.announce_to_ipni, + md.url, + md.url_headers, + md.error, + FALSE AS is_ddo -- Not from market_direct_deals + FROM market_mk12_deals md + WHERE md.uuid = $1 + + UNION ALL + + -- Query from market_direct_deals (new table) + SELECT + mdd.uuid, + mdd.sp_id, + mdd.created_at, + '' AS signed_proposal_cid, + mdd.offline, + mdd.verified, + mdd.start_epoch, + mdd.end_epoch, + '' AS client_peer_id, + 0 AS chain_deal_id, + '' AS publish_cid, + mdd.piece_cid, + mdd.piece_size, + mdd.fast_retrieval, + mdd.announce_to_ipni, + '' AS url, + '{}' AS url_headers, + '' AS error, + TRUE AS is_ddo -- From market_direct_deals + FROM market_direct_deals mdd + WHERE mdd.uuid = $1 + ) AS deal + LEFT JOIN market_piece_deal mpd + ON mpd.id = deal.uuid AND mpd.sp_id = deal.sp_id + LEFT JOIN market_piece_metadata mpm + ON mpm.piece_cid = deal.piece_cid; + `, id.String()) if err != nil { return &StorageDealSummary{}, xerrors.Errorf("select deal summary: %w", err) diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index fd49f0bc9..735e6e96b 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -115,6 +115,7 @@ type SectorPieceMeta struct { PieceCid string `db:"piece_cid"` PieceSize int64 `db:"piece_size"` + DealID string `db:"deal_id"` DataUrl string `db:"data_url"` DataRawSize int64 `db:"data_raw_size"` DeleteOnFinalize bool `db:"data_delete_on_finalize"` @@ -138,7 +139,10 @@ type SectorPieceMeta struct { PieceParkTaskID *int64 `db:"-"` PieceParkCleanupTaskID *int64 `db:"-"` - IsSnapPiece bool `db:"-"` + IsSnapPiece bool `db:"is_snap"` + + MK12Deal bool `db:"boost_deal"` + LegacyDeal bool `db:"legacy_deal"` } type FileLocations struct { @@ -436,32 +440,64 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto var pieces []SectorPieceMeta - // Fetch PoRep pieces err = a.deps.DB.Select(ctx, &pieces, `SELECT piece_index, piece_cid, piece_size, - data_url, data_raw_size, data_delete_on_finalize, - f05_publish_cid, f05_deal_id, direct_piece_activation_manifest FROM sectors_sdr_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid) + data_url, data_raw_size, data_delete_on_finalize, + f05_publish_cid, f05_deal_id, direct_piece_activation_manifest, + mpd.id AS deal_id, -- Extracted id from market_piece_deal + mpd.boost_deal, -- Retrieved boost_deal from market_piece_deal + mpd.legacy_deal, -- Retrieved legacy_deal from market_piece_deal + is_snap -- New column indicating whether the piece is a snap deal + FROM ( + -- Meta table entries (permanent, prioritized) + SELECT meta.piece_num AS piece_index, meta.piece_cid, meta.piece_size, + NULL AS data_url, meta.raw_data_size AS data_raw_size, + NOT meta.requested_keep_data AS data_delete_on_finalize, + meta.f05_publish_cid, meta.f05_deal_id, meta.ddo_pam AS direct_piece_activation_manifest, + meta.sp_id, + NOT sm.is_cc AS is_snap -- is_snap based on is_cc from sectors_meta + FROM sectors_meta_pieces meta + JOIN sectors_meta sm ON meta.sp_id = sm.sp_id AND meta.sector_num = sm.sector_num + WHERE meta.sp_id = $1 AND meta.sector_num = $2 + + UNION ALL + + -- SDR pipeline entries (temporary, non-snap pieces) + SELECT sdr.piece_index, sdr.piece_cid, sdr.piece_size, + sdr.data_url, sdr.data_raw_size, sdr.data_delete_on_finalize, + sdr.f05_publish_cid, sdr.f05_deal_id, sdr.direct_piece_activation_manifest, + sdr.sp_id, + FALSE AS is_snap -- SDR pipeline pieces are never snap deals + FROM sectors_sdr_initial_pieces sdr + WHERE sdr.sp_id = $1 AND sdr.sector_number = $2 + AND NOT EXISTS ( + SELECT 1 + FROM sectors_meta_pieces meta + WHERE meta.sp_id = sdr.sp_id AND meta.piece_cid = sdr.piece_cid + ) + + UNION ALL + + -- Snap pipeline entries (temporary, always snap deals) + SELECT snap.piece_index, snap.piece_cid, snap.piece_size, + snap.data_url, snap.data_raw_size, snap.data_delete_on_finalize, + NULL AS f05_publish_cid, NULL AS f05_deal_id, snap.direct_piece_activation_manifest, + snap.sp_id, + TRUE AS is_snap -- Snap pipeline pieces are always snap deals + FROM sectors_snap_initial_pieces snap + WHERE snap.sp_id = $1 AND snap.sector_number = $2 + AND NOT EXISTS ( + SELECT 1 + FROM sectors_meta_pieces meta + WHERE meta.sp_id = snap.sp_id AND meta.piece_cid = snap.piece_cid + ) + ) AS combined + LEFT JOIN market_piece_deal mpd + ON combined.sp_id = mpd.sp_id AND combined.piece_cid = mpd.piece_cid; + `, spid, intid) if err != nil { return nil, xerrors.Errorf("failed to fetch sector pieces: %w", err) } - // Fetch SnapDeals pieces - var snapPieces []SectorPieceMeta - - err = a.deps.DB.Select(ctx, &snapPieces, `SELECT piece_index, piece_cid, piece_size, - data_url, data_raw_size, data_delete_on_finalize, - NULL as f05_publish_cid, NULL as f05_deal_id, direct_piece_activation_manifest FROM sectors_snap_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid) - if err != nil { - return nil, xerrors.Errorf("failed to fetch snap sector pieces: %w", err) - } - - // Mark SnapDeals pieces - for i := range snapPieces { - snapPieces[i].IsSnapPiece = true - } - - // Combine both slices - pieces = append(pieces, snapPieces...) - for i := range pieces { pieces[i].StrPieceSize = types.SizeStr(types.NewInt(uint64(pieces[i].PieceSize))) pieces[i].StrDataRawSize = types.SizeStr(types.NewInt(uint64(pieces[i].DataRawSize))) diff --git a/web/static/pages/mk12-deal/deal.mjs b/web/static/pages/mk12-deal/deal.mjs index 3f9e458ab..35bee2068 100644 --- a/web/static/pages/mk12-deal/deal.mjs +++ b/web/static/pages/mk12-deal/deal.mjs @@ -33,6 +33,7 @@ class DealDetails extends LitElement { {property: 'Offline', value: entry.offline}, {property: 'Verified', value: entry.verified}, {property: 'Is Legacy', value: entry.is_legacy}, + {property: 'Is DDO', value: entry.is_ddo}, {property: 'Start Epoch', value: html``}, {property: 'End Epoch', value: html``}, {property: 'Client Peer ID', value: entry.client_peer_id}, diff --git a/web/static/pages/sector/sector-info.mjs b/web/static/pages/sector/sector-info.mjs index fc9232893..528260c84 100644 --- a/web/static/pages/sector/sector-info.mjs +++ b/web/static/pages/sector/sector-info.mjs @@ -112,6 +112,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement { Piece Index Piece CID Piece Size + Deal ID Data URL Data Raw Size Delete On Finalize @@ -130,6 +131,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement { ${piece.PieceIndex} ${piece.PieceCid} ${piece.PieceSize} + ${piece.DealID} ${piece.DataUrl} ${piece.DataRawSize} ${piece.DeleteOnFinalize}