Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

jsonrpc autoconnect, indexes client recovery, docker volumes, bugfixes #68

Merged
merged 5 commits into from
Jan 29, 2020
Merged

jsonrpc autoconnect, indexes client recovery, docker volumes, bugfixes #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5eb2b69
rebase
jsign Jan 24, 2020
bad1db8
revert some modified dirty tests
jsign Jan 25, 2020
b09151b
load from datastore for slashing index
jsign Jan 27, 2020
cbf642c
index/miner: better logic for loading ds
jsign Jan 27, 2020
e765a81
extra fixes
jsign Jan 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions chainstore/chainstore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,10 @@ func (s *Store) LoadAndPrune(ctx context.Context, currHead types.TipSetKey, valu
}

func (s *Store) load(tsk types.TipSetKey, v interface{}) error {
buf, err := s.ds.Get(toKeyData(tsk))
key := toKeyData(tsk)
buf, err := s.ds.Get(key)
if err != nil {
return err
return fmt.Errorf("error getting key %s: %s", key, err)
}
if err := cbor.DecodeInto(buf, v); err != nil {
return err
Expand Down
6 changes: 6 additions & 0 deletions docker/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
version: '3.7'

volumes:
textile-fc-data:

services:
textile-fc:
network_mode: host
Expand All @@ -10,7 +13,10 @@ services:
- TEXTILE_LOTUS_TOKEN=${TEXTILE_LOTUS_TOKEN}
ports:
- 8888:8888
- 6060:6060
restart: always
volumes:
- textile-fc-data:/root/.texfc
Comment on lines +18 to +19
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made a change in the docker-compose used to launch the server, prometheus, grafana, etc.
The textile-fc (our service) uses a volume to map the repo folder to a Docker volume. Mostly for saving progress while restarting the compose environment, and tests index persistence.


prometheus:
network_mode: host
Expand Down
106 changes: 95 additions & 11 deletions docker/grafana/provisioning/dashboards/TextileFC.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@
],
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some improvements in clarity and new boxes that helped discover some things.

"timeFrom": null,
"timeShift": null,
"title": "On-Chain Progress",
"title": "On-Chain Refresh",
"type": "gauge"
},
{
Expand Down Expand Up @@ -597,7 +597,7 @@
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Height",
"title": "Index Height",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
Expand Down Expand Up @@ -684,7 +684,7 @@
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Avg Full-Refresh Duration",
"title": "Full-Refresh Duration",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
Expand Down Expand Up @@ -768,7 +768,7 @@
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Avg Delta-Refresh Duration",
"title": "Delta-Refresh Duration",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
Expand Down Expand Up @@ -828,7 +828,7 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Meta Progress",
"title": "Meta Refresh",
"type": "gauge"
},
{
Expand Down Expand Up @@ -884,7 +884,7 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Current #Miners on-chain",
"title": "#Miners Ping Reply",
"type": "bargauge"
},
{
Expand Down Expand Up @@ -948,7 +948,7 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Full refresh progress",
"title": "Full Refresh",
"type": "gauge"
},
{
Expand Down Expand Up @@ -1022,7 +1022,7 @@
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Last Refresh Duration",
"title": "Refresh Duration",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
Expand All @@ -1032,7 +1032,7 @@
"value": "null"
}
],
"valueName": "current"
"valueName": "avg"
},
{
"cacheTimeout": null,
Expand Down Expand Up @@ -1082,7 +1082,7 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Last Query-Ask Response",
"title": "Last Query-Ask Reply",
"type": "bargauge"
},
{
Expand Down Expand Up @@ -1156,8 +1156,92 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Refresh Progress",
"title": "Refresh",
"type": "gauge"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"#299c46",
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": null,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 4,
"w": 2,
"x": 2,
"y": 15
},
"id": 37,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"pluginVersion": "6.5.3",
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": true,
"lineColor": "rgb(31, 120, 193)",
"show": true,
"ymax": null,
"ymin": null
},
"tableColumn": "",
"targets": [
{
"expr": "textilefc_indexslashing_updated_height",
"refId": "A"
}
],
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Index Height",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
}
],
"refresh": "5s",
Expand Down
2 changes: 1 addition & 1 deletion exe/server/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o main cmd/server/main.go

FROM alpine
COPY --from=builder /app/api/server/ip2location-ip4.bin /app/ip2location-ip4.bin
COPY --from=builder /app/main /app/main
COPY --from=builder /app/api/server/ip2location-ip4.bin /app
WORKDIR /app
ENTRYPOINT ["./main"]
9 changes: 9 additions & 0 deletions exe/server/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
logging "github.com/ipfs/go-log"
"github.com/textileio/filecoin/api/server"
"github.com/textileio/filecoin/tests"
_ "net/http/pprof"
)

var (
Expand All @@ -25,7 +26,9 @@ func main() {
logging.SetLogLevel("rpc", "error")
logging.SetLogLevel("dht", "error")
logging.SetLogLevel("swarm2", "error")

instrumentationSetup()
pprofSetup()

// ToDo: Flags for configuration

Expand Down Expand Up @@ -73,3 +76,9 @@ func instrumentationSetup() {
}
}()
}

func pprofSetup() {
go func() {
log.Error(http.ListenAndServe("localhost:6060", nil))
}()
}
34 changes: 2 additions & 32 deletions index/miner/meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"sync"
"time"

"github.com/ipfs/go-datastore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/textileio/filecoin/fchost"
"github.com/textileio/filecoin/iplocation"
Expand All @@ -15,7 +14,7 @@ import (
)

const (
metadataRefreshInterval = time.Second * 30
metadataRefreshInterval = time.Second * 45
pingTimeout = time.Second * 3
pingRateLim = 100
)
Expand Down Expand Up @@ -50,6 +49,7 @@ func (mi *MinerIndex) metaWorker() {
newIndex, err := updateMetaIndex(mi.ctx, mi.api, mi.h, mi.lr, addrs)
if err != nil {
log.Errorf("error when updating meta index: %s", err)
break
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If for some reason generating a new index fails, log the error and break here. If we continue we would override the current-good-state-index with an empty one and lost things. Better to keep it outdated than to lose it.

}
if err := mi.persistMetaIndex(newIndex); err != nil {
log.Errorf("error when persisting meta index: %s", err)
Expand Down Expand Up @@ -151,33 +151,3 @@ func (mi *MinerIndex) persistMetaIndex(index MetaIndex) error {
}
return nil
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved to miners.go since loading from store is a general thing.

// loadFromDS loads persisted indexes to memory datastructures. No locks needed
// since its only called from New().
func (mi *MinerIndex) loadFromDS() error {
buf, err := mi.ds.Get(dsKeyMetaIndex)
if err != nil {
if err == datastore.ErrNotFound {
mi.index = Index{
Meta: MetaIndex{Info: make(map[string]Meta)},
Chain: ChainIndex{Power: make(map[string]Power)},
}
return nil
}
return err
}
var metaIndex MetaIndex
if err := cbor.DecodeInto(buf, &metaIndex); err != nil {
return err
}

var chainIndex ChainIndex
if _, err := mi.store.GetLastCheckpoint(&chainIndex); err != nil {
return err
}

mi.index.Meta = metaIndex
mi.index.Chain = chainIndex

return nil
}
Loading