diff --git a/.gitignore b/.gitignore index df3f25ba..c8ee8b64 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build *.tmp .idea/ +networkhistorydivergencetool/segments diff --git a/networkhistorydivergencetool/README.md b/networkhistorydivergencetool/README.md new file mode 100644 index 00000000..503d0150 --- /dev/null +++ b/networkhistorydivergencetool/README.md @@ -0,0 +1,47 @@ +## Network History Divergence Tool + +A tool to quickly identify the source of network history divergence between 2 nodes. + +For example, to figure where and why the network history started to diverge between two datanodes: + +`networkhistorydivergencetool https://vega.mainnet.stakingcabin.com https://m0.vega.community` + +Would give output like the following: + +``` +Truth Server: https://vega.mainnet.stakingcabin.com +To Compare Server: https://m0.vega.community +IPFS Host: localhost:7001 +Minimum Height: 0 +Maximum Height: 1000000000 +First Different HistorySegmentID values for FromHeight 12601 ToHeight 12900 Truth:QmbVWV9x5y9PVAiabh3ajBeEfUyGut9AEVP3rTM2gwM9wE Compare:QmPETRkkLLGgMRsXL1UHWrXyXqx3oQWMDEwwSpPHBhphEi: +History segment QmbVWV9x5y9PVAiabh3ajBeEfUyGut9AEVP3rTM2gwM9wE sourced successfully. +History segment QmPETRkkLLGgMRsXL1UHWrXyXqx3oQWMDEwwSpPHBhphEi sourced successfully. + +MISMATCHED DATA: currentstate/delegations_current at fromHeight 12601, toHeight 12900, to see differences: diff /home/matthewpendrey/projects/vegatoolstest/vegatools/networkhistorydivergencetool/segments/QmbVWV9x5y9PVAiabh3ajBeEfUyGut9AEVP3rTM2gwM9wE/currentstate/delegations_current /home/matthewpendrey/projects/vegatoolstest/vegatools/networkhistorydivergencetool/segments/QmPETRkkLLGgMRsXL1UHWrXyXqx3oQWMDEwwSpPHBhphEi/currentstate/delegations_current + +MISMATCHED DATA: history/delegations at fromHeight 12601, toHeight 12900, to see differences: diff /home/matthewpendrey/projects/vegatoolstest/vegatools/networkhistorydivergencetool/segments/QmbVWV9x5y9PVAiabh3ajBeEfUyGut9AEVP3rTM2gwM9wE/history/delegations /home/matthewpendrey/projects/vegatoolstest/vegatools/networkhistorydivergencetool/segments/QmPETRkkLLGgMRsXL1UHWrXyXqx3oQWMDEwwSpPHBhphEi/history/delegations +``` + +**Prior to running the tool you must start an IPFS daemon that can connect to and source the segments from the network.** +To do this, install ipfs: https://docs.ipfs.tech/install/command-line/#system-requirements + +Update the ipfs config file (usually in ~/.ipfs) and add the relevant bootstrap peers to connect to the vega network, e.g.: + +``` +"Bootstrap": [ +"/dns/api0.vega.community/tcp/4001/ipfs/12D3KooWAHkKJfX7rt1pAuGebP9g2BGTT5w7peFGyWd2QbpyZwaw", + "/dns/api1.vega.community/tcp/4001/ipfs/12D3KooWDZrusS1p2XyJDbCaWkVDCk2wJaKi6tNb4bjgSHo9yi5Q", + "/dns/api2.vega.community/tcp/4001/ipfs/12D3KooWEH9pQd6P7RgNEpwbRyavWcwrAdiy9etivXqQZzd7Jkrh", + "/dns/m0.vega.community/tcp/4001/ipfs/12D3KooWQvja5nUKkdBR9FdX9p68B8W5ze9TeWjnuVBPPDoXcvW4", + "/dns/m2.vega.community/tcp/4001/ipfs/12D3KooWAdkdG39tiZ7o8CoF4ktPpuxGkfG6at6YrfFwvgAwGHAv", + "/dns/metabase.vega.community/tcp/4001/ipfs/12D3KooWCxuWEb2uGjcZpP87xprJELpZtKDXKkb2AjsYBbXZarG9", + "/ip4/78.141.194.186/tcp/4001/p2p/12D3KooWKjggJGpS5kzQaNz7JPfGBHHQocP3qGYWhquFw8zQLZGv", + "/dns/vega.mainnet.stakingcabin.com/tcp/4001/p2p/12D3KooWJiJoL3Ua6QYPhV7Q227r4hQvAArxLuY6NqkpMn3gFtLf", + "/dns/vega-data.nodes.guru/tcp/4001/p2p/12D3KooWBwQLm9ZskZPDveeNMF42bskZ3M3HXPyVRJ5XtmG1todg", + "/dns/vega-mainnet.anyvalid.com/tcp/4001/p2p/12D3KooWLxQHbaWrtW4XuzzC2DHp3qGBKhUrSscmW59FPXwtw5DW" + ], +``` +Add the `swarm.key` file for the vega network to the ipfs home directory (again, usually ~/.ipfs). You can usually grab this from `/networkhistory/store/ipfs/swarm.key` + +Once this is done, try executing `ipfs get ` where `` matches the id of one segment from each server, this will confirm that your IPFS setup is correct. Once this is done you can run the tool. \ No newline at end of file diff --git a/networkhistorydivergencetool/go.mod b/networkhistorydivergencetool/go.mod new file mode 100644 index 00000000..0d4fbd63 --- /dev/null +++ b/networkhistorydivergencetool/go.mod @@ -0,0 +1,33 @@ +module networkhistorydivergencetool + +go 1.19 + +require ( + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/ipfs/boxo v0.8.0 // indirect + github.com/ipfs/go-cid v0.4.0 // indirect + github.com/ipfs/go-ipfs-api v0.6.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/libp2p/go-libp2p v0.26.3 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.8.0 // indirect + github.com/multiformats/go-multibase v0.1.1 // indirect + github.com/multiformats/go-multicodec v0.8.1 // indirect + github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multistream v0.4.1 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect + golang.org/x/crypto v0.6.0 // indirect + golang.org/x/sys v0.6.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + lukechampine.com/blake3 v1.1.7 // indirect +) diff --git a/networkhistorydivergencetool/go.sum b/networkhistorydivergencetool/go.sum new file mode 100644 index 00000000..fbdbd75d --- /dev/null +++ b/networkhistorydivergencetool/go.sum @@ -0,0 +1,61 @@ +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= +github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= +github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-ipfs-api v0.6.0 h1:JARgG0VTbjyVhO5ZfesnbXv9wTcMvoKRBLF1SzJqzmg= +github.com/ipfs/go-ipfs-api v0.6.0/go.mod h1:iDC2VMwN9LUpQV/GzEeZ2zNqd8NUdRmWcFM+K/6odf0= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= +github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= +github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= +github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/networkhistorydivergencetool/main.go b/networkhistorydivergencetool/main.go new file mode 100644 index 00000000..3709b999 --- /dev/null +++ b/networkhistorydivergencetool/main.go @@ -0,0 +1,417 @@ +package main + +import ( + "archive/zip" + "crypto/md5" + "encoding/json" + "errors" + "fmt" + shell "github.com/ipfs/go-ipfs-api" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Define the original Response struct to parse JSON +type Response struct { + Segments []struct { + FromHeight string `json:"fromHeight"` + ToHeight string `json:"toHeight"` + HistorySegmentID string `json:"historySegmentId"` + PreviousHistorySegmentID string `json:"previousHistorySegmentId"` + DatabaseVersion string `json:"databaseVersion"` + ChainID string `json:"chainId"` + } `json:"segments"` +} + +// Define the modified struct with int type for ToHeight and FromHeight +type Segments struct { + Segments []Segment `json:"segments"` +} + +type Segment struct { + FromHeight int `json:"fromHeight"` + ToHeight int `json:"toHeight"` + HistorySegmentID string `json:"historySegmentId"` + PreviousHistorySegmentID string `json:"previousHistorySegmentId"` + DatabaseVersion string `json:"databaseVersion"` + ChainID string `json:"chainId"` +} + +var ipfsShell *shell.Shell + +const defaultIPFSHost = "localhost:7001" +const defaultMinHeight = 0 +const defaultMaxHeight = 1_000_000_000 + +func main() { + + // Check if the correct number of arguments is provided + if len(os.Args) < 3 || len(os.Args) > 6 { + printHelp() + os.Exit(1) + } + + // Extract the server addresses + truthServer := os.Args[1] + toCompareServer := os.Args[2] + + // Set default values for ipfsHost, minHeight, and maxHeight + ipfsHost := defaultIPFSHost + minHeight := defaultMinHeight + maxHeight := defaultMaxHeight + + // Parse optional parameters if provided + if len(os.Args) > 3 { + ipfsHost = os.Args[3] + } + if len(os.Args) > 4 { + minHeight = parseInt(os.Args[4]) + } + if len(os.Args) > 5 { + maxHeight = parseInt(os.Args[5]) + } + + // Print the provided values + fmt.Printf("Truth Server: %s\n", truthServer) + fmt.Printf("To Compare Server: %s\n", toCompareServer) + fmt.Printf("IPFS Host: %s\n", ipfsHost) + fmt.Printf("Minimum Height: %d\n", minHeight) + fmt.Printf("Maximum Height: %d\n", maxHeight) + + ipfsShell = shell.NewShell(ipfsHost) + + theTruthResponse := getSegments(truthServer) + toCompareResponse := getSegments(toCompareServer) + + toComparetoHeightToSegment := map[int]Segment{} + for _, segment := range toCompareResponse.Segments { + toComparetoHeightToSegment[segment.ToHeight] = segment + } + + theTruthToHeightToSegment := map[int]Segment{} + for _, segment := range theTruthResponse.Segments { + theTruthToHeightToSegment[segment.ToHeight] = segment + } + + var truthSegments []Segment + var toCompareSegments []Segment + + for toHeight, segment := range toComparetoHeightToSegment { + if toHeight > maxHeight { + continue + } + + if toHeight < minHeight { + continue + } + + toCompareSegments = append(toCompareSegments, segment) + if truthSegment, ok := theTruthToHeightToSegment[toHeight]; ok { + truthSegments = append(truthSegments, truthSegment) + } else { + log.Panicf("no truth segment found for height %d", toHeight) + } + + } + + sort.SliceStable(toCompareSegments, func(j, k int) bool { + return toCompareSegments[j].ToHeight < toCompareSegments[k].ToHeight + }) + + sort.SliceStable(truthSegments, func(j, k int) bool { + return truthSegments[j].ToHeight < truthSegments[k].ToHeight + }) + + for i := 0; i < len(truthSegments); i++ { + truth := truthSegments[i] + compare := toCompareSegments[i] + + if truth.HistorySegmentID != compare.HistorySegmentID { + fmt.Printf("First Different HistorySegmentID values for FromHeight %d ToHeight %d Truth:%s Compare:%s:\n", truth.FromHeight, truth.ToHeight, + truth.HistorySegmentID, compare.HistorySegmentID) + + truthSegmentDir, err := sourceHistorySegment(truth.HistorySegmentID) + if err != nil { + log.Panicf("failed to source truth history segment: %s", err) + } + + compareSegmentDir, err := sourceHistorySegment(compare.HistorySegmentID) + if err != nil { + log.Panicf("failed to source compare history segment: %s", err) + } + + mismatchedFiles, err := compareDirectories(truthSegmentDir, compareSegmentDir) + if err != nil { + if errors.Is(err, ERR_DIFF_FILE_NUM) { + fmt.Printf("Failed to compare segments, different number of files in dir") + break + } else { + log.Panicf("failed to compare directories: %s", err) + } + } + + if len(mismatchedFiles) == 0 { + fmt.Printf("No mismatched files found, contents are identical but different history segment IDs\n") + } + + for _, file := range mismatchedFiles { + absTruthDir, err := filepath.Abs(truthSegmentDir) + if err != nil { + log.Panicf("failed to get abs path: %s", err) + } + + absCompareDir, err := filepath.Abs(compareSegmentDir) + if err != nil { + log.Panicf("failed to get compare path: %s", err) + } + + fmt.Printf("\nMISMATCHED DATA: %s at fromHeight %d, toHeight %d, to see differences: diff %s %s \n", file, truth.FromHeight, + truth.ToHeight, filepath.Join(absTruthDir, file), filepath.Join(absCompareDir, file)) + } + + break + } + } + +} + +func printHelp() { + fmt.Println("Usage: go run main.go [ipfsHost] [minHeight] [maxHeight]") + fmt.Println("Parameters:") + fmt.Println("\t\tThe first server address.") + fmt.Println("\t\tThe second server address.") + fmt.Println("[ipfsHost]\t\t(Optional) The IPFS host. Default: " + defaultIPFSHost) + fmt.Println("[minHeight]\t\t(Optional) The minimum height. Default: " + + strconv.Itoa(defaultMinHeight)) + fmt.Println("[maxHeight]\t\t(Optional) The maximum height. Default: " + strconv.Itoa(defaultMaxHeight)) +} + +func parseInt(arg string) int { + value := 0 + _, err := fmt.Sscanf(arg, "%d", &value) + if err != nil { + fmt.Printf("Failed to parse integer from %s. Using default value.\n", arg) + } + return value +} + +var ERR_DIFF_FILE_NUM = errors.New("Different number of files") + +func compareDirectories(dir1, dir2 string) ([]string, error) { + var nonMatchingFiles []string + + // Read the file names from the first directory + files1, err := getFileNames(dir1) + if err != nil { + return nil, err + } + + // Read the file names from the second directory + files2, err := getFileNames(dir2) + if err != nil { + return nil, err + } + + // Compare the number of files + if len(files1) != len(files2) { + return nil, ERR_DIFF_FILE_NUM + } + + // Compare the file names and hashes + for i := range files1 { + if files1[i] != files2[i] { + nonMatchingFiles = append(nonMatchingFiles, files1[i]) + continue + } + + // Read the contents of the file from the first directory + filePath1 := filepath.Join(dir1, files1[i]) + content1, err := ioutil.ReadFile(filePath1) + if err != nil { + return nil, err + } + + // Read the contents of the file from the second directory + filePath2 := filepath.Join(dir2, files2[i]) + content2, err := ioutil.ReadFile(filePath2) + if err != nil { + return nil, err + } + + // Calculate the MD5 hash of the contents + hash1 := md5.Sum(content1) + hash2 := md5.Sum(content2) + + // Compare the hashes + if hash1 != hash2 { + nonMatchingFiles = append(nonMatchingFiles, files1[i]) + } + } + + return nonMatchingFiles, nil +} + +func getFileNames(dir string) ([]string, error) { + var files []string + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // Skip directories + if info.IsDir() { + return nil + } + + // Get the relative file path + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + files = append(files, relPath) + return nil + }) + + if err != nil { + return nil, err + } + + return files, nil +} + +func sourceHistorySegment(segmentID string) (string, error) { + + err := os.MkdirAll(fmt.Sprintf("./segments/%s", segmentID), os.ModePerm) + if err != nil { + return "", fmt.Errorf("Error creating segment directory: %w\n", err) + } + + zipFileName := fmt.Sprintf("./segments/%s.zip", segmentID) + err = ipfsShell.Get(segmentID, zipFileName) + if err != nil { + return "", fmt.Errorf("Error sourcing history segment from IPFS: %v\n", err) + } + fmt.Printf("History segment %s sourced successfully.\n", segmentID) + + segmentDir := fmt.Sprintf("./segments/%s", segmentID) + os.MkdirAll(segmentDir, os.ModePerm) + UnzipSource(zipFileName, segmentDir) + + return segmentDir, nil +} + +func UnzipSource(source, destination string) error { + reader, err := zip.OpenReader(source) + if err != nil { + return err + } + defer reader.Close() + + destination, err = filepath.Abs(destination) + if err != nil { + return err + } + + for _, f := range reader.File { + err := unzipFile(f, destination) + if err != nil { + return err + } + } + + return nil +} + +func unzipFile(f *zip.File, destination string) error { + filePath := filepath.Join(destination, f.Name) + if !strings.HasPrefix(filePath, filepath.Clean(destination)+string(os.PathSeparator)) { + return fmt.Errorf("invalid file path: %s", filePath) + } + + if f.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, os.ModePerm); err != nil { + return err + } + return nil + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return err + } + + destinationFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer destinationFile.Close() + + zippedFile, err := f.Open() + if err != nil { + return err + } + defer zippedFile.Close() + + if _, err := io.Copy(destinationFile, zippedFile); err != nil { + return err + } + return nil +} + +func getSegments(server string) Segments { + + server += "/api/v2/networkhistory/segments" + response, err := http.Get(server) + if err != nil { + log.Panicf("Error accessing %s: %v\n", server, err) + } + defer response.Body.Close() + + if response.StatusCode != 200 { + log.Panicf("Error accessing %s: %s\n", server, response.Status) + } + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Panicf("Error reading response body from %s: %v\n", server, err) + } + + var parsedResponse Response + err = json.Unmarshal(body, &parsedResponse) + if err != nil { + log.Panicf("Error parsing JSON from %s: %v\n", server, err) + } + + modifiedResponse := convertToModifiedResponse(parsedResponse) + return modifiedResponse +} + +// Helper function to convert Response to Segments +func convertToModifiedResponse(response Response) Segments { + modifiedResponse := Segments{} + for _, segment := range response.Segments { + fromHeight, _ := strconv.Atoi(segment.FromHeight) + toHeight, _ := strconv.Atoi(segment.ToHeight) + modifiedSegment := struct { + FromHeight int `json:"fromHeight"` + ToHeight int `json:"toHeight"` + HistorySegmentID string `json:"historySegmentId"` + PreviousHistorySegmentID string `json:"previousHistorySegmentId"` + DatabaseVersion string `json:"databaseVersion"` + ChainID string `json:"chainId"` + }{ + FromHeight: fromHeight, + ToHeight: toHeight, + HistorySegmentID: segment.HistorySegmentID, + PreviousHistorySegmentID: segment.PreviousHistorySegmentID, + DatabaseVersion: segment.DatabaseVersion, + ChainID: segment.ChainID, + } + modifiedResponse.Segments = append(modifiedResponse.Segments, modifiedSegment) + } + return modifiedResponse +}