Annotate and hash test data, include README, download from local mirror of p2pool

p2pool/sidechain/sidechain_test.go

@@ -1,6 +1,7 @@
 package sidechain
 
 import (
+	"compress/gzip"
 	"encoding/binary"
 	"git.gammaspectra.live/P2Pool/consensus/v3/monero/client"
 	"git.gammaspectra.live/P2Pool/consensus/v3/types"
@@ -62,7 +63,7 @@ func (c *SideChain) LoadTestData(reader io.Reader, patchedBlocks ...[]byte) erro
 		blocks = append(blocks, b)
 	}
 
-	// Shuffle blocks
+	// Shuffle blocks. This allows testing proper reorg
 	unsafeRandom.Shuffle(len(blocks), func(i, j int) {
 		blocks[i], blocks[j] = blocks[j], blocks[i]
 	})
@@ -132,13 +133,19 @@ func TestSideChainDefault(t *testing.T) {
 
 	s := NewSideChain(GetFakeTestServer(ConsensusDefault))
 
-	f, err := os.Open("testdata/sidechain_dump.dat")
+	f, err := os.Open("testdata/sidechain_dump.dat.gz")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer f.Close()
 
-	testSideChain(s, t, f, 4957203, 2870010)
+	r, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	testSideChain(s, t, r, 4957203, 2870010)
 }
 
 func TestSideChainDefaultPreFork(t *testing.T) {
@@ -158,13 +165,19 @@ func TestSideChainMini(t *testing.T) {
 
 	s := NewSideChain(GetFakeTestServer(ConsensusMini))
 
-	f, err := os.Open("testdata/sidechain_dump_mini.dat")
+	f, err := os.Open("testdata/sidechain_dump_mini.dat.gz")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer f.Close()
 
-	testSideChain(s, t, f, 4414446, 2870010)
+	r, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	testSideChain(s, t, r, 4414446, 2870010)
 }
 
 func TestSideChainMiniPreFork(t *testing.T) {

testdata/README.md

@@ -0,0 +1,44 @@
+# p2pool test data
+
+Test data from upstream p2pool.
+
+File checksums generated via `$ sha256sum --check *.dat *.gz *.txt` are stored under [testdata.sha256](testdata.sha256)
+
+## Sidechain dumps
+
+### _old_sidechain_dump.dat_
+Dump of the blocks from P2Pool Main, including heights [4952872](https://p2pool.observer/share/4952872) to [522805](https://p2pool.observer/share/522805).
+
+This was done before P2Pool v2 hardfork and is not compressed.
+
+### _old_sidechain_dump_mini.dat_
+Dump of the blocks from P2Pool Mini, including heights [2420029](https://mini.p2pool.observer/share/2420029) to [2424349](https://mini.p2pool.observer/share/2424349).
+
+This was done before P2Pool v2 hardfork and is not compressed.
+
+Due to a bug before commit on [P2Pool b498084388](https://github.com/SChernykh/p2pool/commit/b4980843884d01fd1070710b2b7c08f5f6faca91) / [consensus c438622558](https://git.gammaspectra.live/P2Pool/consensus/commit/c438622558adf71698335af7a3eca818c540ffe8) several blocks are missing to finish a proper SideChain sync.
+
+These blocks are included as `old_sidechain_dump_mini_2420027.dat` and `old_sidechain_dump_mini_2420028.dat` as obtained from Observer, and are required to do proper sync.
+
+### _sidechain_dump.dat.gz_
+Dump of the blocks from P2Pool Main, including heights [4952872](https://p2pool.observer/share/4952872) to [4957203](https://p2pool.observer/share/4957203).
+
+This was done after P2Pool v2 hardfork and is compressed.
+
+### _sidechain_dump_mini.dat.gz_
+Dump of the blocks from P2Pool Mini, including heights [4410115](https://mini.p2pool.observer/share/4410115) to [4414446](https://mini.p2pool.observer/share/4414446).
+
+This was done after P2Pool v2 hardfork and is compressed.
+
+### _block.dat_
+Dump of the block height [4674483](https://p2pool.observer/share/4674483) from P2Pool Main.
+
+This was done after P2Pool v2 hardfork.
+
+### _mainnet_test2_block.dat_
+Dump of the block height [53450](https://p2pool.observer/share/53450) from P2Pool Main.
+
+This was done before P2Pool v2 hardfork.
+
+### _crypto_tests.txt_
+Several cryptography tests to verify derivations and transaction key generation

testdata/setup.sh

@@ -3,6 +3,8 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 pushd "${SCRIPT_DIR}"
 
+ARCHIVE_URL="https://git.gammaspectra.live/P2Pool/p2pool/raw/commit/"
+
 # Pre-v2 p2pool hardfork
 OLD_TESTS_COMMIT_ID=b9eb66e2b3e02a5ec358ff8a0c5169a5606d9fde
 
@@ -10,31 +12,26 @@ function download_old_test() {
     if [ -f "./old_${1}" ]; then
       return
     fi
-    curl --progress-bar --output "./old_${1}" "https://raw.githubusercontent.com/SChernykh/p2pool/${OLD_TESTS_COMMIT_ID}/tests/src/${1}"
+    curl --progress-bar --output "./old_${1}" "${ARCHIVE_URL}${OLD_TESTS_COMMIT_ID}/tests/src/${1}"
 }
 
+# Post-v2 p2pool hardfork
 TESTS_COMMIT_ID=f455ce398c20137a92a67b062c6311580939abea
 
 function download_test() {
     if [ -f "./${1}" ]; then
       return
     fi
-    curl --progress-bar --output "./${1}" "https://raw.githubusercontent.com/SChernykh/p2pool/${TESTS_COMMIT_ID}/tests/src/${1}"
-}
-
-function download_compressed_test() {
-    if [ -f "./${1}" ]; then
-      return
-    fi
-    curl --progress-bar --output "./${1}.gz" "https://raw.githubusercontent.com/SChernykh/p2pool/${TESTS_COMMIT_ID}/tests/src/${1}.gz" && gzip --decompress "${1}.gz"
-
+    curl --progress-bar --output "./${1}" "${ARCHIVE_URL}${TESTS_COMMIT_ID}/tests/src/${1}"
 }
 
 download_test block.dat
 download_test crypto_tests.txt
-download_compressed_test sidechain_dump.dat
-download_compressed_test sidechain_dump_mini.dat
+download_test sidechain_dump.dat.gz
+download_test sidechain_dump_mini.dat.gz
 
 download_old_test mainnet_test2_block.dat
 download_old_test sidechain_dump.dat
 download_old_test sidechain_dump_mini.dat
+
+sha256sum -c testdata.sha256

testdata/testdata.sha256

@@ -0,0 +1,9 @@
+8a28493a965895eebbf49eb5a635f2387bfebc4a428e97dc7162aa3ae98124bd  block.dat
+dd172e23c018624b3585cb72f07166ebed1888bb5dd1bf9a85b1cbd7a2882c96  old_mainnet_test2_block.dat
+b49c2a6dbf25d3cb55746d753e13a474a5741a93a233a9fa3871fe6458398da2  old_sidechain_dump.dat
+896503d62f33443e7b59a7360ca34d26026e2fb5f5cfc0edca03883325928df6  old_sidechain_dump_mini_2420027.dat
+03d62b29d10eccdd285e90b62fdd25956441359ee1d82ab403a1f6c1c41aef48  old_sidechain_dump_mini_2420028.dat
+da6bb956cc2940f96096b8cd071475e1415f13f6b0deaf6e00648f45760a313d  old_sidechain_dump_mini.dat
+980a66f05b6f2482b11515abba3690620c8cabb9f1f42a44ccdeebd23561aeaf  sidechain_dump.dat.gz
+ab5d47f68dc85a180d6df8e586b11c4987f061e9194aec3a490bd33dd3f02bf2  sidechain_dump_mini.dat.gz
+93c1fabe1cfd552545fc031759742d20523d383331e6aa372d7d94fbdffc383b  crypto_tests.txt