Feature/no more anuvu/squashfs dep

.github/workflows/build.yaml

@@ -53,8 +53,6 @@ jobs:
           GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
           curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
           sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse
-          (cd /tmp && git clone https://github.com/AgentD/squashfs-tools-ng && cd squashfs-tools-ng && ./autogen.sh && ./configure --prefix=/usr && make -j2 && sudo make -j2 install && sudo ldconfig -v)
-          (cd /tmp && git clone https://github.com/anuvu/squashfs && cd squashfs && make && sudo cp squashtool/squashtool /usr/bin)
           echo "running kernel is: $(uname -a)"
       - name: Go-download
         run: |

build.yaml

@@ -22,7 +22,6 @@ build-env:
       xz \
       gettext-dev \
       lvm2-dev util-linux-dev \
-      squashfs-tools-ng-dev \
       linux-headers
 
     # json-c doesn't have static binaries in alpine

doc/install.md

@@ -37,7 +37,7 @@ packages:
 #### **Ubuntu 22.04**
 
     sudo apt install lxc-dev libacl1-dev libgpgme-dev libcap-dev libseccomp-dev
-    sudo apt install libpam0g-dev libselinux-dev libssl-dev libzstd-dev libcryptsetup-dev libdevmapper-dev cryptsetup-bin pkg-config libsquashfs1 libsquashfs-dev
+    sudo apt install libpam0g-dev libselinux-dev libssl-dev libzstd-dev libcryptsetup-dev libdevmapper-dev cryptsetup-bin pkg-config
 
 
 **To run `make check` you will also need:**
@@ -46,18 +46,6 @@ packages:
 
 **umoci** - https://github.com/opencontainers/umoci
 
-**squashtool**, but with a slightly different config than what is mentioned in the install guide (see below) - https://github.com/anuvu/squashfs
-
-Contrary to what the documentation in squashfs implies, squashtool and
-libsquash from squash-tools-ng need to be installed globally, as user specific
-path overrides aren't propagated into `make check`'s test envs.
-
-Thus, when you reach the step **install into mylocal="$HOME/lib"** from the squashfs guide, use the config below. You can put them at the end of your .bashrc file so you don't need to run them every time.
-
-    mylocal="/usr/local"
-    export LD_LIBRARY_PATH=$mylocal/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
-    export PKG_CONFIG_PATH=$mylocal/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}
-
 Since the path **/usr/local** is owned by root, when you reach the step to run **make install**, you need to run it as **sudo**.
 
 `make check`  requires the **golangci-lint** binary to be present in $GOPATH/bin

go.mod

@@ -4,7 +4,6 @@ go 1.20
 
 require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
-	github.com/anuvu/squashfs v0.0.0-20220404153901-d496132b2781
 	github.com/apex/log v1.9.0
 	github.com/apparentlymart/go-shquot v0.0.1
 	github.com/cheggaaa/pb/v3 v3.1.2

go.sum

@@ -85,8 +85,6 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/anuvu/squashfs v0.0.0-20220404153901-d496132b2781 h1:GI4SoPum6MHsm6nwTG1fueiJQkMOZkQagMKZ4Z5pAaY=
-github.com/anuvu/squashfs v0.0.0-20220404153901-d496132b2781/go.mod h1:t4fv4HFEgFR4DdlVcCBeQCvinbf4GJrR4G5xgx0w7qc=
 github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
 github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
 github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@@ -926,7 +924,6 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8=
 github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/urfave/cli/v2 v2.10.3/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo=
 github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8=
 github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
@@ -1164,7 +1161,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/squashfs/superblock.go

@@ -0,0 +1,263 @@
+/*
+This file was initially copied from go-diskfs [1].  The copied portion is
+Copyright (c) 2017 Avi Deitcher and licensed under the terms of the MIT
+license [2].
+
+	[1] https://github.com/diskfs/go-diskfs/filesystem/squashfs/superblock.go
+	[2] https://opensource.org/licenses/MIT
+*/
+package squashfs
+
+import (
+	"encoding/binary"
+	"io"
+	"math"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	superblockMagic        uint32 = 0x73717368
+	superblockMajorVersion uint16 = 4
+	superblockMinorVersion uint16 = 0
+)
+
+type compression uint16
+
+// nolint:unused // copied from diskfs
+const (
+	compressionNone compression = 0
+	compressionGzip compression = 1
+	compressionLzma compression = 2
+	compressionLzo  compression = 3
+	compressionXz   compression = 4
+	compressionLz4  compression = 5
+	compressionZstd compression = 6
+)
+
+const (
+	superblockSize = 96
+)
+
+type inodeRef struct {
+	block  uint32
+	offset uint16
+}
+
+// nolint:unused // copied from diskfs
+func (i *inodeRef) toUint64() uint64 {
+	var u uint64
+	u |= (uint64(i.block) << 16)
+	u |= uint64(i.offset)
+	return u
+}
+
+func parseRootInode(u uint64) *inodeRef {
+	i := &inodeRef{
+		block:  uint32((u >> 16) & 0xffffffff),
+		offset: uint16(u & 0xffff),
+	}
+	return i
+}
+
+type superblockFlags struct {
+	uncompressedInodes    bool
+	uncompressedData      bool
+	uncompressedFragments bool
+	noFragments           bool
+	alwaysFragments       bool
+	dedup                 bool
+	exportable            bool
+	uncompressedXattrs    bool
+	noXattrs              bool
+	compressorOptions     bool
+	uncompressedIDs       bool
+}
+
+type superblock struct {
+	inodes              uint32
+	modTime             time.Time
+	blocksize           uint32
+	fragmentCount       uint32
+	compression         compression
+	idCount             uint16
+	versionMajor        uint16
+	versionMinor        uint16
+	rootInode           *inodeRef
+	size                uint64
+	idTableStart        uint64
+	xattrTableStart     uint64
+	inodeTableStart     uint64
+	directoryTableStart uint64
+	fragmentTableStart  uint64
+	exportTableStart    uint64
+	superblockFlags
+}
+
+// nolint:unused // copied from diskfs
+func (s *superblock) equal(a *superblock) bool {
+	// to compare, need to extract the rootInode
+	inodeEql := *a.rootInode == *s.rootInode
+	s1 := &superblock{}
+	a1 := &superblock{}
+	*s1 = *s
+	*a1 = *a
+	s1.rootInode = nil
+	a1.rootInode = nil
+	modTime := time.Now()
+	s1.modTime = modTime
+	a1.modTime = modTime
+	sblockEql := *s1 == *a1
+	return inodeEql && sblockEql
+}
+
+// nolint:unused // copied from diskfs
+func (s *superblockFlags) bytes() []byte {
+	var flags uint16
+	if s.uncompressedInodes {
+		flags |= 0x0001
+	}
+	if s.uncompressedData {
+		flags |= 0x0002
+	}
+	if s.uncompressedFragments {
+		flags |= 0x0008
+	}
+	if s.noFragments {
+		flags |= 0x0010
+	}
+	if s.alwaysFragments {
+		flags |= 0x0020
+	}
+	if s.dedup {
+		flags |= 0x0040
+	}
+	if s.exportable {
+		flags |= 0x0080
+	}
+	if s.uncompressedXattrs {
+		flags |= 0x0100
+	}
+	if s.noXattrs {
+		flags |= 0x0200
+	}
+	if s.compressorOptions {
+		flags |= 0x0400
+	}
+	if s.uncompressedIDs {
+		flags |= 0x0800
+	}
+	b := make([]byte, 2)
+	binary.LittleEndian.PutUint16(b, flags)
+	return b
+}
+
+func parseFlags(b []byte) (*superblockFlags, error) {
+	targetLength := 2
+	if len(b) != targetLength {
+		return nil, errors.Errorf("received %d bytes instead of expected %d", len(b), targetLength)
+	}
+	flags := binary.LittleEndian.Uint16(b)
+	s := &superblockFlags{
+		uncompressedInodes:    flags&0x0001 == 0x0001,
+		uncompressedData:      flags&0x0002 == 0x0002,
+		uncompressedFragments: flags&0x0008 == 0x0008,
+		noFragments:           flags&0x0010 == 0x0010,
+		alwaysFragments:       flags&0x0020 == 0x0020,
+		dedup:                 flags&0x0040 == 0x0040,
+		exportable:            flags&0x0080 == 0x0080,
+		uncompressedXattrs:    flags&0x0100 == 0x0100,
+		noXattrs:              flags&0x0200 == 0x0200,
+		compressorOptions:     flags&0x0400 == 0x0400,
+		uncompressedIDs:       flags&0x0800 == 0x0800,
+	}
+	return s, nil
+}
+
+// nolint:unused // copied from diskfs
+func (s *superblock) toBytes() []byte {
+	b := make([]byte, superblockSize)
+	binary.LittleEndian.PutUint32(b[0:4], superblockMagic)
+	binary.LittleEndian.PutUint32(b[4:8], s.inodes)
+	binary.LittleEndian.PutUint32(b[8:12], uint32(s.modTime.Unix()))
+	binary.LittleEndian.PutUint32(b[12:16], s.blocksize)
+	binary.LittleEndian.PutUint32(b[16:20], s.fragmentCount)
+	binary.LittleEndian.PutUint16(b[20:22], uint16(s.compression))
+	binary.LittleEndian.PutUint16(b[22:24], uint16(math.Log2(float64(s.blocksize))))
+	copy(b[24:26], s.superblockFlags.bytes())
+	binary.LittleEndian.PutUint16(b[26:28], s.idCount)
+	binary.LittleEndian.PutUint16(b[28:30], superblockMajorVersion)
+	binary.LittleEndian.PutUint16(b[30:32], superblockMinorVersion)
+	binary.LittleEndian.PutUint64(b[32:40], s.rootInode.toUint64())
+	binary.LittleEndian.PutUint64(b[40:48], s.size)
+	binary.LittleEndian.PutUint64(b[48:56], s.idTableStart)
+	binary.LittleEndian.PutUint64(b[56:64], s.xattrTableStart)
+	binary.LittleEndian.PutUint64(b[64:72], s.inodeTableStart)
+	binary.LittleEndian.PutUint64(b[72:80], s.directoryTableStart)
+	binary.LittleEndian.PutUint64(b[80:88], s.fragmentTableStart)
+	binary.LittleEndian.PutUint64(b[88:96], s.exportTableStart)
+	return b
+}
+
+func parseSuperblock(b []byte) (*superblock, error) {
+	if len(b) != superblockSize {
+		return nil, errors.Errorf("superblock had %d bytes instead of expected %d", len(b), superblockSize)
+	}
+	magic := binary.LittleEndian.Uint32(b[0:4])
+	if magic != superblockMagic {
+		return nil, errors.Errorf("superblock had magic of %d instead of expected %d", magic, superblockMagic)
+	}
+	majorVersion := binary.LittleEndian.Uint16(b[28:30])
+	minorVersion := binary.LittleEndian.Uint16(b[30:32])
+	if majorVersion != superblockMajorVersion || minorVersion != superblockMinorVersion {
+		return nil, errors.Errorf("superblock version mismatch, received %d.%d instead of expected %d.%d", majorVersion, minorVersion, superblockMajorVersion, superblockMinorVersion)
+	}
+
+	blocksize := binary.LittleEndian.Uint32(b[12:16])
+	blocklog := binary.LittleEndian.Uint16(b[22:24])
+	expectedLog := uint16(math.Log2(float64(blocksize)))
+	if expectedLog != blocklog {
+		return nil, errors.Errorf("superblock block log mismatch, actual %d expected %d", blocklog, expectedLog)
+	}
+	flags, err := parseFlags(b[24:26])
+	if err != nil {
+		return nil, errors.Errorf("error parsing flags bytes: %v", err)
+	}
+	s := &superblock{
+		inodes:              binary.LittleEndian.Uint32(b[4:8]),
+		modTime:             time.Unix(int64(binary.LittleEndian.Uint32(b[8:12])), 0),
+		blocksize:           blocksize,
+		fragmentCount:       binary.LittleEndian.Uint32(b[16:20]),
+		compression:         compression(binary.LittleEndian.Uint16(b[20:22])),
+		idCount:             binary.LittleEndian.Uint16(b[26:28]),
+		versionMajor:        binary.LittleEndian.Uint16(b[28:30]),
+		versionMinor:        binary.LittleEndian.Uint16(b[30:32]),
+		rootInode:           parseRootInode(binary.LittleEndian.Uint64(b[32:40])),
+		size:                binary.LittleEndian.Uint64(b[40:48]),
+		idTableStart:        binary.LittleEndian.Uint64(b[48:56]),
+		xattrTableStart:     binary.LittleEndian.Uint64(b[56:64]),
+		inodeTableStart:     binary.LittleEndian.Uint64(b[64:72]),
+		directoryTableStart: binary.LittleEndian.Uint64(b[72:80]),
+		fragmentTableStart:  binary.LittleEndian.Uint64(b[80:88]),
+		exportTableStart:    binary.LittleEndian.Uint64(b[88:96]),
+		superblockFlags:     *flags,
+	}
+	return s, nil
+}
+
+func readSuperblock(path string) (*superblock, error) {
+	reader, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	buf := make([]byte, superblockSize)
+	if _, err := io.ReadFull(reader, buf); err != nil {
+		return nil, err
+	}
+
+	return parseSuperblock(buf)
+}

pkg/squashfs/verity.go

@@ -1,6 +1,6 @@
 package squashfs
 
-// #cgo pkg-config: libcryptsetup libsquashfs1 devmapper --static
+// #cgo pkg-config: libcryptsetup devmapper --static
 // #include <libcryptsetup.h>
 // #include <stdlib.h>
 // #include <errno.h>
@@ -73,7 +73,6 @@ import (
 	"syscall"
 	"unsafe"
 
-	"github.com/anuvu/squashfs"
 	"github.com/freddierice/go-losetup"
 	"github.com/martinjungblut/go-cryptsetup"
 	"github.com/pkg/errors"
@@ -180,13 +179,8 @@ func appendVerityData(file string) (string, error) {
 	return fmt.Sprintf("%x", rootHash), errors.WithStack(err)
 }
 
-func verityDataLocation(file string) (uint64, error) {
-	sqfs, err := squashfs.OpenSquashfs(file)
-	if err != nil {
-		return 0, errors.WithStack(err)
-	}
-	defer sqfs.Close()
-	squashLen := sqfs.BytesUsed()
+func verityDataLocation(sblock *superblock) (uint64, error) {
+	squashLen := sblock.size
 
 	// squashfs is padded out to the nearest 4k
 	if squashLen%4096 != 0 {
@@ -292,7 +286,12 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error {
 		return errors.WithStack(err)
 	}
 
-	verityOffset, err := verityDataLocation(squashfs)
+	sblock, err := readSuperblock(squashfs)
+	if err != nil {
+		return err
+	}
+
+	verityOffset, err := verityDataLocation(sblock)
 	if err != nil {
 		return err
 	}

pkg/squashfs/verity_test.go

@@ -78,7 +78,10 @@ func TestVerityMetadata(t *testing.T) {
 	err = os.WriteFile(squashfsFile, content, 0600)
 	assert.NoError(err)
 
-	verityOffset, err := verityDataLocation(squashfsFile)
+	sblock, err := readSuperblock(squashfsFile)
+	assert.NoError(err)
+
+	verityOffset, err := verityDataLocation(sblock)
 	assert.NoError(err)
 
 	// now let's try to verify it at least in userspace. exec cryptsetup