feat: meta=eof for IPIP-431; ask for and expect (but not require) from http fetches

cmd/lassie/fetch.go

@@ -291,6 +291,9 @@ func defaultFetchRun(
 		blockCount,
 		humanize.IBytes(stats.Size),
 	)
+	if stats.CarProperties != nil {
+		fmt.Fprintf(msgWriter, "\tChecksum: %x\n", stats.CarProperties.ChecksumMultihash)
+	}
 
 	return nil
 }

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/ipfs/go-unixfsnode v1.7.1
 	github.com/ipld/go-car/v2 v2.10.1
 	github.com/ipld/go-codec-dagpb v1.6.0
-	github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0
+	github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de
 	github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff
 	github.com/libp2p/go-libp2p v0.27.8
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.0

go.sum

@@ -126,7 +126,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
-github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -336,8 +336,8 @@ github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI
 github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
-github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0 h1:iJTl9tx5DEsnKpppX5PmfdoQ3ITuBmkh3yyEpHWY2SI=
-github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0/go.mod h1:wmOtdy70ajP48iZITH8uLsGJVMqA4EJM61/bSfYYGhs=
+github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de h1:N6Wfk6dvcBjF4AJJDSmti6CkgHWZPDZ0fuqSQL+kKnU=
+github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8=
 github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
@@ -616,7 +616,7 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU=
 github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
-github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
+github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=

pkg/httputil/constants.go

@@ -0,0 +1,19 @@
+package httputil
+
+import "fmt"
+
+const (
+	MimeTypeCar                = "application/vnd.ipld.car"            // The only accepted MIME type
+	MimeTypeCarVersion         = "1"                                   // We only accept version 1 of the MIME type
+	ResponseAcceptRangesHeader = "none"                                // We currently don't accept range requests
+	ResponseCacheControlHeader = "public, max-age=29030400, immutable" // Magic cache control values
+	FilenameExtCar             = ".car"                                // The only valid filename extension
+	FormatParameterCar         = "car"                                 // The only valid format parameter value
+	DefaultIncludeDupes        = true                                  // The default value for an unspecified "dups" parameter. See https://github.com/ipfs/specs/pull/412.
+)
+
+var (
+	ResponseChunkDelimeter    = []byte("0\r\n") // An http/1.1 chunk delimeter, used for specifying an early end to the response
+	ResponseContentTypeHeader = fmt.Sprintf("%s; version=%s; order=dfs; dups=y", MimeTypeCar, MimeTypeCarVersion)
+	RequestAcceptHeader       = fmt.Sprintf("%s; version=%s; order=dfs; dups=y; meta=eof", MimeTypeCar, MimeTypeCarVersion)
+)

pkg/httputil/metadata/metadata.go

@@ -0,0 +1,70 @@
+package metadata
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/filecoin-project/lassie/pkg/types"
+	"github.com/ipfs/go-cid"
+	"github.com/ipld/go-ipld-prime/codec/dagjson"
+	bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry"
+	mh "github.com/multiformats/go-multihash"
+
+	_ "embed"
+)
+
+//go:embed metadata.ipldsch
+var schema []byte
+
+var BindnodeRegistry = bindnoderegistry.NewRegistry()
+
+type CarMetadata struct {
+	Metadata *Metadata
+}
+
+func (cm CarMetadata) Serialize(w io.Writer) error {
+	// TODO: do the same checks we do on Deserialize()
+	return BindnodeRegistry.TypeToWriter(&cm, w, dagjson.Encode)
+}
+
+func (cm *CarMetadata) Deserialize(r io.Reader) error {
+	cmIface, err := BindnodeRegistry.TypeFromReader(r, &CarMetadata{}, dagjson.Decode)
+	if err != nil {
+		return fmt.Errorf("invalid CarMetadata: %w", err)
+	}
+	cmm := cmIface.(*CarMetadata) // safe to assume type
+	if cmm.Metadata.Properties == nil && cmm.Metadata.Error == nil {
+		return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields")
+	}
+	if (cmm.Metadata.Properties == nil) == (cmm.Metadata.Error == nil) {
+		return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields, not both")
+	}
+	if cmm.Metadata.Properties != nil {
+		if _, err := mh.Decode(cmm.Metadata.Properties.ChecksumMultihash); err != nil {
+			return fmt.Errorf("invalid CarMetadata: checksum multihash: %w", err)
+		}
+	}
+	// TODO: parse and check EntityBytes format
+	*cm = *cmm
+	return nil
+}
+
+type Metadata struct {
+	Request    Request
+	Properties *types.CarProperties
+	Error      *string
+}
+
+type Request struct {
+	Root        cid.Cid
+	Path        *string
+	Scope       types.DagScope
+	Duplicates  bool
+	EntityBytes *string
+}
+
+func init() {
+	if err := BindnodeRegistry.RegisterType((*CarMetadata)(nil), string(schema), "CarMetadata"); err != nil {
+		panic(err.Error())
+	}
+}

pkg/httputil/metadata/metadata.ipldsch

@@ -0,0 +1,31 @@
+type CarMetadata union {
+  | Metadata "car-metadata/v1"
+} representation keyed
+
+type Metadata struct {
+  request Request
+  # must contain either a properties or an error
+  properties optional CarProperties
+  error optional String
+}
+
+type Request struct {
+  root &Any
+  path optional String
+  scope DagScope
+  duplicates Bool (rename "dups")
+  entityBytes optional String (rename "entity-bytes") # Must be a valid entity-bytes param: "from:to"
+}
+
+type DagScope enum {
+  | all
+  | entity
+  | block
+}
+
+type CarProperties struct {
+  carBytes Int (rename "car_bytes")
+  dataBytes Int (rename "data_bytes")
+  blockCount Int (rename "block_count")
+  checksumMultihash optional Bytes (rename "checksum") # Must be a valid multihash
+}

pkg/httputil/metadata/metadata_test.go

@@ -0,0 +1,118 @@
+package metadata_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/filecoin-project/lassie/pkg/httputil/metadata"
+	"github.com/filecoin-project/lassie/pkg/types"
+	"github.com/ipfs/go-cid"
+	"github.com/stretchr/testify/require"
+)
+
+var testCid = cid.MustParse("bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4")
+
+func TestCarMetadataRoundtrip(t *testing.T) {
+	path := "/birb.mp4"
+	orig := metadata.CarMetadata{
+		Metadata: &metadata.Metadata{
+			Request: metadata.Request{
+				Root:       testCid,
+				Path:       &path,
+				Scope:      types.DagScopeAll,
+				Duplicates: true,
+			},
+			Properties: &types.CarProperties{
+				CarBytes:          202020,
+				DataBytes:         101010,
+				BlockCount:        303,
+				ChecksumMultihash: testCid.Hash(),
+			},
+		},
+	}
+	var buf bytes.Buffer
+	require.NoError(t, orig.Serialize(&buf))
+
+	t.Log("metadata dag-json:", buf.String())
+
+	var roundtrip metadata.CarMetadata
+	require.NoError(t, roundtrip.Deserialize(&buf))
+	require.Equal(t, orig, roundtrip)
+	require.NotNil(t, roundtrip.Metadata)
+	require.Equal(t, testCid, roundtrip.Metadata.Request.Root)
+	require.NotNil(t, roundtrip.Metadata.Request.Path)
+	require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path)
+	require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope)
+	require.True(t, roundtrip.Metadata.Request.Duplicates)
+	require.NotNil(t, roundtrip.Metadata.Properties)
+	require.Nil(t, roundtrip.Metadata.Error)
+	require.Equal(t, int64(202020), roundtrip.Metadata.Properties.CarBytes)
+	require.Equal(t, int64(101010), roundtrip.Metadata.Properties.DataBytes)
+	require.Equal(t, int64(303), roundtrip.Metadata.Properties.BlockCount)
+	require.Equal(t, []byte(testCid.Hash()), roundtrip.Metadata.Properties.ChecksumMultihash)
+}
+
+func TestCarMetadataErrorRoundtrip(t *testing.T) {
+	path := "/birb.mp4"
+	msg := "something bad happened"
+	orig := metadata.CarMetadata{
+		Metadata: &metadata.Metadata{
+			Request: metadata.Request{
+				Root:       testCid,
+				Path:       &path,
+				Scope:      types.DagScopeAll,
+				Duplicates: true,
+			},
+			Error: &msg,
+		},
+	}
+	var buf bytes.Buffer
+	require.NoError(t, orig.Serialize(&buf))
+
+	t.Log("metadata dag-json:", buf.String())
+
+	var roundtrip metadata.CarMetadata
+	require.NoError(t, roundtrip.Deserialize(&buf))
+	require.Equal(t, orig, roundtrip)
+	require.NotNil(t, roundtrip.Metadata)
+	require.Equal(t, testCid, roundtrip.Metadata.Request.Root)
+	require.NotNil(t, roundtrip.Metadata.Request.Path)
+	require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path)
+	require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope)
+	require.True(t, roundtrip.Metadata.Request.Duplicates)
+	require.Nil(t, roundtrip.Metadata.Properties)
+	require.NotNil(t, roundtrip.Metadata.Error)
+	require.Equal(t, "something bad happened", *roundtrip.Metadata.Error)
+}
+
+func TestBadMetadata(t *testing.T) {
+	testCases := []struct {
+		name string
+		byts string
+		err  string
+	}{
+		{"empty", `{}`, `union structure constraints for CarMetadata caused rejection: a union must have exactly one entry`},
+		{"bad key", `{"not metadata":true}`, `union structure constraints for CarMetadata caused rejection: no member named "not metadata"`},
+		{
+			"bad multihash",
+			`{"car-metadata/v1":{"properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"bm90IGEgbXVsdGloYXNo"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
+			`invalid CarMetadata: checksum multihash:`,
+		},
+		{
+			"no properties or error",
+			`{"car-metadata/v1":{"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
+			`invalid CarMetadata: must contain either properties or error fields`,
+		},
+		{
+			"both properties and error",
+			`{"car-metadata/v1":{"error":"something bad happened","properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"EiBd9neBCasGxUmysJN7nGza4ylHikmbsP2+nXs6BlIpvw"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
+			`invalid CarMetadata: must contain either properties or error fields, not both`,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var roundtrip metadata.CarMetadata
+			require.ErrorContains(t, roundtrip.Deserialize(bytes.NewBuffer([]byte(tc.byts))), tc.err)
+		})
+	}
+}

pkg/server/http/util.go → pkg/httputil/server.go

@@ -1,4 +1,4 @@
-package httpserver
+package httputil
 
 import (
 	"errors"
@@ -74,33 +74,34 @@ func ParseFilename(req *http.Request) (string, error) {
 //
 // Lassie only allows the "car" format query parameter
 // https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
-func CheckFormat(req *http.Request) (bool, error) {
+func CheckFormat(req *http.Request) (bool, bool, error) {
 	hasAccept := req.Header.Get("Accept") != ""
 	// check if Accept header includes application/vnd.ipld.car
-	validAccept, includeDupes := ParseAccept(req.Header.Get("Accept"))
+	validAccept, includeDupes, includeMeta := ParseAccept(req.Header.Get("Accept"))
 	if hasAccept && !validAccept {
-		return false, fmt.Errorf("no acceptable content type")
+		return false, false, fmt.Errorf("no acceptable content type")
 	}
 
 	// check if format is "car"
 	hasFormat := req.URL.Query().Has("format")
 	if hasFormat && req.URL.Query().Get("format") != FormatParameterCar {
-		return false, fmt.Errorf("requested non-supported format %s", req.URL.Query().Get("format"))
+		return false, false, fmt.Errorf("requested non-supported format %s", req.URL.Query().Get("format"))
 	}
 
 	// if neither are provided return
 	// one of them has to be given with a CAR type since we only return CAR data
 	if !validAccept && !hasFormat {
-		return false, fmt.Errorf("neither a valid accept header or format parameter were provided")
+		return false, false, fmt.Errorf("neither a valid accept header or format parameter were provided")
 	}
 
-	return includeDupes, nil
+	return includeDupes, includeMeta, nil
 }
 
 // ParseAccept validates that the request Accept header is of the type CAR and
 // returns whether or not duplicate blocks are allowed in the response via
-// IPIP-412: https://github.com/ipfs/specs/pull/412.
-func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool) {
+// IPIP-412: https://github.com/ipfs/specs/pull/412, and whether or not
+// metadata is requested via IPIP-431: https://github.com/ipfs/specs/pull/431.
+func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool, includeMeta bool) {
 	acceptTypes := strings.Split(acceptHeader, ",")
 	validAccept = false
 	includeDupes = DefaultIncludeDupes
@@ -140,6 +141,14 @@ func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool) {
 								// we only do dfs, which also satisfies unk, future extensions are not yet supported
 								validAccept = false
 							}
+						case "meta":
+							switch value {
+							case "eof":
+								includeMeta = true
+							default:
+								// we only support eof, future extensions are not yet supported
+								validAccept = false
+							}
 						default:
 							// ignore others
 						}

pkg/internal/itest/http_fetch_test.go

@@ -1072,7 +1072,7 @@ func TestHttpFetch(t *testing.T) {
 					req.Equal(fmt.Sprintf(`attachment; filename="%s.car"`, srcData[i].Root.String()), resp.Header.Get("Content-Disposition"))
 					req.Equal("none", resp.Header.Get("Accept-Ranges"))
 					req.Equal("public, max-age=29030400, immutable", resp.Header.Get("Cache-Control"))
-					req.Equal("application/vnd.ipld.car; version=1", resp.Header.Get("Content-Type"))
+					req.Equal("application/vnd.ipld.car; version=1; order=dfs; dups=y", resp.Header.Get("Content-Type"))
 					req.Equal("nosniff", resp.Header.Get("X-Content-Type-Options"))
 					etagStart := fmt.Sprintf(`"%s.car.`, srcData[i].Root.String())
 					etagGot := resp.Header.Get("ETag")