playback: improve /list response time (#3637)

Response times of the /list endpoint were slow because the duration of
each segment was computed from scratch by summing the duration of each
of its parts.

This is improved by storing the duration of the overall segment in the
header and using that, if available.

internal/playback/on_get.go

@@ -57,7 +57,7 @@ func seekAndMux(
 		}
 		defer f.Close()
 
-		firstInit, err = segmentFMP4ReadInit(f)
+		firstInit, _, err = segmentFMP4ReadHeader(f)
 		if err != nil {
 			return err
 		}
@@ -81,7 +81,7 @@ func seekAndMux(
 			defer f.Close()
 
 			var init *fmp4.Init
-			init, err = segmentFMP4ReadInit(f)
+			init, _, err = segmentFMP4ReadHeader(f)
 			if err != nil {
 				return err
 			}

internal/playback/on_list.go

@@ -29,7 +29,7 @@ type listEntry struct {
 	URL      string            `json:"url"`
 }
 
-func computeDurationAndConcatenate(
+func readDurationAndConcatenate(
 	recordFormat conf.RecordFormat,
 	segments []*recordstore.Segment,
 ) ([]listEntry, error) {
@@ -45,19 +45,23 @@ func computeDurationAndConcatenate(
 				}
 				defer f.Close()
 
-				init, err := segmentFMP4ReadInit(f)
+				init, duration, err := segmentFMP4ReadHeader(f)
 				if err != nil {
 					return err
 				}
 
-				_, err = f.Seek(0, io.SeekStart)
-				if err != nil {
-					return err
-				}
-
-				maxDuration, err := segmentFMP4ReadDuration(f, init)
-				if err != nil {
-					return err
+				// if duration is not present in the header, compute it
+				// by parsing each part
+				if duration == 0 {
+					_, err = f.Seek(0, io.SeekStart)
+					if err != nil {
+						return err
+					}
+
+					duration, err = segmentFMP4ReadDurationFromParts(f, init)
+					if err != nil {
+						return err
+					}
 				}
 
 				if len(out) != 0 && segmentFMP4CanBeConcatenated(
@@ -66,12 +70,12 @@ func computeDurationAndConcatenate(
 					init,
 					seg.Start) {
 					prevStart := out[len(out)-1].Start
-					curEnd := seg.Start.Add(maxDuration)
+					curEnd := seg.Start.Add(duration)
 					out[len(out)-1].Duration = listEntryDuration(curEnd.Sub(prevStart))
 				} else {
 					out = append(out, listEntry{
 						Start:    seg.Start,
-						Duration: listEntryDuration(maxDuration),
+						Duration: listEntryDuration(duration),
 					})
 				}
 
@@ -137,7 +141,7 @@ func (s *Server) onList(ctx *gin.Context) {
 		return
 	}
 
-	entries, err := computeDurationAndConcatenate(pathConf.RecordFormat, segments)
+	entries, err := readDurationAndConcatenate(pathConf.RecordFormat, segments)
 	if err != nil {
 		s.writeError(ctx, http.StatusInternalServerError, err)
 		return

internal/playback/on_list_test.go

@@ -92,7 +92,7 @@ func TestOnListFiltered(t *testing.T) {
 
 	s := &Server{
 		Address:     "127.0.0.1:9996",
-		ReadTimeout: conf.StringDuration(10 * time.Second),
+		ReadTimeout: conf.Duration(10 * time.Second),
 		PathConfs: map[string]*conf.Path{
 			"mypath": {
 				Name:       "mypath",

internal/playback/segment_fmp4.go

@@ -60,61 +60,73 @@ func segmentFMP4CanBeConcatenated(
 		!curStart.After(prevEnd.Add(concatenationTolerance))
 }
 
-func segmentFMP4ReadInit(r io.ReadSeeker) (*fmp4.Init, error) {
+func segmentFMP4ReadHeader(r io.ReadSeeker) (*fmp4.Init, time.Duration, error) {
 	buf := make([]byte, 8)
 	_, err := io.ReadFull(r, buf)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	// find ftyp
+	// find and skip ftyp
 
 	if !bytes.Equal(buf[4:], []byte{'f', 't', 'y', 'p'}) {
-		return nil, fmt.Errorf("ftyp box not found")
+		return nil, 0, fmt.Errorf("ftyp box not found")
 	}
 
 	ftypSize := uint32(buf[0])<<24 | uint32(buf[1])<<16 | uint32(buf[2])<<8 | uint32(buf[3])
 
 	_, err = r.Seek(int64(ftypSize), io.SeekStart)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	// find moov
+	// find and skip moov
 
 	_, err = io.ReadFull(r, buf)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	if !bytes.Equal(buf[4:], []byte{'m', 'o', 'o', 'v'}) {
-		return nil, fmt.Errorf("moov box not found")
+		return nil, 0, fmt.Errorf("moov box not found")
 	}
 
 	moovSize := uint32(buf[0])<<24 | uint32(buf[1])<<16 | uint32(buf[2])<<8 | uint32(buf[3])
 
-	_, err = r.Seek(0, io.SeekStart)
+	_, err = r.Seek(8, io.SeekCurrent)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	buf = make([]byte, ftypSize+moovSize)
+	// read mvhd
+
+	var mvhd mp4.Mvhd
+	mvhdSize, err := mp4.Unmarshal(r, uint64(moovSize-8), &mvhd, mp4.Context{})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	d := time.Duration(mvhd.DurationV0) * time.Second / time.Duration(mvhd.Timescale)
+
+	// read tracks
+
+	buf = make([]byte, uint64(moovSize)-16-mvhdSize)
 
 	_, err = io.ReadFull(r, buf)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	var init fmp4.Init
 	err = init.Unmarshal(bytes.NewReader(buf))
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	return &init, nil
+	return &init, d, nil
 }
 
-func segmentFMP4ReadDuration(
+func segmentFMP4ReadDurationFromParts(
 	r io.ReadSeeker,
 	init *fmp4.Init,
 ) (time.Duration, error) {

internal/playback/segment_fmp4_test.go

@@ -66,7 +66,7 @@ func BenchmarkFMP4ReadInit(b *testing.B) {
 			}
 			defer f.Close()
 
-			_, err = segmentFMP4ReadInit(f)
+			_, _, err = segmentFMP4ReadHeader(f)
 			if err != nil {
 				panic(err)
 			}

internal/recorder/format_fmp4_segment.go

@@ -1,10 +1,13 @@
 package recorder
 
 import (
+	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"time"
 
+	"github.com/abema/go-mp4"
 	"github.com/bluenviron/mediacommon/pkg/formats/fmp4"
 	"github.com/bluenviron/mediacommon/pkg/formats/fmp4/seekablebuffer"
 
@@ -31,6 +34,70 @@ func writeInit(f io.Writer, tracks []*formatFMP4Track) error {
 	return err
 }
 
+func writeDuration(f *os.File, d time.Duration) error {
+	_, err := f.Seek(0, io.SeekStart)
+	if err != nil {
+		return err
+	}
+
+	// find and skip ftyp
+
+	buf := make([]byte, 8)
+	_, err = io.ReadFull(f, buf)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Equal(buf[4:], []byte{'f', 't', 'y', 'p'}) {
+		return fmt.Errorf("ftyp box not found")
+	}
+
+	ftypSize := uint32(buf[0])<<24 | uint32(buf[1])<<16 | uint32(buf[2])<<8 | uint32(buf[3])
+
+	_, err = f.Seek(int64(ftypSize), io.SeekStart)
+	if err != nil {
+		return err
+	}
+
+	// find and skip moov
+
+	_, err = io.ReadFull(f, buf)
+	if err != nil {
+		return err
+	}
+
+	if !bytes.Equal(buf[4:], []byte{'m', 'o', 'o', 'v'}) {
+		return fmt.Errorf("moov box not found")
+	}
+
+	moovSize := uint32(buf[0])<<24 | uint32(buf[1])<<16 | uint32(buf[2])<<8 | uint32(buf[3])
+
+	moovPos, err := f.Seek(8, io.SeekCurrent)
+	if err != nil {
+		return err
+	}
+
+	var mvhd mp4.Mvhd
+	_, err = mp4.Unmarshal(f, uint64(moovSize-8), &mvhd, mp4.Context{})
+	if err != nil {
+		return err
+	}
+
+	mvhd.DurationV0 = uint32(d / time.Millisecond)
+
+	_, err = f.Seek(moovPos, io.SeekStart)
+	if err != nil {
+		return err
+	}
+
+	_, err = mp4.Marshal(f, &mvhd, mp4.Context{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 type formatFMP4Segment struct {
 	f        *formatFMP4
 	startDTS time.Duration
@@ -55,13 +122,19 @@ func (s *formatFMP4Segment) close() error {
 
 	if s.fi != nil {
 		s.f.ri.Log(logger.Debug, "closing segment %s", s.path)
-		err2 := s.fi.Close()
+
+		duration := s.lastDTS - s.startDTS
+		err2 := writeDuration(s.fi, duration)
+		if err == nil {
+			err = err2
+		}
+
+		err2 = s.fi.Close()
 		if err == nil {
 			err = err2
 		}
 
 		if err2 == nil {
-			duration := s.lastDTS - s.startDTS
 			s.f.ri.rec.OnSegmentComplete(s.path, duration)
 		}
 	}