interprers, reporter: refactor towards statelessness

Due to legacy reasons, each interpreter kept their own state of
which dynamic metadata should be sent to the reporter. Several
of these caches would never expire, causing caching issues in
the otlp reporter module.

This removes the caching state from all interpreters and pushes
it to the reporter module. A new reporter API call FrameNeeded
is added to query if a specific Frame is in the cache or not.
Not all interpreter modules use the call as all the information
might be available with little overhead. FrameMetadata is also
updated to use the FrameID type for symmetry.

Improved are:
 - reduced memory overhead as per-interpreter caches are removed
 - reporter module can now control which frames need resolving
 - fixes otlp to get the frames re-symbolized if its internal
   lru already forgot about the earlier symbolization information

ref open-telemetry#121

interpreter/dotnet/data.go

@@ -144,11 +144,6 @@ func (d *dotnetData) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, bias li
 		return nil, err
 	}
 
-	symbolizedLRU, err := freelru.New[symbolizedKey, libpf.Void](1024, symbolizedKey.Hash32)
-	if err != nil {
-		return nil, err
-	}
-
 	procInfo := C.DotnetProcInfo{
 		version: C.uint(d.version),
 	}
@@ -163,7 +158,6 @@ func (d *dotnetData) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, bias li
 		ranges:         make(map[libpf.Address]dotnetRangeSection),
 		moduleToPEInfo: make(map[libpf.Address]*peInfo),
 		addrToMethod:   addrToMethod,
-		symbolizedLRU:  symbolizedLRU,
 	}, nil
 }
 

interpreter/dotnet/instance.go

@@ -121,15 +121,6 @@ type dotnetRangeSection struct {
 	prefixes []lpm.Prefix
 }
 
-type symbolizedKey struct {
-	fileID libpf.FileID
-	lineID libpf.AddressOrLineno
-}
-
-func (key symbolizedKey) Hash32() uint32 {
-	return key.fileID.Hash32() + uint32(key.lineID)
-}
-
 type dotnetInstance struct {
 	interpreter.InstanceStubs
 
@@ -167,23 +158,24 @@ type dotnetInstance struct {
 	moduleToPEInfo map[libpf.Address]*peInfo
 
 	addrToMethod *freelru.LRU[libpf.Address, *dotnetMethod]
-
-	symbolizedLRU *freelru.LRU[symbolizedKey, libpf.Void]
 }
 
 // calculateAndSymbolizeStubID calculates a stub LineID, and symbolizes it if needed
-func (i *dotnetInstance) calculateAndSymbolizeStubID(symbolReporter reporter.SymbolReporter,
-	codeType uint) {
-	if i.codeTypeMethodIDs[codeType] != 0 {
-		return
-	}
+func (i *dotnetInstance) insertAndSymbolizeStubFrame(symbolReporter reporter.SymbolReporter,
+	trace *libpf.Trace, codeType uint) {
 	name := "[stub: " + codeName[codeType] + "]"
-	h := fnv.New128a()
-	_, _ = h.Write([]byte(name))
-	nameHash := h.Sum(nil)
-	lineID := libpf.AddressOrLineno(npsr.Uint64(nameHash, 0))
-	i.codeTypeMethodIDs[codeType] = lineID
-	symbolReporter.FrameMetadata(stubsFileID, lineID, 0, 0, name, "")
+	lineID := i.codeTypeMethodIDs[codeType]
+	if lineID == 0 {
+		h := fnv.New128a()
+		_, _ = h.Write([]byte(name))
+		nameHash := h.Sum(nil)
+		lineID = libpf.AddressOrLineno(npsr.Uint64(nameHash, 0))
+		i.codeTypeMethodIDs[codeType] = lineID
+	}
+
+	frameID := libpf.NewFrameID(stubsFileID, lineID)
+	trace.AppendFrameID(libpf.DotnetFrame, frameID)
+	symbolReporter.FrameMetadata(frameID, 0, 0, name, "")
 }
 
 // addRange inserts a known memory mapping along with the needed data of it to ebpf maps
@@ -764,19 +756,16 @@ func (i *dotnetInstance) Symbolize(symbolReporter reporter.SymbolReporter,
 		if err != nil {
 			return err
 		}
-		fileID := module.fileID
+		// The Line ID is the Relative Virtual Address (RVA) within into the PE file
+		// where PC is executing. On non-leaf frames it points to the return address.
+		// The instruction after the CALL machine opcode.
 		lineID := libpf.AddressOrLineno(pcOffset)
-
-		if _, ok := i.symbolizedLRU.Get(symbolizedKey{fileID, lineID}); !ok {
+		frameID := libpf.NewFrameID(module.fileID, lineID)
+		trace.AppendFrameID(libpf.DotnetFrame, frameID)
+		if !symbolReporter.FrameKnown(frameID) {
 			methodName := module.resolveR2RMethodName(pcOffset)
-			symbolReporter.FrameMetadata(fileID, lineID, 0, 0,
-				methodName, module.simpleName)
-			i.symbolizedLRU.Add(symbolizedKey{fileID, lineID}, libpf.Void{})
+			symbolReporter.FrameMetadata(frameID, 0, 0, methodName, module.simpleName)
 		}
-		// The Line ID is the Relative Virtual Address (RVA) within into the PE file
-		// where PC is executing. On non-leaf frames it points to the return address.
-		// The instructionafter the CALL machine opcode.
-		trace.AppendFrame(libpf.DotnetFrame, fileID, lineID)
 	case codeJIT:
 		// JITted frame in anonymous mapping
 		method, err := i.getMethod(codeHeaderPtr)
@@ -795,21 +784,19 @@ func (i *dotnetInstance) Symbolize(symbolReporter reporter.SymbolReporter,
 			libpf.AddressOrLineno(ilOffset)
 
 		if method.index == 0 || method.classification == mcDynamic {
-			i.calculateAndSymbolizeStubID(symbolReporter, codeDynamic)
-			trace.AppendFrame(libpf.DotnetFrame, stubsFileID, i.codeTypeMethodIDs[codeDynamic])
+			i.insertAndSymbolizeStubFrame(symbolReporter, trace, codeDynamic)
 		} else {
-			if _, ok := i.symbolizedLRU.Get(symbolizedKey{fileID, lineID}); !ok {
+			frameID := libpf.NewFrameID(fileID, lineID)
+			trace.AppendFrameID(libpf.DotnetFrame, frameID)
+			if !symbolReporter.FrameKnown(frameID) {
 				methodName := method.module.resolveMethodName(method.index)
-				symbolReporter.FrameMetadata(fileID, lineID, 0, ilOffset,
+				symbolReporter.FrameMetadata(frameID, 0, ilOffset,
 					methodName, method.module.simpleName)
-				i.symbolizedLRU.Add(symbolizedKey{fileID, lineID}, libpf.Void{})
 			}
-			trace.AppendFrame(libpf.DotnetFrame, fileID, lineID)
 		}
 	default:
 		// Stub code
-		i.calculateAndSymbolizeStubID(symbolReporter, frameType)
-		trace.AppendFrame(libpf.DotnetFrame, stubsFileID, i.codeTypeMethodIDs[frameType])
+		i.insertAndSymbolizeStubFrame(symbolReporter, trace, frameType)
 	}
 
 	sfCounter.ReportSuccess()

interpreter/hotspot/instance.go

@@ -235,7 +235,7 @@ func (d *hotspotInstance) getPoolSymbol(addr libpf.Address, ndx uint16) string {
 }
 
 // getStubNameID read the stub name from the code blob at given address and generates a ID.
-func (d *hotspotInstance) getStubNameID(symbolReporter reporter.SymbolReporter, ripOrBci int32,
+func (d *hotspotInstance) getStubNameID(symbolReporter reporter.SymbolReporter, ripOrBci uint32,
 	addr libpf.Address, _ uint32) (libpf.AddressOrLineno, error) {
 	if value, ok := d.addrToStubNameID.Get(addr); ok {
 		return value, nil
@@ -256,10 +256,9 @@ func (d *hotspotInstance) getStubNameID(symbolReporter reporter.SymbolReporter,
 	_, _ = h.Write([]byte(stubName))
 	nameHash := h.Sum(nil)
 	stubID := libpf.AddressOrLineno(npsr.Uint64(nameHash, 0))
-
-	symbolReporter.FrameMetadata(hotspotStubsFileID, stubID, 0, 0, stubName, "")
-
+	symbolReporter.FrameMetadata(libpf.NewFrameID(hotspotStubsFileID, stubID), 0, 0, stubName, "")
 	d.addrToStubNameID.Add(addr, stubID)
+
 	return stubID, nil
 }
 
@@ -399,7 +398,6 @@ func (d *hotspotInstance) getMethod(addr libpf.Address, _ uint32) (*hotspotMetho
 		bytecodeSize:   bytecodeSize,
 		lineTable:      lineTable,
 		startLineNo:    uint16(startLine),
-		bciSeen:        make(libpf.Set[uint16]),
 	}
 	d.addrToMethod.Add(addr, sym)
 	return sym, nil
@@ -785,7 +783,7 @@ func (d *hotspotInstance) Symbolize(symbolReporter reporter.SymbolReporter,
 	// Extract the HotSpot frame bitfields from the file and line variables
 	ptr := libpf.Address(frame.File)
 	subtype := uint32(frame.Lineno>>60) & 0xf
-	ripOrBci := int32(frame.Lineno>>32) & 0x0fffffff
+	ripOrBci := uint32(frame.Lineno>>32) & 0x0fffffff
 	ptrCheck := uint32(frame.Lineno)
 
 	var err error
@@ -804,15 +802,15 @@ func (d *hotspotInstance) Symbolize(symbolReporter reporter.SymbolReporter,
 	case C.FRAME_HOTSPOT_INTERPRETER:
 		method, err1 := d.getMethod(ptr, ptrCheck)
 		if err1 != nil {
-			return err
+			return err1
 		}
-		err = method.symbolize(symbolReporter, ripOrBci, d, trace)
+		method.symbolize(symbolReporter, ripOrBci, d, trace)
 	case C.FRAME_HOTSPOT_NATIVE:
 		jitinfo, err1 := d.getJITInfo(ptr, ptrCheck)
 		if err1 != nil {
 			return err1
 		}
-		err = jitinfo.symbolize(symbolReporter, ripOrBci, d, trace)
+		err = jitinfo.symbolize(symbolReporter, int32(ripOrBci), d, trace)
 	default:
 		return fmt.Errorf("hotspot frame subtype %v is not supported", subtype)
 	}

interpreter/hotspot/method.go

@@ -7,8 +7,6 @@ import (
 	"bytes"
 	"fmt"
 
-	log "github.com/sirupsen/logrus"
-
 	"go.opentelemetry.io/ebpf-profiler/libpf"
 	npsr "go.opentelemetry.io/ebpf-profiler/nopanicslicereader"
 	"go.opentelemetry.io/ebpf-profiler/reporter"
@@ -29,46 +27,30 @@ type hotspotMethod struct {
 	bytecodeSize   uint16
 	startLineNo    uint16
 	lineTable      []byte
-	bciSeen        libpf.Set[uint16]
 }
 
 // Symbolize generates symbolization information for given hotspot method and
 // a Byte Code Index (BCI)
-func (m *hotspotMethod) symbolize(symbolReporter reporter.SymbolReporter, bci int32,
-	ii *hotspotInstance, trace *libpf.Trace) error {
+func (m *hotspotMethod) symbolize(symbolReporter reporter.SymbolReporter, bci uint32,
+	ii *hotspotInstance, trace *libpf.Trace) {
 	// Make sure the BCI is within the method range
-	if bci < 0 || bci >= int32(m.bytecodeSize) {
+	if bci >= uint32(m.bytecodeSize) {
 		bci = 0
 	}
-	trace.AppendFrame(libpf.HotSpotFrame, m.objectID, libpf.AddressOrLineno(bci))
-
-	// Check if this is already symbolized
-	if _, ok := m.bciSeen[uint16(bci)]; ok {
-		return nil
-	}
 
-	dec := ii.d.newUnsigned5Decoder(bytes.NewReader(m.lineTable))
-	lineNo := dec.mapByteCodeIndexToLine(bci)
-	functionOffset := uint32(0)
-	if lineNo > libpf.SourceLineno(m.startLineNo) {
-		functionOffset = uint32(lineNo) - uint32(m.startLineNo)
+	// Check if this is already known
+	frameID := libpf.NewFrameID(m.objectID, libpf.AddressOrLineno(bci))
+	trace.AppendFrameID(libpf.HotSpotFrame, frameID)
+	if !symbolReporter.FrameKnown(frameID) {
+		dec := ii.d.newUnsigned5Decoder(bytes.NewReader(m.lineTable))
+		lineNo := dec.mapByteCodeIndexToLine(bci)
+		functionOffset := uint32(0)
+		if lineNo > uint32(m.startLineNo) {
+			functionOffset = lineNo - uint32(m.startLineNo)
+		}
+		symbolReporter.FrameMetadata(frameID, libpf.SourceLineno(lineNo), functionOffset,
+			m.methodName, m.sourceFileName)
 	}
-
-	symbolReporter.FrameMetadata(m.objectID,
-		libpf.AddressOrLineno(bci), lineNo, functionOffset,
-		m.methodName, m.sourceFileName)
-
-	// FIXME: The above FrameMetadata call might fail, but we have no idea of it
-	// due to the requests being queued and send attempts being done asynchronously.
-	// Until the reporting API gets a way to notify failures, just assume it worked.
-	m.bciSeen[uint16(bci)] = libpf.Void{}
-
-	log.Debugf("[%d] [%x] %v+%v at %v:%v", len(trace.FrameTypes),
-		m.objectID,
-		m.methodName, functionOffset,
-		m.sourceFileName, lineNo)
-
-	return nil
 }
 
 // hotspotJITInfo contains symbolization and debug information for one JIT compiled
@@ -125,7 +107,8 @@ func (ji *hotspotJITInfo) symbolize(symbolReporter reporter.SymbolReporter, ripD
 		// It is possible that there is no debug info, or no scope information,
 		// for the given RIP. In this case we can provide the method name
 		// from the metadata.
-		return ji.method.symbolize(symbolReporter, 0, ii, trace)
+		ji.method.symbolize(symbolReporter, 0, ii, trace)
+		return nil
 	}
 
 	// Found scope data. Expand the inlined scope information from it.
@@ -167,10 +150,7 @@ func (ji *hotspotJITInfo) symbolize(symbolReporter reporter.SymbolReporter, ripD
 			if err != nil {
 				return err
 			}
-			err = method.symbolize(symbolReporter, int32(byteCodeIndex), ii, trace)
-			if err != nil {
-				return err
-			}
+			method.symbolize(symbolReporter, byteCodeIndex, ii, trace)
 		}
 	}
 	return nil

interpreter/hotspot/unsigned5.go

@@ -6,8 +6,6 @@ package hotspot // import "go.opentelemetry.io/ebpf-profiler/interpreter/hotspot
 import (
 	"fmt"
 	"io"
-
-	"go.opentelemetry.io/ebpf-profiler/libpf"
 )
 
 // unsigned5Decoder is a decoder for UNSIGNED5 based byte streams.
@@ -88,19 +86,19 @@ func (d *unsigned5Decoder) decodeLineTableEntry(bci, line *uint32) error {
 
 // mapByteCodeIndexToLine decodes a line table to map a given Byte Code Index (BCI)
 // to a line number
-func (d *unsigned5Decoder) mapByteCodeIndexToLine(bci int32) libpf.SourceLineno {
+func (d *unsigned5Decoder) mapByteCodeIndexToLine(bci uint32) uint32 {
 	// The line numbers array is a short array of 2-tuples [start_pc, line_number].
 	// Not necessarily sorted. Encoded as delta-encoded numbers.
 	var curBci, curLine, bestBci, bestLine uint32
 
 	for d.decodeLineTableEntry(&curBci, &curLine) == nil {
-		if curBci == uint32(bci) {
-			return libpf.SourceLineno(curLine)
+		if curBci == bci {
+			return curLine
 		}
-		if curBci >= bestBci && curBci < uint32(bci) {
+		if curBci >= bestBci && curBci < bci {
 			bestBci = curBci
 			bestLine = curLine
 		}
 	}
-	return libpf.SourceLineno(bestLine)
+	return bestLine
 }