prog.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. package ebpf
  2. import (
  3. "bytes"
  4. "encoding/binary"
  5. "errors"
  6. "fmt"
  7. "math"
  8. "path/filepath"
  9. "runtime"
  10. "strings"
  11. "time"
  12. "github.com/cilium/ebpf/asm"
  13. "github.com/cilium/ebpf/btf"
  14. "github.com/cilium/ebpf/internal"
  15. "github.com/cilium/ebpf/internal/sys"
  16. "github.com/cilium/ebpf/internal/unix"
  17. )
  18. // ErrNotSupported is returned whenever the kernel doesn't support a feature.
  19. var ErrNotSupported = internal.ErrNotSupported
  20. // ProgramID represents the unique ID of an eBPF program.
  21. type ProgramID uint32
  22. const (
  23. // Number of bytes to pad the output buffer for BPF_PROG_TEST_RUN.
  24. // This is currently the maximum of spare space allocated for SKB
  25. // and XDP programs, and equal to XDP_PACKET_HEADROOM + NET_IP_ALIGN.
  26. outputPad = 256 + 2
  27. )
  28. // DefaultVerifierLogSize is the default number of bytes allocated for the
  29. // verifier log.
  30. const DefaultVerifierLogSize = 64 * 1024
  31. // ProgramOptions control loading a program into the kernel.
  32. type ProgramOptions struct {
  33. // Controls the detail emitted by the kernel verifier. Set to non-zero
  34. // to enable logging.
  35. LogLevel uint32
  36. // Controls the output buffer size for the verifier. Defaults to
  37. // DefaultVerifierLogSize.
  38. LogSize int
  39. // Type information used for CO-RE relocations and when attaching to
  40. // kernel functions.
  41. //
  42. // This is useful in environments where the kernel BTF is not available
  43. // (containers) or where it is in a non-standard location. Defaults to
  44. // use the kernel BTF from a well-known location if nil.
  45. KernelTypes *btf.Spec
  46. }
  47. // ProgramSpec defines a Program.
  48. type ProgramSpec struct {
  49. // Name is passed to the kernel as a debug aid. Must only contain
  50. // alpha numeric and '_' characters.
  51. Name string
  52. // Type determines at which hook in the kernel a program will run.
  53. Type ProgramType
  54. // AttachType of the program, needed to differentiate allowed context
  55. // accesses in some newer program types like CGroupSockAddr.
  56. //
  57. // Available on kernels 4.17 and later.
  58. AttachType AttachType
  59. // Name of a kernel data structure or function to attach to. Its
  60. // interpretation depends on Type and AttachType.
  61. AttachTo string
  62. // The program to attach to. Must be provided manually.
  63. AttachTarget *Program
  64. // The name of the ELF section this program orininated from.
  65. SectionName string
  66. Instructions asm.Instructions
  67. // Flags is passed to the kernel and specifies additional program
  68. // load attributes.
  69. Flags uint32
  70. // License of the program. Some helpers are only available if
  71. // the license is deemed compatible with the GPL.
  72. //
  73. // See https://www.kernel.org/doc/html/latest/process/license-rules.html#id1
  74. License string
  75. // Version used by Kprobe programs.
  76. //
  77. // Deprecated on kernels 5.0 and later. Leave empty to let the library
  78. // detect this value automatically.
  79. KernelVersion uint32
  80. // The BTF associated with this program. Changing Instructions
  81. // will most likely invalidate the contained data, and may
  82. // result in errors when attempting to load it into the kernel.
  83. BTF *btf.Spec
  84. // The byte order this program was compiled for, may be nil.
  85. ByteOrder binary.ByteOrder
  86. }
  87. // Copy returns a copy of the spec.
  88. func (ps *ProgramSpec) Copy() *ProgramSpec {
  89. if ps == nil {
  90. return nil
  91. }
  92. cpy := *ps
  93. cpy.Instructions = make(asm.Instructions, len(ps.Instructions))
  94. copy(cpy.Instructions, ps.Instructions)
  95. return &cpy
  96. }
  97. // Tag calculates the kernel tag for a series of instructions.
  98. //
  99. // Use asm.Instructions.Tag if you need to calculate for non-native endianness.
  100. func (ps *ProgramSpec) Tag() (string, error) {
  101. return ps.Instructions.Tag(internal.NativeEndian)
  102. }
  103. type VerifierError = internal.VerifierError
  104. // Program represents BPF program loaded into the kernel.
  105. //
  106. // It is not safe to close a Program which is used by other goroutines.
  107. type Program struct {
  108. // Contains the output of the kernel verifier if enabled,
  109. // otherwise it is empty.
  110. VerifierLog string
  111. fd *sys.FD
  112. name string
  113. pinnedPath string
  114. typ ProgramType
  115. }
  116. // NewProgram creates a new Program.
  117. //
  118. // See NewProgramWithOptions for details.
  119. func NewProgram(spec *ProgramSpec) (*Program, error) {
  120. return NewProgramWithOptions(spec, ProgramOptions{})
  121. }
  122. // NewProgramWithOptions creates a new Program.
  123. //
  124. // Loading a program for the first time will perform
  125. // feature detection by loading small, temporary programs.
  126. //
  127. // Returns an error wrapping VerifierError if the program or its BTF is rejected
  128. // by the kernel.
  129. func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
  130. if spec == nil {
  131. return nil, errors.New("can't load a program from a nil spec")
  132. }
  133. handles := newHandleCache()
  134. defer handles.close()
  135. prog, err := newProgramWithOptions(spec, opts, handles)
  136. if errors.Is(err, asm.ErrUnsatisfiedMapReference) {
  137. return nil, fmt.Errorf("cannot load program without loading its whole collection: %w", err)
  138. }
  139. return prog, err
  140. }
  141. func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, handles *handleCache) (*Program, error) {
  142. if len(spec.Instructions) == 0 {
  143. return nil, errors.New("instructions cannot be empty")
  144. }
  145. if spec.Type == UnspecifiedProgram {
  146. return nil, errors.New("can't load program of unspecified type")
  147. }
  148. if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian {
  149. return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian)
  150. }
  151. // Kernels before 5.0 (6c4fc209fcf9 "bpf: remove useless version check for prog load")
  152. // require the version field to be set to the value of the KERNEL_VERSION
  153. // macro for kprobe-type programs.
  154. // Overwrite Kprobe program version if set to zero or the magic version constant.
  155. kv := spec.KernelVersion
  156. if spec.Type == Kprobe && (kv == 0 || kv == internal.MagicKernelVersion) {
  157. v, err := internal.KernelVersion()
  158. if err != nil {
  159. return nil, fmt.Errorf("detecting kernel version: %w", err)
  160. }
  161. kv = v.Kernel()
  162. }
  163. attr := &sys.ProgLoadAttr{
  164. ProgType: sys.ProgType(spec.Type),
  165. ProgFlags: spec.Flags,
  166. ExpectedAttachType: sys.AttachType(spec.AttachType),
  167. License: sys.NewStringPointer(spec.License),
  168. KernVersion: kv,
  169. }
  170. if haveObjName() == nil {
  171. attr.ProgName = sys.NewObjName(spec.Name)
  172. }
  173. kernelTypes := opts.KernelTypes
  174. insns := make(asm.Instructions, len(spec.Instructions))
  175. copy(insns, spec.Instructions)
  176. var btfDisabled bool
  177. if spec.BTF != nil {
  178. if err := applyRelocations(insns, spec.BTF, kernelTypes); err != nil {
  179. return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
  180. }
  181. handle, err := handles.btfHandle(spec.BTF)
  182. btfDisabled = errors.Is(err, btf.ErrNotSupported)
  183. if err != nil && !btfDisabled {
  184. return nil, fmt.Errorf("load BTF: %w", err)
  185. }
  186. if handle != nil {
  187. attr.ProgBtfFd = uint32(handle.FD())
  188. fib, lib, err := btf.MarshalExtInfos(insns, spec.BTF.TypeID)
  189. if err != nil {
  190. return nil, err
  191. }
  192. attr.FuncInfoRecSize = btf.FuncInfoSize
  193. attr.FuncInfoCnt = uint32(len(fib)) / btf.FuncInfoSize
  194. attr.FuncInfo = sys.NewSlicePointer(fib)
  195. attr.LineInfoRecSize = btf.LineInfoSize
  196. attr.LineInfoCnt = uint32(len(lib)) / btf.LineInfoSize
  197. attr.LineInfo = sys.NewSlicePointer(lib)
  198. }
  199. }
  200. if err := fixupAndValidate(insns); err != nil {
  201. return nil, err
  202. }
  203. buf := bytes.NewBuffer(make([]byte, 0, insns.Size()))
  204. err := insns.Marshal(buf, internal.NativeEndian)
  205. if err != nil {
  206. return nil, err
  207. }
  208. bytecode := buf.Bytes()
  209. attr.Insns = sys.NewSlicePointer(bytecode)
  210. attr.InsnCnt = uint32(len(bytecode) / asm.InstructionSize)
  211. if spec.AttachTarget != nil {
  212. targetID, err := findTargetInProgram(spec.AttachTarget, spec.AttachTo, spec.Type, spec.AttachType)
  213. if err != nil {
  214. return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
  215. }
  216. attr.AttachBtfId = uint32(targetID)
  217. attr.AttachProgFd = uint32(spec.AttachTarget.FD())
  218. defer runtime.KeepAlive(spec.AttachTarget)
  219. } else if spec.AttachTo != "" {
  220. targetID, err := findTargetInKernel(kernelTypes, spec.AttachTo, spec.Type, spec.AttachType)
  221. if err != nil && !errors.Is(err, errUnrecognizedAttachType) {
  222. // We ignore errUnrecognizedAttachType since AttachTo may be non-empty
  223. // for programs that don't attach anywhere.
  224. return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
  225. }
  226. attr.AttachBtfId = uint32(targetID)
  227. }
  228. logSize := DefaultVerifierLogSize
  229. if opts.LogSize > 0 {
  230. logSize = opts.LogSize
  231. }
  232. var logBuf []byte
  233. if opts.LogLevel > 0 {
  234. logBuf = make([]byte, logSize)
  235. attr.LogLevel = opts.LogLevel
  236. attr.LogSize = uint32(len(logBuf))
  237. attr.LogBuf = sys.NewSlicePointer(logBuf)
  238. }
  239. fd, err := sys.ProgLoad(attr)
  240. if err == nil {
  241. return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil
  242. }
  243. if opts.LogLevel == 0 && opts.LogSize >= 0 {
  244. // Re-run with the verifier enabled to get better error messages.
  245. logBuf = make([]byte, logSize)
  246. attr.LogLevel = 1
  247. attr.LogSize = uint32(len(logBuf))
  248. attr.LogBuf = sys.NewSlicePointer(logBuf)
  249. _, _ = sys.ProgLoad(attr)
  250. }
  251. switch {
  252. case errors.Is(err, unix.EPERM):
  253. if len(logBuf) > 0 && logBuf[0] == 0 {
  254. // EPERM due to RLIMIT_MEMLOCK happens before the verifier, so we can
  255. // check that the log is empty to reduce false positives.
  256. return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
  257. }
  258. fallthrough
  259. case errors.Is(err, unix.EINVAL):
  260. if hasFunctionReferences(spec.Instructions) {
  261. if err := haveBPFToBPFCalls(); err != nil {
  262. return nil, fmt.Errorf("load program: %w", err)
  263. }
  264. }
  265. }
  266. err = internal.ErrorWithLog(err, logBuf)
  267. if btfDisabled {
  268. return nil, fmt.Errorf("load program: %w (BTF disabled)", err)
  269. }
  270. return nil, fmt.Errorf("load program: %w", err)
  271. }
  272. // NewProgramFromFD creates a program from a raw fd.
  273. //
  274. // You should not use fd after calling this function.
  275. //
  276. // Requires at least Linux 4.10.
  277. func NewProgramFromFD(fd int) (*Program, error) {
  278. f, err := sys.NewFD(fd)
  279. if err != nil {
  280. return nil, err
  281. }
  282. return newProgramFromFD(f)
  283. }
  284. // NewProgramFromID returns the program for a given id.
  285. //
  286. // Returns ErrNotExist, if there is no eBPF program with the given id.
  287. func NewProgramFromID(id ProgramID) (*Program, error) {
  288. fd, err := sys.ProgGetFdById(&sys.ProgGetFdByIdAttr{
  289. Id: uint32(id),
  290. })
  291. if err != nil {
  292. return nil, fmt.Errorf("get program by id: %w", err)
  293. }
  294. return newProgramFromFD(fd)
  295. }
  296. func newProgramFromFD(fd *sys.FD) (*Program, error) {
  297. info, err := newProgramInfoFromFd(fd)
  298. if err != nil {
  299. fd.Close()
  300. return nil, fmt.Errorf("discover program type: %w", err)
  301. }
  302. return &Program{"", fd, "", "", info.Type}, nil
  303. }
  304. func (p *Program) String() string {
  305. if p.name != "" {
  306. return fmt.Sprintf("%s(%s)#%v", p.typ, p.name, p.fd)
  307. }
  308. return fmt.Sprintf("%s(%v)", p.typ, p.fd)
  309. }
  310. // Type returns the underlying type of the program.
  311. func (p *Program) Type() ProgramType {
  312. return p.typ
  313. }
  314. // Info returns metadata about the program.
  315. //
  316. // Requires at least 4.10.
  317. func (p *Program) Info() (*ProgramInfo, error) {
  318. return newProgramInfoFromFd(p.fd)
  319. }
  320. // Handle returns a reference to the program's type information in the kernel.
  321. //
  322. // Returns ErrNotSupported if the kernel has no BTF support, or if there is no
  323. // BTF associated with the program.
  324. func (p *Program) Handle() (*btf.Handle, error) {
  325. info, err := p.Info()
  326. if err != nil {
  327. return nil, err
  328. }
  329. id, ok := info.BTFID()
  330. if !ok {
  331. return nil, fmt.Errorf("program %s: retrieve BTF ID: %w", p, ErrNotSupported)
  332. }
  333. return btf.NewHandleFromID(id)
  334. }
  335. // FD gets the file descriptor of the Program.
  336. //
  337. // It is invalid to call this function after Close has been called.
  338. func (p *Program) FD() int {
  339. return p.fd.Int()
  340. }
  341. // Clone creates a duplicate of the Program.
  342. //
  343. // Closing the duplicate does not affect the original, and vice versa.
  344. //
  345. // Cloning a nil Program returns nil.
  346. func (p *Program) Clone() (*Program, error) {
  347. if p == nil {
  348. return nil, nil
  349. }
  350. dup, err := p.fd.Dup()
  351. if err != nil {
  352. return nil, fmt.Errorf("can't clone program: %w", err)
  353. }
  354. return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil
  355. }
  356. // Pin persists the Program on the BPF virtual file system past the lifetime of
  357. // the process that created it
  358. //
  359. // Calling Pin on a previously pinned program will overwrite the path, except when
  360. // the new path already exists. Re-pinning across filesystems is not supported.
  361. //
  362. // This requires bpffs to be mounted above fileName. See https://docs.cilium.io/en/k8s-doc/admin/#admin-mount-bpffs
  363. func (p *Program) Pin(fileName string) error {
  364. if err := internal.Pin(p.pinnedPath, fileName, p.fd); err != nil {
  365. return err
  366. }
  367. p.pinnedPath = fileName
  368. return nil
  369. }
  370. // Unpin removes the persisted state for the Program from the BPF virtual filesystem.
  371. //
  372. // Failed calls to Unpin will not alter the state returned by IsPinned.
  373. //
  374. // Unpinning an unpinned Program returns nil.
  375. func (p *Program) Unpin() error {
  376. if err := internal.Unpin(p.pinnedPath); err != nil {
  377. return err
  378. }
  379. p.pinnedPath = ""
  380. return nil
  381. }
  382. // IsPinned returns true if the Program has a non-empty pinned path.
  383. func (p *Program) IsPinned() bool {
  384. return p.pinnedPath != ""
  385. }
  386. // Close the Program's underlying file descriptor, which could unload
  387. // the program from the kernel if it is not pinned or attached to a
  388. // kernel hook.
  389. func (p *Program) Close() error {
  390. if p == nil {
  391. return nil
  392. }
  393. return p.fd.Close()
  394. }
  395. // Various options for Run'ing a Program
  396. type RunOptions struct {
  397. // Program's data input. Required field.
  398. Data []byte
  399. // Program's data after Program has run. Caller must allocate. Optional field.
  400. DataOut []byte
  401. // Program's context input. Optional field.
  402. Context interface{}
  403. // Program's context after Program has run. Must be a pointer or slice. Optional field.
  404. ContextOut interface{}
  405. // Number of times to run Program. Optional field. Defaults to 1.
  406. Repeat uint32
  407. // Optional flags.
  408. Flags uint32
  409. // CPU to run Program on. Optional field.
  410. // Note not all program types support this field.
  411. CPU uint32
  412. // Called whenever the syscall is interrupted, and should be set to testing.B.ResetTimer
  413. // or similar. Typically used during benchmarking. Optional field.
  414. Reset func()
  415. }
  416. // Test runs the Program in the kernel with the given input and returns the
  417. // value returned by the eBPF program. outLen may be zero.
  418. //
  419. // Note: the kernel expects at least 14 bytes input for an ethernet header for
  420. // XDP and SKB programs.
  421. //
  422. // This function requires at least Linux 4.12.
  423. func (p *Program) Test(in []byte) (uint32, []byte, error) {
  424. // Older kernels ignore the dataSizeOut argument when copying to user space.
  425. // Combined with things like bpf_xdp_adjust_head() we don't really know what the final
  426. // size will be. Hence we allocate an output buffer which we hope will always be large
  427. // enough, and panic if the kernel wrote past the end of the allocation.
  428. // See https://patchwork.ozlabs.org/cover/1006822/
  429. var out []byte
  430. if len(in) > 0 {
  431. out = make([]byte, len(in)+outputPad)
  432. }
  433. opts := RunOptions{
  434. Data: in,
  435. DataOut: out,
  436. Repeat: 1,
  437. }
  438. ret, _, err := p.testRun(&opts)
  439. if err != nil {
  440. return ret, nil, fmt.Errorf("can't test program: %w", err)
  441. }
  442. return ret, opts.DataOut, nil
  443. }
  444. // Run runs the Program in kernel with given RunOptions.
  445. //
  446. // Note: the same restrictions from Test apply.
  447. func (p *Program) Run(opts *RunOptions) (uint32, error) {
  448. ret, _, err := p.testRun(opts)
  449. if err != nil {
  450. return ret, fmt.Errorf("can't test program: %w", err)
  451. }
  452. return ret, nil
  453. }
  454. // Benchmark runs the Program with the given input for a number of times
  455. // and returns the time taken per iteration.
  456. //
  457. // Returns the result of the last execution of the program and the time per
  458. // run or an error. reset is called whenever the benchmark syscall is
  459. // interrupted, and should be set to testing.B.ResetTimer or similar.
  460. //
  461. // Note: profiling a call to this function will skew it's results, see
  462. // https://github.com/cilium/ebpf/issues/24
  463. //
  464. // This function requires at least Linux 4.12.
  465. func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.Duration, error) {
  466. if uint(repeat) > math.MaxUint32 {
  467. return 0, 0, fmt.Errorf("repeat is too high")
  468. }
  469. opts := RunOptions{
  470. Data: in,
  471. Repeat: uint32(repeat),
  472. Reset: reset,
  473. }
  474. ret, total, err := p.testRun(&opts)
  475. if err != nil {
  476. return ret, total, fmt.Errorf("can't benchmark program: %w", err)
  477. }
  478. return ret, total, nil
  479. }
  480. var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() error {
  481. prog, err := NewProgram(&ProgramSpec{
  482. // SocketFilter does not require privileges on newer kernels.
  483. Type: SocketFilter,
  484. Instructions: asm.Instructions{
  485. asm.LoadImm(asm.R0, 0, asm.DWord),
  486. asm.Return(),
  487. },
  488. License: "MIT",
  489. })
  490. if err != nil {
  491. // This may be because we lack sufficient permissions, etc.
  492. return err
  493. }
  494. defer prog.Close()
  495. // Programs require at least 14 bytes input
  496. in := make([]byte, 14)
  497. attr := sys.ProgRunAttr{
  498. ProgFd: uint32(prog.FD()),
  499. DataSizeIn: uint32(len(in)),
  500. DataIn: sys.NewSlicePointer(in),
  501. }
  502. err = sys.ProgRun(&attr)
  503. switch {
  504. case errors.Is(err, unix.EINVAL):
  505. // Check for EINVAL specifically, rather than err != nil since we
  506. // otherwise misdetect due to insufficient permissions.
  507. return internal.ErrNotSupported
  508. case errors.Is(err, unix.EINTR):
  509. // We know that PROG_TEST_RUN is supported if we get EINTR.
  510. return nil
  511. case errors.Is(err, unix.ENOTSUPP):
  512. // The first PROG_TEST_RUN patches shipped in 4.12 didn't include
  513. // a test runner for SocketFilter. ENOTSUPP means PROG_TEST_RUN is
  514. // supported, but not for the program type used in the probe.
  515. return nil
  516. }
  517. return err
  518. })
  519. func (p *Program) testRun(opts *RunOptions) (uint32, time.Duration, error) {
  520. if uint(len(opts.Data)) > math.MaxUint32 {
  521. return 0, 0, fmt.Errorf("input is too long")
  522. }
  523. if err := haveProgTestRun(); err != nil {
  524. return 0, 0, err
  525. }
  526. var ctxBytes []byte
  527. if opts.Context != nil {
  528. ctx := new(bytes.Buffer)
  529. if err := binary.Write(ctx, internal.NativeEndian, opts.Context); err != nil {
  530. return 0, 0, fmt.Errorf("cannot serialize context: %v", err)
  531. }
  532. ctxBytes = ctx.Bytes()
  533. }
  534. var ctxOut []byte
  535. if opts.ContextOut != nil {
  536. ctxOut = make([]byte, binary.Size(opts.ContextOut))
  537. }
  538. attr := sys.ProgRunAttr{
  539. ProgFd: p.fd.Uint(),
  540. DataSizeIn: uint32(len(opts.Data)),
  541. DataSizeOut: uint32(len(opts.DataOut)),
  542. DataIn: sys.NewSlicePointer(opts.Data),
  543. DataOut: sys.NewSlicePointer(opts.DataOut),
  544. Repeat: uint32(opts.Repeat),
  545. CtxSizeIn: uint32(len(ctxBytes)),
  546. CtxSizeOut: uint32(len(ctxOut)),
  547. CtxIn: sys.NewSlicePointer(ctxBytes),
  548. CtxOut: sys.NewSlicePointer(ctxOut),
  549. Flags: opts.Flags,
  550. Cpu: opts.CPU,
  551. }
  552. for {
  553. err := sys.ProgRun(&attr)
  554. if err == nil {
  555. break
  556. }
  557. if errors.Is(err, unix.EINTR) {
  558. if opts.Reset != nil {
  559. opts.Reset()
  560. }
  561. continue
  562. }
  563. if errors.Is(err, unix.ENOTSUPP) {
  564. return 0, 0, fmt.Errorf("kernel doesn't support testing program type %s: %w", p.Type(), ErrNotSupported)
  565. }
  566. return 0, 0, fmt.Errorf("can't run test: %w", err)
  567. }
  568. if opts.DataOut != nil {
  569. if int(attr.DataSizeOut) > cap(opts.DataOut) {
  570. // Houston, we have a problem. The program created more data than we allocated,
  571. // and the kernel wrote past the end of our buffer.
  572. panic("kernel wrote past end of output buffer")
  573. }
  574. opts.DataOut = opts.DataOut[:int(attr.DataSizeOut)]
  575. }
  576. if len(ctxOut) != 0 {
  577. b := bytes.NewReader(ctxOut)
  578. if err := binary.Read(b, internal.NativeEndian, opts.ContextOut); err != nil {
  579. return 0, 0, fmt.Errorf("failed to decode ContextOut: %v", err)
  580. }
  581. }
  582. total := time.Duration(attr.Duration) * time.Nanosecond
  583. return attr.Retval, total, nil
  584. }
  585. func unmarshalProgram(buf []byte) (*Program, error) {
  586. if len(buf) != 4 {
  587. return nil, errors.New("program id requires 4 byte value")
  588. }
  589. // Looking up an entry in a nested map or prog array returns an id,
  590. // not an fd.
  591. id := internal.NativeEndian.Uint32(buf)
  592. return NewProgramFromID(ProgramID(id))
  593. }
  594. func marshalProgram(p *Program, length int) ([]byte, error) {
  595. if length != 4 {
  596. return nil, fmt.Errorf("can't marshal program to %d bytes", length)
  597. }
  598. buf := make([]byte, 4)
  599. internal.NativeEndian.PutUint32(buf, p.fd.Uint())
  600. return buf, nil
  601. }
  602. // LoadPinnedProgram loads a Program from a BPF file.
  603. //
  604. // Requires at least Linux 4.11.
  605. func LoadPinnedProgram(fileName string, opts *LoadPinOptions) (*Program, error) {
  606. fd, err := sys.ObjGet(&sys.ObjGetAttr{
  607. Pathname: sys.NewStringPointer(fileName),
  608. FileFlags: opts.Marshal(),
  609. })
  610. if err != nil {
  611. return nil, err
  612. }
  613. info, err := newProgramInfoFromFd(fd)
  614. if err != nil {
  615. _ = fd.Close()
  616. return nil, fmt.Errorf("info for %s: %w", fileName, err)
  617. }
  618. return &Program{"", fd, filepath.Base(fileName), fileName, info.Type}, nil
  619. }
  620. // SanitizeName replaces all invalid characters in name with replacement.
  621. // Passing a negative value for replacement will delete characters instead
  622. // of replacing them. Use this to automatically generate valid names for maps
  623. // and programs at runtime.
  624. //
  625. // The set of allowed characters depends on the running kernel version.
  626. // Dots are only allowed as of kernel 5.2.
  627. func SanitizeName(name string, replacement rune) string {
  628. return strings.Map(func(char rune) rune {
  629. if invalidBPFObjNameChar(char) {
  630. return replacement
  631. }
  632. return char
  633. }, name)
  634. }
  635. // ProgramGetNextID returns the ID of the next eBPF program.
  636. //
  637. // Returns ErrNotExist, if there is no next eBPF program.
  638. func ProgramGetNextID(startID ProgramID) (ProgramID, error) {
  639. attr := &sys.ProgGetNextIdAttr{Id: uint32(startID)}
  640. return ProgramID(attr.NextId), sys.ProgGetNextId(attr)
  641. }
  642. // BindMap binds map to the program and is only released once program is released.
  643. //
  644. // This may be used in cases where metadata should be associated with the program
  645. // which otherwise does not contain any references to the map.
  646. func (p *Program) BindMap(m *Map) error {
  647. attr := &sys.ProgBindMapAttr{
  648. ProgFd: uint32(p.FD()),
  649. MapFd: uint32(m.FD()),
  650. }
  651. return sys.ProgBindMap(attr)
  652. }
  653. var errUnrecognizedAttachType = errors.New("unrecognized attach type")
  654. // find an attach target type in the kernel.
  655. //
  656. // spec may be nil and defaults to the canonical kernel BTF. name together with
  657. // progType and attachType determine which type we need to attach to.
  658. //
  659. // Returns errUnrecognizedAttachType.
  660. func findTargetInKernel(spec *btf.Spec, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
  661. type match struct {
  662. p ProgramType
  663. a AttachType
  664. }
  665. var (
  666. typeName, featureName string
  667. isBTFTypeFunc = true
  668. )
  669. switch (match{progType, attachType}) {
  670. case match{LSM, AttachLSMMac}:
  671. typeName = "bpf_lsm_" + name
  672. featureName = name + " LSM hook"
  673. case match{Tracing, AttachTraceIter}:
  674. typeName = "bpf_iter_" + name
  675. featureName = name + " iterator"
  676. case match{Tracing, AttachTraceFEntry}:
  677. typeName = name
  678. featureName = fmt.Sprintf("fentry %s", name)
  679. case match{Tracing, AttachTraceFExit}:
  680. typeName = name
  681. featureName = fmt.Sprintf("fexit %s", name)
  682. case match{Tracing, AttachModifyReturn}:
  683. typeName = name
  684. featureName = fmt.Sprintf("fmod_ret %s", name)
  685. case match{Tracing, AttachTraceRawTp}:
  686. typeName = fmt.Sprintf("btf_trace_%s", name)
  687. featureName = fmt.Sprintf("raw_tp %s", name)
  688. isBTFTypeFunc = false
  689. default:
  690. return 0, errUnrecognizedAttachType
  691. }
  692. spec, err := maybeLoadKernelBTF(spec)
  693. if err != nil {
  694. return 0, fmt.Errorf("load kernel spec: %w", err)
  695. }
  696. var target btf.Type
  697. if isBTFTypeFunc {
  698. var targetFunc *btf.Func
  699. err = spec.TypeByName(typeName, &targetFunc)
  700. target = targetFunc
  701. } else {
  702. var targetTypedef *btf.Typedef
  703. err = spec.TypeByName(typeName, &targetTypedef)
  704. target = targetTypedef
  705. }
  706. if err != nil {
  707. if errors.Is(err, btf.ErrNotFound) {
  708. return 0, &internal.UnsupportedFeatureError{
  709. Name: featureName,
  710. }
  711. }
  712. return 0, fmt.Errorf("find target for %s: %w", featureName, err)
  713. }
  714. return spec.TypeID(target)
  715. }
  716. // find an attach target type in a program.
  717. //
  718. // Returns errUnrecognizedAttachType.
  719. func findTargetInProgram(prog *Program, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
  720. type match struct {
  721. p ProgramType
  722. a AttachType
  723. }
  724. var typeName string
  725. switch (match{progType, attachType}) {
  726. case match{Extension, AttachNone}:
  727. typeName = name
  728. default:
  729. return 0, errUnrecognizedAttachType
  730. }
  731. btfHandle, err := prog.Handle()
  732. if err != nil {
  733. return 0, fmt.Errorf("load target BTF: %w", err)
  734. }
  735. defer btfHandle.Close()
  736. spec, err := btfHandle.Spec(nil)
  737. if err != nil {
  738. return 0, err
  739. }
  740. var targetFunc *btf.Func
  741. err = spec.TypeByName(typeName, &targetFunc)
  742. if err != nil {
  743. return 0, fmt.Errorf("find target %s: %w", typeName, err)
  744. }
  745. return spec.TypeID(targetFunc)
  746. }