From 73316581a056a20e51dc36eb96673acfee7d90b0 Mon Sep 17 00:00:00 2001 From: efim Date: Fri, 23 Aug 2024 12:44:11 +0000 Subject: [PATCH] still not getting data from buffer but the error is in gir::map-c-array length is NIL, not sure what's up with that --- gstreamer/capturing-frames.lisp | 13 +- notes.org | 246 ++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 8 deletions(-) diff --git a/gstreamer/capturing-frames.lisp b/gstreamer/capturing-frames.lisp index 65fb4fe..7dfb1f6 100644 --- a/gstreamer/capturing-frames.lisp +++ b/gstreamer/capturing-frames.lisp @@ -6,9 +6,9 @@ (:use :cl)) (in-package :capturing-frames) -(gir:require-namespace "GLib") + (defvar *gstreamer* (gir:require-namespace "Gst")) -(defvar *gstapp* (gir:require-namespace "GstApp")) +;(defvar *gstapp* (gir:require-namespace "GstApp")) (defun cast-with-videobox (&optional stream-num) (gir:invoke (*gstreamer* 'init) '()) @@ -50,7 +50,6 @@ (gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing)) - ;; Example: Move the video 10 pixels down and 20 pixels to the right (values pipeline appsink))) (defun process-frame (sample) @@ -61,6 +60,7 @@ (caps-struct (gir:invoke (caps 'get-structure) 0)) (width (gir:invoke ((gir:invoke (caps-struct 'get-value) "width") 'get-int))) (height (gir:invoke ((gir:invoke (caps-struct 'get-value) "height") 'get-int)))) + (multiple-value-bind (found map-info) (gir:invoke (buffer 'map) (gir:nget *gstreamer* "MapFlags" :read)) (format t ">>>> the buffer has offset ~A~%" (gir:field buffer 'offset)) (format t ">>>> what's in struct? ~A~%" (gir:invoke (caps-struct 'to-string))) @@ -70,11 +70,8 @@ (let ((data (gir:field map-info 'data)) (size (gir:field map-info 'size))) (format t ">>>> and struct info? ~A~%" map-info) -; (format t ">>>> and data has size ~A and buf ~A~%" size (or data "empty data")) - ))) - ;; Unmap the memory when done - ;; (gir:invoke (info 'unmap) map-info) - )) + (gir:invoke (buffer 'unmap) map-info) + (format t ">>>> and data has size ~A and data is ~A~%" size (or data "empty data"))))))) (defun start-processing (appsink) (gir:connect appsink "new-sample" diff --git a/notes.org b/notes.org index 768846f..2fc55cd 100644 --- a/notes.org +++ b/notes.org @@ -386,3 +386,249 @@ i see CAPTURING-FRAMES> so. there are data and size fields. + +maybe i can try with + +** well, for some reason data is not avaiable? +(let* ((buffer (gir:invoke (sample 'get-buffer))) + (info (gir:invoke (sample 'get-info))) + (caps (gir:invoke (sample 'get-caps))) + (caps-struct (gir:invoke (caps 'get-structure) 0)) + (width (gir:invoke ((gir:invoke (caps-struct 'get-value) "width") 'get-int))) + (height (gir:invoke ((gir:invoke (caps-struct 'get-value) "height") 'get-int))) + (buf-size (gir:invoke (buffer 'get-size))) + (data-copy (gir:invoke (buffer 'extract-dup) 0 buf-size))) + (format t ">>>> at buffer got size ~A and tried to copy ~A~%" buf-size data-copy) + ;; ok, we have buffer, let's try to get copy + + ;; (multiple-value-bind (found map-info) (gir:invoke (buffer 'map) (gir:nget *gstreamer* "MapFlags" :read)) + ;; (format t ">>>> the buffer has offset ~A~%" (gir:field buffer 'offset)) + ;; (format t ">>>> what's in struct? ~A~%" (gir:invoke (caps-struct 'to-string))) + ;; (format t ">>>> the fields are width ~A, height ~A~%" width height) + ;; (format t ">>>> and map result are ~A and ~A~%" found map-info) + ;; (when found + ;; (let ((data (gir:field map-info 'memory)) + ;; (size (gir:field map-info 'size))) + ;; (format t ">>>> and struct info? ~A~%" map-info) + ;; (gir:invoke (buffer 'unmap) map-info) + ;; (format t ">>>> and data has size ~A and memory is ~A~%" size (or data "empty data"))))) + ) + +data-copy is NIL +yep. mda. + +** well, trying to copy as in python example didn't work either + (buf-size (gir:invoke (buffer 'get-size))) + (data-copy (gir:invoke (buffer 'extract-dup) 0 buf-size)) + +https://github.com/jackersson/gst-python-tutorials/blob/master/launch_pipeline/run_appsink.py#L49 + +yeah, it's possible that gir library doesn't work well? even if on c level all is ok? + +* [2024-08-23 Fri] + +** well, problem is in trying to read gir:map-c-array +it gets length nil and fails + +GIR::MAP-C-ARRAY + +so what i'm using: +https://github.com/andy128k/cl-gobject-introspection +and there's wrapper around it +https://github.com/bohonghuang/cl-gobject-introspection-wrapper + +but there is also +https://github.com/kat-co/gir2cl +maybe it will work? +or maybe i'll figure out what's up + +** well, trying to check available slots on that array type +(mapcar #'c2mop:slot-definition-name (c2mop:class-slots (class-of c-array-type))) +=> (PARAM-TYPE FIXED-SIZE ZERO-TERMINATED? LENGTH) + +and how do i get their values? if i don't know name of the :reader ? +( from article on https://riptutorial.com/common-lisp/example/9826/obtain-the-slot-names-of-a-class ) + +but yeah. huh. + +** let's write up the problem? +I'm trying to use gstreamer - a framework for video\audio pipelines with gobject api + +the hope is to use their 'appsink' to get video frame data into the program i'm writing +https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstbuffer.html?gi-language=python#gst_buffer_map + +i'll link the code in case it will be needed for replication, +right now it seems that by doing gir calls i've successfully set up pipeline +and receiving callbacks with the buffers + +the problem i get is when i try to access 'data +and it seems to come from GIR:MAP-C-ARRAY +where length of the parameter c-array-type is NIL + +in #'map-c-array it comes by calling +#+begin_src lisp +(defun c-array-length (c-array-type) + (with-slots (param-type fixed-size zero-terminated? length) + c-array-type + (if fixed-size + (if (and length (< length fixed-size)) + length + fixed-size) + (if length + (+ length (if zero-terminated? 1 0)))))) +#+end_src + +and manually checking slots of the 'c-array-type +it has NIL in length + +right now I don't know whether it's a problem on the side of cl-gobject-introspection +in how it's initializing the c-array-type +or maybe it's somehow problem on the side of the gstreamer library, +that provides bad responses, i'll try to use other languages \ libraries + +but could also really use some help in figuring this out + +fuller log: +#+begin_src +The value + NIL +is not of type + REAL + [Condition of type TYPE-ERROR] + +Restarts: + 0: [ABORT] abort thread (#) + +Backtrace: + 0: (SB-KERNEL:TWO-ARG->= 0 NIL) + 1: (GIR::MAP-C-ARRAY # #.(SB-SYS:INT-SAP #X7F1EC80EBBD8) #) + Locals: + ARRAY = #.(SB-SYS:INT-SAP #X7F1EC80EBBD8) + C-ARRAY-TYPE = # + FUNC = # + I = 0 + LENGTH = NIL + PARAM-SIZE = 1 + POS = #.(SB-SYS:INT-SAP #X7F1EC80EBBD8) + 2: ((:METHOD GIR::MEM-GET (T GIR::POINTER-TYPE)) #.(SB-SYS:INT-SAP #X7F1F97F7FFF8) #) [fast-method] + Locals: + GIR::POS = #.(SB-SYS:INT-SAP #X7F1F97F7FFF8) + TYPE = # + 3: (GIR.FIELD:GET #.(SB-SYS:INT-SAP #X7F1ECC0036F0) #) + 4: (PROCESS-FRAME #) + 5: ((LAMBDA (APPSINK) :IN START-PROCESSING) #) + 6: ((LAMBDA (GIR::CLOSURE RETURN GIR::N-VALUES GIR::PARAMS GIR::HINT GIR::DATA) :IN "/nix/store/hijk9y2lcv24s3a5kv56cyxqcn3frgs4-source-patched/src/signal.lisp") #.(SB-SYS:INT-SAP #X7F1FCC470B30) #.(SB-S.. + 7: ((LAMBDA (SB-ALIEN::ARGS-POINTER SB-ALIEN::RESULT-POINTER FUNCTION) :IN "/nix/store/hijk9y2lcv24s3a5kv56cyxqcn3frgs4-source-patched/src/signal.lisp") # # #<.. + 8: ((FLET SB-UNIX::BODY :IN SB-THREAD::RUN)) + 9: ((FLET "WITHOUT-INTERRUPTS-BODY-" :IN SB-THREAD::RUN)) +10: ((FLET SB-UNIX::BODY :IN SB-THREAD::RUN)) +11: ((FLET "WITHOUT-INTERRUPTS-BODY-" :IN SB-THREAD::RUN)) +12: (SB-THREAD::RUN) +13: (SB-THREAD::ENTER-FOREIGN-CALLBACK 87 69885224809132 69885224809124) +14: ("foreign function: call_into_lisp_") +15: ("foreign function: funcall3") +16: ("foreign function: callback_wrapper_trampoline") +17: ("foreign function: #x501053AB") + --more-- + +#+end_src + +the source: +#+begin_src lisp +;(load (sb-ext:posix-getenv "ASDF")) +(asdf:load-system 'cl-cffi-gtk) +(asdf:load-system 'cl-gobject-introspection) + +(defpackage :capturing-frames + (:use :cl)) + +(in-package :capturing-frames) + +(defvar *gstreamer* (gir:require-namespace "Gst")) +;(defvar *gstapp* (gir:require-namespace "GstApp")) + +(defun cast-with-videobox (&optional stream-num) + (gir:invoke (*gstreamer* 'init) '()) + (let* ((pipeline (gir:invoke (*gstreamer* "Pipeline" 'new) "video-pipeline")) + (source (gir:invoke (*gstreamer* "ElementFactory" 'make) (if stream-num + "pipewiresrc" + "videotestsrc") "source")) + (convert (gir:invoke (*gstreamer* "ElementFactory" 'make) "videoconvert" "convert")) + (tee (gir:invoke (*gstreamer* "ElementFactory" 'make) "tee" "tee")) + (queue1 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue1")) + (queue2 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue2")) + (sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink")) + (appsink (gir:invoke (*gstreamer* "ElementFactory" 'make) "appsink" "appsink")) + ) + (when stream-num + ;; Set pipewiresrc properties, such as the path + (setf (gir:property source 'path) (format nil "~A" stream-num))) + + ;; Add and link elements in the pipeline + (gir:invoke (pipeline 'add) source) + (gir:invoke (pipeline 'add) convert) + (gir:invoke (pipeline 'add) tee) + (gir:invoke (pipeline 'add) queue1) + (gir:invoke (pipeline 'add) queue2) + (gir:invoke (pipeline 'add) sink) + (gir:invoke (pipeline 'add) appsink) + + (gir:invoke (source 'link) convert) + (gir:invoke (convert 'link) tee) + (gir:invoke (tee 'link) queue1) + (gir:invoke (tee 'link) queue2) + (gir:invoke (queue1 'link) sink) + (gir:invoke (queue2 'link) appsink) + + ;; Configure appsink + (setf (gir:property appsink "emit-signals") t + (gir:property appsink "max-buffers") 1 + (gir:property appsink "drop") t) + + (gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing)) + + (values pipeline appsink))) + +(defun process-frame (sample) + (format t ">>> got to process sample with ~A~%" sample) + (let* ((buffer (gir:invoke (sample 'get-buffer))) + (info (gir:invoke (sample 'get-info))) + (caps (gir:invoke (sample 'get-caps))) + (caps-struct (gir:invoke (caps 'get-structure) 0)) + (width (gir:invoke ((gir:invoke (caps-struct 'get-value) "width") 'get-int))) + (height (gir:invoke ((gir:invoke (caps-struct 'get-value) "height") 'get-int)))) + + (multiple-value-bind (found map-info) (gir:invoke (buffer 'map) (gir:nget *gstreamer* "MapFlags" :read)) + (format t ">>>> the buffer has offset ~A~%" (gir:field buffer 'offset)) + (format t ">>>> what's in struct? ~A~%" (gir:invoke (caps-struct 'to-string))) + (format t ">>>> the fields are width ~A, height ~A~%" width height) + (format t ">>>> and map result are ~A and ~A~%" found map-info) + (when found + (let ((data (gir:field map-info 'data)) + (size (gir:field map-info 'size))) + (format t ">>>> and struct info? ~A~%" map-info) + (gir:invoke (buffer 'unmap) map-info) + (format t ">>>> and data has size ~A and data is ~A~%" size (or data "empty data"))))))) + +(defun start-processing (appsink) + (gir:connect appsink "new-sample" + (lambda (appsink) + (format t ">> got printed object ~A~%" appsink) + (format t ">> got printed class ~A~%" (gir:gir-class-of appsink)) + (format t ">> trying to get method on sink ~A~%" (gir:nget appsink 'pull-sample)) + (let ((sample (gir:invoke (appsink 'pull-sample)))) + (when sample + (process-frame sample))) + + + (gir:nget *gstreamer* "FlowReturn" :ok)))) + +(defun try-run () + (multiple-value-bind (pipeline appsink) (cast-with-videobox) + (format t "got appsink ~A" appsink) + (start-processing appsink) + (loop + (sleep 1)))) + +;(try-run) +#+end_src