From 5575db018e4486d83a6a1ef7356905591fe9fbb1 Mon Sep 17 00:00:00 2001 From: efim Date: Thu, 22 Aug 2024 15:56:42 +0000 Subject: [PATCH] getting something out of buffer gir:field to get stuff out of structs but getting 'data still doesn't work null in length --- gstreamer/capturing-frames.lisp | 60 ++++++++++++++++-------- notes.org | 82 +++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 19 deletions(-) diff --git a/gstreamer/capturing-frames.lisp b/gstreamer/capturing-frames.lisp index 697572c..65fb4fe 100644 --- a/gstreamer/capturing-frames.lisp +++ b/gstreamer/capturing-frames.lisp @@ -8,6 +8,7 @@ (in-package :capturing-frames) (gir:require-namespace "GLib") (defvar *gstreamer* (gir:require-namespace "Gst")) +(defvar *gstapp* (gir:require-namespace "GstApp")) (defun cast-with-videobox (&optional stream-num) (gir:invoke (*gstreamer* 'init) '()) @@ -20,7 +21,8 @@ (queue1 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue1")) (queue2 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue2")) (sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink")) - (appsink (gir:invoke (*gstreamer* "ElementFactory" 'make) "appsink" "appsink"))) + (appsink (gir:invoke (*gstreamer* "ElementFactory" 'make) "appsink" "appsink")) + ) (when stream-num ;; Set pipewiresrc properties, such as the path (setf (gir:property source 'path) (format nil "~A" stream-num))) @@ -47,31 +49,51 @@ (gir:property appsink "drop") t) (gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing)) - + ;; Example: Move the video 10 pixels down and 20 pixels to the right - pipeline)) + (values pipeline appsink))) (defun process-frame (sample) + (format t ">>> got to process sample with ~A~%" sample) (let* ((buffer (gir:invoke (sample 'get-buffer))) - (info (gir:invoke (buffer 'get-memory) 0)) - (map-info (gir:invoke (info 'map) (gir:nget *gstreamer* "MapFlags" :read))) - (data (gir:property map-info 'data)) - (size (gir:property map-info 'size))) - ;; Process the frame data here - (format t "Received frame of size ~A bytes~%" size) - ;; Example: Calculate average brightness - (let ((sum 0)) - (dotimes (i size) - (incf sum (aref data i))) - (format t "Average brightness: ~A~%" (/ sum size))) - ;; Unmap the memory when done - (gir:invoke (info 'unmap) map-info))) + (info (gir:invoke (sample 'get-info))) + (caps (gir:invoke (sample 'get-caps))) + (caps-struct (gir:invoke (caps 'get-structure) 0)) + (width (gir:invoke ((gir:invoke (caps-struct 'get-value) "width") 'get-int))) + (height (gir:invoke ((gir:invoke (caps-struct 'get-value) "height") 'get-int)))) + (multiple-value-bind (found map-info) (gir:invoke (buffer 'map) (gir:nget *gstreamer* "MapFlags" :read)) + (format t ">>>> the buffer has offset ~A~%" (gir:field buffer 'offset)) + (format t ">>>> what's in struct? ~A~%" (gir:invoke (caps-struct 'to-string))) + (format t ">>>> the fields are width ~A, height ~A~%" width height) + (format t ">>>> and map result are ~A and ~A~%" found map-info) + (when found + (let ((data (gir:field map-info 'data)) + (size (gir:field map-info 'size))) + (format t ">>>> and struct info? ~A~%" map-info) +; (format t ">>>> and data has size ~A and buf ~A~%" size (or data "empty data")) + ))) + ;; Unmap the memory when done + ;; (gir:invoke (info 'unmap) map-info) + )) (defun start-processing (appsink) (gir:connect appsink "new-sample" (lambda (appsink) - (let ((sample (gir:invoke (appsink 'pull-sample)))) + (format t ">> got printed object ~A~%" appsink) + (format t ">> got printed class ~A~%" (gir:gir-class-of appsink)) + (format t ">> trying to get method on sink ~A~%" (gir:nget appsink 'pull-sample)) + (let ((sample (gir:invoke (appsink 'pull-sample)))) (when sample - (process-frame sample) - (gir:invoke (sample 'unref)))) + (process-frame sample))) + + (gir:nget *gstreamer* "FlowReturn" :ok)))) + +(defun try-run () + (multiple-value-bind (pipeline appsink) (cast-with-videobox) + (format t "got appsink ~A" appsink) + (start-processing appsink) + (loop + (sleep 1)))) + +;(try-run) diff --git a/notes.org b/notes.org index 99bf3fe..768846f 100644 --- a/notes.org +++ b/notes.org @@ -304,3 +304,85 @@ last time i got stuck on video displaying frozen when i tried to add appsink just re-checking my basic gstreamer pipeline and screencast start - thing works but i want to clean it up i guess, to allow for simpler retries? +** so, for the appsnk there are signals +https://gstreamer.freedesktop.org/documentation/app/appsink.html?gi-language=python#appsink::new-sample + + new-sample + +#+begin_src python +def new_sample_callback (appsink, udata): + #python callback for the 'new-sample' signal +#+end_src + +Signal that a new sample is available. + +but how do i connect to a callback from common lisp and gobject introspection +https://github.com/andy128k/cl-gobject-introspection +does the gitlab repo has info on callbacks? + +can't seem to call method 'pull-sample' +maybe article will help? +https://gstreamer.freedesktop.org/documentation/application-development/advanced/pipeline-manipulation.html?gi-language=python#grabbing-data-with-appsink + +well, this looks like method on gst namespace? "gst_app_sink_pull_sample()" +https://gstreamer.freedesktop.org/documentation/applib/gstappsink.html?gi-language=python#gst_app_sink_pull_sample + +there's article for python bindings +https://lifestyletransfer.com/how-to-use-gstreamer-appsink-in-python/ +* [2024-08-22 Thu] +** hey, i think this is progress +i am able to call appsink 'pull-sample after all (for some reason, i don't know what i was doing differently) + +new error: +>>> got to process sample with # +but in the code for processing of the sample some type errors + +Backtrace: + 0: ((:METHOD GIR::THIS-OF (T)) T) [fast-method] + 1: (GIR:PROPERTY T DATA) + 2: (PROCESS-FRAME #) + 3: ((LAMBDA (APPSINK) :IN START-PROCESSING) #) + +ok, but when i have struct instance, how do i introspect it? +** let's just go and read documentation for the type that's returned from pull-sample, sure +also there's that python code, right? +doc 1: get sample on appsink https://people.freedesktop.org/~tsaunier/documentation/gst-plugins-base-app-1.0/gstappsink.html?gi-language=c#gst_app_sink_pull_sample +doc 2: GstSample class https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstsample.html?gi-language=c#GstSample + +so on GstSample everything is taken as results of funciton calls? let's try that + +well, maybe i don't need get-info on sample, it returns null, maybe i need it on buffer? +doc 3: get-buffer on sample https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstsample.html?gi-language=python#gst_sample_get_buffer +doc 4: buffer class info https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstbuffer.html?gi-language=python#gst_buffer_peek_memory + +** i'm recommended to look into GstBuffer.map +https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstbuffer.html?gi-language=python#gst_buffer_map + +it returns some https://people.freedesktop.org/~tsaunier/documentation/gstreamer-1.0/gstmemory.html?gi-language=python#GstMapInfo + +** struggling, searching for answers +https://github.com/andy128k/cl-gobject-introspection/issues/73#issuecomment-755316735 +Examples of inspection of a repository can be found in test files. + +https://github.com/andy128k/cl-gobject-introspection/blob/master/test/test1.lisp + +and i mean it's possible that referencing members of a structure is not implemented? + +so, huh +using internal functions +#+begin_src lisp +(data (gir:fields-dict-of (gir:struct-class-of map-info))) +#+end_src +i see +>>>> and data has size NIL and buf ((memory . #) + (flags . #) + (data . #) + (size . #) + (maxsize . #) + (user_data . #) + (_gst_reserved + . #)) +; Evaluation aborted on T +CAPTURING-FRAMES> + +so. there are data and size fields.