cleanup and trying to catch frames

This commit is contained in:
efim 2024-08-21 11:39:57 +00:00
parent 6465818d3b
commit acd41dd67e
6 changed files with 351 additions and 3 deletions

View File

@ -0,0 +1,47 @@
;(load (sb-ext:posix-getenv "ASDF"))
(asdf:load-system 'cl-gobject-introspection)
(defpackage :gstreamer-simple-pipeline
(:use :cl)
(:export cast-with-videobox))
(in-package :gstreamer-simple-pipeline)
;; name wildly from
;; https://stackoverflow.com/questions/36296165/python-bindings-for-gstreamer-how-to-import-typelib
(defvar *gstreamer* (gir:require-namespace "Gst"))
;; let's generalize, when stream is not passed, use the testsrc
(defun cast-with-videobox (&optional stream-num)
(gir:invoke (*gstreamer* 'init) '())
(let* ((pipeline (gir:invoke (*gstreamer* "Pipeline" 'new) "video-pipeline"))
(source (gir:invoke (*gstreamer* "ElementFactory" 'make) (if stream-num
"pipewiresrc"
"videotestsrc") "source"))
(convert (gir:invoke (*gstreamer* "ElementFactory" 'make) "videoconvert" "convert"))
(box (gir:invoke (*gstreamer* "ElementFactory" 'make) "videobox" "box"))
(sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink")))
(when stream-num
;; Set pipewiresrc properties, such as the path
(setf (gir:property source 'path) (format nil "~A" stream-num)))
;; Add and link elements in the pipeline
(gir:invoke (pipeline 'add) source)
(gir:invoke (pipeline 'add) convert)
(gir:invoke (pipeline 'add) box)
(gir:invoke (pipeline 'add) sink)
(gir:invoke (source 'link) convert)
(gir:invoke (convert 'link) box)
(gir:invoke (box 'link) sink)
(gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing))
;(format t ">> arhg ~A" (gir:property box 'top))
(defun move-video (top left)
(setf (gir:property box 'top) top
(gir:property box 'left) left))
;; Example: Move the video 10 pixels down and 20 pixels to the right
(move-video 100 200)
pipeline))
;; now this works, but padding is symmetrical

174
cleaned-up/start-share.lisp Normal file
View File

@ -0,0 +1,174 @@
;(load (sb-ext:posix-getenv "ASDF"))
(asdf:load-system 'dbus)
(defpackage
#:screencasting
(:use #:cl)
(:export call-with-all-predefined))
(in-package #:screencasting)
;; https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.ScreenCast.html
(defconstant +screen-cast-interface+ "org.freedesktop.portal.ScreenCast")
(defconstant +request-interface+ "org.freedesktop.portal.Request")
;; some iolib tutorial https://pages.cs.wisc.edu/~psilord/blog/data/iolib-tutorial/tutorial.html
;; i hoped it would help me understand the 'publish-events'
;; i'm not sure how to do interactive calls,
;; since maybe i need to "publish objects"
;; and that means the callbacks are set in stone?
;; i guess i could try to do a Notification call in the callback on screenshot request?
(defun send-notification (bus message)
(dbus:with-introspected-object (notification bus "/org/freedesktop/Notifications" "org.freedesktop.Notifications")
(notification "org.freedesktop.Notifications" "Notify"
"Test" 0 "" "Test" (or message "This is a test; I repeat, this is a test.") '() '() -1)))
;; well, i suppose i would need to open multiple connections?
;; let's do something like this https://github.com/Lautaro-Garcia/cl-notify/blob/main/src/dbus-protocol.lisp#L27
;; (defmacro with-dbus-method-inovaction ((result-var method &rest args) &body body)
;; (alexandria:with-gensyms (bus notifications-object)
;; `(dbus:with-open-bus (,bus (dbus:session-server-addresses))
;; (dbus:with-introspected-object (,notifications-object ,bus "/org/freedesktop/Notifications" "org.freedesktop.Notifications")
;; (let ((,result-var (,notifications-object "org.freedesktop.Notifications" ,method ,@args)))
;; ,@body)))))
;; but. this supposes result-var being a resulting value
;; and i'd want to what? pass in a callback to register?
;; alright, let's have function startup new bus connection for each call
;; but then try to do these multiple calls
(defun call-screencast-method (method-name params)
(handler-case
(dbus:with-open-bus (bus (dbus:session-server-addresses))
(let*
((requester-name (cl-ppcre:regex-replace "\\." (dbus:bus-name bus) "_" :start 1))
(request-name "yayay")
(resp-path (concatenate 'string "/org/freedesktop/portal/desktop/request/"
requester-name
"/"
request-name)))
(dbus:define-dbus-object request-object
(:path resp-path))
(dbus:define-dbus-signal-handler (request-object response) ((id :uint32) (results (:ARRAY (:DICT-ENTRY :STRING :VARIANT))))
(:interface +request-interface+)
(format t "Got response ~S with results ~S~%" id results)
(format t "before sending notification")
(send-notification bus "yayaya from the first response")
(format t "ending signal handler")
(force-output))
(format T "Will try to listen on ~A~%" resp-path)
(format T "Bus connection name ~A~%" (dbus:bus-name bus))
(dbus:add-match bus :type :signal
:interface +request-interface+)
(dbus:with-introspected-object (desktop bus "/org/freedesktop/portal/desktop" "org.freedesktop.portal.Desktop")
(desktop +screen-cast-interface+ method-name ; "CreateSession"
params
;; '(("handle_token" ((:string) "yayay"))
;; ("session_handle_token" ((:string) "hohoyyy")))
))
(dbus:publish-objects bus '(request-object) )))
(end-of-file ()
:disconnected-by-bus)))
'(call-screencast-method "CreateSession"
'(("handle_token" ((:string) "yayay"))
("session_handle_token" ((:string) "hohoyyy"))))
;; let's define ALL beforehand?
(defun call-with-all-predefined ()
(handler-case
(dbus:with-open-bus (bus (dbus:session-server-addresses))
(let*
((requester-name (cl-ppcre:regex-replace "\\." (dbus:bus-name bus) "_" :start 1))
(request-create-session-name "createSessionReq")
(resp-path (concatenate 'string "/org/freedesktop/portal/desktop/request/"
requester-name
"/"
request-create-session-name))
(request-select-sources "selectSourcesReq")
(select-sources-resp-path (concatenate 'string "/org/freedesktop/portal/desktop/request/"
requester-name
"/"
request-select-sources))
(request-start "startReq")
(start-resp-path (concatenate 'string "/org/freedesktop/portal/desktop/request/"
requester-name
"/"
request-start))
(session-handle-hardcoded "yayay")
(session-handle-path (concatenate 'string "/org/freedesktop/portal/desktop/session/"
requester-name
"/"
session-handle-hardcoded)))
(dbus:define-dbus-object request-object
(:path resp-path))
(dbus:define-dbus-signal-handler (request-object response) ((id :uint32) (results (:ARRAY (:DICT-ENTRY :STRING :VARIANT))))
(:interface +request-interface+)
(format t "Got response ~S with results ~S~%" id results)
(send-notification bus "yayaya from the first response")
(format t "About sto send SelectSources with path ~S~%" session-handle-path)
(force-output)
(dbus:with-introspected-object (desktop bus "/org/freedesktop/portal/desktop" "org.freedesktop.portal.Desktop")
(desktop +screen-cast-interface+ "SelectSources"
session-handle-path ; hardcoded session-handle
`(("handle_token" ((:string) ,request-select-sources)))))
(format t "Still first callback, after calling SelectSources~%")
(force-output))
(dbus:define-dbus-object select-sources-request-obj
(:path select-sources-resp-path))
(dbus:define-dbus-signal-handler (select-sources-request-obj response) ((id :uint32) (results (:ARRAY (:DICT-ENTRY :STRING :VARIANT))))
(:interface +request-interface+)
(format t ">> Got inside of SelectSources callback ~A ~A~%" id results)
(dbus:with-introspected-object (desktop bus "/org/freedesktop/portal/desktop" "org.freedesktop.portal.Desktop")
(desktop +screen-cast-interface+ "Start"
session-handle-path
"parent-window"
`(("handle_token" ((:string) , request-start)))))
(format t ">> Still inside SelectSources callback, after calling Start~%")
(force-output))
(dbus:define-dbus-object start-request-obj
(:path start-resp-path))
(dbus:define-dbus-signal-handler (start-request-obj response) ((id :uint32) (results (:ARRAY (:DICT-ENTRY :STRING :VARIANT))))
(:interface +request-interface+)
(format t ">> Got inside of Start callback ~A ~A~%" id results))
(format T "Will try to listen on ~A~%" resp-path)
(format T "Bus connection name ~A~%" (dbus:bus-name bus))
(dbus:add-match bus :type :signal
:interface +request-interface+)
(dbus:with-introspected-object (desktop bus "/org/freedesktop/portal/desktop" "org.freedesktop.portal.Desktop")
(desktop +screen-cast-interface+ "CreateSession"
`(("handle_token" ((:string) ,request-create-session-name))
("session_handle_token" ((:string) ,session-handle-hardcoded)))))
(dbus:publish-objects bus)))
(end-of-file ()
:disconnected-by-bus)))
;; ok
;; >> Got inside of Start callback 0 ((streams
;; ((43
;; ((position (0 0)) (size (1920 1080)))))))
;; interesting. now what do do with the streams?
;; An array of PipeWire streams. Each stream consists of a PipeWire node ID (the first element in the tuple, and a Vardict of properties.
;; ok, now i need to figure out how
;; yeah, nix shell nixpkgs#qpwgraph
;; with this tool i see new 'out' node, maybe i can even already connect to it with
;; some program?
;; yes. enefedov@LLF33A87M:~/Documents/personal/learning-screen-share$ gst-launch-1.0 pipewiresrc path=43 ! videoconvert ! autovideosink
;; cool.
;; now i want to have a window with this video stream created by my program
;; this is new level of "i have no idea where to go"
;; https://github.com/death/dbus/issues/31
;; asked my question about multiple async dbus requests

View File

@ -0,0 +1,77 @@
;(load (sb-ext:posix-getenv "ASDF"))
(asdf:load-system 'cl-cffi-gtk)
(asdf:load-system 'cl-gobject-introspection)
(defpackage :capturing-frames
(:use :cl))
(in-package :capturing-frames)
(gir:require-namespace "GLib")
(defvar *gstreamer* (gir:require-namespace "Gst"))
(defun cast-with-videobox (&optional stream-num)
(gir:invoke (*gstreamer* 'init) '())
(let* ((pipeline (gir:invoke (*gstreamer* "Pipeline" 'new) "video-pipeline"))
(source (gir:invoke (*gstreamer* "ElementFactory" 'make) (if stream-num
"pipewiresrc"
"videotestsrc") "source"))
(convert (gir:invoke (*gstreamer* "ElementFactory" 'make) "videoconvert" "convert"))
(tee (gir:invoke (*gstreamer* "ElementFactory" 'make) "tee" "tee"))
(queue1 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue1"))
(queue2 (gir:invoke (*gstreamer* "ElementFactory" 'make) "queue" "queue2"))
(sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink"))
(appsink (gir:invoke (*gstreamer* "ElementFactory" 'make) "appsink" "appsink")))
(when stream-num
;; Set pipewiresrc properties, such as the path
(setf (gir:property source 'path) (format nil "~A" stream-num)))
;; Add and link elements in the pipeline
(gir:invoke (pipeline 'add) source)
(gir:invoke (pipeline 'add) convert)
(gir:invoke (pipeline 'add) tee)
(gir:invoke (pipeline 'add) queue1)
(gir:invoke (pipeline 'add) queue2)
(gir:invoke (pipeline 'add) sink)
(gir:invoke (pipeline 'add) appsink)
(gir:invoke (source 'link) convert)
(gir:invoke (convert 'link) tee)
(gir:invoke (tee 'link) queue1)
(gir:invoke (tee 'link) queue2)
(gir:invoke (queue1 'link) sink)
(gir:invoke (queue2 'link) appsink)
;; Configure appsink
(setf (gir:property appsink "emit-signals") t
(gir:property appsink "max-buffers") 1
(gir:property appsink "drop") t)
(gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing))
;; Example: Move the video 10 pixels down and 20 pixels to the right
pipeline))
(defun process-frame (sample)
(let* ((buffer (gir:invoke (sample 'get-buffer)))
(info (gir:invoke (buffer 'get-memory) 0))
(map-info (gir:invoke (info 'map) (gir:nget *gstreamer* "MapFlags" :read)))
(data (gir:property map-info 'data))
(size (gir:property map-info 'size)))
;; Process the frame data here
(format t "Received frame of size ~A bytes~%" size)
;; Example: Calculate average brightness
(let ((sum 0))
(dotimes (i size)
(incf sum (aref data i)))
(format t "Average brightness: ~A~%" (/ sum size)))
;; Unmap the memory when done
(gir:invoke (info 'unmap) map-info)))
(defun start-processing (appsink)
(gir:connect appsink "new-sample"
(lambda (appsink)
(let ((sample (gir:invoke (appsink 'pull-sample))))
(when sample
(process-frame sample)
(gir:invoke (sample 'unref))))
(gir:nget *gstreamer* "FlowReturn" :ok))))

View File

@ -11,6 +11,13 @@
(defvar *gtk* (gir:require-namespace "Gtk" "3.0")) (defvar *gtk* (gir:require-namespace "Gtk" "3.0"))
(gir:nget *gtk* "WindowType" :toplevel) ; get enum value (gir:nget *gtk* "WindowType" :toplevel) ; get enum value
(defun huh-window ()
(gir:invoke (*gtk* 'init) nil)
(let ((window (gir:invoke (*gtk* "Window" 'new)
(gir:nget *gtk* "WindowType" :toplevel))))
(gir:invoke (window 'show))
(gir:invoke (*gtk* 'main))))
;; name wildly from ;; name wildly from
;; https://stackoverflow.com/questions/36296165/python-bindings-for-gstreamer-how-to-import-typelib ;; https://stackoverflow.com/questions/36296165/python-bindings-for-gstreamer-how-to-import-typelib
(defvar *gstreamer* (gir:require-namespace "Gst")) (defvar *gstreamer* (gir:require-namespace "Gst"))
@ -68,9 +75,10 @@
(format nil "pipewiresrc path=~A ! videoconvert ! videobox name=move ! autovideosink" stream-num) (format nil "pipewiresrc path=~A ! videoconvert ! videobox name=move ! autovideosink" stream-num)
;"playbin uri=https://gstreamer.freedesktop.org/data/media/sintel_trailer-480p.webm" ;"playbin uri=https://gstreamer.freedesktop.org/data/media/sintel_trailer-480p.webm"
)) ))
(move-element (gir:invoke (pipeline 'get-by-name) "move"))) ;(move-element (gir:invoke (pipeline 'get-by-name) "move"))
)
(gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing)) (gir:invoke (pipeline 'set-state) (gir:nget *gstreamer* "State" :playing))
(values pipeline move-element))) pipeline))
;; well, pipewire is not part of what's added to env ;; well, pipewire is not part of what's added to env
;; not sure if i can add it ;; not sure if i can add it
@ -113,7 +121,7 @@
(sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink"))) (sink (gir:invoke (*gstreamer* "ElementFactory" 'make) "autovideosink" "sink")))
(when stream-num (when stream-num
;; Set pipewiresrc properties, such as the path ;; Set pipewiresrc properties, such as the path
(gir:invoke (source 'set-property) 'path stream-num)) (setf (gir:property source 'path) (format nil "~A" stream-num)))
;; Add and link elements in the pipeline ;; Add and link elements in the pipeline
(gir:invoke (pipeline 'add) source) (gir:invoke (pipeline 'add) source)

View File

@ -21,3 +21,13 @@
;; yay, this works. good. ;; yay, this works. good.
;; this https://www.crategus.com/books/cl-gtk/gtk-tutorial.html#idp3
;; and how could i integrate with out node of pipewire?
(defun start-ffmpeg ()
(uiop:run-program '("ffmpeg" "-f" "pipewire" "-i" "out-node" "-f" "rawvideo" "-pix_fmt" "rgb24" "-")
:output :lines
:error-output :lines
:input :interactive
:wait nil
))

View File

@ -272,3 +272,35 @@ looks like this is about changing elements of the pipeline
and it's possible that parameters of the pipeline elements (pads?) are also static and it's possible that parameters of the pipeline elements (pads?) are also static
** options for transforming frames?
*** videotransform : A combination of scaling, 90-degree-step rotation,
https://github.com/Freescale/gstreamer-imx/blob/master/README.md
horizontal/vertical flipping, and color space conversion
operations. These elements are able to render video overlay
compositions from GstVideoOverlayCompositionMeta data. They also
implement the GstVideoDirection interface and have a "video-direction"
property that handles rotation and flipping. Through this property, it
is possible to configure these elements to auto-rotate images
according to the information in image-orientation tags.
*** glimagesink
https://gstreamer.freedesktop.org/documentation/opengl/glimagesink.html?gi-language=python
gst-launch-1.0 -v videotestsrc ! video/x-raw ! glimagesink
has error
ERROR: from element /GstPipeline:pipeline0/GstGLImageSinkBin:glimagesinkbin0/GstGLImageSink:sink: Failed to create EGLDisplay from native display
Additional debug info:
../ext/gl/gstglimagesink.c(1136): _ensure_gl_setup (): /GstPipeline:pipeline0/GstGLImageSinkBin:glimagesinkbin0/GstGLImageSink:sink
ERROR: pipeline doesn't want to preroll.
so maybe opengl is not available? how to check?
*** gltransformation
https://gstreamer.freedesktop.org/documentation/opengl/gltransformation.html?gi-language=python
here examples seem to be for images, not video
and same error, let's figure out the opengl check
** somewhat categorized things?
* [2024-08-21 Wed]
** visiting sister, returning back
last time i got stuck on video displaying frozen when i tried to add appsink
just re-checking my basic gstreamer pipeline and screencast start - thing works
but i want to clean it up i guess, to allow for simpler retries?